freifunk/ffffng
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Refactoring of sorting and filtering in API.

Browse source
This commit is contained in:
baldo 2022-06-23 14:26:15 +02:00
parent 3590ba048d
commit 975d5288a3
6 changed files with 140 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
server/resources/nodeResource.ts
View file

@ -10,7 +10,7 @@ import {forConstraint, forConstraints} from "../validation/validator";
import * as Resources from "../utils/resources";
import {Entity} from "../utils/resources";
import {Request, Response} from "express";
import {EnhancedNode, Node} from "../types";
import {EnhancedNode, isNodeSortField, Node} from "../types";
const nodeFields = ['hostname', 'key', 'email', 'nickname', 'mac', 'coords', 'monitoring'];
@ -130,19 +130,7 @@ async function doGetAll(req: Request): Promise<{ total: number; pageNodes: any }
const sortedNodes = Resources.sort(
filteredNodes,
[
'hostname',
'nickname',
'email',
'token',
'mac',
'key',
'site',
'domain',
'coords',
'onlineState',
'monitoringState'
],
isNodeSortField,
restParams
);
const pageNodes = Resources.getPageEntities(sortedNodes, restParams);

3
server/resources/taskResource.ts
View file

@ -8,6 +8,7 @@ import {getTasks, Task, TaskState} from "../jobs/scheduler";
import {normalizeString} from "../utils/strings";
import {forConstraint} from "../validation/validator";
import {Request, Response} from "express";
import {isTaskSortField} from "../types";
const isValidId = forConstraint(CONSTRAINTS.id, false);
@ -81,7 +82,7 @@ async function doGetAll(req: Request): Promise<{total: number, pageTasks: Entity
const tasks = Resources.sort(
_.values(getTasks()),
['id', 'name', 'schedule', 'state', 'runningSince', 'lastRunStarted'],
isTaskSortField,
restParams
);
const filteredTasks = Resources.filter(

6
server/services/mailService.ts
View file

@ -9,7 +9,7 @@ import Logger from "../logger";
import * as MailTemplateService from "./mailTemplateService";
import * as Resources from "../utils/resources";
import {RestParams} from "../utils/resources";
import {Mail, MailData, MailId, MailType} from "../types";
import {isMailSortField, Mail, MailData, MailId, MailSortField, MailType} from "../types";
const MAIL_QUEUE_DB_BATCH_SIZE = 50;
@ -128,8 +128,8 @@ export async function getPendingMails (restParams: RestParams): Promise<{mails:
const filter = Resources.filterClause(
restParams,
'id',
['id', 'failures', 'sender', 'recipient', 'email', 'created_at', 'modified_at'],
MailSortField.ID,
isMailSortField,
['id', 'failures', 'sender', 'recipient', 'email']
);

46
server/services/monitoringService.ts
View file

@ -16,7 +16,17 @@ import {normalizeMac} from "../utils/strings";
import {monitoringDisableUrl} from "../utils/urlBuilder";
import CONSTRAINTS from "../validation/constraints";
import {forConstraint} from "../validation/validator";
import {MAC, MailType, Node, NodeId, OnlineState, NodeStateData, UnixTimestampSeconds} from "../types";
import {
isMonitoringSortField,
MAC,
MailType,
MonitoringSortField,
Node,
NodeId,
NodeStateData,
OnlineState,
UnixTimestampSeconds
} from "../types";
const MONITORING_STATE_MACS_CHUNK_SIZE = 100;
const NEVER_ONLINE_NODES_DELETION_CHUNK_SIZE = 20;
@ -255,8 +265,7 @@ export function parseNodesJson (body: string): NodesParsingResult {
const parsedNode = parseNode(result.importTimestamp, nodeData);
Logger.tag('monitoring', 'parsing-nodes-json').debug(`Parsing node successful: ${parsedNode.mac}`);
result.nodes.push(parsedNode);
}
catch (error) {
} catch (error) {
result.failedNodesCount += 1;
Logger.tag('monitoring', 'parsing-nodes-json').error("Could not parse node.", error, nodeData);
}
@ -532,21 +541,6 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
}
export async function getAll(restParams: RestParams): Promise<{ total: number, monitoringStates: any[] }> {
const sortFields = [
'id',
'hostname',
'mac',
'site',
'domain',
'monitoring_state',
'state',
'last_seen',
'import_timestamp',
'last_status_mail_type',
'last_status_mail_sent',
'created_at',
'modified_at'
];
const filterFields = [
'hostname',
'mac',
@ -566,8 +560,8 @@ export async function getAll(restParams: RestParams): Promise<{total: number, mo
const filter = Resources.filterClause(
restParams,
'id',
sortFields,
MonitoringSortField.ID,
isMonitoringSortField,
filterFields
);
@ -648,8 +642,7 @@ export async function sendMonitoringMails(): Promise<void> {
try {
await sendOnlineAgainMails(startTime);
}
catch (error) {
} catch (error) {
// only logging an continuing with next type
Logger
.tag('monitoring', 'mail-sending')
@ -659,8 +652,7 @@ export async function sendMonitoringMails(): Promise<void> {
for (let mailNumber = 1; mailNumber <= 3; mailNumber++) {
try {
await sendOfflineMails(startTime, mailNumber);
}
catch (error) {
} catch (error) {
// only logging an continuing with next type
Logger
.tag('monitoring', 'mail-sending')
@ -786,8 +778,7 @@ async function deleteNodeByMac(mac: MAC): Promise<void> {
try {
node = await NodeService.getNodeDataByMac(mac);
}
catch (error) {
} catch (error) {
// Only log error. We try to delete the nodes state anyways.
Logger.tag('nodes', 'delete-offline').error('Could not find node to delete: ' + mac, error);
}
@ -801,8 +792,7 @@ async function deleteNodeByMac(mac: MAC): Promise<void> {
'DELETE FROM node_state WHERE mac = ? AND state = ?',
[mac, 'OFFLINE'],
);
}
catch (error) {
} catch (error) {
// Only log error and continue with next node.
Logger.tag('nodes', 'delete-offline').error('Could not delete node state: ' + mac, error);
}

73
server/types/shared.ts
View file

@ -3,6 +3,9 @@ import {ArrayField, Field, RawJsonField} from "sparkson";
// Types shared with the client.
export type TypeGuard<T> = (arg: unknown) => arg is T;
export type EnumValue<E> = E[keyof E];
export type EnumTypeGuard<E> = TypeGuard<EnumValue<E>>;
export function isObject(arg: unknown): arg is object {
return arg !== null && typeof arg === "object";
}
@ -39,8 +42,8 @@ export function toIsArray<T>(isT: TypeGuard<T>): TypeGuard<T[]> {
return (arg): arg is T[] => isArray(arg, isT);
}
export function toIsEnum<E>(enumDef: E): TypeGuard<E> {
return (arg): arg is E => Object.values(enumDef).includes(arg as [keyof E]);
export function toIsEnum<E>(enumDef: E): EnumTypeGuard<E> {
return (arg): arg is EnumValue<E> => Object.values(enumDef).includes(arg as [keyof E]);
}
export function isOptional<T>(arg: unknown, isT: TypeGuard<T>): arg is (T | undefined) {
@ -333,6 +336,22 @@ export function isEnhancedNode(arg: unknown): arg is EnhancedNode {
);
}
export enum NodeSortField {
HOSTNAME = 'hostname',
NICKNAME = 'nickname',
EMAIL = 'email',
TOKEN = 'token',
MAC = 'mac',
KEY = 'key',
SITE = 'site',
DOMAIN = 'domain',
COORDS = 'coords',
ONLINE_STATE = 'onlineState',
MONITORING_STATE = 'monitoringState',
}
export const isNodeSortField = toIsEnum(NodeSortField);
export interface NodesFilter {
hasKey?: boolean;
hasCoords?: boolean;
@ -365,3 +384,53 @@ export function isNodesFilter(arg: unknown): arg is NodesFilter {
isOptional(filter.onlineState, isOnlineState)
);
}
export enum MonitoringSortField {
ID = 'id',
HOSTNAME = 'hostname',
MAC = 'mac',
SITE = 'site',
DOMAIN = 'domain',
MONITORING_STATE = 'monitoring_state',
STATE = 'state',
LAST_SEEN = 'last_seen',
IMPORT_TIMESTAMP = 'import_timestamp',
LAST_STATUS_MAIL_TYPE = 'last_status_mail_type',
LAST_STATUS_MAIL_SENT = 'last_status_mail_sent',
CREATED_AT = 'created_at',
MODIFIED_AT = 'modified_at',
}
export const isMonitoringSortField = toIsEnum(MonitoringSortField);
export enum TaskSortField {
ID = 'id',
NAME = 'name',
SCHEDULE = 'schedule',
STATE = 'state',
RUNNING_SINCE = 'runningSince',
LAST_RUN_STARTED = 'lastRunStarted',
}
export const isTaskSortField = toIsEnum(TaskSortField);
export enum MailSortField {
ID = 'id',
FAILURES = 'failures',
SENDER = 'sender',
RECIPIENT = 'recipient',
EMAIL = 'email',
CREATED_AT = 'created_at',
MODIFIED_AT = 'modified_at',
}
export const isMailSortField = toIsEnum(MailSortField);
export type GenericSortField = string;
export enum SortDirection {
ASCENDING = "ASC",
DESCENDING = "DESC",
}
export const isSortDirection = toIsEnum(SortDirection);

31
server/utils/resources.ts
View file

@ -5,14 +5,15 @@ import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger";
import {Constraints, forConstraints, isConstraints} from "../validation/validator";
import {Request, Response} from "express";
import {EnumTypeGuard, EnumValue, type GenericSortField, SortDirection, TypeGuard} from "../types";
export type Entity = { [key: string]: any };
export type RestParams = {
q?: string;
_sortField?: string;
_sortDir?: string;
_sortField?: GenericSortField;
_sortDir?: SortDirection;
_page: number;
_perPage: number;
@ -38,18 +39,18 @@ function respond(res: Response, httpCode: number, data: any, type: string): void
}
}
function orderByClause(
function orderByClause<S>(
restParams: RestParams,
defaultSortField: string,
allowedSortFields: string[]
defaultSortField: EnumValue<S>,
isSortField: EnumTypeGuard<S>,
): OrderByClause {
let sortField = _.includes(allowedSortFields, restParams._sortField) ? restParams._sortField : undefined;
let sortField: EnumValue<S> | undefined = isSortField(restParams._sortField) ? restParams._sortField : undefined;
if (!sortField) {
sortField = defaultSortField;
}
return {
query: 'ORDER BY ' + sortField + ' ' + (restParams._sortDir === 'ASC' ? 'ASC' : 'DESC'),
query: 'ORDER BY LOWER(' + sortField + ') ' + (restParams._sortDir === SortDirection.ASCENDING ? 'ASC' : 'DESC'),
params: []
};
}
@ -208,14 +209,14 @@ export function filter<E>(entities: ArrayLike<E>, allowedFilterFields: string[],
});
}
export function sort<T>(entities: ArrayLike<T>, allowedSortFields: string[], restParams: RestParams): ArrayLike<T> {
const sortField = _.includes(allowedSortFields, restParams._sortField) ? restParams._sortField : undefined;
export function sort<T, S>(entities: ArrayLike<T>, isSortField: TypeGuard<S>, restParams: RestParams): ArrayLike<T> {
const sortField: S | undefined = isSortField(restParams._sortField) ? restParams._sortField : undefined;
if (!sortField) {
return entities;
}
const sorted: T[] = _.sortBy(entities, [sortField]);
if (restParams._sortDir === 'ASC') {
if (restParams._sortDir === SortDirection.ASCENDING) {
return sorted;
} else {
return _.reverse(sorted);
@ -231,16 +232,16 @@ export function getPageEntities (entities: ArrayLike<Entity>, restParams: RestPa
export {filterCondition as whereCondition};
export function filterClause (
export function filterClause<S>(
restParams: RestParams,
defaultSortField: string,
allowedSortFields: string[],
defaultSortField: EnumValue<S>,
isSortField: EnumTypeGuard<S>,
filterFields: string[],
): FilterClause {
const orderBy = orderByClause(
const orderBy = orderByClause<S>(
restParams,
defaultSortField,
allowedSortFields
isSortField,
);
const limitOffset = limitOffsetClause(restParams);