freifunk/ffffng
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Major refactoring and fixes.

Browse source 
* Split Node into multiple types and make sure fields are actually set
  when type says so.
* Refactor request handling.
* Start getting rid of moment as a dependency by using
  UnixTimestampSeconds instead.
This commit is contained in:
baldo 2022-07-21 18:39:33 +02:00
parent cfa784dfe2
commit 250353edbf
16 changed files with 676 additions and 455 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
server/config.ts
View file

@ -72,7 +72,7 @@ export function parseCommandLine(): void {
function stripTrailingSlash(url: Url): Url { function stripTrailingSlash(url: Url): Url {
return url.endsWith("/") return url.endsWith("/")
? url.substr(0, url.length - 1) as Url ? url.substring(0, url.length - 1) as Url
: url; : url;
} }

10
server/resources/configResource.ts
View file

@ -1,10 +1,4 @@
import {success} from "../utils/resources"; import {handleJSON} from "../utils/resources";
import {config} from "../config"; import {config} from "../config";
import {Request, Response} from "express";
export function get (req: Request, res: Response): void { export const get = handleJSON(async () => config.client);
success(
res,
config.client
);
}

51
server/resources/mailResource.ts
View file

@ -2,15 +2,20 @@ import CONSTRAINTS from "../validation/constraints";
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
import * as MailService from "../services/mailService"; import * as MailService from "../services/mailService";
import * as Resources from "../utils/resources"; import * as Resources from "../utils/resources";
import {handleJSONWithData, RequestData} from "../utils/resources";
import {normalizeString, parseInteger} from "../utils/strings"; import {normalizeString, parseInteger} from "../utils/strings";
import {forConstraint} from "../validation/validator"; import {forConstraint} from "../validation/validator";
import {Request, Response} from "express"; import {Request, Response} from "express";
import {Mail, MailId} from "../types"; import {isString, Mail, MailId} from "../types";
const isValidId = forConstraint(CONSTRAINTS.id, false); const isValidId = forConstraint(CONSTRAINTS.id, false);
async function withValidMailId(req: Request): Promise<MailId> { async function withValidMailId(data: RequestData): Promise<MailId> {
const id = normalizeString(Resources.getData(req).id); if (!isString(data.id)) {
throw {data: 'Missing mail id.', type: ErrorTypes.badRequest};
}
const id = normalizeString(data.id);
if (!isValidId(id)) { if (!isValidId(id)) {
throw {data: 'Invalid mail id.', type: ErrorTypes.badRequest}; throw {data: 'Invalid mail id.', type: ErrorTypes.badRequest};
@ -19,23 +24,17 @@ async function withValidMailId(req: Request): Promise<MailId> {
return parseInteger(id) as MailId; return parseInteger(id) as MailId;
} }
async function doGet(req: Request): Promise<Mail> { export const get = handleJSONWithData(async data => {
const id = await withValidMailId(req); const id = await withValidMailId(data);
return await MailService.getMail(id); return await MailService.getMail(id);
} });
export function get(req: Request, res: Response): void { async function doGetAll(req: Request): Promise<{ total: number, mails: Mail[] }> {
doGet(req)
.then(mail => Resources.success(res, mail))
.catch(err => Resources.error(res, err))
}
async function doGetAll(req: Request): Promise<{total: number, mails: Mail[]}> {
const restParams = await Resources.getValidRestParams('list', null, req); const restParams = await Resources.getValidRestParams('list', null, req);
return await MailService.getPendingMails(restParams); return await MailService.getPendingMails(restParams);
} }
export function getAll (req: Request, res: Response): void { export function getAll(req: Request, res: Response): void {
doGetAll(req) doGetAll(req)
.then(({total, mails}) => { .then(({total, mails}) => {
res.set('X-Total-Count', total.toString(10)); res.set('X-Total-Count', total.toString(10));
@ -44,24 +43,12 @@ export function getAll (req: Request, res: Response): void {
.catch(err => Resources.error(res, err)) .catch(err => Resources.error(res, err))
} }
async function doRemove(req: Request): Promise<void> { export const remove = handleJSONWithData(async data => {
const id = await withValidMailId(req); const id = await withValidMailId(data);
await MailService.deleteMail(id); await MailService.deleteMail(id);
} });
export function remove (req: Request, res: Response): void { export const resetFailures = handleJSONWithData(async data => {
doRemove(req) const id = await withValidMailId(data);
.then(() => Resources.success(res, {}))
.catch(err => Resources.error(res, err));
}
async function doResetFailures(req: Request): Promise<Mail> {
const id = await withValidMailId(req);
return await MailService.resetFailures(id); return await MailService.resetFailures(id);
} });
export function resetFailures (req: Request, res: Response): void {
doResetFailures(req)
.then(mail => Resources.success(res, mail))
.catch(err => Resources.error(res, err));
}

42
server/resources/monitoringResource.ts
View file

@ -4,14 +4,15 @@ import CONSTRAINTS from "../validation/constraints";
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
import * as MonitoringService from "../services/monitoringService"; import * as MonitoringService from "../services/monitoringService";
import * as Resources from "../utils/resources"; import * as Resources from "../utils/resources";
import {handleJSONWithData} from "../utils/resources";
import {normalizeString} from "../utils/strings"; import {normalizeString} from "../utils/strings";
import {forConstraint} from "../validation/validator"; import {forConstraint} from "../validation/validator";
import {Request, Response} from "express"; import {Request, Response} from "express";
import {MonitoringToken} from "../types"; import {MonitoringResponse, MonitoringToken, toMonitoringResponse} from "../types";
const isValidToken = forConstraint(CONSTRAINTS.token, false); const isValidToken = forConstraint(CONSTRAINTS.token, false);
async function doGetAll(req: Request): Promise<{total: number, result: any}> { async function doGetAll(req: Request): Promise<{ total: number, result: any }> {
const restParams = await Resources.getValidRestParams('list', null, req); const restParams = await Resources.getValidRestParams('list', null, req);
const {monitoringStates, total} = await MonitoringService.getAll(restParams); const {monitoringStates, total} = await MonitoringService.getAll(restParams);
return { return {
@ -32,41 +33,24 @@ export function getAll(req: Request, res: Response): void {
.catch(err => Resources.error(res, err)); .catch(err => Resources.error(res, err));
} }
export function confirm(req: Request, res: Response): void { export const confirm = handleJSONWithData<MonitoringResponse>(async data => {
const data = Resources.getData(req);
const token = normalizeString(data.token); const token = normalizeString(data.token);
if (!isValidToken(token)) { if (!isValidToken(token)) {
return Resources.error(res, {data: 'Invalid token.', type: ErrorTypes.badRequest}); throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
} }
const validatedToken: MonitoringToken = token as MonitoringToken; const validatedToken: MonitoringToken = token as MonitoringToken;
MonitoringService.confirm(validatedToken) const node = await MonitoringService.confirm(validatedToken);
.then(node => Resources.success(res, { return toMonitoringResponse(node);
hostname: node.hostname, });
mac: node.mac,
email: node.email,
monitoring: node.monitoring,
monitoringConfirmed: node.monitoringConfirmed
}))
.catch(err => Resources.error(res, err));
}
export function disable(req: Request, res: Response): void {
const data = Resources.getData(req);
export const disable = handleJSONWithData<MonitoringResponse>(async data => {
const token = normalizeString(data.token); const token = normalizeString(data.token);
if (!isValidToken(token)) { if (!isValidToken(token)) {
return Resources.error(res, {data: 'Invalid token.', type: ErrorTypes.badRequest}); throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
} }
const validatedToken: MonitoringToken = token as MonitoringToken; const validatedToken: MonitoringToken = token as MonitoringToken;
MonitoringService.disable(validatedToken) const node = await MonitoringService.disable(validatedToken);
.then(node => Resources.success(res, { return toMonitoringResponse(node);
hostname: node.hostname, });
mac: node.mac,
email: node.email,
monitoring: node.monitoring
}))
.catch(err => Resources.error(res, err));
}

123
server/resources/nodeResource.ts
View file

@ -1,5 +1,4 @@
import _ from "lodash"; import _ from "lodash";
import deepExtend from "deep-extend";
import Constraints from "../validation/constraints"; import Constraints from "../validation/constraints";
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
@ -8,13 +7,28 @@ import * as NodeService from "../services/nodeService";
import {normalizeMac, normalizeString} from "../utils/strings"; import {normalizeMac, normalizeString} from "../utils/strings";
import {forConstraint, forConstraints} from "../validation/validator"; import {forConstraint, forConstraints} from "../validation/validator";
import * as Resources from "../utils/resources"; import * as Resources from "../utils/resources";
import {handleJSONWithData} from "../utils/resources";
import {Request, Response} from "express"; import {Request, Response} from "express";
import {EnhancedNode, isNodeSortField, MAC, Node, Token} from "../types"; import {
CreateOrUpdateNode,
DomainSpecificNodeResponse,
isNodeSortField,
isToken, JSONObject,
MAC,
NodeResponse,
NodeStateData,
NodeTokenResponse,
StoredNode,
toDomainSpecificNodeResponse,
Token,
toNodeResponse,
toNodeTokenResponse
} from "../types";
const nodeFields = ['hostname', 'key', 'email', 'nickname', 'mac', 'coords', 'monitoring']; const nodeFields = ['hostname', 'key', 'email', 'nickname', 'mac', 'coords', 'monitoring'];
function getNormalizedNodeData(reqData: any): Node { function getNormalizedNodeData(reqData: any): CreateOrUpdateNode {
const node: {[key: string]: any} = {}; const node: { [key: string]: any } = {};
_.each(nodeFields, function (field) { _.each(nodeFields, function (field) {
let value = normalizeString(reqData[field]); let value = normalizeString(reqData[field]);
if (field === 'mac') { if (field === 'mac') {
@ -22,69 +36,54 @@ function getNormalizedNodeData(reqData: any): Node {
} }
node[field] = value; node[field] = value;
}); });
return node as Node; return node as CreateOrUpdateNode;
} }
const isValidNode = forConstraints(Constraints.node, false); const isValidNode = forConstraints(Constraints.node, false);
const isValidToken = forConstraint(Constraints.token, false); const isValidToken = forConstraint(Constraints.token, false);
export function create (req: Request, res: Response): void { function getValidatedToken(data: JSONObject): Token {
const data = Resources.getData(req); if (!isToken(data.token)) {
throw {data: 'Missing token.', type: ErrorTypes.badRequest};
const node = getNormalizedNodeData(data);
if (!isValidNode(node)) {
return Resources.error(res, {data: 'Invalid node data.', type: ErrorTypes.badRequest});
} }
NodeService.createNode(node)
.then(result => Resources.success(res, result))
.catch(err => Resources.error(res, err));
}
export function update (req: Request, res: Response): void {
const data = Resources.getData(req);
const token = normalizeString(data.token); const token = normalizeString(data.token);
if (!isValidToken(token)) { if (!isValidToken(token)) {
return Resources.error(res, {data: 'Invalid token.', type: ErrorTypes.badRequest}); throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
} }
const validatedToken: Token = token as Token; return token as Token;
const node = getNormalizedNodeData(data);
if (!isValidNode(node)) {
return Resources.error(res, {data: 'Invalid node data.', type: ErrorTypes.badRequest});
}
NodeService.updateNode(validatedToken, node)
.then(result => Resources.success(res, result))
.catch(err => Resources.error(res, err));
} }
export function remove(req: Request, res: Response): void { export const create = handleJSONWithData<NodeTokenResponse>(async data => {
const data = Resources.getData(req); const baseNode = getNormalizedNodeData(data);
if (!isValidNode(baseNode)) {
const token = normalizeString(data.token); throw {data: 'Invalid node data.', type: ErrorTypes.badRequest};
if (!isValidToken(token)) {
return Resources.error(res, {data: 'Invalid token.', type: ErrorTypes.badRequest});
} }
const validatedToken: Token = token as Token;
NodeService.deleteNode(validatedToken) const node = await NodeService.createNode(baseNode);
.then(() => Resources.success(res, {})) return toNodeTokenResponse(node);
.catch(err => Resources.error(res, err)); });
}
export function get(req: Request, res: Response): void { export const update = handleJSONWithData<NodeTokenResponse>(async data => {
const token = normalizeString(Resources.getData(req).token); const validatedToken: Token = getValidatedToken(data);
if (!isValidToken(token)) { const baseNode = getNormalizedNodeData(data);
return Resources.error(res, {data: 'Invalid token.', type: ErrorTypes.badRequest}); if (!isValidNode(baseNode)) {
throw {data: 'Invalid node data.', type: ErrorTypes.badRequest};
} }
const validatedToken: Token = token as Token;
NodeService.getNodeDataByToken(validatedToken) const node = await NodeService.updateNode(validatedToken, baseNode);
.then(node => Resources.success(res, node)) return toNodeTokenResponse(node);
.catch(err => Resources.error(res, err)); });
}
export const remove = handleJSONWithData<void>(async data => {
const validatedToken = getValidatedToken(data);
await NodeService.deleteNode(validatedToken);
});
export const get = handleJSONWithData<NodeResponse>(async data => {
const validatedToken: Token = getValidatedToken(data);
const node = await NodeService.getNodeDataByToken(validatedToken);
return toNodeResponse(node);
});
async function doGetAll(req: Request): Promise<{ total: number; pageNodes: any }> { async function doGetAll(req: Request): Promise<{ total: number; pageNodes: any }> {
const restParams = await Resources.getValidRestParams('list', 'node', req); const restParams = await Resources.getValidRestParams('list', 'node', req);
@ -96,24 +95,16 @@ async function doGetAll(req: Request): Promise<{ total: number; pageNodes: any }
!!node.token !!node.token
); );
const macs: MAC[] = _.map(realNodes, (node: Node): MAC => node.mac); const macs: MAC[] = _.map(realNodes, (node: StoredNode): MAC => node.mac);
const nodeStateByMac = await MonitoringService.getByMacs(macs); const nodeStateByMac = await MonitoringService.getByMacs(macs);
const enhancedNodes: EnhancedNode[] = _.map(realNodes, (node: Node): EnhancedNode => { const domainSpecificNodes: DomainSpecificNodeResponse[] = _.map(realNodes, (node: StoredNode): DomainSpecificNodeResponse => {
const nodeState = nodeStateByMac[node.mac]; const nodeState: NodeStateData = nodeStateByMac[node.mac] || {};
if (nodeState) { return toDomainSpecificNodeResponse(node, nodeState);
return deepExtend({}, node, {
site: nodeState.site,
domain: nodeState.domain,
onlineState: nodeState.state
});
}
return node as EnhancedNode;
}); });
const filteredNodes = Resources.filter<EnhancedNode>( const filteredNodes = Resources.filter<DomainSpecificNodeResponse>(
enhancedNodes, domainSpecificNodes,
[ [
'hostname', 'hostname',
'nickname', 'nickname',
@ -142,7 +133,7 @@ async function doGetAll(req: Request): Promise<{ total: number; pageNodes: any }
export function getAll(req: Request, res: Response): void { export function getAll(req: Request, res: Response): void {
doGetAll(req) doGetAll(req)
.then((result: {total: number, pageNodes: any[]}) => { .then((result: { total: number, pageNodes: any[] }) => {
res.set('X-Total-Count', result.total.toString(10)); res.set('X-Total-Count', result.total.toString(10));
return Resources.success(res, result.pageNodes); return Resources.success(res, result.pageNodes);
}) })

27
server/resources/statisticsResource.ts
View file

@ -1,19 +1,16 @@
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger"; import Logger from "../logger";
import {getNodeStatistics} from "../services/nodeService"; import {getNodeStatistics} from "../services/nodeService";
import * as Resources from "../utils/resources"; import {handleJSON} from "../utils/resources";
import {Request, Response} from "express";
export function get (req: Request, res: Response): void { export const get = handleJSON(async () => {
getNodeStatistics() try {
.then(nodeStatistics => Resources.success( const nodeStatistics = await getNodeStatistics();
res, return {
{ nodes: nodeStatistics
nodes: nodeStatistics };
} } catch (error) {
)) Logger.tag('statistics').error('Error getting statistics:', error);
.catch(err => { throw {data: 'Internal error.', type: ErrorTypes.internalError};
Logger.tag('statistics').error('Error getting statistics:', err); }
return Resources.error(res, {data: 'Internal error.', type: ErrorTypes.internalError}); });
});
}

69
server/resources/taskResource.ts
View file

@ -3,16 +3,16 @@ import _ from "lodash";
import CONSTRAINTS from "../validation/constraints"; import CONSTRAINTS from "../validation/constraints";
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
import * as Resources from "../utils/resources"; import * as Resources from "../utils/resources";
import {Entity} from "../utils/resources"; import {Entity, handleJSONWithData, RequestData} from "../utils/resources";
import {getTasks, Task, TaskState} from "../jobs/scheduler"; import {getTasks, Task, TaskState} from "../jobs/scheduler";
import {normalizeString} from "../utils/strings"; import {normalizeString} from "../utils/strings";
import {forConstraint} from "../validation/validator"; import {forConstraint} from "../validation/validator";
import {Request, Response} from "express"; import {Request, Response} from "express";
import {isTaskSortField} from "../types"; import {isString, isTaskSortField} from "../types";
const isValidId = forConstraint(CONSTRAINTS.id, false); const isValidId = forConstraint(CONSTRAINTS.id, false);
interface ExternalTask { interface TaskResponse {
id: number, id: number,
name: string, name: string,
description: string, description: string,
@ -26,7 +26,7 @@ interface ExternalTask {
enabled: boolean, enabled: boolean,
} }
function toExternalTask(task: Task): ExternalTask { function toTaskResponse(task: Task): TaskResponse {
return { return {
id: task.id, id: task.id,
name: task.name, name: task.name,
@ -37,13 +37,16 @@ function toExternalTask(task: Task): ExternalTask {
lastRunDuration: task.lastRunDuration || null, lastRunDuration: task.lastRunDuration || null,
state: task.state, state: task.state,
result: task.state !== TaskState.RUNNING && task.result ? task.result.state : null, result: task.state !== TaskState.RUNNING && task.result ? task.result.state : null,
message:task.state !== TaskState.RUNNING && task.result ? task.result.message || null : null, message: task.state !== TaskState.RUNNING && task.result ? task.result.message || null : null,
enabled: task.enabled enabled: task.enabled
}; };
} }
async function withValidTaskId(req: Request): Promise<string> { async function withValidTaskId(data: RequestData): Promise<string> {
const id = normalizeString(Resources.getData(req).id); if (!isString(data.id)) {
throw {data: 'Missing task id.', type: ErrorTypes.badRequest};
}
const id = normalizeString(data.id);
if (!isValidId(id)) { if (!isValidId(id)) {
throw {data: 'Invalid task id.', type: ErrorTypes.badRequest}; throw {data: 'Invalid task id.', type: ErrorTypes.badRequest};
@ -63,21 +66,18 @@ async function getTask(id: string): Promise<Task> {
return task; return task;
} }
async function withTask(req: Request): Promise<Task> { async function withTask(data: RequestData): Promise<Task> {
const id = await withValidTaskId(req); const id = await withValidTaskId(data);
return await getTask(id); return await getTask(id);
} }
function setTaskEnabled(req: Request, res: Response, enable: boolean) { async function setTaskEnabled(data: RequestData, enable: boolean): Promise<TaskResponse> {
withTask(req) const task = await withTask(data);
.then(task => { task.enabled = enable;
task.enabled = enable; return toTaskResponse(task);
Resources.success(res, toExternalTask(task))
})
.catch(err => Resources.error(res, err))
} }
async function doGetAll(req: Request): Promise<{total: number, pageTasks: Entity[]}> { async function doGetAll(req: Request): Promise<{ total: number, pageTasks: Entity[] }> {
const restParams = await Resources.getValidRestParams('list', null, req); const restParams = await Resources.getValidRestParams('list', null, req);
const tasks = Resources.sort( const tasks = Resources.sort(
@ -100,33 +100,30 @@ async function doGetAll(req: Request): Promise<{total: number, pageTasks: Entity
}; };
} }
export function getAll (req: Request, res: Response): void { export function getAll(req: Request, res: Response): void {
doGetAll(req) doGetAll(req)
.then(({total, pageTasks}) => { .then(({total, pageTasks}) => {
res.set('X-Total-Count', total.toString(10)); res.set('X-Total-Count', total.toString(10));
Resources.success(res, _.map(pageTasks, toExternalTask)); Resources.success(res, _.map(pageTasks, toTaskResponse));
}) })
.catch(err => Resources.error(res, err)); .catch(err => Resources.error(res, err));
} }
export function run (req: Request, res: Response): void { export const run = handleJSONWithData(async data => {
withTask(req) const task = await withTask(data);
.then(task => {
if (task.runningSince) {
return Resources.error(res, {data: 'Task already running.', type: ErrorTypes.conflict});
}
task.run(); if (task.runningSince) {
throw {data: 'Task already running.', type: ErrorTypes.conflict};
}
Resources.success(res, toExternalTask(task)); task.run();
}) return toTaskResponse(task);
.catch(err => Resources.error(res, err)); });
}
export function enable (req: Request, res: Response): void { export const enable = handleJSONWithData(async data => {
setTaskEnabled(req, res, true); await setTaskEnabled(data, true);
} });
export function disable (req: Request, res: Response): void { export const disable = handleJSONWithData(async data => {
setTaskEnabled(req, res, false); await setTaskEnabled(data, false);
} });

14
server/resources/versionResource.ts
View file

@ -1,12 +1,6 @@
import {success} from "../utils/resources"; import {handleJSON} from "../utils/resources";
import {version} from "../config"; import {version} from "../config";
import {Request, Response} from "express";
export function get (req: Request, res: Response): void { export const get = handleJSON(async () => ({
success( version
res, }));
{
version
}
);
}

93
server/services/monitoringService.test.ts
View file

@ -1,8 +1,9 @@
import moment from 'moment'; import moment from 'moment';
import {ParsedNode, parseNode, parseNodesJson, parseTimestamp} from "./monitoringService"; import {ParsedNode, parseNode, parseNodesJson} from "./monitoringService";
import {Domain, MAC, OnlineState, Site} from "../types"; import {Domain, MAC, OnlineState, Site, UnixTimestampSeconds} from "../types";
import Logger from '../logger'; import Logger from '../logger';
import {MockLogger} from "../__mocks__/logger"; import {MockLogger} from "../__mocks__/logger";
import {now, parseTimestamp} from "../utils/time";
const mockedLogger = Logger as MockLogger; const mockedLogger = Logger as MockLogger;
@ -15,60 +16,19 @@ const NODES_JSON_VALID_VERSION = 2;
const TIMESTAMP_INVALID_STRING = "2020-01-02T42:99:23.000Z"; const TIMESTAMP_INVALID_STRING = "2020-01-02T42:99:23.000Z";
const TIMESTAMP_VALID_STRING = "2020-01-02T12:34:56.000Z"; const TIMESTAMP_VALID_STRING = "2020-01-02T12:34:56.000Z";
const PARSED_TIMESTAMP_VALID = parseTimestamp(TIMESTAMP_VALID_STRING);
if (PARSED_TIMESTAMP_VALID === null) {
fail("Should not happen: Parsed valid timestamp as invalid.");
}
const TIMESTAMP_VALID: UnixTimestampSeconds = PARSED_TIMESTAMP_VALID;
beforeEach(() => { beforeEach(() => {
mockedLogger.reset(); mockedLogger.reset();
}); });
test('parseTimestamp() should fail parsing non-string timestamp', () => {
// given
const timestamp = {};
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp.isValid()).toBe(false);
});
test('parseTimestamp() should fail parsing empty timestamp string', () => {
// given
const timestamp = "";
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp.isValid()).toBe(false);
});
test('parseTimestamp() should fail parsing invalid timestamp string', () => {
// given
// noinspection UnnecessaryLocalVariableJS
const timestamp = TIMESTAMP_INVALID_STRING;
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp.isValid()).toBe(false);
});
test('parseTimestamp() should succeed parsing valid timestamp string', () => {
// given
const timestamp = TIMESTAMP_VALID_STRING;
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp.isValid()).toBe(true);
expect(parsedTimestamp.toISOString()).toEqual(timestamp);
});
test('parseNode() should fail parsing node for undefined node data', () => { test('parseNode() should fail parsing node for undefined node data', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = undefined; const nodeData = undefined;
// then // then
@ -77,7 +37,7 @@ test('parseNode() should fail parsing node for undefined node data', () => {
test('parseNode() should fail parsing node for empty node data', () => { test('parseNode() should fail parsing node for empty node data', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = {}; const nodeData = {};
// then // then
@ -86,7 +46,7 @@ test('parseNode() should fail parsing node for empty node data', () => {
test('parseNode() should fail parsing node for empty node info', () => { test('parseNode() should fail parsing node for empty node info', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: {} nodeinfo: {}
}; };
@ -97,7 +57,7 @@ test('parseNode() should fail parsing node for empty node info', () => {
test('parseNode() should fail parsing node for non-string node id', () => { test('parseNode() should fail parsing node for non-string node id', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: 42 node_id: 42
@ -110,7 +70,7 @@ test('parseNode() should fail parsing node for non-string node id', () => {
test('parseNode() should fail parsing node for empty node id', () => { test('parseNode() should fail parsing node for empty node id', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "" node_id: ""
@ -123,7 +83,7 @@ test('parseNode() should fail parsing node for empty node id', () => {
test('parseNode() should fail parsing node for empty network info', () => { test('parseNode() should fail parsing node for empty network info', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -137,7 +97,7 @@ test('parseNode() should fail parsing node for empty network info', () => {
test('parseNode() should fail parsing node for invalid mac', () => { test('parseNode() should fail parsing node for invalid mac', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -153,7 +113,7 @@ test('parseNode() should fail parsing node for invalid mac', () => {
test('parseNode() should fail parsing node for missing flags', () => { test('parseNode() should fail parsing node for missing flags', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -169,7 +129,7 @@ test('parseNode() should fail parsing node for missing flags', () => {
test('parseNode() should fail parsing node for empty flags', () => { test('parseNode() should fail parsing node for empty flags', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -186,7 +146,7 @@ test('parseNode() should fail parsing node for empty flags', () => {
test('parseNode() should fail parsing node for missing last seen timestamp', () => { test('parseNode() should fail parsing node for missing last seen timestamp', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -205,7 +165,7 @@ test('parseNode() should fail parsing node for missing last seen timestamp', ()
test('parseNode() should fail parsing node for invalid last seen timestamp', () => { test('parseNode() should fail parsing node for invalid last seen timestamp', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -225,7 +185,7 @@ test('parseNode() should fail parsing node for invalid last seen timestamp', ()
test('parseNode() should succeed parsing node without site and domain', () => { test('parseNode() should succeed parsing node without site and domain', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -244,7 +204,7 @@ test('parseNode() should succeed parsing node without site and domain', () => {
mac: "12:34:56:78:90:AB" as MAC, mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: importTimestamp, importTimestamp: importTimestamp,
state: OnlineState.ONLINE, state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING), lastSeen: TIMESTAMP_VALID,
site: "<unknown-site>" as Site, site: "<unknown-site>" as Site,
domain: "<unknown-domain>" as Domain, domain: "<unknown-domain>" as Domain,
}; };
@ -253,7 +213,7 @@ test('parseNode() should succeed parsing node without site and domain', () => {
test('parseNode() should succeed parsing node with site and domain', () => { test('parseNode() should succeed parsing node with site and domain', () => {
// given // given
const importTimestamp = moment(); const importTimestamp = now();
const nodeData = { const nodeData = {
nodeinfo: { nodeinfo: {
node_id: "1234567890ab", node_id: "1234567890ab",
@ -276,7 +236,7 @@ test('parseNode() should succeed parsing node with site and domain', () => {
mac: "12:34:56:78:90:AB" as MAC, mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: importTimestamp, importTimestamp: importTimestamp,
state: OnlineState.ONLINE, state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING), lastSeen: TIMESTAMP_VALID,
site: "test-site" as Site, site: "test-site" as Site,
domain: "test-domain" as Domain, domain: "test-domain" as Domain,
}; };
@ -388,7 +348,6 @@ test('parseNodesJson() should succeed parsing no nodes', () => {
const result = parseNodesJson(json); const result = parseNodesJson(json);
// then // then
expect(result.importTimestamp.isValid()).toBe(true);
expect(result.nodes).toEqual([]); expect(result.nodes).toEqual([]);
expect(result.failedNodesCount).toEqual(0); expect(result.failedNodesCount).toEqual(0);
expect(result.totalNodesCount).toEqual(0); expect(result.totalNodesCount).toEqual(0);
@ -424,7 +383,6 @@ test('parseNodesJson() should skip parsing invalid nodes', () => {
const result = parseNodesJson(json); const result = parseNodesJson(json);
// then // then
expect(result.importTimestamp.isValid()).toBe(true);
expect(result.nodes).toEqual([]); expect(result.nodes).toEqual([]);
expect(result.failedNodesCount).toEqual(2); expect(result.failedNodesCount).toEqual(2);
expect(result.totalNodesCount).toEqual(2); expect(result.totalNodesCount).toEqual(2);
@ -463,14 +421,13 @@ test('parseNodesJson() should parse valid nodes', () => {
// then // then
const expectedParsedNode: ParsedNode = { const expectedParsedNode: ParsedNode = {
mac: "12:34:56:78:90:AB" as MAC, mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: parseTimestamp(TIMESTAMP_VALID_STRING), importTimestamp: TIMESTAMP_VALID,
state: OnlineState.ONLINE, state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING), lastSeen: TIMESTAMP_VALID,
site: "test-site" as Site, site: "test-site" as Site,
domain: "test-domain" as Domain, domain: "test-domain" as Domain,
}; };
expect(result.importTimestamp.isValid()).toBe(true);
expect(result.nodes).toEqual([expectedParsedNode]); expect(result.nodes).toEqual([expectedParsedNode]);
expect(result.failedNodesCount).toEqual(1); expect(result.failedNodesCount).toEqual(1);
expect(result.totalNodesCount).toEqual(2); expect(result.totalNodesCount).toEqual(2);

172
server/services/monitoringService.ts
View file

@ -1,5 +1,4 @@
import _ from "lodash"; import _ from "lodash";
import moment, {Moment, unitOfTime} from "moment";
import request from "request"; import request from "request";
import {config} from "../config"; import {config} from "../config";
@ -18,21 +17,25 @@ import CONSTRAINTS from "../validation/constraints";
import {forConstraint} from "../validation/validator"; import {forConstraint} from "../validation/validator";
import { import {
Domain, Domain,
DurationSeconds,
Hostname, Hostname,
isMonitoringSortField, isMonitoringSortField,
isOnlineState, isOnlineState,
MAC, MAC,
MailType, MailType,
MonitoringSortField, MonitoringSortField,
MonitoringState,
MonitoringToken, MonitoringToken,
Node,
NodeId, NodeId,
NodeStateData, NodeStateData,
OnlineState, OnlineState,
RunResult, RunResult,
Site, Site,
StoredNode,
toCreateOrUpdateNode,
UnixTimestampSeconds UnixTimestampSeconds
} from "../types"; } from "../types";
import {days, formatTimestamp, hours, now, parseTimestamp, subtract, weeks} from "../utils/time";
type NodeStateRow = { type NodeStateRow = {
id: number, id: number,
@ -56,27 +59,24 @@ const MONITORING_MAILS_DB_BATCH_SIZE = 50;
/** /**
* Defines the intervals emails are sent if a node is offline * Defines the intervals emails are sent if a node is offline
*/ */
const MONITORING_OFFLINE_MAILS_SCHEDULE: { [key: number]: { amount: number, unit: unitOfTime.DurationConstructor } } = { const MONITORING_OFFLINE_MAILS_SCHEDULE: Record<number, DurationSeconds> = {
1: {amount: 3, unit: 'hours'}, 1: hours(3),
2: {amount: 1, unit: 'days'}, 2: days(1),
3: {amount: 7, unit: 'days'} 3: weeks(1),
};
const DELETE_OFFLINE_NODES_AFTER_DURATION: { amount: number, unit: unitOfTime.DurationConstructor } = {
amount: 100,
unit: 'days'
}; };
const DELETE_OFFLINE_NODES_AFTER_DURATION: DurationSeconds = days(100);
export type ParsedNode = { export type ParsedNode = {
mac: MAC, mac: MAC,
importTimestamp: Moment, importTimestamp: UnixTimestampSeconds,
state: OnlineState, state: OnlineState,
lastSeen: Moment, lastSeen: UnixTimestampSeconds,
site: Site, site: Site,
domain: Domain, domain: Domain,
}; };
export type NodesParsingResult = { export type NodesParsingResult = {
importTimestamp: Moment, importTimestamp: UnixTimestampSeconds,
nodes: ParsedNode[], nodes: ParsedNode[],
failedNodesCount: number, failedNodesCount: number,
totalNodesCount: number, totalNodesCount: number,
@ -87,9 +87,9 @@ export type RetrieveNodeInformationResult = {
totalNodesCount: number, totalNodesCount: number,
}; };
let previousImportTimestamp: Moment | null = null; let previousImportTimestamp: UnixTimestampSeconds | null = null;
async function insertNodeInformation(nodeData: ParsedNode, node: Node): Promise<void> { async function insertNodeInformation(nodeData: ParsedNode, node: StoredNode): Promise<void> {
Logger Logger
.tag('monitoring', 'information-retrieval') .tag('monitoring', 'information-retrieval')
.debug('Node is new in monitoring, creating data: %s', nodeData.mac); .debug('Node is new in monitoring, creating data: %s', nodeData.mac);
@ -105,20 +105,20 @@ async function insertNodeInformation(nodeData: ParsedNode, node: Node): Promise<
nodeData.domain, nodeData.domain,
node.monitoringState, node.monitoringState,
nodeData.state, nodeData.state,
nodeData.lastSeen.unix(), nodeData.lastSeen,
nodeData.importTimestamp.unix(), nodeData.importTimestamp,
null, // new node so we haven't send a mail yet null, // new node so we haven't send a mail yet
null // new node so we haven't send a mail yet null // new node so we haven't send a mail yet
] ]
); );
} }
async function updateNodeInformation(nodeData: ParsedNode, node: Node, row: any): Promise<void> { async function updateNodeInformation(nodeData: ParsedNode, node: StoredNode, row: any): Promise<void> {
Logger Logger
.tag('monitoring', 'informacallbacktion-retrieval') .tag('monitoring', 'informacallbacktion-retrieval')
.debug('Node is known in monitoring: %s', nodeData.mac); .debug('Node is known in monitoring: %s', nodeData.mac);
if (!moment(row.import_timestamp).isBefore(nodeData.importTimestamp)) { if (row.import_timestamp >= nodeData.importTimestamp) {
Logger Logger
.tag('monitoring', 'information-retrieval') .tag('monitoring', 'information-retrieval')
.debug('No new data for node, skipping: %s', nodeData.mac); .debug('No new data for node, skipping: %s', nodeData.mac);
@ -147,9 +147,9 @@ async function updateNodeInformation(nodeData: ParsedNode, node: Node, row: any)
nodeData.domain || row.domain, nodeData.domain || row.domain,
node.monitoringState, node.monitoringState,
nodeData.state, nodeData.state,
nodeData.lastSeen.unix(), nodeData.lastSeen,
nodeData.importTimestamp.unix(), nodeData.importTimestamp,
moment().unix(), now(),
row.id, row.id,
node.mac node.mac
@ -157,7 +157,7 @@ async function updateNodeInformation(nodeData: ParsedNode, node: Node, row: any)
); );
} }
async function storeNodeInformation(nodeData: ParsedNode, node: Node): Promise<void> { async function storeNodeInformation(nodeData: ParsedNode, node: StoredNode): Promise<void> {
Logger.tag('monitoring', 'information-retrieval').debug('Storing status for node: %s', nodeData.mac); Logger.tag('monitoring', 'information-retrieval').debug('Storing status for node: %s', nodeData.mac);
const row = await db.get('SELECT * FROM node_state WHERE mac = ?', [node.mac]); const row = await db.get('SELECT * FROM node_state WHERE mac = ?', [node.mac]);
@ -171,15 +171,8 @@ async function storeNodeInformation(nodeData: ParsedNode, node: Node): Promise<v
const isValidMac = forConstraint(CONSTRAINTS.node.mac, false); const isValidMac = forConstraint(CONSTRAINTS.node.mac, false);
export function parseTimestamp(timestamp: any): Moment {
if (!_.isString(timestamp)) {
return moment.invalid();
}
return moment.utc(timestamp);
}
// TODO: Use sparkson for JSON parsing. // TODO: Use sparkson for JSON parsing.
export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode { export function parseNode(importTimestamp: UnixTimestampSeconds, nodeData: any): ParsedNode {
if (!_.isPlainObject(nodeData)) { if (!_.isPlainObject(nodeData)) {
throw new Error( throw new Error(
'Unexpected node type: ' + (typeof nodeData) 'Unexpected node type: ' + (typeof nodeData)
@ -225,7 +218,7 @@ export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode {
const isOnline = nodeData.flags.online; const isOnline = nodeData.flags.online;
const lastSeen = parseTimestamp(nodeData.lastseen); const lastSeen = parseTimestamp(nodeData.lastseen);
if (!lastSeen.isValid()) { if (lastSeen === null) {
throw new Error( throw new Error(
'Node ' + nodeId + ': Invalid lastseen timestamp: ' + nodeData.lastseen 'Node ' + nodeId + ': Invalid lastseen timestamp: ' + nodeData.lastseen
); );
@ -245,7 +238,7 @@ export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode {
mac, mac,
importTimestamp: importTimestamp, importTimestamp: importTimestamp,
state: isOnline ? OnlineState.ONLINE : OnlineState.OFFLINE, state: isOnline ? OnlineState.ONLINE : OnlineState.OFFLINE,
lastSeen: lastSeen, lastSeen,
site, site,
domain, domain,
}; };
@ -266,17 +259,18 @@ export function parseNodesJson(body: string): NodesParsingResult {
throw new Error(`Unexpected nodes.json version "${json.version}". Expected: "${expectedVersion}"`); throw new Error(`Unexpected nodes.json version "${json.version}". Expected: "${expectedVersion}"`);
} }
const importTimestamp = parseTimestamp(json.timestamp);
if (importTimestamp === null) {
throw new Error('Invalid timestamp: ' + json.timestamp);
}
const result: NodesParsingResult = { const result: NodesParsingResult = {
importTimestamp: parseTimestamp(json.timestamp), importTimestamp,
nodes: [], nodes: [],
failedNodesCount: 0, failedNodesCount: 0,
totalNodesCount: 0, totalNodesCount: 0,
}; };
if (!result.importTimestamp.isValid()) {
throw new Error('Invalid timestamp: ' + json.timestamp);
}
if (!_.isArray(json.nodes)) { if (!_.isArray(json.nodes)) {
throw new Error('Invalid nodes array type: ' + (typeof json.nodes)); throw new Error('Invalid nodes array type: ' + (typeof json.nodes));
} }
@ -296,13 +290,13 @@ export function parseNodesJson(body: string): NodesParsingResult {
return result; return result;
} }
async function updateSkippedNode(id: NodeId, node?: Node): Promise<RunResult> { async function updateSkippedNode(id: NodeId, node?: StoredNode): Promise<RunResult> {
return await db.run( return await db.run(
'UPDATE node_state ' + 'UPDATE node_state ' +
'SET hostname = ?, monitoring_state = ?, modified_at = ?' + 'SET hostname = ?, monitoring_state = ?, modified_at = ?' +
'WHERE id = ?', 'WHERE id = ?',
[ [
node ? node.hostname : '', node ? node.monitoringState : '', moment().unix(), node ? node.hostname : '', node ? node.monitoringState : '', now(),
id id
] ]
); );
@ -328,7 +322,7 @@ async function sendMonitoringMailsBatched(
const mac = nodeState.mac; const mac = nodeState.mac;
Logger.tag('monitoring', 'mail-sending').debug('Loading node data for: %s', mac); Logger.tag('monitoring', 'mail-sending').debug('Loading node data for: %s', mac);
const result = await NodeService.getNodeDataWithSecretsByMac(mac); const result = await NodeService.findNodeDataWithSecretsByMac(mac);
if (!result) { if (!result) {
Logger Logger
.tag('monitoring', 'mail-sending') .tag('monitoring', 'mail-sending')
@ -341,11 +335,11 @@ async function sendMonitoringMailsBatched(
const {node, nodeSecrets} = result; const {node, nodeSecrets} = result;
if (!(node.monitoring && node.monitoringConfirmed)) { if (node.monitoringState !== MonitoringState.ACTIVE) {
Logger Logger
.tag('monitoring', 'mail-sending') .tag('monitoring', 'mail-sending')
.debug('Monitoring disabled, skipping "%s" mail for: %s', name, mac); .debug('Monitoring disabled, skipping "%s" mail for: %s', name, mac);
await updateSkippedNode(nodeState.id); await updateSkippedNode(nodeState.id, node);
continue; continue;
} }
@ -354,7 +348,7 @@ async function sendMonitoringMailsBatched(
Logger Logger
.tag('monitoring', 'mail-sending') .tag('monitoring', 'mail-sending')
.error('Node has no monitoring token. Cannot send mail "%s" for: %s', name, mac); .error('Node has no monitoring token. Cannot send mail "%s" for: %s', name, mac);
await updateSkippedNode(nodeState.id); await updateSkippedNode(nodeState.id, node);
continue; continue;
} }
@ -377,13 +371,13 @@ async function sendMonitoringMailsBatched(
.tag('monitoring', 'mail-sending') .tag('monitoring', 'mail-sending')
.debug('Updating node state: ', mac); .debug('Updating node state: ', mac);
const now = moment().unix(); const timestamp = now();
await db.run( await db.run(
'UPDATE node_state ' + 'UPDATE node_state ' +
'SET hostname = ?, monitoring_state = ?, modified_at = ?, last_status_mail_sent = ?, last_status_mail_type = ?' + 'SET hostname = ?, monitoring_state = ?, modified_at = ?, last_status_mail_sent = ?, last_status_mail_type = ?' +
'WHERE id = ?', 'WHERE id = ?',
[ [
node.hostname, node.monitoringState, now, now, mailType, node.hostname, node.monitoringState, timestamp, timestamp, mailType,
nodeState.id nodeState.id
] ]
); );
@ -391,7 +385,7 @@ async function sendMonitoringMailsBatched(
} }
} }
async function sendOnlineAgainMails(startTime: Moment): Promise<void> { async function sendOnlineAgainMails(startTime: UnixTimestampSeconds): Promise<void> {
await sendMonitoringMailsBatched( await sendMonitoringMailsBatched(
'online again', 'online again',
MailType.MONITORING_ONLINE_AGAIN, MailType.MONITORING_ONLINE_AGAIN,
@ -402,7 +396,7 @@ async function sendOnlineAgainMails(startTime: Moment): Promise<void> {
')' + ')' +
'ORDER BY id ASC LIMIT ?', 'ORDER BY id ASC LIMIT ?',
[ [
startTime.unix(), startTime,
'ONLINE', 'ONLINE',
MONITORING_MAILS_DB_BATCH_SIZE MONITORING_MAILS_DB_BATCH_SIZE
@ -411,7 +405,7 @@ async function sendOnlineAgainMails(startTime: Moment): Promise<void> {
); );
} }
async function sendOfflineMails(startTime: Moment, mailType: MailType): Promise<void> { async function sendOfflineMails(startTime: UnixTimestampSeconds, mailType: MailType): Promise<void> {
const mailNumber = parseInteger(mailType.split("-")[2]); const mailNumber = parseInteger(mailType.split("-")[2]);
await sendMonitoringMailsBatched( await sendMonitoringMailsBatched(
'offline ' + mailNumber, 'offline ' + mailNumber,
@ -424,7 +418,7 @@ async function sendOfflineMails(startTime: Moment, mailType: MailType): Promise<
const allowNull = mailNumber === 1 ? ' OR last_status_mail_type IS NULL' : ''; const allowNull = mailNumber === 1 ? ' OR last_status_mail_type IS NULL' : '';
const schedule = MONITORING_OFFLINE_MAILS_SCHEDULE[mailNumber]; const schedule = MONITORING_OFFLINE_MAILS_SCHEDULE[mailNumber];
const scheduledTimeBefore = moment().subtract(schedule.amount, schedule.unit); const scheduledTimeBefore = subtract(now(), schedule);
return await db.all( return await db.all(
'SELECT * FROM node_state ' + 'SELECT * FROM node_state ' +
@ -432,11 +426,11 @@ async function sendOfflineMails(startTime: Moment, mailType: MailType): Promise<
'last_seen <= ? AND (last_status_mail_sent <= ? OR last_status_mail_sent IS NULL) ' + 'last_seen <= ? AND (last_status_mail_sent <= ? OR last_status_mail_sent IS NULL) ' +
'ORDER BY id ASC LIMIT ?', 'ORDER BY id ASC LIMIT ?',
[ [
startTime.unix(), startTime,
'OFFLINE', 'OFFLINE',
previousType, previousType,
scheduledTimeBefore.unix(), scheduledTimeBefore,
scheduledTimeBefore.unix(), scheduledTimeBefore,
MONITORING_MAILS_DB_BATCH_SIZE MONITORING_MAILS_DB_BATCH_SIZE
], ],
@ -487,10 +481,10 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
let totalNodesCount = 0; let totalNodesCount = 0;
for (const data of datas) { for (const data of datas) {
if (data.importTimestamp.isAfter(maxTimestamp)) { if (data.importTimestamp >= maxTimestamp) {
maxTimestamp = data.importTimestamp; maxTimestamp = data.importTimestamp;
} }
if (data.importTimestamp.isBefore(minTimestamp)) { if (data.importTimestamp <= minTimestamp) {
minTimestamp = data.importTimestamp; minTimestamp = data.importTimestamp;
} }
@ -498,13 +492,13 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
totalNodesCount += data.totalNodesCount; totalNodesCount += data.totalNodesCount;
} }
if (previousImportTimestamp !== null && !maxTimestamp.isAfter(previousImportTimestamp)) { if (previousImportTimestamp !== null && maxTimestamp >= previousImportTimestamp) {
Logger Logger
.tag('monitoring', 'information-retrieval') .tag('monitoring', 'information-retrieval')
.debug( .debug(
'No new data, skipping. Current timestamp: %s, previous timestamp: %s', 'No new data, skipping. Current timestamp: %s, previous timestamp: %s',
maxTimestamp.format(), formatTimestamp(maxTimestamp),
previousImportTimestamp.format() formatTimestamp(previousImportTimestamp)
); );
return { return {
failedParsingNodesCount, failedParsingNodesCount,
@ -518,7 +512,7 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
const allNodes = _.flatMap(datas, data => data.nodes); const allNodes = _.flatMap(datas, data => data.nodes);
// Get rid of duplicates from different nodes.json files. Always use the one with the newest // Get rid of duplicates from different nodes.json files. Always use the one with the newest
const sortedNodes = _.orderBy(allNodes, [node => node.lastSeen.unix()], ['desc']); const sortedNodes = _.orderBy(allNodes, [node => node.lastSeen], ['desc']);
const uniqueNodes = _.uniqBy(sortedNodes, function (node) { const uniqueNodes = _.uniqBy(sortedNodes, function (node) {
return node.mac; return node.mac;
}); });
@ -526,7 +520,7 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
for (const nodeData of uniqueNodes) { for (const nodeData of uniqueNodes) {
Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac); Logger.tag('monitoring', 'information-retrieval').debug('Importing: %s', nodeData.mac);
const result = await NodeService.getNodeDataByMac(nodeData.mac); const result = await NodeService.findNodeDataByMac(nodeData.mac);
if (!result) { if (!result) {
Logger Logger
.tag('monitoring', 'information-retrieval') .tag('monitoring', 'information-retrieval')
@ -551,8 +545,8 @@ async function retrieveNodeInformationForUrls(urls: string[]): Promise<RetrieveN
'SET state = ?, modified_at = ?' + 'SET state = ?, modified_at = ?' +
'WHERE import_timestamp < ?', 'WHERE import_timestamp < ?',
[ [
OnlineState.OFFLINE, moment().unix(), OnlineState.OFFLINE, now(),
minTimestamp.unix() minTimestamp
] ]
); );
@ -627,34 +621,40 @@ export async function getByMacs(macs: MAC[]): Promise<Record<MAC, NodeStateData>
return nodeStateByMac; return nodeStateByMac;
} }
export async function confirm(token: MonitoringToken): Promise<Node> { export async function confirm(token: MonitoringToken): Promise<StoredNode> {
const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token); const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
if (!node.monitoring || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) { if (node.monitoringState === MonitoringState.DISABLED || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) {
throw {data: 'Invalid token.', type: ErrorTypes.badRequest}; throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
} }
if (node.monitoringConfirmed) { if (node.monitoringState === MonitoringState.ACTIVE) {
return node; return node;
} }
node.monitoringConfirmed = true; node.monitoringState = MonitoringState.ACTIVE;
return await NodeService.internalUpdateNode(
const {node: newNode} = await NodeService.internalUpdateNode(node.token, node, nodeSecrets); node.token,
return newNode; toCreateOrUpdateNode(node),
node.monitoringState,
nodeSecrets
);
} }
export async function disable(token: MonitoringToken): Promise<Node> { export async function disable(token: MonitoringToken): Promise<StoredNode> {
const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token); const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
if (!node.monitoring || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) { if (node.monitoringState === MonitoringState.DISABLED || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) {
throw {data: 'Invalid token.', type: ErrorTypes.badRequest}; throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
} }
node.monitoring = false; node.monitoringState = MonitoringState.DISABLED;
node.monitoringConfirmed = false;
nodeSecrets.monitoringToken = undefined; nodeSecrets.monitoringToken = undefined;
const {node: newNode} = await NodeService.internalUpdateNode(node.token, node, nodeSecrets); return await NodeService.internalUpdateNode(
return newNode; node.token,
toCreateOrUpdateNode(node),
node.monitoringState,
nodeSecrets
);
} }
export async function retrieveNodeInformation(): Promise<RetrieveNodeInformationResult> { export async function retrieveNodeInformation(): Promise<RetrieveNodeInformationResult> {
@ -669,7 +669,7 @@ export async function retrieveNodeInformation(): Promise<RetrieveNodeInformation
export async function sendMonitoringMails(): Promise<void> { export async function sendMonitoringMails(): Promise<void> {
Logger.tag('monitoring', 'mail-sending').debug('Sending monitoring mails...'); Logger.tag('monitoring', 'mail-sending').debug('Sending monitoring mails...');
const startTime = moment(); const startTime = now();
try { try {
await sendOnlineAgainMails(startTime); await sendOnlineAgainMails(startTime);
@ -696,24 +696,18 @@ export async function sendMonitoringMails(): Promise<void> {
} }
} }
function toUnixTimestamp(moment: Moment): UnixTimestampSeconds {
return moment.unix() as UnixTimestampSeconds;
}
export async function deleteOfflineNodes(): Promise<void> { export async function deleteOfflineNodes(): Promise<void> {
Logger Logger
.tag('nodes', 'delete-offline') .tag('nodes', 'delete-offline')
.info( .info(
'Deleting offline nodes older than ' + `Deleting offline nodes older than ${DELETE_OFFLINE_NODES_AFTER_DURATION} seconds.`
DELETE_OFFLINE_NODES_AFTER_DURATION.amount + ' ' +
DELETE_OFFLINE_NODES_AFTER_DURATION.unit
); );
const deleteBefore = const deleteBefore =
toUnixTimestamp(moment().subtract( subtract(
DELETE_OFFLINE_NODES_AFTER_DURATION.amount, now(),
DELETE_OFFLINE_NODES_AFTER_DURATION.unit DELETE_OFFLINE_NODES_AFTER_DURATION,
)); );
await deleteNeverOnlineNodesBefore(deleteBefore); await deleteNeverOnlineNodesBefore(deleteBefore);
await deleteNodesOfflineSinceBefore(deleteBefore); await deleteNodesOfflineSinceBefore(deleteBefore);
@ -727,7 +721,7 @@ async function deleteNeverOnlineNodesBefore(deleteBefore: UnixTimestampSeconds):
deleteBefore deleteBefore
); );
const deletionCandidates: Node[] = await NodeService.findNodesModifiedBefore(deleteBefore); const deletionCandidates: StoredNode[] = await NodeService.findNodesModifiedBefore(deleteBefore);
Logger Logger
.tag('nodes', 'delete-never-online') .tag('nodes', 'delete-never-online')
@ -816,7 +810,7 @@ async function deleteNodeByMac(mac: MAC): Promise<void> {
let node; let node;
try { try {
node = await NodeService.getNodeDataByMac(mac); node = await NodeService.findNodeDataByMac(mac);
} catch (error) { } catch (error) {
// Only log error. We try to delete the nodes state anyways. // Only log error. We try to delete the nodes state anyways.
Logger.tag('nodes', 'delete-offline').error('Could not find node to delete: ' + mac, error); Logger.tag('nodes', 'delete-offline').error('Could not find node to delete: ' + mac, error);

167
server/services/nodeService.ts
View file

@ -7,22 +7,26 @@ import glob from "glob";
import {config} from "../config"; import {config} from "../config";
import ErrorTypes from "../utils/errorTypes"; import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger"; import Logger from "../logger";
import logger from "../logger";
import * as MailService from "../services/mailService"; import * as MailService from "../services/mailService";
import {normalizeString} from "../utils/strings"; import {normalizeString} from "../utils/strings";
import {monitoringConfirmUrl, monitoringDisableUrl} from "../utils/urlBuilder"; import {monitoringConfirmUrl, monitoringDisableUrl} from "../utils/urlBuilder";
import { import {
BaseNode,
Coordinates, Coordinates,
CreateOrUpdateNode,
EmailAddress, EmailAddress,
FastdKey, FastdKey,
Hostname, Hostname,
isStoredNode,
MAC, MAC,
MailType, MailType,
MonitoringState, MonitoringState,
MonitoringToken, MonitoringToken,
Nickname, Nickname,
Node,
NodeSecrets, NodeSecrets,
NodeStatistics, NodeStatistics,
StoredNode,
Token, Token,
toUnixTimestampSeconds, toUnixTimestampSeconds,
unhandledEnumField, unhandledEnumField,
@ -117,7 +121,7 @@ function parseNodeFilename(filename: string): NodeFilenameParsed {
return parsed; return parsed;
} }
function isDuplicate(filter: NodeFilter, token: Token | null): boolean { function isDuplicate(filter: NodeFilter, token?: Token): boolean {
const files = findNodeFilesSync(filter); const files = findNodeFilesSync(filter);
if (files.length === 0) { if (files.length === 0) {
return false; return false;
@ -130,7 +134,7 @@ function isDuplicate(filter: NodeFilter, token: Token | null): boolean {
return parseNodeFilename(files[0]).token !== token; return parseNodeFilename(files[0]).token !== token;
} }
function checkNoDuplicates(token: Token | null, node: Node, nodeSecrets: NodeSecrets): void { function checkNoDuplicates(token: Token | undefined, node: BaseNode, nodeSecrets: NodeSecrets): void {
if (isDuplicate({hostname: node.hostname}, token)) { if (isDuplicate({hostname: node.hostname}, token)) {
throw {data: {msg: 'Already exists.', field: 'hostname'}, type: ErrorTypes.conflict}; throw {data: {msg: 'Already exists.', field: 'hostname'}, type: ErrorTypes.conflict};
} }
@ -150,7 +154,7 @@ function checkNoDuplicates(token: Token | null, node: Node, nodeSecrets: NodeSec
} }
} }
function toNodeFilename(token: Token, node: Node, nodeSecrets: NodeSecrets): string { function toNodeFilename(token: Token, node: BaseNode, nodeSecrets: NodeSecrets): string {
return config.server.peersPath + '/' + return config.server.peersPath + '/' +
( (
(node.hostname || '') + '@' + (node.hostname || '') + '@' +
@ -161,7 +165,13 @@ function toNodeFilename(token: Token, node: Node, nodeSecrets: NodeSecrets): str
).toLowerCase(); ).toLowerCase();
} }
function getNodeValue(prefix: LINE_PREFIX, node: Node, nodeSecrets: NodeSecrets): string { function getNodeValue(
prefix: LINE_PREFIX,
token: Token,
node: CreateOrUpdateNode,
monitoringState: MonitoringState,
nodeSecrets: NodeSecrets
): string {
switch (prefix) { switch (prefix) {
case LINE_PREFIX.HOSTNAME: case LINE_PREFIX.HOSTNAME:
return node.hostname; return node.hostname;
@ -174,11 +184,11 @@ function getNodeValue(prefix: LINE_PREFIX, node: Node, nodeSecrets: NodeSecrets)
case LINE_PREFIX.MAC: case LINE_PREFIX.MAC:
return node.mac; return node.mac;
case LINE_PREFIX.TOKEN: case LINE_PREFIX.TOKEN:
return node.token; return token;
case LINE_PREFIX.MONITORING: case LINE_PREFIX.MONITORING:
if (node.monitoring && node.monitoringConfirmed) { if (node.monitoring && monitoringState === MonitoringState.ACTIVE) {
return "aktiv"; return "aktiv";
} else if (node.monitoring && !node.monitoringConfirmed) { } else if (node.monitoring && monitoringState === MonitoringState.PENDING) {
return "pending"; return "pending";
} }
return ""; return "";
@ -192,22 +202,22 @@ function getNodeValue(prefix: LINE_PREFIX, node: Node, nodeSecrets: NodeSecrets)
async function writeNodeFile( async function writeNodeFile(
isUpdate: boolean, isUpdate: boolean,
token: Token, token: Token,
node: Node, node: CreateOrUpdateNode,
monitoringState: MonitoringState,
nodeSecrets: NodeSecrets, nodeSecrets: NodeSecrets,
): Promise<{ token: Token, node: Node }> { ): Promise<StoredNode> {
const filename = toNodeFilename(token, node, nodeSecrets); const filename = toNodeFilename(token, node, nodeSecrets);
let data = ''; let data = '';
for (const prefix of Object.values(LINE_PREFIX)) { for (const prefix of Object.values(LINE_PREFIX)) {
data += `${prefix}${getNodeValue(prefix, node, nodeSecrets)}\n`; data += `${prefix}${getNodeValue(prefix, token, node, monitoringState, nodeSecrets)}\n`;
} }
if (node.key) { if (node.key) {
data += `key "${node.key}";\n`; data += `key "${node.key}";\n`;
} }
// since node.js is single threaded we don't need a lock // since node.js is single threaded we don't need a lock when working with synchronous operations
if (isUpdate) { if (isUpdate) {
const files = findNodeFilesSync({token: token}); const files = findNodeFilesSync({token: token});
if (files.length !== 1) { if (files.length !== 1) {
@ -224,12 +234,13 @@ async function writeNodeFile(
throw {data: 'Could not remove old node data.', type: ErrorTypes.internalError}; throw {data: 'Could not remove old node data.', type: ErrorTypes.internalError};
} }
} else { } else {
checkNoDuplicates(null, node, nodeSecrets); checkNoDuplicates(undefined, node, nodeSecrets);
} }
try { try {
oldFs.writeFileSync(filename, data, 'utf8'); oldFs.writeFileSync(filename, data, 'utf8');
return {token, node}; const {node: storedNode} = await parseNodeFile(filename);
return storedNode;
} catch (error) { } catch (error) {
Logger.tag('node', 'save').error('Could not write node file: ' + filename, error); Logger.tag('node', 'save').error('Could not write node file: ' + filename, error);
throw {data: 'Could not write node data.', type: ErrorTypes.internalError}; throw {data: 'Could not write node data.', type: ErrorTypes.internalError};
@ -257,7 +268,7 @@ async function deleteNodeFile(token: Token): Promise<void> {
} }
} }
class NodeBuilder { class StoredNodeBuilder {
public token: Token = "" as Token; // FIXME: Either make token optional in Node or handle this! public token: Token = "" as Token; // FIXME: Either make token optional in Node or handle this!
public nickname: Nickname = "" as Nickname; public nickname: Nickname = "" as Nickname;
public email: EmailAddress = "" as EmailAddress; public email: EmailAddress = "" as EmailAddress;
@ -265,8 +276,6 @@ class NodeBuilder {
public coords?: Coordinates; public coords?: Coordinates;
public key?: FastdKey; public key?: FastdKey;
public mac: MAC = "" as MAC; // FIXME: Either make mac optional in Node or handle this! public mac: MAC = "" as MAC; // FIXME: Either make mac optional in Node or handle this!
public monitoring: boolean = false;
public monitoringConfirmed: boolean = false;
public monitoringState: MonitoringState = MonitoringState.DISABLED; public monitoringState: MonitoringState = MonitoringState.DISABLED;
constructor( constructor(
@ -274,8 +283,8 @@ class NodeBuilder {
) { ) {
} }
public build(): Node { public build(): StoredNode {
return { const node = {
token: this.token, token: this.token,
nickname: this.nickname, nickname: this.nickname,
email: this.email, email: this.email,
@ -283,15 +292,20 @@ class NodeBuilder {
coords: this.coords, coords: this.coords,
key: this.key, key: this.key,
mac: this.mac, mac: this.mac,
monitoring: this.monitoring,
monitoringConfirmed: this.monitoringConfirmed,
monitoringState: this.monitoringState, monitoringState: this.monitoringState,
modifiedAt: this.modifiedAt, modifiedAt: this.modifiedAt,
};
if (!isStoredNode(node)) {
logger.tag("NodeService").error("Not a valid StoredNode:", node);
throw {data: "Could not build StoredNode.", type: ErrorTypes.internalError};
} }
return node;
} }
} }
function setNodeValue(prefix: LINE_PREFIX, node: NodeBuilder, nodeSecrets: NodeSecrets, value: string) { function setNodeValue(prefix: LINE_PREFIX, node: StoredNodeBuilder, nodeSecrets: NodeSecrets, value: string) {
switch (prefix) { switch (prefix) {
case LINE_PREFIX.HOSTNAME: case LINE_PREFIX.HOSTNAME:
node.hostname = value as Hostname; node.hostname = value as Hostname;
@ -314,8 +328,6 @@ function setNodeValue(prefix: LINE_PREFIX, node: NodeBuilder, nodeSecrets: NodeS
case LINE_PREFIX.MONITORING: case LINE_PREFIX.MONITORING:
const active = value === 'aktiv'; const active = value === 'aktiv';
const pending = value === 'pending'; const pending = value === 'pending';
node.monitoring = active || pending;
node.monitoringConfirmed = active;
node.monitoringState = node.monitoringState =
active ? MonitoringState.ACTIVE : (pending ? MonitoringState.PENDING : MonitoringState.DISABLED); active ? MonitoringState.ACTIVE : (pending ? MonitoringState.PENDING : MonitoringState.DISABLED);
break; break;
@ -332,13 +344,13 @@ async function getModifiedAt(file: string): Promise<UnixTimestampSeconds> {
return toUnixTimestampSeconds(modifiedAtMs); return toUnixTimestampSeconds(modifiedAtMs);
} }
async function parseNodeFile(file: string): Promise<{ node: Node, nodeSecrets: NodeSecrets }> { async function parseNodeFile(file: string): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets }> {
const contents = await fs.readFile(file); const contents = await fs.readFile(file);
const modifiedAt = await getModifiedAt(file); const modifiedAt = await getModifiedAt(file);
const lines = contents.toString().split("\n"); const lines = contents.toString().split("\n");
const node = new NodeBuilder(modifiedAt); const node = new StoredNodeBuilder(modifiedAt);
const nodeSecrets: NodeSecrets = {}; const nodeSecrets: NodeSecrets = {};
for (const line of lines) { for (const line of lines) {
@ -361,7 +373,7 @@ async function parseNodeFile(file: string): Promise<{ node: Node, nodeSecrets: N
}; };
} }
async function findNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: Node, nodeSecrets: NodeSecrets } | null> { async function findNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets } | null> {
const files = await findNodeFiles(filter); const files = await findNodeFiles(filter);
if (files.length !== 1) { if (files.length !== 1) {
@ -372,7 +384,7 @@ async function findNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: No
return await parseNodeFile(file); return await parseNodeFile(file);
} }
async function getNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: Node, nodeSecrets: NodeSecrets }> { async function getNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets }> {
const result = await findNodeDataByFilePattern(filter); const result = await findNodeDataByFilePattern(filter);
if (!result) { if (!result) {
throw {data: 'Node not found.', type: ErrorTypes.notFound}; throw {data: 'Node not found.', type: ErrorTypes.notFound};
@ -381,7 +393,7 @@ async function getNodeDataByFilePattern(filter: NodeFilter): Promise<{ node: Nod
return result; return result;
} }
async function sendMonitoringConfirmationMail(node: Node, nodeSecrets: NodeSecrets): Promise<void> { async function sendMonitoringConfirmationMail(node: StoredNode, nodeSecrets: NodeSecrets): Promise<void> {
const monitoringToken = nodeSecrets.monitoringToken; const monitoringToken = nodeSecrets.monitoringToken;
if (!monitoringToken) { if (!monitoringToken) {
Logger Logger
@ -405,76 +417,81 @@ async function sendMonitoringConfirmationMail(node: Node, nodeSecrets: NodeSecre
); );
} }
export async function createNode(node: Node): Promise<{ token: Token, node: Node }> { export async function createNode(node: CreateOrUpdateNode): Promise<StoredNode> {
const token: Token = generateToken(); const token: Token = generateToken();
const nodeSecrets: NodeSecrets = {}; const nodeSecrets: NodeSecrets = {};
node.monitoringConfirmed = false; const monitoringState = node.monitoring ? MonitoringState.PENDING : MonitoringState.DISABLED;
if (node.monitoring) { if (node.monitoring) {
nodeSecrets.monitoringToken = generateToken<MonitoringToken>(); nodeSecrets.monitoringToken = generateToken<MonitoringToken>();
} }
const written = await writeNodeFile(false, token, node, nodeSecrets); const createdNode = await writeNodeFile(false, token, node, monitoringState, nodeSecrets);
if (written.node.monitoring && !written.node.monitoringConfirmed) { if (createdNode.monitoringState == MonitoringState.PENDING) {
await sendMonitoringConfirmationMail(written.node, nodeSecrets) await sendMonitoringConfirmationMail(createdNode, nodeSecrets);
} }
return written; return createdNode;
} }
export async function updateNode(token: Token, node: Node): Promise<{ token: Token, node: Node }> { export async function updateNode(token: Token, node: CreateOrUpdateNode): Promise<StoredNode> {
const {node: currentNode, nodeSecrets} = await getNodeDataWithSecretsByToken(token); const {node: currentNode, nodeSecrets} = await getNodeDataWithSecretsByToken(token);
let monitoringConfirmed = false; let monitoringState = MonitoringState.DISABLED;
let monitoringToken: MonitoringToken | undefined; let monitoringToken: MonitoringToken | undefined = undefined;
if (node.monitoring) { if (node.monitoring) {
if (!currentNode.monitoring) { switch (currentNode.monitoringState) {
// monitoring just has been enabled case MonitoringState.DISABLED:
monitoringConfirmed = false; // monitoring just has been enabled
monitoringToken = generateToken<MonitoringToken>(); monitoringState = MonitoringState.PENDING;
} else {
// monitoring is still enabled
if (currentNode.email !== node.email) {
// new email so we need a new token and a reconfirmation
monitoringConfirmed = false;
monitoringToken = generateToken<MonitoringToken>(); monitoringToken = generateToken<MonitoringToken>();
break;
} else { case MonitoringState.PENDING:
// email unchanged, keep token (fix if not set) and confirmation state case MonitoringState.ACTIVE:
monitoringConfirmed = currentNode.monitoringConfirmed; if (currentNode.email !== node.email) {
monitoringToken = nodeSecrets.monitoringToken || generateToken<MonitoringToken>(); // new email so we need a new token and a reconfirmation
} monitoringState = MonitoringState.PENDING;
monitoringToken = generateToken<MonitoringToken>();
} else {
// email unchanged, keep token (fix if not set) and confirmation state
monitoringState = currentNode.monitoringState;
monitoringToken = nodeSecrets.monitoringToken || generateToken<MonitoringToken>();
}
break;
default:
unhandledEnumField(currentNode.monitoringState);
} }
} }
node.monitoringConfirmed = monitoringConfirmed;
nodeSecrets.monitoringToken = monitoringToken; nodeSecrets.monitoringToken = monitoringToken;
const written = await writeNodeFile(true, token, node, nodeSecrets); const storedNode = await writeNodeFile(true, token, node, monitoringState, nodeSecrets);
if (written.node.monitoring && !written.node.monitoringConfirmed) { if (storedNode.monitoringState === MonitoringState.PENDING) {
await sendMonitoringConfirmationMail(written.node, nodeSecrets) await sendMonitoringConfirmationMail(storedNode, nodeSecrets)
} }
return written; return storedNode;
} }
export async function internalUpdateNode( export async function internalUpdateNode(
token: Token, token: Token,
node: Node, nodeSecrets: NodeSecrets node: CreateOrUpdateNode,
): Promise<{ token: Token, node: Node }> { monitoringState: MonitoringState,
return await writeNodeFile(true, token, node, nodeSecrets); nodeSecrets: NodeSecrets,
): Promise<StoredNode> {
return await writeNodeFile(true, token, node, monitoringState, nodeSecrets);
} }
export async function deleteNode(token: Token): Promise<void> { export async function deleteNode(token: Token): Promise<void> {
await deleteNodeFile(token); await deleteNodeFile(token);
} }
export async function getAllNodes(): Promise<Node[]> { export async function getAllNodes(): Promise<StoredNode[]> {
let files; let files;
try { try {
files = await findNodeFiles({}); files = await findNodeFiles({});
@ -483,7 +500,7 @@ export async function getAllNodes(): Promise<Node[]> {
throw {data: 'Internal error.', type: ErrorTypes.internalError}; throw {data: 'Internal error.', type: ErrorTypes.internalError};
} }
const nodes: Node[] = []; const nodes: StoredNode[] = [];
for (const file of files) { for (const file of files) {
try { try {
const {node} = await parseNodeFile(file); const {node} = await parseNodeFile(file);
@ -497,33 +514,33 @@ export async function getAllNodes(): Promise<Node[]> {
return nodes; return nodes;
} }
export async function getNodeDataWithSecretsByMac(mac: MAC): Promise<{ node: Node, nodeSecrets: NodeSecrets } | null> { export async function findNodeDataWithSecretsByMac(mac: MAC): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets } | null> {
return await findNodeDataByFilePattern({mac}); return await findNodeDataByFilePattern({mac});
} }
export async function getNodeDataByMac(mac: MAC): Promise<Node | null> { export async function findNodeDataByMac(mac: MAC): Promise<StoredNode | null> {
const result = await findNodeDataByFilePattern({mac}); const result = await findNodeDataByFilePattern({mac});
return result ? result.node : null; return result ? result.node : null;
} }
export async function getNodeDataWithSecretsByToken(token: Token): Promise<{ node: Node, nodeSecrets: NodeSecrets }> { export async function getNodeDataWithSecretsByToken(token: Token): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets }> {
return await getNodeDataByFilePattern({token: token}); return await getNodeDataByFilePattern({token: token});
} }
export async function getNodeDataByToken(token: Token): Promise<Node> { export async function getNodeDataByToken(token: Token): Promise<StoredNode> {
const {node} = await getNodeDataByFilePattern({token: token}); const {node} = await getNodeDataByFilePattern({token: token});
return node; return node;
} }
export async function getNodeDataWithSecretsByMonitoringToken( export async function getNodeDataWithSecretsByMonitoringToken(
monitoringToken: MonitoringToken monitoringToken: MonitoringToken
): Promise<{ node: Node, nodeSecrets: NodeSecrets }> { ): Promise<{ node: StoredNode, nodeSecrets: NodeSecrets }> {
return await getNodeDataByFilePattern({monitoringToken: monitoringToken}); return await getNodeDataByFilePattern({monitoringToken: monitoringToken});
} }
export async function getNodeDataByMonitoringToken( export async function getNodeDataByMonitoringToken(
monitoringToken: MonitoringToken monitoringToken: MonitoringToken
): Promise<Node> { ): Promise<StoredNode> {
const {node} = await getNodeDataByFilePattern({monitoringToken: monitoringToken}); const {node} = await getNodeDataByFilePattern({monitoringToken: monitoringToken});
return node; return node;
} }
@ -547,7 +564,7 @@ export async function fixNodeFilenames(): Promise<void> {
} }
} }
export async function findNodesModifiedBefore(timestamp: UnixTimestampSeconds): Promise<Node[]> { export async function findNodesModifiedBefore(timestamp: UnixTimestampSeconds): Promise<StoredNode[]> {
const nodes = await getAllNodes(); const nodes = await getAllNodes();
return _.filter(nodes, node => node.modifiedAt < timestamp); return _.filter(nodes, node => node.modifiedAt < timestamp);
} }
@ -565,7 +582,7 @@ export async function getNodeStatistics(): Promise<NodeStatistics> {
} }
}; };
_.each(nodes, function (node: Node): void { for (const node of nodes) {
if (node.key) { if (node.key) {
nodeStatistics.withVPN += 1; nodeStatistics.withVPN += 1;
} }
@ -589,7 +606,7 @@ export async function getNodeStatistics(): Promise<NodeStatistics> {
default: default:
unhandledEnumField(monitoringState); unhandledEnumField(monitoringState);
} }
}); }
return nodeStatistics; return nodeStatistics;
} }

71
server/types/index.ts
View file

@ -1,4 +1,19 @@
import {Domain, EmailAddress, JSONObject, MonitoringToken, OnlineState, Site, toIsEnum} from "./shared"; import {
CreateOrUpdateNode,
Domain,
DomainSpecificNodeResponse,
EmailAddress,
JSONObject,
MonitoringResponse,
MonitoringState,
MonitoringToken,
NodeResponse,
NodeTokenResponse,
OnlineState,
Site,
StoredNode,
toIsEnum,
} from "./shared";
export * from "./config"; export * from "./config";
export * from "./database"; export * from "./database";
@ -11,6 +26,60 @@ export type NodeStateData = {
state: OnlineState, state: OnlineState,
} }
export function toCreateOrUpdateNode(node: StoredNode): CreateOrUpdateNode {
return {
nickname: node.nickname,
email: node.email,
hostname: node.hostname,
coords: node.coords,
key: node.key,
mac: node.mac,
monitoring: node.monitoringState !== MonitoringState.DISABLED,
}
}
export function toNodeResponse(node: StoredNode): NodeResponse {
return {
token: node.token,
nickname: node.nickname,
email: node.email,
hostname: node.hostname,
coords: node.coords,
key: node.key,
mac: node.mac,
monitoring: node.monitoringState !== MonitoringState.DISABLED,
monitoringConfirmed: node.monitoringState === MonitoringState.ACTIVE,
monitoringState: node.monitoringState,
modifiedAt: node.modifiedAt,
}
}
export function toNodeTokenResponse(node: StoredNode): NodeTokenResponse {
return {
token: node.token,
node: toNodeResponse(node),
}
}
export function toDomainSpecificNodeResponse(node: StoredNode, nodeStateData: NodeStateData): DomainSpecificNodeResponse {
return {
...toNodeResponse(node),
site: nodeStateData.site,
domain: nodeStateData.domain,
onlineState: nodeStateData.state,
}
}
export function toMonitoringResponse(node: StoredNode): MonitoringResponse {
return {
hostname: node.hostname,
mac: node.mac,
email: node.email,
monitoring: node.monitoringState !== MonitoringState.DISABLED,
monitoringConfirmed: node.monitoringState === MonitoringState.ACTIVE,
};
}
// TODO: Complete interface / class declaration. // TODO: Complete interface / class declaration.
export type NodeSecrets = { export type NodeSecrets = {
monitoringToken?: MonitoringToken, monitoringToken?: MonitoringToken,

133
server/types/shared.ts
View file

@ -1,5 +1,4 @@
import {ArrayField, Field, RawJsonField} from "sparkson"; import {ArrayField, Field, RawJsonField} from "sparkson";
import exp from "constants";
// Types shared with the client. // Types shared with the client.
export type TypeGuard<T> = (arg: unknown) => arg is T; export type TypeGuard<T> = (arg: unknown) => arg is T;
@ -140,7 +139,7 @@ export function isNodeStatistics(arg: unknown): arg is NodeStatistics {
); );
} }
export interface Statistics { export type Statistics = {
nodes: NodeStatistics; nodes: NodeStatistics;
} }
@ -324,6 +323,9 @@ export const isFastdKey = isString;
export type MAC = string & { readonly __tag: unique symbol }; export type MAC = string & { readonly __tag: unique symbol };
export const isMAC = isString; export const isMAC = isString;
export type DurationSeconds = number & { readonly __tag: unique symbol };
export const isDurationSeconds = isNumber;
export type UnixTimestampSeconds = number & { readonly __tag: unique symbol }; export type UnixTimestampSeconds = number & { readonly __tag: unique symbol };
export const isUnixTimestampSeconds = isNumber; export const isUnixTimestampSeconds = isNumber;
@ -355,41 +357,105 @@ export const isNickname = isString;
export type Coordinates = string & { readonly __tag: unique symbol }; export type Coordinates = string & { readonly __tag: unique symbol };
export const isCoordinates = isString; export const isCoordinates = isString;
// TODO: More Newtypes /**
export type Node = { * Basic node data.
token: Token; */
export type BaseNode = {
nickname: Nickname; nickname: Nickname;
email: EmailAddress; email: EmailAddress;
hostname: Hostname; hostname: Hostname;
coords?: Coordinates; coords?: Coordinates;
key?: FastdKey; key?: FastdKey;
mac: MAC; mac: MAC;
monitoring: boolean; }
monitoringConfirmed: boolean;
monitoringState: MonitoringState;
modifiedAt: UnixTimestampSeconds;
};
export function isNode(arg: unknown): arg is Node { export function isBaseNode(arg: unknown): arg is BaseNode {
if (!isObject(arg)) { if (!isObject(arg)) {
return false; return false;
} }
const node = arg as Node; const node = arg as BaseNode;
return ( return (
isToken(node.token) &&
isNickname(node.nickname) && isNickname(node.nickname) &&
isEmailAddress(node.email) && isEmailAddress(node.email) &&
isHostname(node.hostname) && isHostname(node.hostname) &&
isOptional(node.coords, isCoordinates) && isOptional(node.coords, isCoordinates) &&
isOptional(node.key, isFastdKey) && isOptional(node.key, isFastdKey) &&
isMAC(node.mac) && isMAC(node.mac)
isBoolean(node.monitoring) && );
isBoolean(node.monitoringConfirmed) && }
/**
* Node data used for creating or updating a node.
*/
export type CreateOrUpdateNode = BaseNode & {
monitoring: boolean;
}
export function isCreateOrUpdateNode(arg: unknown): arg is CreateOrUpdateNode {
if (!isBaseNode(arg)) {
return false;
}
const node = arg as CreateOrUpdateNode;
return (
isBoolean(node.monitoring)
);
}
/**
* Representation of a stored node.
*/
export type StoredNode = BaseNode & {
token: Token;
monitoringState: MonitoringState;
modifiedAt: UnixTimestampSeconds;
}
export function isStoredNode(arg: unknown): arg is StoredNode {
if (!isObject(arg)) {
return false;
}
const node = arg as StoredNode;
return (
isBaseNode(node) &&
isToken(node.token) &&
isMonitoringState(node.monitoringState) && isMonitoringState(node.monitoringState) &&
isUnixTimestampSeconds(node.modifiedAt) isUnixTimestampSeconds(node.modifiedAt)
); );
} }
export type NodeResponse = StoredNode & {
monitoring: boolean;
monitoringConfirmed: boolean;
}
export function isNodeResponse(arg: unknown): arg is NodeResponse {
if (!isStoredNode(arg)) {
return false;
}
const node = arg as NodeResponse;
return (
isBoolean(node.monitoring) &&
isBoolean(node.monitoringConfirmed)
);
}
export type NodeTokenResponse = {
token: Token;
node: NodeResponse;
}
export function isNodeTokenResponse(arg: unknown): arg is NodeTokenResponse {
if (!isObject(arg)) {
return false;
}
const response = arg as NodeTokenResponse;
return (
isToken(response.token) &&
isNodeResponse(response.node) &&
response.token === response.node.token
);
}
export enum OnlineState { export enum OnlineState {
ONLINE = "ONLINE", ONLINE = "ONLINE",
OFFLINE = "OFFLINE", OFFLINE = "OFFLINE",
@ -403,17 +469,20 @@ export const isSite = isString;
export type Domain = string & { readonly __tag: unique symbol }; export type Domain = string & { readonly __tag: unique symbol };
export const isDomain = isString; export const isDomain = isString;
export interface EnhancedNode extends Node { /**
* Represents a node in the context of a Freifunk site and domain.
*/
export type DomainSpecificNodeResponse = NodeResponse & {
site?: Site, site?: Site,
domain?: Domain, domain?: Domain,
onlineState?: OnlineState, onlineState?: OnlineState,
} }
export function isEnhancedNode(arg: unknown): arg is EnhancedNode { export function isDomainSpecificNodeResponse(arg: unknown): arg is DomainSpecificNodeResponse {
if (!isNode(arg)) { if (!isNodeResponse(arg)) {
return false; return false;
} }
const node = arg as EnhancedNode; const node = arg as DomainSpecificNodeResponse;
return ( return (
isOptional(node.site, isSite) && isOptional(node.site, isSite) &&
isOptional(node.domain, isDomain) && isOptional(node.domain, isDomain) &&
@ -421,6 +490,28 @@ export function isEnhancedNode(arg: unknown): arg is EnhancedNode {
); );
} }
export type MonitoringResponse = {
hostname: Hostname,
mac: MAC,
email: EmailAddress,
monitoring: boolean,
monitoringConfirmed: boolean,
}
export function isMonitoringResponse(arg: unknown): arg is MonitoringResponse {
if (!Object(arg)) {
return false;
}
const response = arg as MonitoringResponse;
return (
isHostname(response.hostname) &&
isMAC(response.mac) &&
isEmailAddress(response.email) &&
isBoolean(response.monitoring) &&
isBoolean(response.monitoringConfirmed)
);
}
export enum NodeSortField { export enum NodeSortField {
HOSTNAME = 'hostname', HOSTNAME = 'hostname',
NICKNAME = 'nickname', NICKNAME = 'nickname',
@ -437,7 +528,7 @@ export enum NodeSortField {
export const isNodeSortField = toIsEnum(NodeSortField); export const isNodeSortField = toIsEnum(NodeSortField);
export interface NodesFilter { export type NodesFilter = {
hasKey?: boolean; hasKey?: boolean;
hasCoords?: boolean; hasCoords?: boolean;
monitoringState?: MonitoringState; monitoringState?: MonitoringState;

47
server/utils/resources.ts
View file

@ -5,7 +5,18 @@ import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger"; import Logger from "../logger";
import {Constraints, forConstraints, isConstraints} from "../validation/validator"; import {Constraints, forConstraints, isConstraints} from "../validation/validator";
import {Request, Response} from "express"; import {Request, Response} from "express";
import {EnumTypeGuard, EnumValue, type GenericSortField, SortDirection, TypeGuard} from "../types"; import {
EnumTypeGuard,
EnumValue,
type GenericSortField,
isJSONObject,
JSONObject,
SortDirection,
TypeGuard
} from "../types";
export type RequestData = JSONObject;
export type RequestHandler = (request: Request, response: Response) => void;
export type Entity = { [key: string]: any }; export type Entity = { [key: string]: any };
@ -110,8 +121,20 @@ function getConstrainedValues(data: { [key: string]: any }, constraints: Constra
return values; return values;
} }
export function getData(req: Request): any { function normalize(data: any): JSONObject {
return _.extend({}, req.body, req.params, req.query); return isJSONObject(data) ? data : {};
}
export function getData(req: Request): RequestData {
const body = normalize(req.body);
const params = normalize(req.params);
const query = normalize(req.query);
return {
...body,
...params,
...query,
};
} }
export async function getValidRestParams( export async function getValidRestParams(
@ -197,7 +220,7 @@ export function filter<E>(entities: ArrayLike<E>, allowedFilterFields: string[],
return true; return true;
} }
if (_.startsWith(key, 'has')) { if (_.startsWith(key, 'has')) {
const entityKey = key.substr(3, 1).toLowerCase() + key.substr(4); const entityKey = key.substring(3, 4).toLowerCase() + key.substring(4);
return _.isEmpty(entity[entityKey]).toString() !== value; return _.isEmpty(entity[entityKey]).toString() !== value;
} }
return entity[key] === value; return entity[key] === value;
@ -267,3 +290,19 @@ export function successHtml(res: Response, html: string) {
export function error(res: Response, err: { data: any, type: { code: number } }) { export function error(res: Response, err: { data: any, type: { code: number } }) {
respond(res, err.type.code, err.data, 'json'); respond(res, err.type.code, err.data, 'json');
} }
export function handleJSON<Response>(handler: () => Promise<Response>): RequestHandler {
return (request, response) => {
handler()
.then(data => success(response, data || {}))
.catch(error => error(response, error));
};
}
export function handleJSONWithData<Response>(handler: (data: RequestData) => Promise<Response>): RequestHandler {
return (request, response) => {
handler(getData(request))
.then(data => success(response, data || {}))
.catch(error => error(response, error));
};
}

53
server/utils/time.test.ts Normal file
View file

@ -0,0 +1,53 @@
import {parseTimestamp} from "./time";
import moment from "moment";
const TIMESTAMP_INVALID_STRING = "2020-01-02T42:99:23.000Z";
const TIMESTAMP_VALID_STRING = "2020-01-02T12:34:56.000Z";
test('parseTimestamp() should fail parsing non-string timestamp', () => {
// given
const timestamp = {};
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp).toEqual(null);
});
test('parseTimestamp() should fail parsing empty timestamp string', () => {
// given
const timestamp = "";
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp).toEqual(null);
});
test('parseTimestamp() should fail parsing invalid timestamp string', () => {
// given
// noinspection UnnecessaryLocalVariableJS
const timestamp = TIMESTAMP_INVALID_STRING;
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
expect(parsedTimestamp).toEqual(null);
});
test('parseTimestamp() should succeed parsing valid timestamp string', () => {
// given
const timestamp = TIMESTAMP_VALID_STRING;
// when
const parsedTimestamp = parseTimestamp(timestamp);
// then
if (parsedTimestamp === null) {
fail('timestamp should not be null');
}
expect(moment.unix(parsedTimestamp).toISOString()).toEqual(timestamp);
});

57
server/utils/time.ts Normal file
View file

@ -0,0 +1,57 @@
import {DurationSeconds, UnixTimestampSeconds} from "../types";
import _ from "lodash";
import moment, {Moment} from "moment";
export function now(): UnixTimestampSeconds {
return Math.round(Date.now() / 1000.0) as UnixTimestampSeconds;
}
export function subtract(timestamp: UnixTimestampSeconds, duration: DurationSeconds): UnixTimestampSeconds {
return (timestamp - duration) as UnixTimestampSeconds;
}
const SECOND: DurationSeconds = 1 as DurationSeconds;
const MINUTE: DurationSeconds = (60 * SECOND) as DurationSeconds;
const HOUR: DurationSeconds = (60 * MINUTE) as DurationSeconds;
const DAY: DurationSeconds = (24 * HOUR) as DurationSeconds;
const WEEK: DurationSeconds = (7 * DAY) as DurationSeconds;
export function seconds(n: number): DurationSeconds {
return (n * SECOND) as DurationSeconds;
}
export function minutes(n: number): DurationSeconds {
return (n * MINUTE) as DurationSeconds;
}
export function hours(n: number): DurationSeconds {
return (n * HOUR) as DurationSeconds;
}
export function days(n: number): DurationSeconds {
return (n * DAY) as DurationSeconds;
}
export function weeks(n: number): DurationSeconds {
return (n * WEEK) as DurationSeconds;
}
export function unix(moment: Moment): UnixTimestampSeconds {
return moment.unix() as UnixTimestampSeconds;
}
export function formatTimestamp(timestamp: UnixTimestampSeconds): string {
return moment.unix(timestamp).format();
}
export function parseTimestamp(timestamp: any): UnixTimestampSeconds | null {
if (!_.isString(timestamp)) {
return null;
}
const parsed = moment.utc(timestamp);
if (!parsed.isValid()) {
return null;
}
return unix(parsed);
}