Migrate to version 2 of nodes.json and start adding tests

server/services/mailService.ts

@@ -1,10 +1,10 @@
 import _ from "lodash";
 import deepExtend from "deep-extend";
 import moment, {Moment} from "moment";
-import {createTransport} from "nodemailer";
+import {createTransport, Transporter} from "nodemailer";
 
 import {config} from "../config";
-import {db, Statement} from "../db/database";
+import {db} from "../db/database";
 import Logger from "../logger";
 import * as MailTemplateService from "./mailTemplateService";
 import * as Resources from "../utils/resources";
@@ -13,16 +13,25 @@ import {Mail, MailData, MailId, MailType} from "../types";
 
 const MAIL_QUEUE_DB_BATCH_SIZE = 50;
 
-const transporter = createTransport(deepExtend(
-    {},
-    config.server.email.smtp,
-    {
-        transport: 'smtp',
-        pool: true
-    }
-));
+// TODO: Extract transporter into own module and initialize during main().
+let transporterSingleton: Transporter | null = null;
 
-MailTemplateService.configureTransporter(transporter);
+function transporter() {
+    if (!transporterSingleton) {
+        transporterSingleton = createTransport(deepExtend(
+            {},
+            config.server.email.smtp,
+            {
+                transport: 'smtp',
+                pool: true
+            }
+        ));
+
+        MailTemplateService.configureTransporter(transporterSingleton);
+    }
+
+    return transporterSingleton;
+}
 
 async function sendMail(options: Mail): Promise<void> {
     Logger
@@ -42,7 +51,7 @@ async function sendMail(options: Mail): Promise<void> {
         html: renderedTemplate.body
     };
 
-    await transporter.sendMail(mailOptions);
+    await transporter().sendMail(mailOptions);
 
     Logger.tag('mail', 'queue').info('Mail[%d] has been send.', options.id);
 }

server/services/monitoringService.test.ts

@@ -0,0 +1,471 @@
+import moment from 'moment';
+import {ParsedNode, parseNode, parseNodesJson, parseTimestamp} from "./monitoringService";
+import {NodeState} from "../types";
+import Logger from '../logger';
+import {MockLogger} from "../__mocks__/logger";
+
+const mockedLogger = Logger as MockLogger;
+
+jest.mock('../logger');
+jest.mock('../db/database');
+
+const NODES_JSON_INVALID_VERSION = 1;
+const NODES_JSON_VALID_VERSION = 2;
+
+const TIMESTAMP_INVALID_STRING = "2020-01-02T42:99:23.000Z";
+const TIMESTAMP_VALID_STRING = "2020-01-02T12:34:56.000Z";
+
+
+beforeEach(() => {
+    mockedLogger.reset();
+});
+
+test('parseTimestamp() should fail parsing non-string timestamp', () => {
+    // given
+    const timestamp = {};
+
+    // when
+    const parsedTimestamp = parseTimestamp(timestamp);
+
+    // then
+    expect(parsedTimestamp.isValid()).toBe(false);
+});
+
+test('parseTimestamp() should fail parsing empty timestamp string', () => {
+    // given
+    const timestamp = "";
+
+    // when
+    const parsedTimestamp = parseTimestamp(timestamp);
+
+    // then
+    expect(parsedTimestamp.isValid()).toBe(false);
+});
+
+test('parseTimestamp() should fail parsing invalid timestamp string', () => {
+    // given
+    const timestamp = TIMESTAMP_INVALID_STRING;
+
+    // when
+    const parsedTimestamp = parseTimestamp(timestamp);
+
+    // then
+    expect(parsedTimestamp.isValid()).toBe(false);
+});
+
+test('parseTimestamp() should succeed parsing valid timestamp string', () => {
+    // given
+    const timestamp = TIMESTAMP_VALID_STRING;
+
+    // when
+    const parsedTimestamp = parseTimestamp(timestamp);
+
+    // then
+    expect(parsedTimestamp.isValid()).toBe(true);
+    expect(parsedTimestamp.toISOString()).toEqual(timestamp);
+});
+
+test('parseNode() should fail parsing node for undefined node data', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = undefined;
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for empty node data', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {};
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for empty node info', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {}
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for non-string node id', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: 42
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for empty node id', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: ""
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for empty network info', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {}
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for invalid mac', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "xxx"
+            }
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for missing flags', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            }
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for empty flags', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            }
+        },
+        flags: {}
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for missing last seen timestamp', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            }
+        },
+        flags: {
+            online: true
+        }
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should fail parsing node for invalid last seen timestamp', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            }
+        },
+        flags: {
+            online: true
+        },
+        lastseen: 42
+    };
+
+    // then
+    expect(() => parseNode(importTimestamp, nodeData)).toThrowError();
+});
+
+test('parseNode() should succeed parsing node without site and domain', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            }
+        },
+        flags: {
+            online: true
+        },
+        lastseen: TIMESTAMP_VALID_STRING
+    };
+
+    // then
+    const expectedParsedNode: ParsedNode = {
+        mac: "12:34:56:78:90:AB",
+        importTimestamp: importTimestamp,
+        state: NodeState.ONLINE,
+        lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
+        site: '<unknown-site>',
+        domain: '<unknown-domain>'
+    };
+    expect(parseNode(importTimestamp, nodeData)).toEqual(expectedParsedNode);
+});
+
+test('parseNode() should succeed parsing node with site and domain', () => {
+    // given
+    const importTimestamp = moment();
+    const nodeData = {
+        nodeinfo: {
+            node_id: "1234567890ab",
+            network: {
+                mac: "12:34:56:78:90:ab"
+            },
+            system: {
+                site_code: "test-site",
+                domain_code: "test-domain"
+            }
+        },
+        flags: {
+            online: true
+        },
+        lastseen: TIMESTAMP_VALID_STRING,
+    };
+
+    // then
+    const expectedParsedNode: ParsedNode = {
+        mac: "12:34:56:78:90:AB",
+        importTimestamp: importTimestamp,
+        state: NodeState.ONLINE,
+        lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
+        site: 'test-site',
+        domain: 'test-domain'
+    };
+    expect(parseNode(importTimestamp, nodeData)).toEqual(expectedParsedNode);
+});
+
+test('parseNodesJson() should fail parsing empty string', () => {
+    // given
+    const json = "";
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing malformed JSON', () => {
+    // given
+    const json = '{"version": 2]';
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing JSON null', () => {
+    // given
+    const json = JSON.stringify(null);
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing JSON string', () => {
+    // given
+    const json = JSON.stringify("foo");
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing JSON number', () => {
+    // given
+    const json = JSON.stringify(42);
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing empty JSON object', () => {
+    // given
+    const json = JSON.stringify({});
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing for mismatching version', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_INVALID_VERSION
+    });
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing for missing timestamp', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        nodes: []
+    });
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing for invalid timestamp', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        timestamp: TIMESTAMP_INVALID_STRING,
+        nodes: []
+    });
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should fail parsing for nodes object instead of array', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        timestamp: TIMESTAMP_VALID_STRING,
+        nodes: {}
+    });
+
+    // then
+    expect(() => parseNodesJson(json)).toThrowError();
+});
+
+test('parseNodesJson() should succeed parsing no nodes', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        timestamp: TIMESTAMP_VALID_STRING,
+        nodes: []
+    });
+
+    // when
+    const result = parseNodesJson(json);
+
+    // then
+    expect(result.importTimestamp.isValid()).toBe(true);
+    expect(result.nodes).toEqual([]);
+});
+
+test('parseNodesJson() should skip parsing invalid nodes', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        timestamp: TIMESTAMP_VALID_STRING,
+        nodes: [
+            {},
+            {
+                nodeinfo: {
+                    node_id: "1234567890ab",
+                    network: {
+                        mac: "12:34:56:78:90:ab"
+                    },
+                    system: {
+                        site_code: "test-site",
+                        domain_code: "test-domain"
+                    }
+                },
+                flags: {
+                    online: true
+                },
+                lastseen: TIMESTAMP_INVALID_STRING,
+            }
+        ]
+    });
+
+    // when
+    const result = parseNodesJson(json);
+
+    // then
+    expect(result.importTimestamp.isValid()).toBe(true);
+    expect(result.nodes).toEqual([]);
+    expect(mockedLogger.getMessages('error', 'monitoring', 'parsing-nodes-json').length).toEqual(2);
+});
+
+test('parseNodesJson() should parse valid nodes', () => {
+    // given
+    const json = JSON.stringify({
+        version: NODES_JSON_VALID_VERSION,
+        timestamp: TIMESTAMP_VALID_STRING,
+        nodes: [
+            {}, // keep an invalid one for good measure
+            {
+                nodeinfo: {
+                    node_id: "1234567890ab",
+                    network: {
+                        mac: "12:34:56:78:90:ab"
+                    },
+                    system: {
+                        site_code: "test-site",
+                        domain_code: "test-domain"
+                    }
+                },
+                flags: {
+                    online: true
+                },
+                lastseen: TIMESTAMP_VALID_STRING,
+            }
+        ]
+    });
+
+    // when
+    const result = parseNodesJson(json);
+
+    // then
+    const expectedParsedNode: ParsedNode = {
+        mac: "12:34:56:78:90:AB",
+        importTimestamp: parseTimestamp(TIMESTAMP_VALID_STRING),
+        state: NodeState.ONLINE,
+        lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
+        site: 'test-site',
+        domain: 'test-domain'
+    };
+
+    expect(result.importTimestamp.isValid()).toBe(true);
+    expect(result.nodes).toEqual([expectedParsedNode]);
+    expect(mockedLogger.getMessages('error', 'monitoring', 'parsing-nodes-json').length).toEqual(1);
+});

server/services/monitoringService.ts

@@ -33,7 +33,7 @@ const DELETE_OFFLINE_NODES_AFTER_DURATION: {amount: number, unit: unitOfTime.Dur
     unit: 'days'
 };
 
-type ParsedNode = {
+export type ParsedNode = {
     mac: string,
     importTimestamp: Moment,
     state: NodeState,
@@ -42,7 +42,7 @@ type ParsedNode = {
     domain: string,
 };
 
-type NodesParsingResult = {
+export type NodesParsingResult = {
     importTimestamp: Moment,
     nodes: ParsedNode[],
 }
@@ -131,25 +131,34 @@ async function storeNodeInformation(nodeData: ParsedNode, node: Node): Promise<v
 
 const isValidMac = forConstraint(CONSTRAINTS.node.mac, false);
 
-function parseTimestamp(timestamp: any): Moment {
+export function parseTimestamp(timestamp: any): Moment {
     if (!_.isString(timestamp)) {
         return moment.invalid();
     }
     return moment.utc(timestamp);
 }
 
-function parseNode(importTimestamp: Moment, nodeData: any, nodeId: NodeId): ParsedNode {
+// TODO: Use sparkson for JSON parsing.
+export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode {
     if (!_.isPlainObject(nodeData)) {
         throw new Error(
-            'Node ' + nodeId + ': Unexpected node type: ' + (typeof nodeData)
+            'Unexpected node type: ' + (typeof nodeData)
         );
     }
 
     if (!_.isPlainObject(nodeData.nodeinfo)) {
         throw new Error(
-            'Node ' + nodeId + ': Unexpected nodeinfo type: ' + (typeof nodeData.nodeinfo)
+            'Unexpected nodeinfo type: ' + (typeof nodeData.nodeinfo)
         );
     }
+
+    const nodeId = nodeData.nodeinfo.node_id;
+    if (!nodeId || !_.isString(nodeId)) {
+        throw new Error(
+            `Invalid node id of type "${typeof nodeId}": ${nodeId}`
+        );
+    }
+
     if (!_.isPlainObject(nodeData.nodeinfo.network)) {
         throw new Error(
             'Node ' + nodeId + ': Unexpected nodeinfo.network type: ' + (typeof nodeData.nodeinfo.network)
@@ -197,52 +206,51 @@ function parseNode(importTimestamp: Moment, nodeData: any, nodeId: NodeId): Pars
         importTimestamp: importTimestamp,
         state: isOnline ? NodeState.ONLINE : NodeState.OFFLINE,
         lastSeen: lastSeen,
-        site: site,
-        domain: domain
+        site: site || '<unknown-site>',
+        domain: domain || '<unknown-domain>'
     };
 }
 
-function parseNodesJson (body: string): NodesParsingResult {
+// TODO: Use sparkson for JSON parsing.
+export function parseNodesJson (body: string): NodesParsingResult {
     Logger.tag('monitoring', 'information-retrieval').debug('Parsing nodes.json...');
 
-    const data: {[key: string]: any} = {};
-
     const json = JSON.parse(body);
 
-    if (json.version !== 1) {
-        throw new Error('Unexpected nodes.json version: ' + json.version);
+    if (!_.isPlainObject(json)) {
+        throw new Error(`Expecting a JSON object as the nodes.json root, but got: ${typeof json}`);
     }
-    data.importTimestamp = parseTimestamp(json.timestamp);
 
-    if (!data.importTimestamp.isValid()) {
+    const expectedVersion = 2;
+    if (json.version !== expectedVersion) {
+        throw new Error(`Unexpected nodes.json version "${json.version}". Expected: "${expectedVersion}"`);
+    }
+
+    const result: NodesParsingResult = {
+        importTimestamp: parseTimestamp(json.timestamp),
+        nodes: []
+    };
+
+    if (!result.importTimestamp.isValid()) {
         throw new Error('Invalid timestamp: ' + json.timestamp);
     }
 
-    if (!_.isPlainObject(json.nodes)) {
-        throw new Error('Invalid nodes object type: ' + (typeof json.nodes));
+    if (!_.isArray(json.nodes)) {
+        throw new Error('Invalid nodes array type: ' + (typeof json.nodes));
     }
 
-    data.nodes = _.filter(
-        _.values(
-            _.map(
-                json.nodes,
-                function (nodeData, nodeId) {
-                    try {
-                        return parseNode(data.importTimestamp, nodeData, nodeId);
-                    }
-                    catch (error) {
-                        Logger.tag('monitoring', 'information-retrieval').error(error);
-                        return null;
-                    }
-                }
-            )
-        ),
-        function (node) {
-            return node !== null;
+    for (const nodeData of json.nodes) {
+        try {
+            const parsedNode = parseNode(result.importTimestamp, nodeData);
+            Logger.tag('monitoring', 'parsing-nodes-json').debug(`Parsing node successful: ${parsedNode.mac}`);
+            result.nodes.push(parsedNode);
         }
-    );
+        catch (error) {
+            Logger.tag('monitoring', 'parsing-nodes-json').error("Could not parse node.", error, nodeData);
+        }
+    }
 
-    return data as NodesParsingResult;
+    return result;
 }
 
 async function updateSkippedNode(id: NodeId, node?: Node): Promise<Statement> {