ffffng/server/services/monitoringService.ts

import _ from "lodash";
import request from "request";

import { config } from "../config";
import { db } from "../db/database";
import * as DatabaseUtil from "../utils/databaseUtil";
import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger";

import * as MailService from "../services/mailService";
import * as NodeService from "../services/nodeService";
import * as Resources from "../utils/resources";
import type { RestParams } from "../utils/resources";
import { normalizeMac, parseInteger } from "../shared/utils/strings";
import { monitoringDisableUrl } from "../utils/urlBuilder";
import CONSTRAINTS from "../shared/validation/constraints";
import { forConstraint } from "../shared/validation/validator";
import {
    Domain,
    DurationSeconds,
    Hostname,
    isBoolean,
    isDomain,
    isMailType,
    isMonitoringSortField,
    isMonitoringState,
    isOnlineState,
    isPlainObject,
    isSite,
    isString,
    isUndefined,
    JSONValue,
    MAC,
    MailType,
    MonitoringSortField,
    MonitoringSortFieldEnum,
    MonitoringState,
    MonitoringToken,
    NodeMonitoringStateResponse,
    NodeStateData,
    NodeStateId,
    OnlineState,
    Site,
    StoredNode,
    toCreateOrUpdateNode,
    UnixTimestampSeconds,
} from "../types";
import {
    days,
    formatTimestamp,
    hours,
    now,
    parseTimestamp,
    subtract,
    weeks,
} from "../utils/time";
import { filterUndefinedFromJSON, parseJSON } from "../shared/utils/json";
import { mapIdFromMAC } from "../shared/utils/node";

type NodeStateRow = {
    id: NodeStateId;
    created_at: UnixTimestampSeconds;
    domain: Domain | null;
    hostname: Hostname | null;
    import_timestamp: UnixTimestampSeconds;
    last_seen: UnixTimestampSeconds;
    last_status_mail_sent: UnixTimestampSeconds | null;
    last_status_mail_type: string | null;
    mac: MAC;
    modified_at: UnixTimestampSeconds;
    monitoring_state: string | null;
    site: Site | null;
    state: string;
};

const MONITORING_STATE_MACS_CHUNK_SIZE = 100;
const NEVER_ONLINE_NODES_DELETION_CHUNK_SIZE = 20;
const MONITORING_MAILS_DB_BATCH_SIZE = 50;
/**
 * Defines the intervals emails are sent if a node is offline
 */
const MONITORING_OFFLINE_MAILS_SCHEDULE: Record<number, DurationSeconds> = {
    1: hours(3),
    2: days(1),
    3: weeks(1),
};
const DELETE_OFFLINE_NODES_AFTER_DURATION: DurationSeconds = days(100);

export type ParsedNode = {
    mac: MAC;
    importTimestamp: UnixTimestampSeconds;
    state: OnlineState;
    lastSeen: UnixTimestampSeconds;
    site?: Site;
    domain?: Domain;
};

export type NodesParsingResult = {
    importTimestamp: UnixTimestampSeconds;
    nodes: ParsedNode[];
    failedNodesCount: number;
    totalNodesCount: number;
};

export type RetrieveNodeInformationResult = {
    failedParsingNodesCount: number;
    totalNodesCount: number;
};

let previousImportTimestamp: UnixTimestampSeconds | null = null;

async function insertNodeInformation(
    nodeData: ParsedNode,
    node: StoredNode
): Promise<void> {
    Logger.tag("monitoring", "information-retrieval").debug(
        "Node is new in monitoring, creating data: %s",
        nodeData.mac
    );

    await db.run(
        "INSERT INTO node_state " +
            "(hostname, mac, site, domain, monitoring_state, state, last_seen, import_timestamp, last_status_mail_sent, last_status_mail_type) " +
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
        [
            node.hostname,
            node.mac,
            nodeData.site,
            nodeData.domain,
            node.monitoringState,
            nodeData.state,
            nodeData.lastSeen,
            nodeData.importTimestamp,
            null, // new node so we haven't send a mail yet
            null, // new node so we haven't send a mail yet
        ]
    );
}

async function updateNodeInformation(
    nodeData: ParsedNode,
    node: StoredNode,
    row: NodeStateRow
): Promise<void> {
    Logger.tag("monitoring", "informacallbacktion-retrieval").debug(
        "Node is known in monitoring: %s",
        nodeData.mac
    );

    if (row.import_timestamp >= nodeData.importTimestamp) {
        Logger.tag("monitoring", "information-retrieval").debug(
            "No new data for node, skipping: %s",
            nodeData.mac
        );
        return;
    }

    Logger.tag("monitoring", "information-retrieval").debug(
        "New data for node, updating: %s",
        nodeData.mac
    );

    await db.run(
        "UPDATE node_state " +
            "SET " +
            "hostname = ?, " +
            "site = ?, " +
            "domain = ?, " +
            "monitoring_state = ?, " +
            "state = ?, " +
            "last_seen = ?, " +
            "import_timestamp = ?, " +
            "modified_at = ? " +
            "WHERE id = ? AND mac = ?",
        [
            node.hostname,
            nodeData.site || row.site,
            nodeData.domain || row.domain,
            node.monitoringState,
            nodeData.state,
            nodeData.lastSeen,
            nodeData.importTimestamp,
            now(),

            row.id,
            node.mac,
        ]
    );
}

async function storeNodeInformation(
    nodeData: ParsedNode,
    node: StoredNode
): Promise<void> {
    Logger.tag("monitoring", "information-retrieval").debug(
        "Storing status for node: %s",
        nodeData.mac
    );

    const row = await db.get<NodeStateRow>(
        "SELECT * FROM node_state WHERE mac = ?",
        [node.mac]
    );

    if (isUndefined(row)) {
        return await insertNodeInformation(nodeData, node);
    } else {
        return await updateNodeInformation(nodeData, node, row);
    }
}

const isValidMac = forConstraint(CONSTRAINTS.node.mac, false);

export function parseNode(
    importTimestamp: UnixTimestampSeconds,
    nodeData: JSONValue
): ParsedNode {
    if (!isPlainObject(nodeData)) {
        throw new Error("Unexpected node type: " + typeof nodeData);
    }

    if (!isPlainObject(nodeData.nodeinfo)) {
        throw new Error(
            "Unexpected nodeinfo type: " + typeof nodeData.nodeinfo
        );
    }

    const nodeId = nodeData.nodeinfo.node_id;
    if (!nodeId || !isString(nodeId)) {
        throw new Error(
            `Invalid node id of type "${typeof nodeId}": ${nodeId}`
        );
    }

    if (!isPlainObject(nodeData.nodeinfo.network)) {
        throw new Error(
            "Node " +
                nodeId +
                ": Unexpected nodeinfo.network type: " +
                typeof nodeData.nodeinfo.network
        );
    }

    if (!isValidMac(nodeData.nodeinfo.network.mac)) {
        throw new Error(
            "Node " + nodeId + ": Invalid MAC: " + nodeData.nodeinfo.network.mac
        );
    }
    const mac = normalizeMac(nodeData.nodeinfo.network.mac as MAC);

    if (!isPlainObject(nodeData.flags)) {
        throw new Error(
            "Node " +
                nodeId +
                ": Unexpected flags type: " +
                typeof nodeData.flags
        );
    }
    if (!isBoolean(nodeData.flags.online)) {
        throw new Error(
            "Node " +
                nodeId +
                ": Unexpected flags.online type: " +
                typeof nodeData.flags.online
        );
    }
    const isOnline = nodeData.flags.online;

    const lastSeen = parseTimestamp(nodeData.lastseen);
    if (lastSeen === null) {
        throw new Error(
            "Node " +
                nodeId +
                ": Invalid lastseen timestamp: " +
                nodeData.lastseen
        );
    }

    let site: Site | undefined;
    if (
        isPlainObject(nodeData.nodeinfo.system) &&
        isSite(nodeData.nodeinfo.system.site_code)
    ) {
        site = nodeData.nodeinfo.system.site_code;
    }

    let domain: Domain | undefined;
    if (
        isPlainObject(nodeData.nodeinfo.system) &&
        isDomain(nodeData.nodeinfo.system.domain_code)
    ) {
        domain = nodeData.nodeinfo.system.domain_code;
    }

    return {
        mac,
        importTimestamp: importTimestamp,
        state: isOnline ? OnlineState.ONLINE : OnlineState.OFFLINE,
        lastSeen,
        site,
        domain,
    };
}

export function parseNodesJson(body: string): NodesParsingResult {
    Logger.tag("monitoring", "information-retrieval").debug(
        "Parsing nodes.json..."
    );

    const json = parseJSON(body);

    if (!isPlainObject(json)) {
        throw new Error(
            `Expecting a JSON object as the nodes.json root, but got: ${typeof json}`
        );
    }

    const expectedVersion = 2;
    if (json.version !== expectedVersion) {
        throw new Error(
            `Unexpected nodes.json version "${json.version}". Expected: "${expectedVersion}"`
        );
    }

    const importTimestamp = parseTimestamp(json.timestamp);
    if (importTimestamp === null) {
        throw new Error("Invalid timestamp: " + json.timestamp);
    }

    const result: NodesParsingResult = {
        importTimestamp,
        nodes: [],
        failedNodesCount: 0,
        totalNodesCount: 0,
    };

    if (!_.isArray(json.nodes)) {
        throw new Error("Invalid nodes array type: " + typeof json.nodes);
    }

    for (const nodeData of json.nodes) {
        result.totalNodesCount += 1;
        try {
            const parsedNode = parseNode(result.importTimestamp, nodeData);
            Logger.tag("monitoring", "parsing-nodes-json").debug(
                `Parsing node successful: ${parsedNode.mac}`
            );
            result.nodes.push(parsedNode);
        } catch (error) {
            result.failedNodesCount += 1;
            Logger.tag("monitoring", "parsing-nodes-json").error(
                "Could not parse node.",
                error,
                nodeData
            );
        }
    }

    return result;
}

async function updateSkippedNode(
    id: NodeStateId,
    node?: StoredNode
): Promise<void> {
    await db.run(
        "UPDATE node_state " +
            "SET hostname = ?, monitoring_state = ?, modified_at = ?" +
            "WHERE id = ?",
        [node ? node.hostname : "", node ? node.monitoringState : "", now(), id]
    );
}

async function sendMonitoringMailsBatched(
    name: string,
    mailType: MailType,
    findBatchFun: () => Promise<NodeStateRow[]>
): Promise<void> {
    Logger.tag("monitoring", "mail-sending").debug(
        'Sending "%s" mails...',
        name
    );

    let nodeStates = await findBatchFun();
    while (nodeStates.length > 0) {
        await sendMonitoringMailsBatch(name, mailType, nodeStates);
        nodeStates = await findBatchFun();
    }

    Logger.tag("monitoring", "mail-sending").debug(
        'Done sending "%s" mails.',
        name
    );
}

async function sendMonitoringMailsBatch(
    name: string,
    mailType: MailType,
    nodeStates: NodeStateRow[]
) {
    Logger.tag("monitoring", "mail-sending").debug("Sending next batch...");

    for (const nodeState of nodeStates) {
        await sendMonitoringMail(name, mailType, nodeState);
    }
}

async function sendMonitoringMail(
    name: string,
    mailType: MailType,
    nodeState: NodeStateRow
) {
    const mac = nodeState.mac;
    Logger.tag("monitoring", "mail-sending").debug(
        "Loading node data for: %s",
        mac
    );

    const result = await NodeService.findNodeDataWithSecretsByMac(mac);
    if (!result) {
        Logger.tag("monitoring", "mail-sending").debug(
            `Node not found. Skipping sending of "${name}" mail: ${mac}`
        );
        await updateSkippedNode(nodeState.id);
        return;
    }

    const { node, nodeSecrets } = result;

    if (node.monitoringState !== MonitoringState.ACTIVE) {
        Logger.tag("monitoring", "mail-sending").debug(
            'Monitoring disabled, skipping "%s" mail for: %s',
            name,
            mac
        );
        await updateSkippedNode(nodeState.id, node);
        return;
    }

    const monitoringToken = nodeSecrets.monitoringToken;
    if (!monitoringToken) {
        Logger.tag("monitoring", "mail-sending").error(
            'Node has no monitoring token. Cannot send mail "%s" for: %s',
            name,
            mac
        );
        await updateSkippedNode(nodeState.id, node);
        return;
    }

    await enqueMail(name, mailType, node, nodeState, monitoringToken);
    await updateNodeForSentEmail(mailType, node, nodeState);
}

async function enqueMail(
    name: string,
    mailType: MailType,
    node: StoredNode,
    nodeState: NodeStateRow,
    monitoringToken: MonitoringToken
) {
    Logger.tag("monitoring", "mail-sending").info(
        'Sending "%s" mail for: %s',
        name,
        node.mac
    );

    await MailService.enqueue(
        config.server.email.from,
        node.nickname + " <" + node.email + ">",
        mailType,
        {
            node: filterUndefinedFromJSON(node),
            lastSeen: nodeState.last_seen,
            disableUrl: monitoringDisableUrl(monitoringToken),
        }
    );
}

async function updateNodeForSentEmail(
    mailType: MailType,
    node: StoredNode,
    nodeState: NodeStateRow
) {
    Logger.tag("monitoring", "mail-sending").debug(
        "Updating node state: ",
        node.mac
    );

    const timestamp = now();
    await db.run(
        "UPDATE node_state " +
            "SET hostname = ?, monitoring_state = ?, modified_at = ?, last_status_mail_sent = ?, last_status_mail_type = ?" +
            "WHERE id = ?",
        [
            node.hostname,
            node.monitoringState,
            timestamp,
            timestamp,
            mailType,
            nodeState.id,
        ]
    );
}

async function sendOnlineAgainMails(
    startTime: UnixTimestampSeconds
): Promise<void> {
    await sendMonitoringMailsBatched(
        "online again",
        MailType.MONITORING_ONLINE_AGAIN,
        async (): Promise<NodeStateRow[]> =>
            await db.all<NodeStateRow>(
                "SELECT * FROM node_state " +
                    "WHERE modified_at < ? AND state = ? AND last_status_mail_type IN (" +
                    "'monitoring-offline-1', 'monitoring-offline-2', 'monitoring-offline-3'" +
                    ")" +
                    "ORDER BY id ASC LIMIT ?",
                [startTime, "ONLINE", MONITORING_MAILS_DB_BATCH_SIZE]
            )
    );
}

async function sendOfflineMails(
    startTime: UnixTimestampSeconds,
    mailType: MailType
): Promise<void> {
    const mailNumber = parseInteger(mailType.split("-")[2]);
    await sendMonitoringMailsBatched(
        "offline " + mailNumber,
        mailType,
        async (): Promise<NodeStateRow[]> => {
            const previousType =
                mailNumber === 1
                    ? "monitoring-online-again"
                    : "monitoring-offline-" + (mailNumber - 1);

            // the first time the first offline mail is send, there was no mail before
            const allowNull =
                mailNumber === 1 ? " OR last_status_mail_type IS NULL" : "";

            const schedule = MONITORING_OFFLINE_MAILS_SCHEDULE[mailNumber];
            const scheduledTimeBefore = subtract(now(), schedule);

            return await db.all<NodeStateRow>(
                "SELECT * FROM node_state " +
                    "WHERE modified_at < ? AND state = ? AND (last_status_mail_type = ?" +
                    allowNull +
                    ") AND " +
                    "last_seen <= ? AND (last_status_mail_sent <= ? OR last_status_mail_sent IS NULL) " +
                    "ORDER BY id ASC LIMIT ?",
                [
                    startTime,
                    "OFFLINE",
                    previousType,
                    scheduledTimeBefore,
                    scheduledTimeBefore,

                    MONITORING_MAILS_DB_BATCH_SIZE,
                ]
            );
        }
    );
}

function doRequest(
    url: string
): Promise<{ response: request.Response; body: string }> {
    return new Promise<{ response: request.Response; body: string }>(
        (resolve, reject) => {
            request(url, function (err, response, body) {
                if (err) {
                    return reject(err);
                }

                resolve({ response, body });
            });
        }
    );
}

async function withUrlsData(urls: string[]): Promise<NodesParsingResult[]> {
    const results: NodesParsingResult[] = [];

    for (const url of urls) {
        Logger.tag("monitoring", "information-retrieval").debug(
            "Retrieving nodes.json: %s",
            url
        );

        const { response, body } = await doRequest(url);
        if (response.statusCode !== 200) {
            throw new Error(
                "Could not download nodes.json from " +
                    url +
                    ": " +
                    response.statusCode +
                    " - " +
                    response.statusMessage
            );
        }

        results.push(await parseNodesJson(body));
    }
    return results;
}

async function retrieveNodeInformationForUrls(
    urls: string[]
): Promise<RetrieveNodeInformationResult> {
    const datas = await withUrlsData(urls);

    let maxTimestamp = datas[0].importTimestamp;
    let minTimestamp = maxTimestamp;

    let failedParsingNodesCount = 0;
    let totalNodesCount = 0;

    for (const data of datas) {
        if (data.importTimestamp >= maxTimestamp) {
            maxTimestamp = data.importTimestamp;
        }
        if (data.importTimestamp <= minTimestamp) {
            minTimestamp = data.importTimestamp;
        }

        failedParsingNodesCount += data.failedNodesCount;
        totalNodesCount += data.totalNodesCount;
    }

    if (
        previousImportTimestamp !== null &&
        maxTimestamp >= previousImportTimestamp
    ) {
        Logger.tag("monitoring", "information-retrieval").debug(
            "No new data, skipping. Current timestamp: %s, previous timestamp: %s",
            formatTimestamp(maxTimestamp),
            formatTimestamp(previousImportTimestamp)
        );
        return {
            failedParsingNodesCount,
            totalNodesCount,
        };
    }
    previousImportTimestamp = maxTimestamp;

    // We do not parallelize here as the sqlite will start slowing down and blocking with too many
    // parallel queries. This has resulted in blocking other requests too and thus in a major slowdown.
    const allNodes = _.flatMap(datas, (data) => data.nodes);

    // Get rid of duplicates from different nodes.json files. Always use the one with the newest
    const sortedNodes = _.orderBy(
        allNodes,
        [(node) => node.lastSeen],
        ["desc"]
    );
    const uniqueNodes = _.uniqBy(sortedNodes, function (node) {
        return node.mac;
    });

    for (const nodeData of uniqueNodes) {
        Logger.tag("monitoring", "information-retrieval").debug(
            "Importing: %s",
            nodeData.mac
        );

        const result = await NodeService.findNodeDataByMac(nodeData.mac);
        if (!result) {
            Logger.tag("monitoring", "information-retrieval").debug(
                "Unknown node, skipping: %s",
                nodeData.mac
            );
            continue;
        }

        await storeNodeInformation(nodeData, result);

        Logger.tag("monitoring", "information-retrieval").debug(
            "Updating / deleting node data done: %s",
            nodeData.mac
        );
    }

    Logger.tag("monitoring", "information-retrieval").debug(
        "Marking missing nodes as offline."
    );

    // Mark nodes as offline that haven't been imported in this run.
    await db.run(
        "UPDATE node_state " +
            "SET state = ?, modified_at = ?" +
            "WHERE import_timestamp < ?",
        [OnlineState.OFFLINE, now(), minTimestamp]
    );

    return {
        failedParsingNodesCount,
        totalNodesCount,
    };
}

function toResponse(row: NodeStateRow): NodeMonitoringStateResponse {
    // TODO: Handle conversion errors.
    return {
        id: row.id,
        created_at: row.created_at,
        domain: row.domain || undefined,
        hostname: row.hostname || undefined,
        import_timestamp: row.import_timestamp,
        last_seen: row.last_seen,
        last_status_mail_sent: row.last_status_mail_sent || undefined,
        last_status_mail_type: isMailType(row.last_status_mail_type)
            ? row.last_status_mail_type
            : undefined,
        mac: row.mac,
        modified_at: row.modified_at,
        monitoring_state: isMonitoringState(row.monitoring_state)
            ? row.monitoring_state
            : undefined,
        site: row.site || undefined,
        state: isOnlineState(row.state) ? row.state : OnlineState.OFFLINE,
        mapId: mapIdFromMAC(row.mac),
    };
}

export async function getAll(
    restParams: RestParams
): Promise<{ total: number; monitoringStates: NodeMonitoringStateResponse[] }> {
    const filterFields = [
        "hostname",
        "mac",
        "monitoring_state",
        "state",
        "last_status_mail_type",
    ];

    const where = Resources.whereCondition(restParams, filterFields);

    const row = await db.get<{ total: number }>(
        "SELECT count(*) AS total FROM node_state WHERE " + where.query,
        where.params
    );

    const total = row?.total || 0;

    const filter = Resources.filterClause<MonitoringSortField>(
        restParams,
        MonitoringSortFieldEnum.ID,
        isMonitoringSortField,
        filterFields
    );

    const monitoringStates = await db.all<NodeStateRow>(
        "SELECT * FROM node_state WHERE " + filter.query,
        filter.params
    );

    return {
        monitoringStates: monitoringStates.map(toResponse),
        total,
    };
}

export async function getByMacs(
    macs: MAC[]
): Promise<Record<MAC, NodeStateData>> {
    if (_.isEmpty(macs)) {
        return {};
    }

    const nodeStateByMac: { [key: string]: NodeStateData } = {};

    for (const subMacs of _.chunk(macs, MONITORING_STATE_MACS_CHUNK_SIZE)) {
        const inCondition = DatabaseUtil.inCondition("mac", subMacs);

        const rows = await db.all<NodeStateRow>(
            "SELECT * FROM node_state WHERE " + inCondition.query,
            inCondition.params
        );

        for (const row of rows) {
            const onlineState = row.state;
            if (!isOnlineState(onlineState)) {
                throw new Error(
                    `Invalid online state in database: "${onlineState}"`
                );
            }

            nodeStateByMac[row.mac] = {
                site: row.site || undefined,
                domain: row.domain || undefined,
                state: onlineState,
            };
        }
    }

    return nodeStateByMac;
}

export async function confirm(token: MonitoringToken): Promise<StoredNode> {
    const { node, nodeSecrets } =
        await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
    if (
        node.monitoringState === MonitoringState.DISABLED ||
        !nodeSecrets.monitoringToken ||
        nodeSecrets.monitoringToken !== token
    ) {
        throw { data: "Invalid token.", type: ErrorTypes.badRequest };
    }

    if (node.monitoringState === MonitoringState.ACTIVE) {
        return node;
    }

    node.monitoringState = MonitoringState.ACTIVE;
    return await NodeService.internalUpdateNode(
        node.token,
        toCreateOrUpdateNode(node),
        node.monitoringState,
        nodeSecrets
    );
}

export async function disable(token: MonitoringToken): Promise<StoredNode> {
    const { node, nodeSecrets } =
        await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
    if (
        node.monitoringState === MonitoringState.DISABLED ||
        !nodeSecrets.monitoringToken ||
        nodeSecrets.monitoringToken !== token
    ) {
        throw { data: "Invalid token.", type: ErrorTypes.badRequest };
    }

    node.monitoringState = MonitoringState.DISABLED;
    nodeSecrets.monitoringToken = undefined;

    return await NodeService.internalUpdateNode(
        node.token,
        toCreateOrUpdateNode(node),
        node.monitoringState,
        nodeSecrets
    );
}

export async function retrieveNodeInformation(): Promise<RetrieveNodeInformationResult> {
    const urls = config.server.map.nodesJsonUrl;
    if (_.isEmpty(urls)) {
        throw new Error(
            "No nodes.json-URLs set. Please adjust config.json: server.map.nodesJsonUrl"
        );
    }

    return await retrieveNodeInformationForUrls(urls);
}

export async function sendMonitoringMails(): Promise<void> {
    Logger.tag("monitoring", "mail-sending").debug(
        "Sending monitoring mails..."
    );

    const startTime = now();

    try {
        await sendOnlineAgainMails(startTime);
    } catch (error) {
        // only logging an continuing with next type
        Logger.tag("monitoring", "mail-sending").error(
            'Error sending "online again" mails.',
            error
        );
    }

    for (const mailType of [
        MailType.MONITORING_OFFLINE_1,
        MailType.MONITORING_OFFLINE_2,
        MailType.MONITORING_OFFLINE_3,
    ]) {
        try {
            await sendOfflineMails(startTime, mailType);
        } catch (error) {
            // only logging an continuing with next type
            Logger.tag("monitoring", "mail-sending").error(
                'Error sending "' + mailType + '" mails.',
                error
            );
        }
    }
}

export async function deleteOfflineNodes(): Promise<void> {
    Logger.tag("nodes", "delete-offline").info(
        `Deleting offline nodes older than ${DELETE_OFFLINE_NODES_AFTER_DURATION} seconds.`
    );

    const deleteBefore = subtract(now(), DELETE_OFFLINE_NODES_AFTER_DURATION);

    await deleteNeverOnlineNodesBefore(deleteBefore);
    await deleteNodesOfflineSinceBefore(deleteBefore);
}

async function deleteNeverOnlineNodesBefore(
    deleteBefore: UnixTimestampSeconds
): Promise<void> {
    Logger.tag("nodes", "delete-never-online").info(
        "Deleting nodes that were never online created before " + deleteBefore
    );

    const deletionCandidates: StoredNode[] =
        await NodeService.findNodesModifiedBefore(deleteBefore);

    Logger.tag("nodes", "delete-never-online").info(
        "Number of nodes created before " +
            deleteBefore +
            ": " +
            deletionCandidates.length
    );

    const deletionCandidateMacs: MAC[] = deletionCandidates.map(
        (node) => node.mac
    );
    const chunks: MAC[][] = _.chunk(
        deletionCandidateMacs,
        NEVER_ONLINE_NODES_DELETION_CHUNK_SIZE
    );

    Logger.tag("nodes", "delete-never-online").info(
        "Number of chunks to check for deletion: " + chunks.length
    );

    for (const macs of chunks) {
        Logger.tag("nodes", "delete-never-online").info(
            "Checking chunk of " + macs.length + " MACs for deletion."
        );

        const placeholders = macs.map(() => "?").join(",");

        const rows = await db.all<NodeStateRow>(
            `SELECT * FROM node_state WHERE mac IN (${placeholders})`,
            macs
        );

        Logger.tag("nodes", "delete-never-online").info(
            "Of the chunk of " +
                macs.length +
                " MACs there were " +
                rows.length +
                " nodes found in monitoring database. Those should be skipped."
        );

        const seenMacs: MAC[] = rows.map((row) => row.mac);
        const neverSeenMacs = _.difference(macs, seenMacs);

        Logger.tag("nodes", "delete-never-online").info(
            "Of the chunk of " +
                macs.length +
                " MACs there are " +
                neverSeenMacs.length +
                " nodes that were never online. Those will be deleted."
        );

        for (const neverSeenMac of neverSeenMacs) {
            await deleteNodeByMac(neverSeenMac);
        }
    }
}

async function deleteNodesOfflineSinceBefore(
    deleteBefore: UnixTimestampSeconds
): Promise<void> {
    const rows = await db.all<NodeStateRow>(
        "SELECT * FROM node_state WHERE state = ? AND last_seen < ?",
        ["OFFLINE", deleteBefore]
    );

    for (const row of rows) {
        await deleteNodeByMac(row.mac);
    }
}

async function deleteNodeByMac(mac: MAC): Promise<void> {
    Logger.tag("nodes", "delete-offline").debug("Deleting node " + mac);

    let node;

    try {
        node = await NodeService.findNodeDataByMac(mac);
    } catch (error) {
        // Only log error. We try to delete the nodes state anyways.
        Logger.tag("nodes", "delete-offline").error(
            "Could not find node to delete: " + mac,
            error
        );
    }

    if (node && node.token) {
        await NodeService.deleteNode(node.token);
    }

    try {
        await db.run("DELETE FROM node_state WHERE mac = ? AND state = ?", [
            mac,
            "OFFLINE",
        ]);
    } catch (error) {
        // Only log error and continue with next node.
        Logger.tag("nodes", "delete-offline").error(
            "Could not delete node state: " + mac,
            error
        );
    }
}