freifunk/ffffng
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Sqlite upgrade and type refactorings

Browse source
This commit is contained in:
baldo 2022-07-18 17:49:42 +02:00
commit 28c8429edd
20 changed files with 873 additions and 663 deletions
Download patch file Download diff file Expand all files Collapse all files

76
server/services/mailService.ts
View file

@ -9,7 +9,29 @@ import Logger from "../logger";
import * as MailTemplateService from "./mailTemplateService";
import * as Resources from "../utils/resources";
import {RestParams} from "../utils/resources";
import {isMailSortField, Mail, MailData, MailId, MailSortField, MailType} from "../types";
import {
EmailAddress, isJSONObject,
isMailSortField, isMailType, JSONObject,
Mail,
MailData,
MailId,
MailSortField,
MailType,
parseJSON,
UnixTimestampSeconds
} from "../types";
import ErrorTypes from "../utils/errorTypes";
type EmaiQueueRow = {
id: MailId,
created_at: UnixTimestampSeconds,
data: string,
email: string,
failures: number,
modified_at: UnixTimestampSeconds,
recipient: EmailAddress,
sender: EmailAddress,
};
const MAIL_QUEUE_DB_BATCH_SIZE = 50;
@ -24,7 +46,7 @@ function transporter() {
{
transport: 'smtp',
pool: true
}
} as JSONObject
));
MailTemplateService.configureTransporter(transporterSingleton);
@ -57,18 +79,29 @@ async function sendMail(options: Mail): Promise<void> {
}
async function findPendingMailsBefore(beforeMoment: Moment, limit: number): Promise<Mail[]> {
const rows = await db.all(
const rows = await db.all<EmaiQueueRow>(
'SELECT * FROM email_queue WHERE modified_at < ? AND failures < ? ORDER BY id ASC LIMIT ?',
[beforeMoment.unix(), 5, limit],
);
return _.map(rows, row => deepExtend(
{},
row,
{
data: JSON.parse(row.data)
return rows.map(row => {
const mailType = row.email;
if (!isMailType(mailType)) {
throw new Error(`Invalid mailtype in database: ${mailType}`);
}
));
const data = parseJSON(row.data);
if (!isJSONObject(data)) {
throw new Error(`Invalid email data in database: ${typeof data}`);
}
return {
id: row.id,
email: mailType,
sender: row.sender,
recipient: row.recipient,
data,
failures: row.failures,
};
});
}
async function removePendingMailFromQueue(id: MailId): Promise<void> {
@ -85,8 +118,7 @@ async function incrementFailureCounterForPendingEmail(id: MailId): Promise<void>
async function sendPendingMail(pendingMail: Mail): Promise<void> {
try {
await sendMail(pendingMail);
}
catch (error) {
} catch (error) {
// we only log the error and increment the failure counter as we want to continue with pending mails
Logger.tag('mail', 'queue').error('Error sending pending mail[' + pendingMail.id + ']:', error);
@ -98,10 +130,14 @@ async function sendPendingMail(pendingMail: Mail): Promise<void> {
}
async function doGetMail(id: MailId): Promise<Mail> {
return await db.get('SELECT * FROM email_queue WHERE id = ?', [id]);
const row = await db.get<Mail>('SELECT * FROM email_queue WHERE id = ?', [id]);
if (row === undefined) {
throw {data: 'Mail not found.', type: ErrorTypes.notFound};
}
return row;
}
export async function enqueue (sender: string, recipient: string, email: MailType, data: MailData): Promise<void> {
export async function enqueue(sender: string, recipient: string, email: MailType, data: MailData): Promise<void> {
if (!_.isPlainObject(data)) {
throw new Error('Unexpected data: ' + data);
}
@ -113,17 +149,17 @@ export async function enqueue (sender: string, recipient: string, email: MailTyp
);
}
export async function getMail (id: MailId): Promise<Mail> {
export async function getMail(id: MailId): Promise<Mail> {
return await doGetMail(id);
}
export async function getPendingMails (restParams: RestParams): Promise<{mails: Mail[], total: number}> {
const row = await db.get(
export async function getPendingMails(restParams: RestParams): Promise<{ mails: Mail[], total: number }> {
const row = await db.get<{ total: number }>(
'SELECT count(*) AS total FROM email_queue',
[],
);
const total = row.total;
const total = row?.total || 0;
const filter = Resources.filterClause(
restParams,
@ -143,11 +179,11 @@ export async function getPendingMails (restParams: RestParams): Promise<{mails:
}
}
export async function deleteMail (id: MailId): Promise<void> {
export async function deleteMail(id: MailId): Promise<void> {
await removePendingMailFromQueue(id);
}
export async function resetFailures (id: MailId): Promise<Mail> {
export async function resetFailures(id: MailId): Promise<Mail> {
const statement = await db.run(
'UPDATE email_queue SET failures = 0, modified_at = ? WHERE id = ?',
[moment().unix(), id],
@ -160,7 +196,7 @@ export async function resetFailures (id: MailId): Promise<Mail> {
return await doGetMail(id);
}
export async function sendPendingMails (): Promise<void> {
export async function sendPendingMails(): Promise<void> {
Logger.tag('mail', 'queue').debug('Start sending pending mails...');
const startTime = moment();

8
server/services/mailTemplateService.ts
View file

@ -13,7 +13,13 @@ import {MailData, Mail} from "../types";
const templateBasePath = __dirname + '/../mailTemplates';
const snippetsBasePath = templateBasePath + '/snippets';
const templateFunctions: {[key: string]: (...data: MailData) => string} = {};
const templateFunctions: {
[key: string]:
| ((name: string, data: MailData) => string)
| ((data: MailData) => string)
| ((href: string, text: string) => string)
| ((unix: number) => string)
} = {};
function renderSnippet(this: any, name: string, data: MailData): string {
const snippetFile = snippetsBasePath + '/' + name + '.html';

21
server/services/monitoringService.test.ts
View file

@ -1,6 +1,6 @@
import moment from 'moment';
import {ParsedNode, parseNode, parseNodesJson, parseTimestamp} from "./monitoringService";
import {MAC, OnlineState, to} from "../types";
import {Domain, MAC, OnlineState, Site} from "../types";
import Logger from '../logger';
import {MockLogger} from "../__mocks__/logger";
@ -44,6 +44,7 @@ test('parseTimestamp() should fail parsing empty timestamp string', () => {
test('parseTimestamp() should fail parsing invalid timestamp string', () => {
// given
// noinspection UnnecessaryLocalVariableJS
const timestamp = TIMESTAMP_INVALID_STRING;
// when
@ -240,12 +241,12 @@ test('parseNode() should succeed parsing node without site and domain', () => {
// then
const expectedParsedNode: ParsedNode = {
mac: to("12:34:56:78:90:AB"),
mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: importTimestamp,
state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
site: to("<unknown-site>"),
domain: to("<unknown-domain>"),
site: "<unknown-site>" as Site,
domain: "<unknown-domain>" as Domain,
};
expect(parseNode(importTimestamp, nodeData)).toEqual(expectedParsedNode);
});
@ -272,12 +273,12 @@ test('parseNode() should succeed parsing node with site and domain', () => {
// then
const expectedParsedNode: ParsedNode = {
mac: to("12:34:56:78:90:AB"),
mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: importTimestamp,
state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
site: to("test-site"),
domain: to("test-domain")
site: "test-site" as Site,
domain: "test-domain" as Domain,
};
expect(parseNode(importTimestamp, nodeData)).toEqual(expectedParsedNode);
});
@ -461,12 +462,12 @@ test('parseNodesJson() should parse valid nodes', () => {
// then
const expectedParsedNode: ParsedNode = {
mac: to("12:34:56:78:90:AB"),
mac: "12:34:56:78:90:AB" as MAC,
importTimestamp: parseTimestamp(TIMESTAMP_VALID_STRING),
state: OnlineState.ONLINE,
lastSeen: parseTimestamp(TIMESTAMP_VALID_STRING),
site: to("test-site"),
domain: to("test-domain"),
site: "test-site" as Site,
domain: "test-domain" as Domain,
};
expect(result.importTimestamp.isValid()).toBe(true);

87
server/services/monitoringService.ts
View file

@ -3,7 +3,7 @@ import moment, {Moment, unitOfTime} from "moment";
import request from "request";
import {config} from "../config";
import {db, Statement} from "../db/database";
import {db, RunResult} from "../db/database";
import * as DatabaseUtil from "../utils/databaseUtil";
import ErrorTypes from "../utils/errorTypes";
import Logger from "../logger";
@ -12,14 +12,15 @@ import * as MailService from "../services/mailService";
import * as NodeService from "../services/nodeService";
import * as Resources from "../utils/resources";
import {RestParams} from "../utils/resources";
import {normalizeMac} from "../utils/strings";
import {normalizeMac, parseInteger} from "../utils/strings";
import {monitoringDisableUrl} from "../utils/urlBuilder";
import CONSTRAINTS from "../validation/constraints";
import {forConstraint} from "../validation/validator";
import {
Domain,
equal,
Hostname,
isMonitoringSortField,
isOnlineState,
MAC,
MailType,
MonitoringSortField,
@ -29,10 +30,25 @@ import {
NodeStateData,
OnlineState,
Site,
to,
UnixTimestampSeconds
} from "../types";
type NodeStateRow = {
id: number,
created_at: UnixTimestampSeconds,
domain: Domain | null,
hostname: Hostname | null,
import_timestamp: UnixTimestampSeconds,
last_seen: UnixTimestampSeconds,
last_status_mail_sent: string | null,
last_status_mail_type: string | null,
mac: MAC,
modified_at: UnixTimestampSeconds,
monitoring_state: string | null,
site: Site | null,
state: string,
};
const MONITORING_STATE_MACS_CHUNK_SIZE = 100;
const NEVER_ONLINE_NODES_DELETION_CHUNK_SIZE = 20;
const MONITORING_MAILS_DB_BATCH_SIZE = 50;
@ -193,7 +209,7 @@ export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode {
'Node ' + nodeId + ': Invalid MAC: ' + nodeData.nodeinfo.network.mac
);
}
const mac = normalizeMac(nodeData.nodeinfo.network.mac);
const mac = normalizeMac(nodeData.nodeinfo.network.mac) as MAC;
if (!_.isPlainObject(nodeData.flags)) {
throw new Error(
@ -214,23 +230,23 @@ export function parseNode(importTimestamp: Moment, nodeData: any): ParsedNode {
);
}
let site = null;
let site = "<unknown-site>" as Site; // FIXME: Handle this
if (_.isPlainObject(nodeData.nodeinfo.system) && _.isString(nodeData.nodeinfo.system.site_code)) {
site = nodeData.nodeinfo.system.site_code;
site = nodeData.nodeinfo.system.site_code as Site;
}
let domain = null;
let domain = "<unknown-domain>" as Domain; // FIXME: Handle this
if (_.isPlainObject(nodeData.nodeinfo.system) && _.isString(nodeData.nodeinfo.system.domain_code)) {
domain = nodeData.nodeinfo.system.domain_code;
domain = nodeData.nodeinfo.system.domain_code as Domain;
}
return {
mac: to(mac),
mac,
importTimestamp: importTimestamp,
state: isOnline ? OnlineState.ONLINE : OnlineState.OFFLINE,
lastSeen: lastSeen,
site: to(site || '<unknown-site>'), // FIXME: Handle this
domain: to(domain || '<unknown-domain>') // FIXME: Handle this
site,
domain,
};
}
@ -279,7 +295,7 @@ export function parseNodesJson(body: string): NodesParsingResult {
return result;
}
async function updateSkippedNode(id: NodeId, node?: Node): Promise<Statement> {
async function updateSkippedNode(id: NodeId, node?: Node): Promise<RunResult> {
return await db.run(
'UPDATE node_state ' +
'SET hostname = ?, monitoring_state = ?, modified_at = ?' +
@ -352,8 +368,7 @@ async function sendMonitoringMailsBatched(
{
node: node,
lastSeen: nodeState.last_seen,
disableUrl: monitoringDisableUrl(monitoringToken)
disableUrl: monitoringDisableUrl(monitoringToken),
}
);
@ -378,7 +393,7 @@ async function sendMonitoringMailsBatched(
async function sendOnlineAgainMails(startTime: Moment): Promise<void> {
await sendMonitoringMailsBatched(
'online again',
'monitoring-online-again',
MailType.MONITORING_ONLINE_AGAIN,
async (): Promise<any[]> => await db.all(
'SELECT * FROM node_state ' +
'WHERE modified_at < ? AND state = ? AND last_status_mail_type IN (' +
@ -395,10 +410,11 @@ async function sendOnlineAgainMails(startTime: Moment): Promise<void> {
);
}
async function sendOfflineMails(startTime: Moment, mailNumber: number): Promise<void> {
async function sendOfflineMails(startTime: Moment, mailType: MailType): Promise<void> {
const mailNumber = parseInteger(mailType.split("-")[2]);
await sendMonitoringMailsBatched(
'offline ' + mailNumber,
'monitoring-offline-' + mailNumber,
mailType,
async (): Promise<any[]> => {
const previousType =
mailNumber === 1 ? 'monitoring-online-again' : ('monitoring-offline-' + (mailNumber - 1));
@ -556,12 +572,12 @@ export async function getAll(restParams: RestParams): Promise<{ total: number, m
const where = Resources.whereCondition(restParams, filterFields);
const row = await db.get(
const row = await db.get<{ total: number }>(
'SELECT count(*) AS total FROM node_state WHERE ' + where.query,
_.concat([], where.params),
);
const total = row.total;
const total = row?.total || 0;
const filter = Resources.filterClause(
restParams,
@ -578,7 +594,7 @@ export async function getAll(restParams: RestParams): Promise<{ total: number, m
return {monitoringStates, total};
}
export async function getByMacs(macs: MAC[]): Promise<Record<string, NodeStateData>> {
export async function getByMacs(macs: MAC[]): Promise<Record<MAC, NodeStateData>> {
if (_.isEmpty(macs)) {
return {};
}
@ -588,13 +604,22 @@ export async function getByMacs(macs: MAC[]): Promise<Record<string, NodeStateDa
for (const subMacs of _.chunk(macs, MONITORING_STATE_MACS_CHUNK_SIZE)) {
const inCondition = DatabaseUtil.inCondition('mac', subMacs);
const rows = await db.all(
const rows = await db.all<NodeStateRow>(
'SELECT * FROM node_state WHERE ' + inCondition.query,
_.concat([], inCondition.params),
);
for (const row of rows) {
nodeStateByMac[row.mac] = row;
const onlineState = row.state;
if (!isOnlineState(onlineState)) {
throw new Error(`Invalid online state in database: "${onlineState}"`);
}
nodeStateByMac[row.mac] = {
site: row.site || "<unknown-site>" as Site, // FIXME: Handle this
domain: row.domain || "<unknown-domain>" as Domain, // FIXME: Handle this
state: onlineState,
};
}
}
@ -603,7 +628,7 @@ export async function getByMacs(macs: MAC[]): Promise<Record<string, NodeStateDa
export async function confirm(token: MonitoringToken): Promise<Node> {
const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
if (!node.monitoring || !nodeSecrets.monitoringToken || !equal(nodeSecrets.monitoringToken, token)) {
if (!node.monitoring || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) {
throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
}
@ -619,7 +644,7 @@ export async function confirm(token: MonitoringToken): Promise<Node> {
export async function disable(token: MonitoringToken): Promise<Node> {
const {node, nodeSecrets} = await NodeService.getNodeDataWithSecretsByMonitoringToken(token);
if (!node.monitoring || !nodeSecrets.monitoringToken || !equal(nodeSecrets.monitoringToken, token)) {
if (!node.monitoring || !nodeSecrets.monitoringToken || nodeSecrets.monitoringToken !== token) {
throw {data: 'Invalid token.', type: ErrorTypes.badRequest};
}
@ -654,14 +679,18 @@ export async function sendMonitoringMails(): Promise<void> {
.error('Error sending "online again" mails.', error);
}
for (let mailNumber = 1; mailNumber <= 3; mailNumber++) {
for (const mailType of [
MailType.MONITORING_OFFLINE_1,
MailType.MONITORING_OFFLINE_2,
MailType.MONITORING_OFFLINE_3,
]) {
try {
await sendOfflineMails(startTime, mailNumber);
await sendOfflineMails(startTime, mailType);
} catch (error) {
// only logging an continuing with next type
Logger
.tag('monitoring', 'mail-sending')
.error('Error sending "offline ' + mailNumber + '" mails.', error);
.error('Error sending "' + mailType + '" mails.', error);
}
}
}
@ -767,7 +796,7 @@ async function deleteNeverOnlineNodesBefore(deleteBefore: UnixTimestampSeconds):
}
async function deleteNodesOfflineSinceBefore(deleteBefore: UnixTimestampSeconds): Promise<void> {
const rows = await db.all(
const rows = await db.all<NodeStateRow>(
'SELECT * FROM node_state WHERE state = ? AND last_seen < ?',
[
'OFFLINE',

57
server/services/nodeService.ts
View file

@ -11,14 +11,18 @@ import * as MailService from "../services/mailService";
import {normalizeString} from "../utils/strings";
import {monitoringConfirmUrl, monitoringDisableUrl} from "../utils/urlBuilder";
import {
Coordinates,
EmailAddress,
FastdKey,
Hostname,
MAC,
MailType,
MonitoringState,
MonitoringToken,
Nickname,
Node,
NodeSecrets,
NodeStatistics,
to,
Token,
toUnixTimestampSeconds,
unhandledEnumField,
@ -60,18 +64,17 @@ enum LINE_PREFIX {
const filenameParts = ['hostname', 'mac', 'key', 'token', 'monitoringToken'];
function generateToken<Type extends { readonly __tag: symbol, value: any } =
{ readonly __tag: unique symbol, value: never }>(): Type {
return to<Type>(crypto.randomBytes(8).toString('hex'));
function generateToken<Type extends string & { readonly __tag: symbol } = never>(): Type {
return crypto.randomBytes(8).toString('hex') as Type;
}
function toNodeFilesPattern(filter: NodeFilter): string {
const fields: (string | undefined)[] = [
filter.hostname,
filter.mac?.value,
filter.key?.value,
filter.token?.value,
filter.monitoringToken?.value,
filter.mac,
filter.key,
filter.token,
filter.monitoringToken,
];
const pattern = fields.map((value) => value || '*').join('@');
@ -124,7 +127,7 @@ function isDuplicate(filter: NodeFilter, token: Token | null): boolean {
return true;
}
return parseNodeFilename(files[0]).token !== token.value;
return parseNodeFilename(files[0]).token !== token;
}
function checkNoDuplicates(token: Token | null, node: Node, nodeSecrets: NodeSecrets): void {
@ -169,9 +172,9 @@ function getNodeValue(prefix: LINE_PREFIX, node: Node, nodeSecrets: NodeSecrets)
case LINE_PREFIX.COORDS:
return node.coords || "";
case LINE_PREFIX.MAC:
return node.mac.value;
return node.mac;
case LINE_PREFIX.TOKEN:
return node.token.value;
return node.token;
case LINE_PREFIX.MONITORING:
if (node.monitoring && node.monitoringConfirmed) {
return "aktiv";
@ -180,7 +183,7 @@ function getNodeValue(prefix: LINE_PREFIX, node: Node, nodeSecrets: NodeSecrets)
}
return "";
case LINE_PREFIX.MONITORING_TOKEN:
return nodeSecrets.monitoringToken?.value || "";
return nodeSecrets.monitoringToken || "";
default:
return unhandledEnumField(prefix);
}
@ -255,13 +258,13 @@ async function deleteNodeFile(token: Token): Promise<void> {
}
class NodeBuilder {
public token: Token = to(""); // FIXME: Either make token optional in Node or handle this!
public nickname: string = "";
public email: string = "";
public hostname: string = ""; // FIXME: Either make hostname optional in Node or handle this!
public coords?: string;
public token: Token = "" as Token; // FIXME: Either make token optional in Node or handle this!
public nickname: Nickname = "" as Nickname;
public email: EmailAddress = "" as EmailAddress;
public hostname: Hostname = "" as Hostname; // FIXME: Either make hostname optional in Node or handle this!
public coords?: Coordinates;
public key?: FastdKey;
public mac: MAC = to(""); // FIXME: Either make mac optional in Node or handle this!
public mac: MAC = "" as MAC; // FIXME: Either make mac optional in Node or handle this!
public monitoring: boolean = false;
public monitoringConfirmed: boolean = false;
public monitoringState: MonitoringState = MonitoringState.DISABLED;
@ -291,22 +294,22 @@ class NodeBuilder {
function setNodeValue(prefix: LINE_PREFIX, node: NodeBuilder, nodeSecrets: NodeSecrets, value: string) {
switch (prefix) {
case LINE_PREFIX.HOSTNAME:
node.hostname = value;
node.hostname = value as Hostname;
break;
case LINE_PREFIX.NICKNAME:
node.nickname = value;
node.nickname = value as Nickname;
break;
case LINE_PREFIX.EMAIL:
node.email = value;
node.email = value as EmailAddress;
break;
case LINE_PREFIX.COORDS:
node.coords = value;
node.coords = value as Coordinates;
break;
case LINE_PREFIX.MAC:
node.mac = to(value);
node.mac = value as MAC;
break;
case LINE_PREFIX.TOKEN:
node.token = to(value);
node.token = value as Token;
break;
case LINE_PREFIX.MONITORING:
const active = value === 'aktiv';
@ -317,7 +320,7 @@ function setNodeValue(prefix: LINE_PREFIX, node: NodeBuilder, nodeSecrets: NodeS
active ? MonitoringState.ACTIVE : (pending ? MonitoringState.PENDING : MonitoringState.DISABLED);
break;
case LINE_PREFIX.MONITORING_TOKEN:
nodeSecrets.monitoringToken = to<MonitoringToken>(value);
nodeSecrets.monitoringToken = value as MonitoringToken;
break;
default:
return unhandledEnumField(prefix);
@ -340,7 +343,7 @@ async function parseNodeFile(file: string): Promise<{ node: Node, nodeSecrets: N
for (const line of lines) {
if (line.substring(0, 5) === 'key "') {
node.key = to<FastdKey>(normalizeString(line.split('"')[1]));
node.key = normalizeString(line.split('"')[1]) as FastdKey;
} else {
for (const prefix of Object.values(LINE_PREFIX)) {
if (line.substring(0, prefix.length) === prefix) {
@ -393,7 +396,7 @@ async function sendMonitoringConfirmationMail(node: Node, nodeSecrets: NodeSecre
await MailService.enqueue(
config.server.email.from,
node.nickname + ' <' + node.email + '>',
'monitoring-confirmation',
MailType.MONITORING_CONFIRMATION,
{
node: node,
confirmUrl: confirmUrl,