#!/bin/env python import argparse import email import json import mimetypes import re import smtplib import ssl import sys from datetime import datetime, timezone, timedelta from email import encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from os import getenv from pathlib import Path from textwrap import dedent from time import sleep import humanize import pgsql class Config: """ Get config from environment variables """ def __init__(self): self.verbose = False self.revision_age = timedelta(days=14) self.note_age = timedelta(days=95) self.upload_age = timedelta(days=1) self.postgres_hostname = getenv('POSTGRES_HOSTNAME', 'localhost') self.postgres_username = getenv('POSTGRES_USERNAME', 'hedgedoc') self.postgres_password = getenv('POSTGRES_PASSWORD', 'geheim') self.postgres_database = getenv('POSTGRES_DATABASE', 'hedgedoc') self.postgres_port = int(getenv('POSTGRES_PORT', '5432')) self.smtp_hostname = getenv('SMTP_HOSTNAME', 'localhost') self.smtp_port = int(getenv('SMTP_PORT', '587')) self.smtp_username = getenv('SMTP_USERNAME', '') self.smtp_password = getenv('SMTP_PASSWORD', '') self.smtp_from = getenv('SMTP_FROM', '') self.url = getenv('URL', 'http://localhost:3000') self.uploads_path = getenv('UPLOADS_PATH', '') self.max_attachment_size = int(getenv('MAX_ATTACHMENT_SIZE', str(20 * 1024 * 1024))) class EmailSender: """ Send email message through SMTP """ def __init__(self, hostname: str, port: int, username: str, password: str, mail_from: str): self.hostname = hostname self.port = port self.username = username self.password = password self.mail_from = mail_from def send(self, message: email.message.Message) -> None: """ Using the configured SMTP coordinates, send the message out. The code assumes the submission protocol with StartTLS enabled, and authentication required. :param message: to be sent :return: """ try: smtp_server = smtplib.SMTP(self.hostname, port=self.port) context = ssl.create_default_context() smtp_server.starttls(context=context) smtp_server.login(self.username, self.password) smtp_server.send_message(message) except Exception as e: print(f'Unable to send mail through {self}: {e}') raise e print(f'Report email to {message["To"]} sent successfully.') def __str__(self): return f'EmailSender<{self.hostname},{self.port},{self.username},{self.mail_from}>' class HedgedocExpire: # matches the file name in links to uploads, e.g. https://example.com/uploads/abc.png or /uploads/abc.png UPLOAD_REFERENCE = re.compile(r'/uploads/([^/\s()\[\]<>"\'?#]+)') def __init__(self, config: Config, email_sender: EmailSender): self.config = config self.email_sender = email_sender @staticmethod def email_from_email_or_profile(row) -> str: """ Get the email address of the creator from a database row. If the email column is populated, use that, otherwise try to extract it from the login profile. The profile is a JSON object that has an emails array. We're using the first address from there. :param row: database row as a dict with email and profile columns :return: email address """ if row['email'] is not None: return row['email'] profile = json.loads(row['profile']) return profile['emails'][0] def notes_to_be_expired(self, db) -> list[any]: """ Get a list of all notes to be expired. :return: """ notes = [] cutoff = datetime.now(timezone.utc) - self.config.note_age with db.prepare('''SELECT "Notes"."alias", "Notes"."content", "Notes"."createdAt", "Notes"."ownerId", "Notes"."shortid", "Notes"."id", "Notes"."title", "Notes"."updatedAt", "Users"."email", "Users"."profile" FROM "Notes", "Users" WHERE "Notes"."updatedAt" < $1 AND "Notes"."ownerId" = "Users"."id" ORDER BY "Notes"."updatedAt" ''') as notes_older_than: for row in notes_older_than(cutoff): notes.append({ 'alias': row.alias if row.alias is not None else row.shortid, 'content': row.content, 'createdAt': row.createdAt, 'email': row.email, "id": row.id, 'ownerId': row.ownerId, 'profile': row.profile, 'shortid': row.shortid, 'title': row.title, 'updatedAt': row.updatedAt }) return notes def revisions_to_be_expired(self, db) -> list[any]: """ Obtain a list of revisions to be expired. :param db: the database connection :return: """ revisions = [] cutoff = datetime.now(timezone.utc) - self.config.revision_age with db.prepare('''SELECT "Notes"."alias", "Revisions"."createdAt", "Users"."email", "Users"."profile", "Revisions"."id" as "revisionId", "Notes"."id" as "noteId", "Notes"."shortid" as "shortid", "Notes"."title" FROM "Revisions", "Notes", "Users" WHERE "Revisions"."createdAt" < $1 AND "Revisions"."noteId" = "Notes"."id" AND "Notes"."ownerId" = "Users"."id" ORDER BY "Notes"."createdAt", "Revisions"."createdAt" ''') as revs_older_than: for row in revs_older_than(cutoff): revisions.append({ 'alias': row.alias, 'createdAt': row.createdAt, 'email': row.email, 'noteId': row.noteId, 'profile': row.profile, 'revisionId': row.revisionId, 'shortid': row.shortid, 'title': row.title }) return revisions def referenced_uploads(self, db, note_cutoff: datetime, revision_cutoff: datetime) -> set[str]: """ Collect the file names of all uploads referenced by notes updated since note_cutoff, and by revisions created since revision_cutoff. :param db: the database connection :param note_cutoff: ignore notes last updated before this time :param revision_cutoff: ignore revisions created before this time :return: set of file names in the uploads directory """ referenced = set() with db.prepare('''SELECT COALESCE("content", '') AS "text" FROM "Notes" WHERE "updatedAt" >= $1 UNION ALL SELECT COALESCE("content", '') || COALESCE("lastContent", '') || COALESCE("patch", '') AS "text" FROM "Revisions" WHERE "createdAt" >= $2 ''') as texts: for row in texts(note_cutoff, revision_cutoff): referenced.update(self.UPLOAD_REFERENCE.findall(row.text)) return referenced def uploads_to_be_expired(self, db, note_cutoff: datetime, revision_cutoff: datetime) -> list[Path]: """ Get a list of all uploaded files that are not referenced by any note or revision, and are older than the configured upload age. The age check avoids removing files that have just been uploaded, but whose note has not been saved to the database yet. :param db: the database connection :param note_cutoff: notes last updated before this time do not count as references :param revision_cutoff: revisions created before this time do not count as references :return: list of paths to unreferenced files """ uploads = Path(self.config.uploads_path) referenced = self.referenced_uploads(db, note_cutoff, revision_cutoff) cutoff = (datetime.now(timezone.utc) - self.config.upload_age).timestamp() files = [] for file in sorted(uploads.iterdir()): if file.name.startswith('.') or not file.is_file() or file.is_symlink(): continue if file.name not in referenced and file.stat().st_mtime < cutoff: files.append(file) return files def check_uploads_to_be_expired(self, db) -> str: """ Return a list of uploads that will be expired, assuming the notes and revisions listed by the other checks have been expired. :param db: the database connection :return: a multi-line text suitable for humans to read """ if not self.config.uploads_path: return '' r = f'Uploads to be deleted not referenced by any note and older than {humanize.naturaldelta(self.config.upload_age)}:\n' now = datetime.now(timezone.utc) for file in self.uploads_to_be_expired(db, now - self.config.note_age, now - self.config.revision_age): stat = file.stat() age = now - datetime.fromtimestamp(stat.st_mtime, timezone.utc) r += f' {file.name} ({humanize.naturalsize(stat.st_size)}, {humanize.naturaldelta(age)})\n' return r def check_notes_to_be_expired(self, db) -> str: """ Return a list of notes that will be expired. :param db: the database connection :return: a multi-line text suitable for humans to read """ r = '' cutoff = datetime.now(timezone.utc) - self.config.note_age r += f'Notes to be deleted not changed since {cutoff} ({humanize.naturaldelta(self.config.note_age)}):\n' for note in self.notes_to_be_expired(db): age = datetime.now(timezone.utc) - datetime.fromisoformat(note['updatedAt']) url = self.config.url + '/' + (note["alias"] if note["alias"] is not None else note["shortid"]) r += f' {self.email_from_email_or_profile(note)} ({humanize.naturaldelta(age)}) {url}: {note["title"]}\n' return r def check_revisions_to_be_expired(self, db) -> str: """ Return a list of revisions that will be expired. :return: a multi-line text suitable for humans to read """ r = '' cutoff = datetime.now(timezone.utc) - self.config.revision_age r += f'Revisions to be deleted created before {cutoff} ({humanize.naturaldelta(self.config.revision_age)}):\n' notes = {} for row in self.revisions_to_be_expired(db): row['age'] = datetime.now(timezone.utc) - datetime.fromisoformat(row['createdAt']) if row['noteId'] not in notes: notes[row['noteId']] = [] notes[row['noteId']].append(row) for revisionId, revisions in notes.items(): addr = self.email_from_email_or_profile(revisions[0]) url = self.config.url + '/' + ( revisions[0]["alias"] if revisions[0]["alias"] is not None else revisions[0]["shortid"]) r += f' {addr} {url}: {revisions[0]["title"]}\n' for rev in revisions: r += f' {humanize.naturaldelta(rev["age"])}: {rev["revisionId"]}\n' return r def upload_attachments(self, content: str) -> tuple[list[MIMEBase], list[str]]: """ Create mail attachments for the uploads referenced in a note, up to the configured total size. :param content: the note content :return: the attachments, and the names of referenced uploads that could not be attached """ attachments = [] skipped = [] if not self.config.uploads_path: return attachments, skipped total = 0 for name in sorted(set(self.UPLOAD_REFERENCE.findall(content))): file = Path(self.config.uploads_path) / name if not file.is_file() or file.is_symlink(): continue size = file.stat().st_size if total + size > self.config.max_attachment_size: skipped.append(name) continue total += size content_type, _ = mimetypes.guess_type(name) maintype, subtype = (content_type or 'application/octet-stream').split('/', 1) attachment = MIMEBase(maintype, subtype) attachment.set_payload(file.read_bytes()) encoders.encode_base64(attachment) attachment.add_header('Content-Disposition', 'attachment', filename=name) attachments.append(attachment) return attachments, skipped def expire_old_notes(self, db) -> None: """ Email old notes to their owners, then delete them. :param db: the database connection :return: """ with db.prepare('DELETE FROM "Notes" WHERE "id" = $1') as delete_statement: for note in self.notes_to_be_expired(db): try: # notes that are empty or only contain whitespace are deleted without sending a backup has_content = len((note["content"] or '').strip()) > 0 if has_content: note_age = datetime.now(timezone.utc) - datetime.fromisoformat(note['updatedAt']) attachments, skipped = self.upload_attachments(note["content"]) msg = MIMEMultipart() msg['From'] = self.email_sender.mail_from msg['To'] = self.email_from_email_or_profile(note) msg['Subject'] = f'Your HedgeDoc Note "{note["title"]}" has expired' msg.attach(MIMEText(dedent(f'''\ You created the note titled "{note["title"]}" on {note["createdAt"]}. It was lasted updated {note['updatedAt']}, {humanize.naturaldelta(note_age)} ago. We expire all notes that have not been updated within {humanize.naturaldelta(self.config.note_age)}. Please find attached the contents of the latest revision of your note, and the files uploaded to it. ''') + (dedent(f'''\ These uploaded files were too large to attach: ''') + ''.join(f' {name}\n' for name in skipped) + '\n' if skipped else '') + dedent(f'''\ The admin team for {self.config.url} '''), 'plain', 'utf-8')) md = MIMEText(note["content"], 'markdown', 'utf-8') filename = note['title'].encode('ascii', 'ignore').decode('utf-8') if len(filename) == 0: filename = 'note' md.add_header('Content-Disposition', f'attachment; filename={filename}.md') msg.attach(md) for attachment in attachments: msg.attach(attachment) self.email_sender.send(msg) # email backup of the note sent, now we can delete it delete_statement(note["id"]) if self.config.verbose: url = self.config.url + '/' + (note["alias"] if note["alias"] is not None else note["shortid"]) if has_content: print(f'Note "{note["title"]}" ({url}) emailed to {self.email_from_email_or_profile(note)}') else: print(f'Empty note "{note["title"]}" ({url}) deleted') except Exception as e: print(f'Unable to send email to {self.email_from_email_or_profile(note)}: {e}', file=sys.stderr) def expire_old_revisions(self, db) -> None: """ Removes all revision on all notes that have been modified earlier than age. :param db: the database connection :return: """ cutoff = datetime.now(timezone.utc) - self.config.revision_age with db.prepare('DELETE FROM "Revisions" WHERE "createdAt" < $1 RETURNING id') as delete: rows = list(delete(cutoff)) if self.config.verbose: print(f'Deleted {len(rows)} old revisions') def expire_unreferenced_uploads(self, db) -> None: """ Remove all uploaded files that are not referenced by any remaining note or revision. :param db: the database connection :return: """ if not self.config.uploads_path: return # count every note and revision that is still in the database, including notes that failed to expire epoch = datetime.fromtimestamp(0, timezone.utc) deleted = 0 for file in self.uploads_to_be_expired(db, epoch, epoch): try: file.unlink() deleted += 1 except Exception as e: print(f'Unable to delete upload {file}: {e}', file=sys.stderr) if self.config.verbose: print(f'Deleted {deleted} unreferenced uploads') def cmd_check(self) -> None: with pgsql.Connection((self.config.postgres_hostname, self.config.postgres_port), self.config.postgres_username, self.config.postgres_password) as db: print(self.check_revisions_to_be_expired(db) + self.check_notes_to_be_expired(db) + self.check_uploads_to_be_expired(db)) def cmd_emailcheck(self) -> None: with pgsql.Connection((self.config.postgres_hostname, self.config.postgres_port), self.config.postgres_username, self.config.postgres_password) as db: report = (self.check_revisions_to_be_expired(db) + self.check_notes_to_be_expired(db) + self.check_uploads_to_be_expired(db)) msg = MIMEMultipart() msg['From'] = self.email_sender.mail_from msg['To'] = self.email_sender.mail_from msg['Subject'] = f'Hedgedoc Expire: Report' msg.attach(MIMEText(dedent(f'''\ This report shows which notes, revisions and uploads would be deleted if expire would be run now. ''') + report + dedent(f'''\ The admin team for {self.config.url} '''))) self.email_sender.send(msg) def cmd_expire(self) -> None: with pgsql.Connection((self.config.postgres_hostname, self.config.postgres_port), self.config.postgres_username, self.config.postgres_password) as db: self.expire_old_revisions(db) self.expire_old_notes(db) self.expire_unreferenced_uploads(db) def main(): parser = argparse.ArgumentParser( prog='hedgedoc-expire', formatter_class=argparse.RawDescriptionHelpFormatter, description=dedent('''\ Remove old notes and revisions from Hedgedoc Notes that have not been updated in the specified time will be emailed to the creator and then deleted. Revisions of notes that have been created before the specified time will be deleted. If UPLOADS_PATH is set, uploaded files no longer referenced by any note or revision will be deleted. '''), epilog=dedent('''\ command is one of: - check: print a list of revisions, notes and uploads to be expired - cron: run expire every 24 hours - emailcheck: send am email from the configured sender to themselves with the the check report - expire: expire old revisions, untouched notes and unreferenced uploads See https://git.hamburg.ccc.de/CCCHH/hedgedoc-expire ''') ) parser.add_argument('-n', '--notes', metavar='DAYS', type=float, default=95, help='remove all notes not changed in these many days') parser.add_argument('-r', '--revisions', metavar='DAYS', type=float, default=14, help='remove all revisions created more than these many days ago') parser.add_argument('-u', '--uploads', metavar='DAYS', type=float, default=1, help='remove unreferenced uploads older than these many days') parser.add_argument('command', choices=['check', 'cron', 'emailcheck', 'expire'], default='check', nargs='?', help='action to perform') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print more info while running') args = parser.parse_args() config = Config() config.note_age = timedelta(days=args.notes) config.revision_age = timedelta(days=args.revisions) config.upload_age = timedelta(days=args.uploads) config.verbose = args.verbose mail = EmailSender(config.smtp_hostname, config.smtp_port, config.smtp_username, config.smtp_password, config.smtp_from) hedgedoc_expire = HedgedocExpire(config, mail) if args.command == 'check': hedgedoc_expire.cmd_check() elif args.command == 'cron': while True: next_expire = datetime.now().replace(hour=2, minute=0, second=0, microsecond=0) + timedelta(days=1) if args.verbose: print(f'Next expire execution: {next_expire}') seconds = (next_expire - datetime.now()).total_seconds() if seconds > 0: sleep(seconds) hedgedoc_expire.cmd_expire() elif args.command == 'emailcheck': hedgedoc_expire.cmd_emailcheck() elif args.command == 'expire': hedgedoc_expire.cmd_expire() else: parser.print_help() if __name__ == '__main__': main()