From 1080932235ef54d875a32dd2beb94f5eb0dfbae2 Mon Sep 17 00:00:00 2001 From: June Date: Thu, 13 Feb 2025 02:00:32 +0100 Subject: [PATCH 01/12] add redis role for ensuring redis is installed from distro packages This is a requirement for a new netbox role. --- roles/redis/README.md | 15 +++++++++++++++ roles/redis/tasks/main.yaml | 5 +++++ 2 files changed, 20 insertions(+) create mode 100644 roles/redis/README.md create mode 100644 roles/redis/tasks/main.yaml diff --git a/roles/redis/README.md b/roles/redis/README.md new file mode 100644 index 0000000..dd30500 --- /dev/null +++ b/roles/redis/README.md @@ -0,0 +1,15 @@ +# Role `redis` + +Ensures `redis` is installed by installing the distributions package. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +None. + +## Optional Arguments + +None. diff --git a/roles/redis/tasks/main.yaml b/roles/redis/tasks/main.yaml new file mode 100644 index 0000000..ad70e44 --- /dev/null +++ b/roles/redis/tasks/main.yaml @@ -0,0 +1,5 @@ +- name: Ensure redis is installed + ansible.builtin.apt: + name: + - redis + become: true From c307e81d9f58054f4656badf40ff36e8ffcd63cb Mon Sep 17 00:00:00 2001 From: June Date: Thu, 13 Feb 2025 03:24:15 +0100 Subject: [PATCH 02/12] add postgresql role for ens. psql and opt. some dbs and users are set up Add postgresql role for ensuring postgresql is installed. Furthermore the role optionally takes some basic configuration to ensure databases with their owners and users are set up as specified. This is a requirement for a new netbox role. --- roles/postgresql/README.md | 37 +++++++++++++++++++++++ roles/postgresql/defaults/main.yaml | 2 ++ roles/postgresql/meta/argument_specs.yaml | 28 +++++++++++++++++ roles/postgresql/tasks/main.yaml | 28 +++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 roles/postgresql/README.md create mode 100644 roles/postgresql/defaults/main.yaml create mode 100644 roles/postgresql/meta/argument_specs.yaml create mode 100644 roles/postgresql/tasks/main.yaml diff --git a/roles/postgresql/README.md b/roles/postgresql/README.md new file mode 100644 index 0000000..6457931 --- /dev/null +++ b/roles/postgresql/README.md @@ -0,0 +1,37 @@ +# Role `postgresql` + +Ensures `postgresql` is installed by installing the distributions package. +Also ensures the optionally given databases and users are set up as specified. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +None. + +## Optional Arguments + +- `postgresql__dbs`: List of databases with their owner to ensure are set up. +- `postgresql__dbs.*.name`: Name of the database. +- `postgresql__dbs.*.owner`: Owner of the database. +- `postgresql__users`: List of users to ensure are set up. +- `postgresql__users.*.name`: Name of the user. +- `postgresql__users.*.password`: Optional password for the user. + If left unset, the user will have no password set, but can still connect using [peer authentication](https://www.postgresql.org/docs/current/auth-peer.html) on the local system. + (Peer authentication works when a password is set as well.) + +## Example Arguments + +```yaml +postgresql__dbs: + - name: netbox + owner: netbox + - name: foo + owner: bar +postgresql__users: + - name: netbox + password: super_secret + - name: bar +``` diff --git a/roles/postgresql/defaults/main.yaml b/roles/postgresql/defaults/main.yaml new file mode 100644 index 0000000..21fcd46 --- /dev/null +++ b/roles/postgresql/defaults/main.yaml @@ -0,0 +1,2 @@ +postgresql__dbs: [ ] +postgresql__users: [ ] diff --git a/roles/postgresql/meta/argument_specs.yaml b/roles/postgresql/meta/argument_specs.yaml new file mode 100644 index 0000000..28e5813 --- /dev/null +++ b/roles/postgresql/meta/argument_specs.yaml @@ -0,0 +1,28 @@ +argument_specs: + main: + options: + postgresql__dbs: + type: list + elements: dict + required: false + default: [ ] + options: + name: + type: str + required: true + owner: + type: str + required: true + postgresql__users: + type: list + elements: dict + required: false + default: [ ] + options: + name: + type: str + required: true + password: + type: str + required: false + default: "" diff --git a/roles/postgresql/tasks/main.yaml b/roles/postgresql/tasks/main.yaml new file mode 100644 index 0000000..fc61857 --- /dev/null +++ b/roles/postgresql/tasks/main.yaml @@ -0,0 +1,28 @@ +- name: Ensure postgresql is installed + ansible.builtin.apt: + name: + - postgresql + become: true + +- name: Ensure Python library for community.postgresql is installed if needed + ansible.builtin.apt: + name: + - python3-psycopg + become: true + when: postgresql__dbs != [ ] or postgresql__users != [ ] + +- name: Ensure users + community.postgresql.postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password | default('') }}" + become: true + become_user: postgres + loop: "{{ postgresql__users }}" + +- name: Ensure dbs with owners + community.postgresql.postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.owner }}" + become: true + become_user: postgres + loop: "{{ postgresql__dbs }}" From 1bae6234ae3267accd663cc4e8ff2d727b134545 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Fri, 14 Feb 2025 19:52:19 +0100 Subject: [PATCH 03/12] redirect to eh22 --- resources/chaosknoten/tickets/nginx/tickets.hamburg.ccc.de.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/chaosknoten/tickets/nginx/tickets.hamburg.ccc.de.conf b/resources/chaosknoten/tickets/nginx/tickets.hamburg.ccc.de.conf index 90be686..f8a802f 100644 --- a/resources/chaosknoten/tickets/nginx/tickets.hamburg.ccc.de.conf +++ b/resources/chaosknoten/tickets/nginx/tickets.hamburg.ccc.de.conf @@ -38,7 +38,7 @@ server { location = / { #return 302 https://wiki.hamburg.ccc.de/infrastructure:service-overview#tickets_pretix; - return 302 https://tickets.hamburg.ccc.de/hackertours/38c3/; + return 302 https://tickets.hamburg.ccc.de/hackertours/eh22/; } location / { From cf33fb2e0ec63f1155b62891d22c7cbdcdfd71bd Mon Sep 17 00:00:00 2001 From: June Date: Fri, 14 Feb 2025 21:43:44 +0100 Subject: [PATCH 04/12] add netbox role for ensuring netbox is deployed as specified The role takes over the deployment of netbox and its dependencies, while still requiring the user to provide the netbox version, db password and config as well as to set up a web server and handle stuff like creating users, etc. --- roles/netbox/README.md | 77 +++++++++++++++++++ roles/netbox/handlers/main.yaml | 24 ++++++ roles/netbox/meta/argument_specs.yaml | 12 +++ roles/netbox/meta/main.yaml | 11 +++ roles/netbox/tasks/main.yaml | 103 ++++++++++++++++++++++++++ 5 files changed, 227 insertions(+) create mode 100644 roles/netbox/README.md create mode 100644 roles/netbox/handlers/main.yaml create mode 100644 roles/netbox/meta/argument_specs.yaml create mode 100644 roles/netbox/meta/main.yaml create mode 100644 roles/netbox/tasks/main.yaml diff --git a/roles/netbox/README.md b/roles/netbox/README.md new file mode 100644 index 0000000..38b7968 --- /dev/null +++ b/roles/netbox/README.md @@ -0,0 +1,77 @@ +# `netbox` role + +A role for setting up NetBox. +It automatically pulls in all required dependencies like Redis and PostgreSQL, deploys the provided systemd services and gunicorn config and sets up a PostgreSQL database named `netbox` with an owner named `netbox` and the specified password. +However providing the [NetBox configuration](#netbox-configuration), [setting up a web server like nginx to proxy to gunicorn](#web-server-setup) and tasks like creating users, etc. you have to do yourself. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +- `netbox__version`: The NetBox version to deploy. +- `netbox__db_password`: The password to use for connection to the database. + This is required since the upgrade script runs as root and therefore peer authentication doesn't work. +- `netbox__config`: The NetBox config to deploy. + See [NetBox Configuration](#netbox-configuration) for more infos. + +## Optional Arguments + +None. + +## NetBox Configuration + +The NetBox configuration should include a connection to Redis as well as a connection to PostgreSQL. +Configuration for the Redis connection: + +```python +REDIS = { + "tasks": { + "HOST": "localhost", + "PORT": 6379, + "USERNAME": "", + "PASSWORD": "", + "DATABASE": 0, + "SSL": False, + }, + "caching": { + "HOST": "localhost", + "PORT": 6379, + "USERNAME": "", + "PASSWORD": "", + "DATABASE": 1, + "SSL": False, + }, +} +``` + +Configuration for the PostgreSQL connection: + +```python +DATABASE = { + "HOST": "localhost", + "NAME": "netbox", + "USER": "netbox", + "PASSWORD": "", +} +``` + +Further configuration should take place. Some relevant resources can be found here: + +- Installation guide configuration docs: +- Configuration docs: +- Example configuration: + +## Web Server Setup + +As this role just sets up gunicorn, but doesn't set up a web server, you need to do that yourself. +The relevant documentation on how to do that can be found here: + +- Web server setup docs: +- Example base nginx config: + +## Links & Resources + +- The NetBox Git Repo: +- The NetBox installation docs: diff --git a/roles/netbox/handlers/main.yaml b/roles/netbox/handlers/main.yaml new file mode 100644 index 0000000..fd7eb62 --- /dev/null +++ b/roles/netbox/handlers/main.yaml @@ -0,0 +1,24 @@ +- name: Run upgrade script + ansible.builtin.command: /opt/netbox/upgrade.sh + become: true + # When it runs, this should always report changed. + changed_when: true + +- name: Ensure netbox systemd services are set up and up-to-date + ansible.builtin.systemd_service: + daemon_reload: true + name: "{{ item }}" + enabled: true + state: restarted + become: true + loop: + - "netbox.service" + - "netbox-rq.service" + +- name: Ensure netbox housekeeping timer is set up and up-to-date + ansible.builtin.systemd_service: + daemon_reload: true + name: "netbox-housekeeping.timer" + enabled: true + state: restarted + become: true diff --git a/roles/netbox/meta/argument_specs.yaml b/roles/netbox/meta/argument_specs.yaml new file mode 100644 index 0000000..f836b43 --- /dev/null +++ b/roles/netbox/meta/argument_specs.yaml @@ -0,0 +1,12 @@ +argument_specs: + main: + options: + netbox__version: + type: str + required: true + netbox__db_password: + type: str + required: true + netbox__config: + type: str + required: true diff --git a/roles/netbox/meta/main.yaml b/roles/netbox/meta/main.yaml new file mode 100644 index 0000000..79c845d --- /dev/null +++ b/roles/netbox/meta/main.yaml @@ -0,0 +1,11 @@ +--- +dependencies: + - role: redis + - role: postgresql + vars: + postgresql__dbs: + - name: netbox + owner: netbox + postgresql__users: + - name: netbox + password: "{{ netbox__db_password }}" diff --git a/roles/netbox/tasks/main.yaml b/roles/netbox/tasks/main.yaml new file mode 100644 index 0000000..89bf4a6 --- /dev/null +++ b/roles/netbox/tasks/main.yaml @@ -0,0 +1,103 @@ +- name: Ensure all dependencies are installed + ansible.builtin.apt: + name: + - python3 + - python3-pip + - python3-venv + - python3-dev + - build-essential + - libxml2-dev + - libxslt1-dev + - libffi-dev + - libpq-dev + - libssl-dev + - zlib1g-dev + - git + become: true + +- name: Ensure NetBox source is present + ansible.builtin.git: + repo: https://github.com/netbox-community/netbox.git + dest: /opt/netbox/ + version: "{{ netbox__version }}" + become: true + notify: + - Run upgrade script + - Ensure netbox systemd services are set up and up-to-date + +- name: Ensure netbox user + block: + - name: Ensure netbox group exists + ansible.builtin.group: + name: netbox + system: true + become: true + + - name: Ensure netbox user exists + ansible.builtin.user: + name: netbox + group: netbox + password: '!' + system: true + become: true + +- name: Ensure relevant directories are owned by netbox user + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: netbox + recurse: true + become: true + loop: + - "/opt/netbox/netbox/media/" + - "/opt/netbox/netbox/reports/" + - "/opt/netbox/netbox/scripts/" + +- name: Deploy configuration.py + ansible.builtin.copy: + content: "{{ netbox__config }}" + dest: "/opt/netbox/netbox/netbox/configuration.py" + mode: "0644" + owner: root + group: root + become: true + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided gunicorn config is copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/gunicorn.py" + dest: "/opt/netbox/gunicorn.py" + mode: "0644" + owner: root + group: root + become: true + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided netbox systemd service files are copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + owner: root + group: root + become: true + loop: + - "netbox.service" + - "netbox-rq.service" + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided housekeeping systemd service and timer are copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + owner: root + group: root + become: true + loop: + - "netbox-housekeeping.service" + - "netbox-housekeeping.timer" + notify: Ensure netbox housekeeping timer is set up and up-to-date From aa9a0cdbce11aa7aaa982dcba9bc6980bfdf7dab Mon Sep 17 00:00:00 2001 From: June Date: Sat, 15 Feb 2025 03:52:02 +0100 Subject: [PATCH 05/12] add license notice to README This is in preparation for extending that notice to point out code that is licensed differently. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 68c92b4..ab14d1d 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,7 @@ Im Ansible-Repo müssen diese Sachen hinzugefügt werden: * Individuelle Config für den Service. Wenn Docker Compose, hier weiterleiten auf den eigentlichen Dienst in Compose. * Cert-Dateinamen anpassen * `resources/chaosknoten/`*host*`/docker_compose/compose.yaml.j2`: Config für Docker Compose (wenn verwendet) + +## License + +This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). From 249a3b753ed991addb9819921303a08c7000a692 Mon Sep 17 00:00:00 2001 From: June Date: Sat, 15 Feb 2025 05:22:21 +0100 Subject: [PATCH 06/12] add option to netbox role for custom pipeline code for OIDC mapping Add option to netbox role for ensuring custom pipeline code for OIDC group and role mapping is either present or not. The custom pipeline code is licensed under the Creative Commons: CC BY-SA 4.0 license. See: https://github.com/goauthentik/authentik/blob/main/LICENSE https://github.com/goauthentik/authentik/blob/main/website/integrations/services/netbox/index.md https://docs.goauthentik.io/integrations/services/netbox/ https://git.hamburg.ccc.de/CCCHH/nix-infra/commit/5676b1a4680dbe706686f38902f5607ec33330ff --- README.md | 3 +- roles/netbox/README.md | 13 ++++- roles/netbox/defaults/main.yaml | 1 + ...om_pipeline_oidc_group_and_role_mapping.py | 55 +++++++++++++++++++ roles/netbox/meta/argument_specs.yaml | 4 ++ roles/netbox/tasks/main.yaml | 21 +++++++ 6 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 roles/netbox/defaults/main.yaml create mode 100644 roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py diff --git a/README.md b/README.md index ab14d1d..6906a7f 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,5 @@ Im Ansible-Repo müssen diese Sachen hinzugefügt werden: ## License -This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). +This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). +[`custom_pipeline_oidc_group_and_role_mapping.py`](./roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py) is licensed under the Creative Commons: CC BY-SA 4.0 license. diff --git a/roles/netbox/README.md b/roles/netbox/README.md index 38b7968..594e8f8 100644 --- a/roles/netbox/README.md +++ b/roles/netbox/README.md @@ -18,7 +18,9 @@ Should work on Debian-based distributions. ## Optional Arguments -None. +- `netbox__custom_pipeline_oidc_group_and_role_mapping`: Whether or not to have custom pipeline code for OIDC group and role mapping present. + See [Custom Pipeline Code for OIDC Group and Role Mapping](#custom-pipeline-code-for-oidc-group-and-role-mapping) for more infos. + Defaults to `false`. ## NetBox Configuration @@ -71,6 +73,15 @@ The relevant documentation on how to do that can be found here: - Web server setup docs: - Example base nginx config: +## Custom Pipeline Code for OIDC Group and Role Mapping + +Setting the option `netbox__custom_pipeline_oidc_group_and_role_mapping` to `true` makes this role ensure custom pipeline code for OIDC group and role mapping is present. +Note that this role uses code for NetBox >= 4.0.0. +The code is available in `files/custom_pipeline_oidc_group_and_role_mapping.py`, licensed under the CC BY-SA 4.0 license and taken from [this authentik NetBox documentation](https://docs.goauthentik.io/integrations/services/netbox/). +The documentation also shows how to use the pipeline code by defining a custom `SOCIAL_AUTH_PIPELINE`, which you also need to do, as the configuration isn't provided by this role. +However instead of under `netbox.custom_pipeline.` the functions are available under `netbox.custom_pipeline_oidc_mapping.` with this role. +See also [the default settings.py](https://github.com/netbox-community/netbox/blob/main/netbox/netbox/settings.py) for the default `SOCIAL_AUTH_PIPELINE`. + ## Links & Resources - The NetBox Git Repo: diff --git a/roles/netbox/defaults/main.yaml b/roles/netbox/defaults/main.yaml new file mode 100644 index 0000000..49b518e --- /dev/null +++ b/roles/netbox/defaults/main.yaml @@ -0,0 +1 @@ +netbox__custom_pipeline_oidc_group_and_role_mapping: false diff --git a/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py b/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py new file mode 100644 index 0000000..470f388 --- /dev/null +++ b/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py @@ -0,0 +1,55 @@ +# Licensed under Creative Commons: CC BY-SA 4.0 license. +# https://github.com/goauthentik/authentik/blob/main/LICENSE +# https://github.com/goauthentik/authentik/blob/main/website/integrations/services/netbox/index.md +# https://docs.goauthentik.io/integrations/services/netbox/ +from netbox.authentication import Group + +class AuthFailed(Exception): + pass + +def add_groups(response, user, backend, *args, **kwargs): + try: + groups = response['groups'] + except KeyError: + pass + + # Add all groups from oAuth token + for group in groups: + group, created = Group.objects.get_or_create(name=group) + user.groups.add(group) + +def remove_groups(response, user, backend, *args, **kwargs): + try: + groups = response['groups'] + except KeyError: + # Remove all groups if no groups in oAuth token + user.groups.clear() + pass + + # Get all groups of user + user_groups = [item.name for item in user.groups.all()] + # Get groups of user which are not part of oAuth token + delete_groups = list(set(user_groups) - set(groups)) + + # Delete non oAuth token groups + for delete_group in delete_groups: + group = Group.objects.get(name=delete_group) + user.groups.remove(group) + + +def set_roles(response, user, backend, *args, **kwargs): + # Remove Roles temporary + user.is_superuser = False + user.is_staff = False + try: + groups = response['groups'] + except KeyError: + # When no groups are set + # save the user without Roles + user.save() + pass + + # Set roles is role (superuser or staff) is in groups + user.is_superuser = True if 'superusers' in groups else False + user.is_staff = True if 'staff' in groups else False + user.save() diff --git a/roles/netbox/meta/argument_specs.yaml b/roles/netbox/meta/argument_specs.yaml index f836b43..0506389 100644 --- a/roles/netbox/meta/argument_specs.yaml +++ b/roles/netbox/meta/argument_specs.yaml @@ -10,3 +10,7 @@ argument_specs: netbox__config: type: str required: true + netbox__custom_pipeline_oidc_group_and_role_mapping: + type: bool + required: false + default: false diff --git a/roles/netbox/tasks/main.yaml b/roles/netbox/tasks/main.yaml index 89bf4a6..dffa746 100644 --- a/roles/netbox/tasks/main.yaml +++ b/roles/netbox/tasks/main.yaml @@ -25,6 +25,27 @@ - Run upgrade script - Ensure netbox systemd services are set up and up-to-date +- name: Ensures custom pipeline code for OIDC group and role mapping is present + ansible.builtin.copy: + src: custom_pipeline_oidc_group_and_role_mapping.py + dest: /opt/netbox/netbox/netbox/custom_pipeline_oidc_mapping.py + mode: "0644" + owner: root + group: root + when: netbox__custom_pipeline_oidc_group_and_role_mapping + become: true + notify: + - Ensure netbox systemd services are set up and up-to-date + +- name: Ensures custom pipeline code for OIDC group and role mapping is not present + ansible.builtin.file: + path: /opt/netbox/netbox/netbox/custom_pipeline_oidc_mapping.py + state: absent + when: not netbox__custom_pipeline_oidc_group_and_role_mapping + become: true + notify: + - Ensure netbox systemd services are set up and up-to-date + - name: Ensure netbox user block: - name: Ensure netbox group exists From ac7e8bb6f22e73e591abec4b2000f11291fa2e59 Mon Sep 17 00:00:00 2001 From: June Date: Sat, 15 Feb 2025 06:05:44 +0100 Subject: [PATCH 07/12] grafana: set dur. for Prom. hyperv. disk rw rate and hdd io aler. to 90m Set duration for Prometheus hypervisor disk rw rate and hard disk io alerts to 90m to account for the very long running (over an hour) backup job. --- .../grafana/docker_compose/prometheus_alerts.rules.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml b/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml index 5cc54e9..f684385 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml @@ -166,7 +166,7 @@ groups: # Longer intervals to account for disk intensive hypervisor tasks (backups, moving VMs, etc.). - alert: HypervisorHostUnusualDiskReadRate expr: (sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="chaosknoten"} - for: 60m + for: 90m labels: severity: warning annotations: @@ -174,7 +174,7 @@ groups: description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}" - alert: HypervisorHostUnusualDiskWriteRate expr: (sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="chaosknoten"} - for: 60m + for: 90m labels: severity: warning annotations: @@ -256,7 +256,7 @@ groups: # Since hard disks on the hypervisor can easily have their IO saturated by hypervisor tasks (backups, moving VMs, etc.), alert when the IO is above the regular threshold for a very long time. - alert: HypervisorHostUnusualHardDiskIo expr: (rate(node_disk_io_time_seconds_total{device=~"s.+"}[1m]) > 0.5) * on(instance) group_left (nodename) node_uname_info{nodename="chaosknoten"} - for: 50m + for: 90m labels: severity: warning annotations: From ca2b9c30681e6290109238dae73456b85a714177 Mon Sep 17 00:00:00 2001 From: June Date: Thu, 13 Feb 2025 02:00:32 +0100 Subject: [PATCH 08/12] add redis role for ensuring redis is installed from distro packages This is a requirement for a new netbox role. --- roles/redis/README.md | 15 +++++++++++++++ roles/redis/tasks/main.yaml | 5 +++++ 2 files changed, 20 insertions(+) create mode 100644 roles/redis/README.md create mode 100644 roles/redis/tasks/main.yaml diff --git a/roles/redis/README.md b/roles/redis/README.md new file mode 100644 index 0000000..dd30500 --- /dev/null +++ b/roles/redis/README.md @@ -0,0 +1,15 @@ +# Role `redis` + +Ensures `redis` is installed by installing the distributions package. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +None. + +## Optional Arguments + +None. diff --git a/roles/redis/tasks/main.yaml b/roles/redis/tasks/main.yaml new file mode 100644 index 0000000..ad70e44 --- /dev/null +++ b/roles/redis/tasks/main.yaml @@ -0,0 +1,5 @@ +- name: Ensure redis is installed + ansible.builtin.apt: + name: + - redis + become: true From 8c240f463e840cc2c1db64ef62bb2309f94e76fb Mon Sep 17 00:00:00 2001 From: June Date: Thu, 13 Feb 2025 03:24:15 +0100 Subject: [PATCH 09/12] add postgresql role for ens. psql and opt. some dbs and users are set up Add postgresql role for ensuring postgresql is installed. Furthermore the role optionally takes some basic configuration to ensure databases with their owners and users are set up as specified. This is a requirement for a new netbox role. --- roles/postgresql/README.md | 37 +++++++++++++++++++++++ roles/postgresql/defaults/main.yaml | 2 ++ roles/postgresql/meta/argument_specs.yaml | 28 +++++++++++++++++ roles/postgresql/tasks/main.yaml | 28 +++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 roles/postgresql/README.md create mode 100644 roles/postgresql/defaults/main.yaml create mode 100644 roles/postgresql/meta/argument_specs.yaml create mode 100644 roles/postgresql/tasks/main.yaml diff --git a/roles/postgresql/README.md b/roles/postgresql/README.md new file mode 100644 index 0000000..6457931 --- /dev/null +++ b/roles/postgresql/README.md @@ -0,0 +1,37 @@ +# Role `postgresql` + +Ensures `postgresql` is installed by installing the distributions package. +Also ensures the optionally given databases and users are set up as specified. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +None. + +## Optional Arguments + +- `postgresql__dbs`: List of databases with their owner to ensure are set up. +- `postgresql__dbs.*.name`: Name of the database. +- `postgresql__dbs.*.owner`: Owner of the database. +- `postgresql__users`: List of users to ensure are set up. +- `postgresql__users.*.name`: Name of the user. +- `postgresql__users.*.password`: Optional password for the user. + If left unset, the user will have no password set, but can still connect using [peer authentication](https://www.postgresql.org/docs/current/auth-peer.html) on the local system. + (Peer authentication works when a password is set as well.) + +## Example Arguments + +```yaml +postgresql__dbs: + - name: netbox + owner: netbox + - name: foo + owner: bar +postgresql__users: + - name: netbox + password: super_secret + - name: bar +``` diff --git a/roles/postgresql/defaults/main.yaml b/roles/postgresql/defaults/main.yaml new file mode 100644 index 0000000..21fcd46 --- /dev/null +++ b/roles/postgresql/defaults/main.yaml @@ -0,0 +1,2 @@ +postgresql__dbs: [ ] +postgresql__users: [ ] diff --git a/roles/postgresql/meta/argument_specs.yaml b/roles/postgresql/meta/argument_specs.yaml new file mode 100644 index 0000000..28e5813 --- /dev/null +++ b/roles/postgresql/meta/argument_specs.yaml @@ -0,0 +1,28 @@ +argument_specs: + main: + options: + postgresql__dbs: + type: list + elements: dict + required: false + default: [ ] + options: + name: + type: str + required: true + owner: + type: str + required: true + postgresql__users: + type: list + elements: dict + required: false + default: [ ] + options: + name: + type: str + required: true + password: + type: str + required: false + default: "" diff --git a/roles/postgresql/tasks/main.yaml b/roles/postgresql/tasks/main.yaml new file mode 100644 index 0000000..fc61857 --- /dev/null +++ b/roles/postgresql/tasks/main.yaml @@ -0,0 +1,28 @@ +- name: Ensure postgresql is installed + ansible.builtin.apt: + name: + - postgresql + become: true + +- name: Ensure Python library for community.postgresql is installed if needed + ansible.builtin.apt: + name: + - python3-psycopg + become: true + when: postgresql__dbs != [ ] or postgresql__users != [ ] + +- name: Ensure users + community.postgresql.postgresql_user: + name: "{{ item.name }}" + password: "{{ item.password | default('') }}" + become: true + become_user: postgres + loop: "{{ postgresql__users }}" + +- name: Ensure dbs with owners + community.postgresql.postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.owner }}" + become: true + become_user: postgres + loop: "{{ postgresql__dbs }}" From 745755604c5a59851623db5571665a2df67c5cdd Mon Sep 17 00:00:00 2001 From: June Date: Fri, 14 Feb 2025 21:43:44 +0100 Subject: [PATCH 10/12] add netbox role for ensuring netbox is deployed as specified The role takes over the deployment of netbox and its dependencies, while still requiring the user to provide the netbox version, db password and config as well as to set up a web server and handle stuff like creating users, etc. --- roles/netbox/README.md | 77 +++++++++++++++++++ roles/netbox/handlers/main.yaml | 24 ++++++ roles/netbox/meta/argument_specs.yaml | 12 +++ roles/netbox/meta/main.yaml | 11 +++ roles/netbox/tasks/main.yaml | 103 ++++++++++++++++++++++++++ 5 files changed, 227 insertions(+) create mode 100644 roles/netbox/README.md create mode 100644 roles/netbox/handlers/main.yaml create mode 100644 roles/netbox/meta/argument_specs.yaml create mode 100644 roles/netbox/meta/main.yaml create mode 100644 roles/netbox/tasks/main.yaml diff --git a/roles/netbox/README.md b/roles/netbox/README.md new file mode 100644 index 0000000..38b7968 --- /dev/null +++ b/roles/netbox/README.md @@ -0,0 +1,77 @@ +# `netbox` role + +A role for setting up NetBox. +It automatically pulls in all required dependencies like Redis and PostgreSQL, deploys the provided systemd services and gunicorn config and sets up a PostgreSQL database named `netbox` with an owner named `netbox` and the specified password. +However providing the [NetBox configuration](#netbox-configuration), [setting up a web server like nginx to proxy to gunicorn](#web-server-setup) and tasks like creating users, etc. you have to do yourself. + +## Supported Distributions + +Should work on Debian-based distributions. + +## Required Arguments + +- `netbox__version`: The NetBox version to deploy. +- `netbox__db_password`: The password to use for connection to the database. + This is required since the upgrade script runs as root and therefore peer authentication doesn't work. +- `netbox__config`: The NetBox config to deploy. + See [NetBox Configuration](#netbox-configuration) for more infos. + +## Optional Arguments + +None. + +## NetBox Configuration + +The NetBox configuration should include a connection to Redis as well as a connection to PostgreSQL. +Configuration for the Redis connection: + +```python +REDIS = { + "tasks": { + "HOST": "localhost", + "PORT": 6379, + "USERNAME": "", + "PASSWORD": "", + "DATABASE": 0, + "SSL": False, + }, + "caching": { + "HOST": "localhost", + "PORT": 6379, + "USERNAME": "", + "PASSWORD": "", + "DATABASE": 1, + "SSL": False, + }, +} +``` + +Configuration for the PostgreSQL connection: + +```python +DATABASE = { + "HOST": "localhost", + "NAME": "netbox", + "USER": "netbox", + "PASSWORD": "", +} +``` + +Further configuration should take place. Some relevant resources can be found here: + +- Installation guide configuration docs: +- Configuration docs: +- Example configuration: + +## Web Server Setup + +As this role just sets up gunicorn, but doesn't set up a web server, you need to do that yourself. +The relevant documentation on how to do that can be found here: + +- Web server setup docs: +- Example base nginx config: + +## Links & Resources + +- The NetBox Git Repo: +- The NetBox installation docs: diff --git a/roles/netbox/handlers/main.yaml b/roles/netbox/handlers/main.yaml new file mode 100644 index 0000000..fd7eb62 --- /dev/null +++ b/roles/netbox/handlers/main.yaml @@ -0,0 +1,24 @@ +- name: Run upgrade script + ansible.builtin.command: /opt/netbox/upgrade.sh + become: true + # When it runs, this should always report changed. + changed_when: true + +- name: Ensure netbox systemd services are set up and up-to-date + ansible.builtin.systemd_service: + daemon_reload: true + name: "{{ item }}" + enabled: true + state: restarted + become: true + loop: + - "netbox.service" + - "netbox-rq.service" + +- name: Ensure netbox housekeeping timer is set up and up-to-date + ansible.builtin.systemd_service: + daemon_reload: true + name: "netbox-housekeeping.timer" + enabled: true + state: restarted + become: true diff --git a/roles/netbox/meta/argument_specs.yaml b/roles/netbox/meta/argument_specs.yaml new file mode 100644 index 0000000..f836b43 --- /dev/null +++ b/roles/netbox/meta/argument_specs.yaml @@ -0,0 +1,12 @@ +argument_specs: + main: + options: + netbox__version: + type: str + required: true + netbox__db_password: + type: str + required: true + netbox__config: + type: str + required: true diff --git a/roles/netbox/meta/main.yaml b/roles/netbox/meta/main.yaml new file mode 100644 index 0000000..79c845d --- /dev/null +++ b/roles/netbox/meta/main.yaml @@ -0,0 +1,11 @@ +--- +dependencies: + - role: redis + - role: postgresql + vars: + postgresql__dbs: + - name: netbox + owner: netbox + postgresql__users: + - name: netbox + password: "{{ netbox__db_password }}" diff --git a/roles/netbox/tasks/main.yaml b/roles/netbox/tasks/main.yaml new file mode 100644 index 0000000..89bf4a6 --- /dev/null +++ b/roles/netbox/tasks/main.yaml @@ -0,0 +1,103 @@ +- name: Ensure all dependencies are installed + ansible.builtin.apt: + name: + - python3 + - python3-pip + - python3-venv + - python3-dev + - build-essential + - libxml2-dev + - libxslt1-dev + - libffi-dev + - libpq-dev + - libssl-dev + - zlib1g-dev + - git + become: true + +- name: Ensure NetBox source is present + ansible.builtin.git: + repo: https://github.com/netbox-community/netbox.git + dest: /opt/netbox/ + version: "{{ netbox__version }}" + become: true + notify: + - Run upgrade script + - Ensure netbox systemd services are set up and up-to-date + +- name: Ensure netbox user + block: + - name: Ensure netbox group exists + ansible.builtin.group: + name: netbox + system: true + become: true + + - name: Ensure netbox user exists + ansible.builtin.user: + name: netbox + group: netbox + password: '!' + system: true + become: true + +- name: Ensure relevant directories are owned by netbox user + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: netbox + recurse: true + become: true + loop: + - "/opt/netbox/netbox/media/" + - "/opt/netbox/netbox/reports/" + - "/opt/netbox/netbox/scripts/" + +- name: Deploy configuration.py + ansible.builtin.copy: + content: "{{ netbox__config }}" + dest: "/opt/netbox/netbox/netbox/configuration.py" + mode: "0644" + owner: root + group: root + become: true + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided gunicorn config is copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/gunicorn.py" + dest: "/opt/netbox/gunicorn.py" + mode: "0644" + owner: root + group: root + become: true + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided netbox systemd service files are copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + owner: root + group: root + become: true + loop: + - "netbox.service" + - "netbox-rq.service" + notify: Ensure netbox systemd services are set up and up-to-date + +- name: Ensure provided housekeeping systemd service and timer are copied + ansible.builtin.copy: + remote_src: true + src: "/opt/netbox/contrib/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + owner: root + group: root + become: true + loop: + - "netbox-housekeeping.service" + - "netbox-housekeeping.timer" + notify: Ensure netbox housekeeping timer is set up and up-to-date From 7fbc240c7bc6cf5f039f2cba8b20ab2537dc4a3b Mon Sep 17 00:00:00 2001 From: June Date: Sat, 15 Feb 2025 03:52:02 +0100 Subject: [PATCH 11/12] add license notice to README This is in preparation for extending that notice to point out code that is licensed differently. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 68c92b4..ab14d1d 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,7 @@ Im Ansible-Repo müssen diese Sachen hinzugefügt werden: * Individuelle Config für den Service. Wenn Docker Compose, hier weiterleiten auf den eigentlichen Dienst in Compose. * Cert-Dateinamen anpassen * `resources/chaosknoten/`*host*`/docker_compose/compose.yaml.j2`: Config für Docker Compose (wenn verwendet) + +## License + +This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). From 4c5eb12712ac72950c617fd892e42e825aecd0a5 Mon Sep 17 00:00:00 2001 From: June Date: Sat, 15 Feb 2025 05:22:21 +0100 Subject: [PATCH 12/12] add option to netbox role for custom pipeline code for OIDC mapping Add option to netbox role for ensuring custom pipeline code for OIDC group and role mapping is either present or not. The custom pipeline code is licensed under the Creative Commons: CC BY-SA 4.0 license. See: https://github.com/goauthentik/authentik/blob/main/LICENSE https://github.com/goauthentik/authentik/blob/main/website/integrations/services/netbox/index.md https://docs.goauthentik.io/integrations/services/netbox/ https://git.hamburg.ccc.de/CCCHH/nix-infra/commit/5676b1a4680dbe706686f38902f5607ec33330ff --- README.md | 3 +- roles/netbox/README.md | 13 ++++- roles/netbox/defaults/main.yaml | 1 + ...om_pipeline_oidc_group_and_role_mapping.py | 55 +++++++++++++++++++ roles/netbox/meta/argument_specs.yaml | 4 ++ roles/netbox/tasks/main.yaml | 21 +++++++ 6 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 roles/netbox/defaults/main.yaml create mode 100644 roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py diff --git a/README.md b/README.md index ab14d1d..6906a7f 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,5 @@ Im Ansible-Repo müssen diese Sachen hinzugefügt werden: ## License -This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). +This CCCHH ansible-ccchh repository is licensed under the [MIT License](./LICENSE). +[`custom_pipeline_oidc_group_and_role_mapping.py`](./roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py) is licensed under the Creative Commons: CC BY-SA 4.0 license. diff --git a/roles/netbox/README.md b/roles/netbox/README.md index 38b7968..594e8f8 100644 --- a/roles/netbox/README.md +++ b/roles/netbox/README.md @@ -18,7 +18,9 @@ Should work on Debian-based distributions. ## Optional Arguments -None. +- `netbox__custom_pipeline_oidc_group_and_role_mapping`: Whether or not to have custom pipeline code for OIDC group and role mapping present. + See [Custom Pipeline Code for OIDC Group and Role Mapping](#custom-pipeline-code-for-oidc-group-and-role-mapping) for more infos. + Defaults to `false`. ## NetBox Configuration @@ -71,6 +73,15 @@ The relevant documentation on how to do that can be found here: - Web server setup docs: - Example base nginx config: +## Custom Pipeline Code for OIDC Group and Role Mapping + +Setting the option `netbox__custom_pipeline_oidc_group_and_role_mapping` to `true` makes this role ensure custom pipeline code for OIDC group and role mapping is present. +Note that this role uses code for NetBox >= 4.0.0. +The code is available in `files/custom_pipeline_oidc_group_and_role_mapping.py`, licensed under the CC BY-SA 4.0 license and taken from [this authentik NetBox documentation](https://docs.goauthentik.io/integrations/services/netbox/). +The documentation also shows how to use the pipeline code by defining a custom `SOCIAL_AUTH_PIPELINE`, which you also need to do, as the configuration isn't provided by this role. +However instead of under `netbox.custom_pipeline.` the functions are available under `netbox.custom_pipeline_oidc_mapping.` with this role. +See also [the default settings.py](https://github.com/netbox-community/netbox/blob/main/netbox/netbox/settings.py) for the default `SOCIAL_AUTH_PIPELINE`. + ## Links & Resources - The NetBox Git Repo: diff --git a/roles/netbox/defaults/main.yaml b/roles/netbox/defaults/main.yaml new file mode 100644 index 0000000..49b518e --- /dev/null +++ b/roles/netbox/defaults/main.yaml @@ -0,0 +1 @@ +netbox__custom_pipeline_oidc_group_and_role_mapping: false diff --git a/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py b/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py new file mode 100644 index 0000000..470f388 --- /dev/null +++ b/roles/netbox/files/custom_pipeline_oidc_group_and_role_mapping.py @@ -0,0 +1,55 @@ +# Licensed under Creative Commons: CC BY-SA 4.0 license. +# https://github.com/goauthentik/authentik/blob/main/LICENSE +# https://github.com/goauthentik/authentik/blob/main/website/integrations/services/netbox/index.md +# https://docs.goauthentik.io/integrations/services/netbox/ +from netbox.authentication import Group + +class AuthFailed(Exception): + pass + +def add_groups(response, user, backend, *args, **kwargs): + try: + groups = response['groups'] + except KeyError: + pass + + # Add all groups from oAuth token + for group in groups: + group, created = Group.objects.get_or_create(name=group) + user.groups.add(group) + +def remove_groups(response, user, backend, *args, **kwargs): + try: + groups = response['groups'] + except KeyError: + # Remove all groups if no groups in oAuth token + user.groups.clear() + pass + + # Get all groups of user + user_groups = [item.name for item in user.groups.all()] + # Get groups of user which are not part of oAuth token + delete_groups = list(set(user_groups) - set(groups)) + + # Delete non oAuth token groups + for delete_group in delete_groups: + group = Group.objects.get(name=delete_group) + user.groups.remove(group) + + +def set_roles(response, user, backend, *args, **kwargs): + # Remove Roles temporary + user.is_superuser = False + user.is_staff = False + try: + groups = response['groups'] + except KeyError: + # When no groups are set + # save the user without Roles + user.save() + pass + + # Set roles is role (superuser or staff) is in groups + user.is_superuser = True if 'superusers' in groups else False + user.is_staff = True if 'staff' in groups else False + user.save() diff --git a/roles/netbox/meta/argument_specs.yaml b/roles/netbox/meta/argument_specs.yaml index f836b43..0506389 100644 --- a/roles/netbox/meta/argument_specs.yaml +++ b/roles/netbox/meta/argument_specs.yaml @@ -10,3 +10,7 @@ argument_specs: netbox__config: type: str required: true + netbox__custom_pipeline_oidc_group_and_role_mapping: + type: bool + required: false + default: false diff --git a/roles/netbox/tasks/main.yaml b/roles/netbox/tasks/main.yaml index 89bf4a6..dffa746 100644 --- a/roles/netbox/tasks/main.yaml +++ b/roles/netbox/tasks/main.yaml @@ -25,6 +25,27 @@ - Run upgrade script - Ensure netbox systemd services are set up and up-to-date +- name: Ensures custom pipeline code for OIDC group and role mapping is present + ansible.builtin.copy: + src: custom_pipeline_oidc_group_and_role_mapping.py + dest: /opt/netbox/netbox/netbox/custom_pipeline_oidc_mapping.py + mode: "0644" + owner: root + group: root + when: netbox__custom_pipeline_oidc_group_and_role_mapping + become: true + notify: + - Ensure netbox systemd services are set up and up-to-date + +- name: Ensures custom pipeline code for OIDC group and role mapping is not present + ansible.builtin.file: + path: /opt/netbox/netbox/netbox/custom_pipeline_oidc_mapping.py + state: absent + when: not netbox__custom_pipeline_oidc_group_and_role_mapping + become: true + notify: + - Ensure netbox systemd services are set up and up-to-date + - name: Ensure netbox user block: - name: Ensure netbox group exists