From e26e50553ba8aa698468e11e5abbde6fe66d9bb9 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Thu, 14 Nov 2024 14:38:17 +0100 Subject: [PATCH 1/4] Add small Mongo docker role, for running on a single node --- provision.yml | 6 ++ roles/mongodbdocker/defaults/main.yml | 2 + roles/mongodbdocker/tasks/main.yml | 87 +++++++++++++++++++ .../templates/backup_mongo.pl.j2 | 37 ++++++++ 4 files changed, 132 insertions(+) create mode 100644 roles/mongodbdocker/defaults/main.yml create mode 100644 roles/mongodbdocker/tasks/main.yml create mode 100644 roles/mongodbdocker/templates/backup_mongo.pl.j2 diff --git a/provision.yml b/provision.yml index 5a21697c3..4d292a291 100644 --- a/provision.yml +++ b/provision.yml @@ -189,5 +189,11 @@ - { role: lifecycle, tags: ["lifecycle"] } - { role: stepuptiqr, tags: ['stepuptiqr' , 'stepup'] } +- hosts: docker_mariadb + become: true + roles: + - { role: mariadbdocker, tags: ['mariadbdocker']} + - { role: mongodbdocker, tags: ['mongodbdocker']} + - import_playbook: "{{ environment_dir }}/playbook.yml" diff --git a/roles/mongodbdocker/defaults/main.yml b/roles/mongodbdocker/defaults/main.yml new file mode 100644 index 000000000..c0095f38b --- /dev/null +++ b/roles/mongodbdocker/defaults/main.yml @@ -0,0 +1,2 @@ +replica_set_name: "{{ instance_name }}" +docker_mongodb_network_range: "172.21.22.0/24" diff --git a/roles/mongodbdocker/tasks/main.yml b/roles/mongodbdocker/tasks/main.yml new file mode 100644 index 000000000..fcc667d54 --- /dev/null +++ b/roles/mongodbdocker/tasks/main.yml @@ -0,0 +1,87 @@ +--- +- name: Install required packages + ansible.builtin.apt: + name: "python3-pymongo" + state: present + +- name: Create MongoDB volume + community.docker.docker_volume: + name: openconext_mongodb + state: present + +- name: Create MongoDB network + community.docker.docker_network: + name: openconext_mongodb + state: present + internal: false + ipam_config: + - subnet: "{{ docker_mongodb_network_range }}" + +- name: Create the MongoDB container + community.docker.docker_container: + name: openconext_mongodb + image: bitnami/mongodb:7.0 + state: started + pull: true + restart_policy: "always" + ports: "127.0.0.1:27017:27017" + networks: + - name: "openconext_mongodb" + mounts: + - type: volume + source: openconext_mongodb + target: /var/lib/mysql + - type: bind + source: /home/backup/mongo/ + target: /home/backup + env: + MONGODB_ROOT_USER: admin + MONGODB_ROOT_PASSWORD: "{{ mongo_admin_password }}" + MONGODB_REPLICA_SET_NAME: "{{ replica_set_name }}" + MONGODB_REPLICA_SET_MODE: primary + MONGODB_REPLICA_SET_KEY: "{{ mongodb_replicateset_key }}" + MONGODB_ADVERTISED_HOSTNAME: openconext_mongodb + volumes: + - openconext_mongodb:/bitnami/mongodb + hostname: openconext_mongodb + +- name: Create mongo database users + community.mongodb.mongodb_user: + login_database: admin + database: "{{ item.db_name }}" + login_user: admin + login_password: "{{ mongo_admin_password }}" + login_host: 127.0.0.1 + name: "{{ item.name }}" + password: "{{ item.password }}" + roles: readWrite + replica_set: "{{ replica_set_name }}" + strict_compatibility: false + no_log: false + run_once: true + with_items: "{{ mongo.users }}" + changed_when: false + tags: mongo_users + +- name: Create the backupdir + ansible.builtin.file: + path: /home/backup/mongo + owner: 1001 + group: 1001 + mode: "0700" + +- name: Install the backup script + ansible.builtin.template: + src: "backup_mongo.pl.j2" + dest: "/usr/local/sbin/backup_mongo.pl" + mode: "0700" + owner: root + group: root + +- name: Create cron symlink for backup script + ansible.builtin.file: + src: "/usr/local/sbin/backup_mongo.pl" + dest: "/etc/cron.daily/mongodb_backup" + state: link + mode: "0700" + owner: root diff --git a/roles/mongodbdocker/templates/backup_mongo.pl.j2 b/roles/mongodbdocker/templates/backup_mongo.pl.j2 new file mode 100644 index 000000000..c8e014742 --- /dev/null +++ b/roles/mongodbdocker/templates/backup_mongo.pl.j2 @@ -0,0 +1,37 @@ +#!/usr/bin/perl +# Variables + +$backupdir = "/home/backup"; +$username = "admin"; +$password = "{{ mongo_admin_password }}"; + +umask 0077; + +# Determine current day +$day = `/bin/date +'%a'`; +chomp($day); + +# Remove old backups if exists +if ( -e "$backupdir/mongo-dump-$day/") { +`rm -rf $backupdir/mongo-dump-$day/`; +} + +# Dump databases +`docker exec openconext_mongodb mongodump --username $username --password $password --authenticationDatabase admin --out $backupdir/mongo-dump-$day`; + +# Gzip dumps +opendir(BDIR, "$backupdir/mongo-dump-$day/"); +my @files = readdir(BDIR); +closedir(BDIR); +chdir("$backupdir/mongo-dump-$day/"); +foreach $dir (@files) { +if ($dir !~ /^\.+$/) { +if ($dir !~ /\.\./g) { +if ( -d "$backupdir/mongo-dump-$day/$dir") { +`tar -cvzf $backupdir/mongo-dump-$day/$dir.tar.gz $dir/`; +`rm -rf $backupdir/mongo-dump-$day/$dir/`; +} +} +} +} +umask 0022; From 268fc153390a206f79afd3869cf5f1cb41dce9ef Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:54:54 +0100 Subject: [PATCH 2/4] Manage: make it possible to connect to a locally available mongo in docker --- roles/manage/defaults/main.yml | 7 +++++-- roles/manage/tasks/main.yml | 10 ++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/roles/manage/defaults/main.yml b/roles/manage/defaults/main.yml index ef5c4271d..bc41cd266 100644 --- a/roles/manage/defaults/main.yml +++ b/roles/manage/defaults/main.yml @@ -1,8 +1,8 @@ --- manage_dir: /config/ -manage_snapshot_timestamp: '' +manage_snapshot_timestamp: "" manage_jar: manage-current.jar -manage_random_source: 'file:///dev/urandom' +manage_random_source: "file:///dev/urandom" manage_cronjobmaster: false manage_disclaimer_background_color: "{{ environment_ribbon_colour }}" manage_disclaimer_content: "{{ environment_shortname }}" @@ -30,3 +30,6 @@ manage_tabs_enabled: - single_tenant_template - provisioning - sram + +manage_docker_networks: + - name: loadbalancer diff --git a/roles/manage/tasks/main.yml b/roles/manage/tasks/main.yml index 9a4132c42..3ab287de4 100644 --- a/roles/manage/tasks/main.yml +++ b/roles/manage/tasks/main.yml @@ -72,6 +72,13 @@ notify: - "restart manageserver" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + manage_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: manageserver @@ -80,8 +87,7 @@ pull: true restart_policy: "always" state: started - networks: - - name: "loadbalancer" + networks: "{{ manage_docker_networks }}" mounts: - source: /opt/openconext/manage/ target: /config/ From 053df90b31706cfbc536307c43b28550fc1883fb Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:56:38 +0100 Subject: [PATCH 3/4] Myconext: make it possible to connect to a locally available mongo in docker --- roles/myconext/defaults/main.yml | 2 ++ roles/myconext/tasks/main.yml | 14 ++++++++++---- roles/myconext/templates/application.yml.j2 | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/roles/myconext/defaults/main.yml b/roles/myconext/defaults/main.yml index 2fa088e32..4082fa28d 100644 --- a/roles/myconext/defaults/main.yml +++ b/roles/myconext/defaults/main.yml @@ -1,2 +1,4 @@ --- myconext_cronjobmaster: true +myconext_docker_networks: + - name: loadbalancer diff --git a/roles/myconext/tasks/main.yml b/roles/myconext/tasks/main.yml index c5d424713..167b35523 100644 --- a/roles/myconext/tasks/main.yml +++ b/roles/myconext/tasks/main.yml @@ -56,7 +56,7 @@ notify: - "restart myconextserver" -- name: copy / create private key +- name: Copy / create private key ansible.builtin.copy: content: "{{ myconext_private_key }}" dest: "/opt/openconext/myconext/myconext_saml.key" @@ -66,7 +66,7 @@ notify: - "restart myconextserver" -- name: copy / create certificate +- name: Copy / create certificate ansible.builtin.copy: src: "{{ inventory_dir }}/files/certs/myconext/myconext_saml.crt" dest: "/opt/openconext/myconext/myconext_saml.crt" @@ -92,6 +92,13 @@ group: "root" mode: "0755" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + myconext_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: myconextserver @@ -102,8 +109,7 @@ env: USE_SYSTEM_CA_CERTS: "1" TZ: "{{ timezone }}" - networks: - - name: "loadbalancer" + networks: "{{ myconext_docker_networks }}" mounts: - source: /opt/openconext/myconext/ target: /config/ diff --git a/roles/myconext/templates/application.yml.j2 b/roles/myconext/templates/application.yml.j2 index 2502621b8..081a8196e 100644 --- a/roles/myconext/templates/application.yml.j2 +++ b/roles/myconext/templates/application.yml.j2 @@ -215,7 +215,7 @@ verify: spring: data: mongodb: - uri: mongodb://{{ myconext.mongo_user }}:{{ myconext.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ myconext.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ myconext.mongo_database }}?ssl=true + uri: mongodb://{{ myconext.mongo_user }}:{{ myconext.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ myconext.mongo_port }}{% if not loop.last %},{% endif %}{% endfor %}/{{ myconext.mongo_database }}?ssl={{ mongodb_ssl | default('true') }} mail: host: {{ smtp_server }} From 6a0d3e1765a7cd41d470283d61e70e8dd32ba561 Mon Sep 17 00:00:00 2001 From: Bart Geesink Date: Tue, 26 Nov 2024 21:57:12 +0100 Subject: [PATCH 4/4] OIDCNG: make it possible to connect to a locally available mongo in docker --- roles/oidcng/defaults/main.yml | 7 ++++--- roles/oidcng/tasks/main.yml | 13 ++++++++----- roles/oidcng/templates/application.yml.j2 | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/roles/oidcng/defaults/main.yml b/roles/oidcng/defaults/main.yml index cc444fba4..44641c6c1 100644 --- a/roles/oidcng/defaults/main.yml +++ b/roles/oidcng/defaults/main.yml @@ -1,8 +1,8 @@ --- oidcng_dir: /opt/openconext/oidcng oidcng_config_dir: /config -oidcng_version: '' -oidcng_snapshot_timestamp: '' +oidcng_version: "" +oidcng_snapshot_timestamp: "" oidcng_cronjobmaster: true oidcng_saml_sp_entityid: https://connect.{{ base_domain }} oidcng_idp_metadata_url: https://engine.{{ base_domain }}/authentication/idp/metadata @@ -19,4 +19,5 @@ oidcng_manage_provision_samlsp_metadata_url: "https://connect.{{ base_domain }}/ oidcng_manage_provision_samlsp_sp_cert: "{{ lookup('file', '{{ inventory_dir }}/files/certs/oidc/oidcsaml.crt') | depem }}" oidcng_manage_provision_samlsp_sign: "True" oidcng_manage_provision_samlsp_trusted_proxy: "True" - +oidcng_docker_networks: + - name: loadbalancer diff --git a/roles/oidcng/tasks/main.yml b/roles/oidcng/tasks/main.yml index fdd8834e8..fa35fac7c 100644 --- a/roles/oidcng/tasks/main.yml +++ b/roles/oidcng/tasks/main.yml @@ -88,6 +88,13 @@ group: "root" mode: "0755" +- name: Add the mongodb docker network to the list of networks when MongoDB runs in Docker + ansible.builtin.set_fact: + oidcng_docker_networks: + - name: loadbalancer + - name: openconext_mongodb + when: mongodb_in_docker | default(false) | bool + - name: Create and start the server container community.docker.docker_container: name: oidcngserver @@ -96,8 +103,7 @@ pull: true restart_policy: "always" state: started - networks: - - name: "loadbalancer" + networks: "{{ oidcng_docker_networks }}" mounts: - source: "{{ oidcng_dir }}" target: /config/ @@ -137,9 +143,6 @@ traefik.http.middlewares.oidcngmw.replacepathregex.regex: "^/.well-known/openid-configuration" traefik.http.middlewares.oidcngmw.replacepathregex.replacement: "/oidc/.well-known/openid-configuration" register: oidcngservercontainer - - - #- name: Include the role manage_provision_entities to provision oidncg to Manage # ansible.builtin.include_role: # name: manage_provision_entities diff --git a/roles/oidcng/templates/application.yml.j2 b/roles/oidcng/templates/application.yml.j2 index f4595497c..c4f4d58d6 100644 --- a/roles/oidcng/templates/application.yml.j2 +++ b/roles/oidcng/templates/application.yml.j2 @@ -76,7 +76,7 @@ idp: spring: data: mongodb: - uri: "mongodb://{{ oidcng.mongo_user }}:{{ oidcng.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ oidcng.mongo_port }}{% if not loop.last %},{% endif %}{%endfor %}/{{ oidcng.mongo_database }}?ssl=true" + uri: "mongodb://{{ oidcng.mongo_user }}:{{ oidcng.mongo_password }}@{% for host in groups['mongo_servers'] %}{{ hostvars[host]['inventory_hostname'] }}:{{ oidcng.mongo_port }}{% if not loop.last %},{% endif %}{%endfor %}/{{ oidcng.mongo_database }}?ssl={{ mongodb_ssl | default('true') }}" thymeleaf: cache: true