From 4dc191d418456af316b3799acbb374813bbaf944 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 21 Jul 2024 22:23:08 +0200 Subject: [PATCH 01/11] =?UTF-8?q?Met=20=C3=A0=20jour=20les=20d=C3=A9pendan?= =?UTF-8?q?ces=20Ruby?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suite au signalement de Dependabot --- Gemfile.lock | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 39dd49e..29e1e23 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,15 +1,16 @@ GEM remote: https://rubygems.org/ specs: - bcrypt_pbkdf (1.1.0) + base64 (0.2.0) + bcrypt_pbkdf (1.1.1) bigdecimal (3.1.8) - builder (3.2.4) - chef-utils (18.4.12) + builder (3.3.0) + chef-utils (18.5.0) concurrent-ruby - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.3) ed25519 (1.3.0) - erubi (1.12.0) - ffi (1.16.3) + erubi (1.13.0) + ffi (1.17.0) gssapi (1.3.1) ffi (>= 1.0.1) gyoku (1.4.0) @@ -27,14 +28,14 @@ GEM tty-box (~> 0.6) tty-prompt (~> 0.20) little-plugger (1.1.4) - logging (2.3.1) + logging (2.4.0) little-plugger (~> 1.1) multi_json (~> 1.14) mixlib-install (3.12.30) mixlib-shellout mixlib-versioning thor - mixlib-shellout (3.2.7) + mixlib-shellout (3.2.8) chef-utils mixlib-versioning (1.2.12) multi_json (1.15.0) @@ -47,9 +48,10 @@ GEM bigdecimal pastel (0.8.0) tty-color (~> 0.5) - rexml (3.2.8) - strscan (>= 3.0.9) - rubyntlm (0.6.3) + rexml (3.3.2) + strscan + rubyntlm (0.6.5) + base64 rubyzip (2.3.2) strings (0.2.1) strings-ansi (~> 0.2) @@ -89,7 +91,7 @@ GEM tty-screen (0.8.2) unicode-display_width (2.5.0) unicode_utils (1.4.0) - winrm (2.3.6) + winrm (2.3.8) builder (>= 2.1.2) erubi (~> 1.8) gssapi (~> 1.2) @@ -97,6 +99,7 @@ GEM httpclient (~> 2.2, >= 2.2.0.2) logging (>= 1.6.1, < 3.0) nori (~> 2.0) + rexml (~> 3.0) rubyntlm (~> 0.6.0, >= 0.6.3) winrm-elevated (1.2.3) erubi (~> 1.8) From 37ce1ac43b94e053b21855dee3189dfa8abbfe74 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Fri, 2 Aug 2024 20:21:19 +0200 Subject: [PATCH 02/11] =?UTF-8?q?Met=20=C3=A0=20jour=20les=20d=C3=A9pendan?= =?UTF-8?q?ces=20Ruby?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suite au signalement de Dependabot --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 29e1e23..d248128 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -44,11 +44,11 @@ GEM net-ssh (7.2.3) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) - nori (2.7.0) + nori (2.7.1) bigdecimal pastel (0.8.0) tty-color (~> 0.5) - rexml (3.3.2) + rexml (3.3.4) strscan rubyntlm (0.6.5) base64 @@ -91,14 +91,14 @@ GEM tty-screen (0.8.2) unicode-display_width (2.5.0) unicode_utils (1.4.0) - winrm (2.3.8) + winrm (2.3.9) builder (>= 2.1.2) erubi (~> 1.8) gssapi (~> 1.2) gyoku (~> 1.0) httpclient (~> 2.2, >= 2.2.0.2) logging (>= 1.6.1, < 3.0) - nori (~> 2.0) + nori (~> 2.0, >= 2.7.1) rexml (~> 3.0) rubyntlm (~> 0.6.0, >= 0.6.3) winrm-elevated (1.2.3) From 1b73a93b51e897eee77829de10eb0cfdab23aab8 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Fri, 2 Aug 2024 20:29:15 +0200 Subject: [PATCH 03/11] =?UTF-8?q?Met=20=C3=A0=20jour=20ansible,=20ansible-?= =?UTF-8?q?lint=20et=20pre-commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87aee08..e27b7ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/ansible-community/ansible-lint.git - rev: v24.2.1 # doit aussi être mis à jour dans requirements.txt + rev: v24.7.0 # doit aussi être mis à jour dans requirements.txt hooks: - id: ansible-lint files: \.(yaml|yml)$ diff --git a/requirements.txt b/requirements.txt index 00fe4e4..fc26c7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -ansible==9.3.0 -ansible-lint==24.2.1 # doit aussi être mis à jour dans .pre-commit-config.yaml -pre-commit==3.6.2 +ansible==10.2.0 +ansible-lint==24.7.0 # doit aussi être mis à jour dans .pre-commit-config.yaml +pre-commit==3.8.0 From c2e7ade18adf543e0ca41ac4fceec30989e1e3f5 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 19 May 2024 11:58:03 +0200 Subject: [PATCH 04/11] =?UTF-8?q?Remplace=20le=20r=C3=B4le=20ElasticSearch?= =?UTF-8?q?=20par=20Typesense?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- playbook.yml | 2 +- roles/app/tasks/fixtures.yml | 2 +- roles/app/tasks/main.yml | 35 ++--- roles/app/templates/config.toml.j2 | 4 +- roles/app/templates/zds-es-index.service.j2 | 8 -- .../zds-search-engine-index.service.j2 | 8 ++ ...er.j2 => zds-search-engine-index.timer.j2} | 2 +- .../templates/beta/restore-from-prod.sh.j2 | 4 +- roles/elasticsearch/files/jvm.options | 94 ------------- roles/elasticsearch/handlers/main.yml | 5 - roles/elasticsearch/tasks/main.yml | 31 ----- roles/munin/files/plugin-conf.d/elasticsearch | 3 - roles/munin/files/plugins/typesense | 124 ++++++++++++++++++ .../files/systemd/munin-node-override.conf | 2 +- roles/munin/tasks/main.yml | 35 ++--- .../templates/plugin-conf.d/typesense.j2 | 2 + roles/munin/vars/main.yml | 11 +- roles/typesense/tasks/main.yml | 27 ++++ .../templates/logrotate/typesense.j2 | 8 ++ 19 files changed, 201 insertions(+), 206 deletions(-) delete mode 100644 roles/app/templates/zds-es-index.service.j2 create mode 100644 roles/app/templates/zds-search-engine-index.service.j2 rename roles/app/templates/{zds-es-index.timer.j2 => zds-search-engine-index.timer.j2} (59%) delete mode 100644 roles/elasticsearch/files/jvm.options delete mode 100644 roles/elasticsearch/handlers/main.yml delete mode 100644 roles/elasticsearch/tasks/main.yml delete mode 100644 roles/munin/files/plugin-conf.d/elasticsearch create mode 100755 roles/munin/files/plugins/typesense create mode 100644 roles/munin/templates/plugin-conf.d/typesense.j2 create mode 100644 roles/typesense/tasks/main.yml create mode 100644 roles/typesense/templates/logrotate/typesense.j2 diff --git a/playbook.yml b/playbook.yml index 9ea7d3d..bf0a7aa 100644 --- a/playbook.yml +++ b/playbook.yml @@ -7,7 +7,7 @@ tags: bootstrap - role: backup tags: bootstrap - - role: elasticsearch + - role: typesense tags: bootstrap - role: mysql tags: bootstrap diff --git a/roles/app/tasks/fixtures.yml b/roles/app/tasks/fixtures.yml index 3267d53..5fed8cc 100644 --- a/roles/app/tasks/fixtures.yml +++ b/roles/app/tasks/fixtures.yml @@ -16,5 +16,5 @@ {{ workdir }}/wrapper load_factory_data {{ appdir }}/fixtures/advanced/aide_tuto_media.yaml && {{ workdir }}/wrapper load_fixtures --size=low --all && touch {{ appdir }}/.loaded_fixtures && - {{ workdir }}/wrapper es_manager index_all; \ + {{ workdir }}/wrapper search_engine_manager index_all; \ fi diff --git a/roles/app/tasks/main.yml b/roles/app/tasks/main.yml index 68c5544..1b5114e 100644 --- a/roles/app/tasks/main.yml +++ b/roles/app/tasks/main.yml @@ -112,6 +112,14 @@ tags: - bootstrap +- name: get Typesense config file + ansible.builtin.slurp: + src: /etc/typesense/typesense-server.ini + register: typesense_file + tags: + - bootstrap + - upgrade + - name: create app config file ansible.builtin.template: src: templates/config.toml.j2 @@ -210,27 +218,6 @@ - bootstrap - upgrade -- name: patch elasticsearch-dsl for Python 3.11 - become: true - become_user: "{{ appuser }}" - ansible.builtin.lineinfile: - path: "{{ virtualenv }}/lib/python3.11/site-packages/elasticsearch_dsl/{{ item }}" - regexp: ^import collections$ - line: import collections.abc as collections - firstmatch: true - with_items: - - search.py - - utils.py - - mapping.py - - field.py - - aggs.py - - document.py - - function.py - - query.py - tags: - - bootstrap - - upgrade - - name: include nodejs installation ansible.builtin.include_role: name: common @@ -352,8 +339,8 @@ - zmd.service - zds.service - zds.socket - - zds-es-index.service - - zds-es-index.timer + - zds-search-engine-index.service + - zds-search-engine-index.timer - zds-watchdog.service tags: - bootstrap @@ -368,7 +355,7 @@ - zmd.service - zds.service - zds.socket - - zds-es-index.timer + - zds-search-engine-index.timer - zds-watchdog.service tags: - bootstrap diff --git a/roles/app/templates/config.toml.j2 b/roles/app/templates/config.toml.j2 index 1a7a3f3..e685ac2 100644 --- a/roles/app/templates/config.toml.j2 +++ b/roles/app/templates/config.toml.j2 @@ -60,8 +60,8 @@ dsn = "{{ sentry_dsn }}" environment = "{{ env }}" {% endif %} -[elasticsearch] -shards = 3 +[typesense] +api_key = "{{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }}" {% if recaptcha is defined %} [recaptcha] diff --git a/roles/app/templates/zds-es-index.service.j2 b/roles/app/templates/zds-es-index.service.j2 deleted file mode 100644 index 6469088..0000000 --- a/roles/app/templates/zds-es-index.service.j2 +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Reindex ES Service - -[Service] -Type=oneshot -User={{ appuser }} -Group={{ appuser }} -ExecStart={{ workdir }}/wrapper es_manager index_flagged diff --git a/roles/app/templates/zds-search-engine-index.service.j2 b/roles/app/templates/zds-search-engine-index.service.j2 new file mode 100644 index 0000000..ce9b2f0 --- /dev/null +++ b/roles/app/templates/zds-search-engine-index.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Reindex new and updated content + +[Service] +Type=oneshot +User={{ appuser }} +Group={{ appuser }} +ExecStart={{ workdir }}/wrapper search_engine_manager index_flagged diff --git a/roles/app/templates/zds-es-index.timer.j2 b/roles/app/templates/zds-search-engine-index.timer.j2 similarity index 59% rename from roles/app/templates/zds-es-index.timer.j2 rename to roles/app/templates/zds-search-engine-index.timer.j2 index c4ed4ed..016185c 100644 --- a/roles/app/templates/zds-es-index.timer.j2 +++ b/roles/app/templates/zds-search-engine-index.timer.j2 @@ -1,5 +1,5 @@ [Unit] -Description=ES reindex flagged contents +Description=Search engine reindex new and updated contents [Timer] OnCalendar=*:30:00 diff --git a/roles/backup/templates/beta/restore-from-prod.sh.j2 b/roles/backup/templates/beta/restore-from-prod.sh.j2 index fa7fc4d..c91297b 100755 --- a/roles/backup/templates/beta/restore-from-prod.sh.j2 +++ b/roles/backup/templates/beta/restore-from-prod.sh.j2 @@ -271,8 +271,8 @@ then $ZDS_WRAPPER migrate print_info "collectstatic..." $ZDS_WRAPPER collectstatic - print_info "es_manager index_all..." - $ZDS_WRAPPER es_manager index_all + print_info "search_engine_manager index_all..." + $ZDS_WRAPPER search_engine_manager index_all fi diff --git a/roles/elasticsearch/files/jvm.options b/roles/elasticsearch/files/jvm.options deleted file mode 100644 index e39fb31..0000000 --- a/roles/elasticsearch/files/jvm.options +++ /dev/null @@ -1,94 +0,0 @@ -## JVM configuration - -################################################################ -## IMPORTANT: JVM heap size -################################################################ -## -## You should always set the min and max JVM heap -## size to the same value. For example, to set -## the heap to 4 GB, set: -## -## -Xms4g -## -Xmx4g -## -## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html -## for more information -## -################################################################ - -# Xms represents the initial size of total heap space -# Xmx represents the maximum size of total heap space - --Xms512m --Xmx1g - -################################################################ -## Expert settings -################################################################ -## -## All settings below this section are considered -## expert settings. Don't tamper with them unless -## you understand what you are doing -## -################################################################ - -## GC configuration --XX:+UseG1GC -# -XX:CMSInitiatingOccupancyFraction=75 -# -XX:+UseCMSInitiatingOccupancyOnly - -## optimizations - -# disable calls to System#gc --XX:+DisableExplicitGC - -# pre-touch memory pages used by the JVM during initialization --XX:+AlwaysPreTouch - -## basic - -# force the server VM --server - -# set to headless, just in case --Djava.awt.headless=true - -# ensure UTF-8 encoding by default (e.g. filenames) --Dfile.encoding=UTF-8 - -# use our provided JNA always versus the system one --Djna.nosys=true - -# flag to explicitly tell Netty to not use unsafe --Dio.netty.noUnsafe=true - -## heap dumps - -# generate a heap dump when an allocation from the Java heap fails -# heap dumps are created in the working directory of the JVM --XX:+HeapDumpOnOutOfMemoryError - -# specify an alternative path for heap dumps -# ensure the directory exists and has sufficient space -#-XX:HeapDumpPath=${heap.dump.path} - -## GC logging - -#-XX:+PrintGCDetails -#-XX:+PrintGCTimeStamps -#-XX:+PrintGCDateStamps -#-XX:+PrintClassHistogram -#-XX:+PrintTenuringDistribution -#-XX:+PrintGCApplicationStoppedTime - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${loggc} - -# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. -# If documents were already indexed with unquoted fields in a previous version -# of Elasticsearch, some operations may throw errors. -# -# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided -# only for migration purposes. -#-Delasticsearch.json.allow_unquoted_field_names=true diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml deleted file mode 100644 index 2937e19..0000000 --- a/roles/elasticsearch/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart elasticsearch - ansible.builtin.systemd: - name: elasticsearch.service - state: restarted diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml deleted file mode 100644 index ff5eb13..0000000 --- a/roles/elasticsearch/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: add elasticsearch repository key - ansible.builtin.apt_key: - id: 46095ACC8548582C1A2699A9D27D666CD88E42B4 - keyserver: hkp://keyserver.ubuntu.com:80 - -- name: add elasticsearch repository - ansible.builtin.apt_repository: - filename: elasticsearch - repo: deb https://artifacts.elastic.co/packages/5.x/apt stable main - state: present - -- name: install openjdk-17-jre-headless and elasticsearch - ansible.builtin.apt: - pkg: - - openjdk-17-jre-headless - - elasticsearch - cache_valid_time: 3600 - -- name: copy elasticsearch config files - ansible.builtin.copy: - src: jvm.options - dest: /etc/elasticsearch/jvm.options - mode: u=rw,g=rw - notify: restart elasticsearch - -- name: start elasticsearch - ansible.builtin.systemd: - name: elasticsearch.service - state: started - enabled: true diff --git a/roles/munin/files/plugin-conf.d/elasticsearch b/roles/munin/files/plugin-conf.d/elasticsearch deleted file mode 100644 index 92b0cdd..0000000 --- a/roles/munin/files/plugin-conf.d/elasticsearch +++ /dev/null @@ -1,3 +0,0 @@ -[elasticsearch_*] -env.host localhost -env.port 9200 diff --git a/roles/munin/files/plugins/typesense b/roles/munin/files/plugins/typesense new file mode 100755 index 0000000..8a097d1 --- /dev/null +++ b/roles/munin/files/plugins/typesense @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import urllib.request + +address = os.environ.get("address", "127.0.0.1") +port = os.environ.get("port", "8108") +api_key = os.environ.get("api_key", "") + +show_config = (len(sys.argv) == 2 and sys.argv[1] == "config") + +headers = { + "X-TYPESENSE-API-KEY": api_key, +} +url = f"http://{address}:{port}" + +# https://typesense.org/docs/26.0/api/cluster-operations.html#cluster-metrics +req_metrics = urllib.request.Request(f"{url}/metrics.json", headers=headers) +with urllib.request.urlopen(req_metrics) as f: + metrics = json.loads(f.read().decode("utf-8")) +# Keep only metrics about Typesense, other metrics are collected through other Munin plugins: +metrics = {k: v for k, v in metrics.items() if k.startswith("typesense")} + +# https://typesense.org/docs/26.0/api/cluster-operations.html#api-stats +req_stats = urllib.request.Request(f"{url}/stats.json", headers=headers) +with urllib.request.urlopen(req_stats) as f: + stats = json.loads(f.read().decode("utf-8")) + + +req_collections = urllib.request.Request(f"{url}/collections", headers=headers) +with urllib.request.urlopen(req_collections) as f: + collections = json.loads(f.read().decode("utf-8")) + + +print("multigraph typesense_memory_bytes") +if show_config: + print("graph_title Memory") + print("graph_args --base 1000") + print("graph_vlabel Memory (MB)") + print("graph_category typesense") + print("graph_scale no") + print() +for k, v in metrics.items(): + if k.startswith("typesense_memory") and k.endswith("bytes"): + name = k[len("typensense_memory"):-len("_bytes")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {int(v)/1024/1024}") + +print() + +print("multigraph typesense_memory_fragmentation_ratio") +if show_config: + print("graph_title Memory fragmentation") + print("graph_vlabel Ratio") + print("graph_category typesense") + print("graph_scale no") + print() + print("fragmentation.label Memory fragmentation ratio") + print("fragmentation.min 0") +print(f"fragmentation.value {metrics['typesense_memory_fragmentation_ratio']}") + +print() + +print("multigraph typesense_latency") +if show_config: + print("graph_title Latency") + print("graph_args --base 1000") + print("graph_vlabel Latency (ms)") + print("graph_category typesense") + print() +for k, v in stats.items(): + if k.endswith("_latency_ms"): + name = k[:-len("_latency_ms")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {v}") + +print() + +print("multigraph typesense_throughput") +if show_config: + print("graph_title Throughput") + print("graph_args --base 1000") + print("graph_vlabel Requests per second") + print("graph_category typesense") + print() +for k, v in stats.items(): + if k.endswith("_requests_per_second"): + name = k[:-len("_requests_per_second")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {v}") + +print() + +print("multigraph typesense_pending_write_batches") +if show_config: + print("graph_title Pending write batches") + print("graph_vlabel Number of pending write batches") + print("graph_category typesense") + print() + print("pending_write_batches.label Pending write batches") + print("pending_write_batches.min 0") +print(f"pending_write_batches.value {stats['pending_write_batches']}") + +print() + +if len(collections) > 0: + print("multigraph typesense_collections") + if show_config: + print("graph_title Number of documents in collections") + print("graph_vlabel Number of documents") + print("graph_category typesense") + print() + for collection in collections: + if show_config: + print(f"{collection['name']}.label {collection['name']}") + print(f"{collection['name']}.min 0") + print(f"{collection['name']}.value {collection['num_documents']}") diff --git a/roles/munin/files/systemd/munin-node-override.conf b/roles/munin/files/systemd/munin-node-override.conf index 8aff50c..7927141 100644 --- a/roles/munin/files/systemd/munin-node-override.conf +++ b/roles/munin/files/systemd/munin-node-override.conf @@ -1,2 +1,2 @@ [Unit] -After=network-online.target mariadb.service elasticsearch.service +After=network-online.target mariadb.service typesense-server.service diff --git a/roles/munin/tasks/main.yml b/roles/munin/tasks/main.yml index b1eae80..c367e91 100644 --- a/roles/munin/tasks/main.yml +++ b/roles/munin/tasks/main.yml @@ -47,23 +47,6 @@ group: root mode: "0644" -- name: get elasticsearch munin plugins - ansible.builtin.get_url: - url: https://raw.githubusercontent.com/y-ken/munin-plugin-elasticsearch/master/elasticsearch_{{ item }} - dest: "{{ munin_available_plugins_dir }}/elasticsearch_{{ item }}" - mode: u=rwx,g=rx,o=rx - with_items: - - cache - - cluster_shards - - docs - - gc_time - - index_size - - index_total - - jvm_memory - - jvm_pools_size - - jvm_threads - - open_files - - name: get memcached_multi munin plugin ansible.builtin.get_url: url: https://raw.githubusercontent.com/mhwest13/Memcached-Munin-Plugin/master/memcached_multi_ @@ -90,6 +73,12 @@ dest: "{{ munin_available_plugins_dir }}/zmd" mode: u=rwx,g=rx,o=rx +- name: copy typesense munin plugin + ansible.builtin.copy: + src: plugins/typesense + dest: "{{ munin_available_plugins_dir }}/typesense" + mode: u=rwx,g=rx,o=rx + - name: create symlinks for munin plugins ansible.builtin.file: src: "{{ munin_available_plugins_dir }}/{{ item.src }}" @@ -115,18 +104,18 @@ - { src: postfix_mailvolume, dest: postfix_mailvolume } when: installed_postfix_check is succeeded -- name: copy configuration file of ElasticSearch munin plugin - ansible.builtin.copy: - src: plugin-conf.d/elasticsearch - dest: /etc/munin/plugin-conf.d/elasticsearch - mode: u=rw,g=r,o=r - - name: copy configuration file of wget_page munin plugin ansible.builtin.template: src: plugin-conf.d/wget_page.j2 dest: /etc/munin/plugin-conf.d/wget_page mode: u=rw,g=r,o=r +- name: copy configuration file of typesense munin plugin + ansible.builtin.template: + src: plugin-conf.d/typesense.j2 + dest: /etc/munin/plugin-conf.d/typesense + mode: '640' # the file contains the Typesense API key + - name: use correct MySQL user for Munin plugin ansible.builtin.lineinfile: path: /etc/munin/plugin-conf.d/munin-node diff --git a/roles/munin/templates/plugin-conf.d/typesense.j2 b/roles/munin/templates/plugin-conf.d/typesense.j2 new file mode 100644 index 0000000..7ce7e47 --- /dev/null +++ b/roles/munin/templates/plugin-conf.d/typesense.j2 @@ -0,0 +1,2 @@ +[typesense] +env.api_key {{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }} diff --git a/roles/munin/vars/main.yml b/roles/munin/vars/main.yml index dbbaafc..217b874 100644 --- a/roles/munin/vars/main.yml +++ b/roles/munin/vars/main.yml @@ -8,16 +8,6 @@ munin_enabled_plugins: - { src: df, dest: df } - { src: df_inode, dest: df_inode } - { src: diskstats, dest: diskstats } - - { src: elasticsearch_cache, dest: elasticsearch_cache } - - { src: elasticsearch_cluster_shards, dest: elasticsearch_cluster_shards } - - { src: elasticsearch_docs, dest: elasticsearch_docs } - - { src: elasticsearch_gc_time, dest: elasticsearch_gc_time } - - { src: elasticsearch_index_size, dest: elasticsearch_index_size } - - { src: elasticsearch_index_total, dest: elasticsearch_index_total } - - { src: elasticsearch_jvm_memory, dest: elasticsearch_jvm_memory } - - { src: elasticsearch_jvm_pools_size, dest: elasticsearch_jvm_pools_size } - - { src: elasticsearch_jvm_threads, dest: elasticsearch_jvm_threads } - - { src: elasticsearch_open_files, dest: elasticsearch_open_files } - { src: entropy, dest: entropy } - { src: forks, dest: forks } - { src: fw_conntrack, dest: fw_conntrack } @@ -88,3 +78,4 @@ munin_enabled_plugins: - { src: zmd, dest: zmd_event_loop_lag } - { src: zmd, dest: zmd_memory } - { src: zmd, dest: zmd_status } + - { src: typesense, dest: typesense } diff --git a/roles/typesense/tasks/main.yml b/roles/typesense/tasks/main.yml new file mode 100644 index 0000000..646f2b2 --- /dev/null +++ b/roles/typesense/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: install Typesense + ansible.builtin.apt: + deb: https://dl.typesense.org/releases/27.1/typesense-server-27.1-amd64.deb + +- name: fix permissions for Typesense configuration file + ansible.builtin.file: + path: /etc/typesense/typesense-server.ini + mode: '640' + +- name: make Typesense listen only on localhost + ansible.builtin.lineinfile: + path: /etc/typesense/typesense-server.ini + search_string: "api-address = 0.0.0.0" + line: "api-address = 127.0.0.1" + +- name: generate logrotate config file + ansible.builtin.template: + src: logrotate/typesense.j2 + dest: /etc/logrotate.d/typesense + mode: u=rw,g=r,o=r + +- name: start Typesense + ansible.builtin.systemd: + name: typesense-server.service + state: started + enabled: true diff --git a/roles/typesense/templates/logrotate/typesense.j2 b/roles/typesense/templates/logrotate/typesense.j2 new file mode 100644 index 0000000..9423844 --- /dev/null +++ b/roles/typesense/templates/logrotate/typesense.j2 @@ -0,0 +1,8 @@ +/var/log/typesense/typesense*.log { + rotate 52 + compress + size 2M + missingok + notifempty + delaycompress +} From f1fa7bfe80f1ab15f2c54253e68ff2b7eb654a8c Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Wed, 2 Oct 2024 22:00:08 +0200 Subject: [PATCH 05/11] =?UTF-8?q?Met=20=C3=A0=20jour=20les=20d=C3=A9pendan?= =?UTF-8?q?ces=20Ruby=20et=20Python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 2 +- Gemfile.lock | 14 ++++++-------- requirements.txt | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e27b7ce..9bb5802 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/ansible-community/ansible-lint.git - rev: v24.7.0 # doit aussi être mis à jour dans requirements.txt + rev: v24.9.2 # doit aussi être mis à jour dans requirements.txt hooks: - id: ansible-lint files: \.(yaml|yml)$ diff --git a/Gemfile.lock b/Gemfile.lock index d248128..0ad94a3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,7 +7,7 @@ GEM builder (3.3.0) chef-utils (18.5.0) concurrent-ruby - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) ed25519 (1.3.0) erubi (1.13.0) ffi (1.17.0) @@ -41,15 +41,14 @@ GEM multi_json (1.15.0) net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) - net-ssh (7.2.3) + net-ssh (7.3.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) nori (2.7.1) bigdecimal pastel (0.8.0) tty-color (~> 0.5) - rexml (3.3.4) - strscan + rexml (3.3.8) rubyntlm (0.6.5) base64 rubyzip (2.3.2) @@ -58,8 +57,7 @@ GEM unicode-display_width (>= 1.5, < 3.0) unicode_utils (~> 1.4) strings-ansi (0.2.0) - strscan (3.1.0) - test-kitchen (3.6.0) + test-kitchen (3.7.0) bcrypt_pbkdf (~> 1.0) chef-utils (>= 16.4.35) ed25519 (~> 1.2) @@ -73,7 +71,7 @@ GEM winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.3.1) + thor (1.3.2) tomlrb (2.0.3) tty-box (0.7.0) pastel (~> 0.8) @@ -89,7 +87,7 @@ GEM tty-screen (~> 0.8) wisper (~> 2.0) tty-screen (0.8.2) - unicode-display_width (2.5.0) + unicode-display_width (2.6.0) unicode_utils (1.4.0) winrm (2.3.9) builder (>= 2.1.2) diff --git a/requirements.txt b/requirements.txt index fc26c7d..5b60ecb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -ansible==10.2.0 -ansible-lint==24.7.0 # doit aussi être mis à jour dans .pre-commit-config.yaml +ansible==10.4.0 +ansible-lint==24.9.2 # doit aussi être mis à jour dans .pre-commit-config.yaml pre-commit==3.8.0 From a9fa8fdcaa6c99018d912cedcee8b4a06a5fc772 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Wed, 23 Oct 2024 22:43:57 +0200 Subject: [PATCH 06/11] Ajoute des timers pour clearsessions et remove_one_year_old_ip_addresses --- roles/app/tasks/main.yml | 6 ++++++ roles/app/templates/zds-clear-sessions.service.j2 | 8 ++++++++ roles/app/templates/zds-clear-sessions.timer.j2 | 10 ++++++++++ roles/app/templates/zds-remove-old-ips.service.j2 | 8 ++++++++ roles/app/templates/zds-remove-old-ips.timer.j2 | 10 ++++++++++ 5 files changed, 42 insertions(+) create mode 100644 roles/app/templates/zds-clear-sessions.service.j2 create mode 100644 roles/app/templates/zds-clear-sessions.timer.j2 create mode 100644 roles/app/templates/zds-remove-old-ips.service.j2 create mode 100644 roles/app/templates/zds-remove-old-ips.timer.j2 diff --git a/roles/app/tasks/main.yml b/roles/app/tasks/main.yml index 1b5114e..32e4f27 100644 --- a/roles/app/tasks/main.yml +++ b/roles/app/tasks/main.yml @@ -339,6 +339,10 @@ - zmd.service - zds.service - zds.socket + - zds-clear-sessions.service + - zds-clear-sessions.timer + - zds-remove-old-ips.service + - zds-remove-old-ips.timer - zds-search-engine-index.service - zds-search-engine-index.timer - zds-watchdog.service @@ -355,6 +359,8 @@ - zmd.service - zds.service - zds.socket + - zds-clear-sessions.timer + - zds-remove-old-ips.timer - zds-search-engine-index.timer - zds-watchdog.service tags: diff --git a/roles/app/templates/zds-clear-sessions.service.j2 b/roles/app/templates/zds-clear-sessions.service.j2 new file mode 100644 index 0000000..a0b6c19 --- /dev/null +++ b/roles/app/templates/zds-clear-sessions.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Clear old sessions from database + +[Service] +Type=oneshot +User={{ appuser }} +Group={{ appuser }} +ExecStart={{ workdir }}/wrapper clearsessions diff --git a/roles/app/templates/zds-clear-sessions.timer.j2 b/roles/app/templates/zds-clear-sessions.timer.j2 new file mode 100644 index 0000000..7a5b35d --- /dev/null +++ b/roles/app/templates/zds-clear-sessions.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Clear old sessions from database + +[Timer] +# between two database backups at 2:00 and 3:15 +OnCalendar=2:30:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/roles/app/templates/zds-remove-old-ips.service.j2 b/roles/app/templates/zds-remove-old-ips.service.j2 new file mode 100644 index 0000000..28f7bbf --- /dev/null +++ b/roles/app/templates/zds-remove-old-ips.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Remove IP informations of old messages + +[Service] +Type=oneshot +User={{ appuser }} +Group={{ appuser }} +ExecStart={{ workdir }}/wrapper remove_one_year_old_ip_addresses diff --git a/roles/app/templates/zds-remove-old-ips.timer.j2 b/roles/app/templates/zds-remove-old-ips.timer.j2 new file mode 100644 index 0000000..714a38d --- /dev/null +++ b/roles/app/templates/zds-remove-old-ips.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Remove IP informations of old messages + +[Timer] +# between two database backups at 2:00 and 3:15 and after clearsessions at 2:30 +OnCalendar=3:05:00 +Persistent=true + +[Install] +WantedBy=timers.target From 2f06eca431d6d1ae852c8834183bbfaa0d4c086b Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Wed, 23 Oct 2024 22:47:00 +0200 Subject: [PATCH 07/11] =?UTF-8?q?Ajoute=20les=20d=C3=A9pendances=20entre?= =?UTF-8?q?=20les=20services=20Systemd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pour éviter que zds démarre avant mariadb et lève de exceptions de base de données inaccessible. --- roles/app/templates/zds-clear-sessions.service.j2 | 2 ++ roles/app/templates/zds-remove-old-ips.service.j2 | 2 ++ roles/app/templates/zds-search-engine-index.service.j2 | 2 ++ roles/app/templates/zds.service.j2 | 4 ++-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/app/templates/zds-clear-sessions.service.j2 b/roles/app/templates/zds-clear-sessions.service.j2 index a0b6c19..ddecbc5 100644 --- a/roles/app/templates/zds-clear-sessions.service.j2 +++ b/roles/app/templates/zds-clear-sessions.service.j2 @@ -1,5 +1,7 @@ [Unit] Description=Clear old sessions from database +Requires=mariadb.service +After=mariadb.service [Service] Type=oneshot diff --git a/roles/app/templates/zds-remove-old-ips.service.j2 b/roles/app/templates/zds-remove-old-ips.service.j2 index 28f7bbf..d1558bc 100644 --- a/roles/app/templates/zds-remove-old-ips.service.j2 +++ b/roles/app/templates/zds-remove-old-ips.service.j2 @@ -1,5 +1,7 @@ [Unit] Description=Remove IP informations of old messages +Requires=mariadb.service +After=mariadb.service [Service] Type=oneshot diff --git a/roles/app/templates/zds-search-engine-index.service.j2 b/roles/app/templates/zds-search-engine-index.service.j2 index ce9b2f0..0428959 100644 --- a/roles/app/templates/zds-search-engine-index.service.j2 +++ b/roles/app/templates/zds-search-engine-index.service.j2 @@ -1,5 +1,7 @@ [Unit] Description=Reindex new and updated content +Requires=typesense-server.service mariadb.service +After=typesense-server.service mariadb.service [Service] Type=oneshot diff --git a/roles/app/templates/zds.service.j2 b/roles/app/templates/zds.service.j2 index 2e59b9a..a92f819 100644 --- a/roles/app/templates/zds.service.j2 +++ b/roles/app/templates/zds.service.j2 @@ -1,7 +1,7 @@ [Unit] Description=Zeste de Savoir -Requires=zds.socket -After=network.target +Requires=zds.socket mariadb.service zmd.service +After=network.target mariadb.service zmd.service [Service] PIDFile={{ workdir }}/run/gunicorn.pid From 5e0c7d60e963df3b92458ecf9947ab23b917f17b Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Wed, 23 Oct 2024 22:48:41 +0200 Subject: [PATCH 08/11] =?UTF-8?q?Ajoute=20des=20s=C3=A9parateurs=20dans=20?= =?UTF-8?q?les=20logs=20des=20sauvegardes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- roles/backup/files/prod/backups.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/backup/files/prod/backups.sh b/roles/backup/files/prod/backups.sh index 6b2d6c2..88e488e 100644 --- a/roles/backup/files/prod/backups.sh +++ b/roles/backup/files/prod/backups.sh @@ -102,8 +102,10 @@ fi # Exception handling: if the first backup fails, we don't want it to stop the others. set +e backup2beta2023; err1=$? +echo # Ajouter ici les autres appels aux fonctions de sauvegarde # backup2toto; err2=? +# echo err=$((err1+err2)) set -e From 02517ea7daa2758344218a57f4c6a00f5e18e94a Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Wed, 23 Oct 2024 22:49:11 +0200 Subject: [PATCH 09/11] =?UTF-8?q?Red=C3=A9marre=20memcached=20=C3=A0=20la?= =?UTF-8?q?=20fin=20de=20la=20restauration=20d'une=20sauvegarde?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pas vraiment sûr pourquoi c'est nécessaire, mais si ce n'est pas fait, des erreurs 400 sont générées si on avait une session active. C'est nécessaire depuis qu'on a changé dans zds-site la façon dont les sessions sont stockées. --- roles/backup/templates/beta/restore-from-prod.sh.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/backup/templates/beta/restore-from-prod.sh.j2 b/roles/backup/templates/beta/restore-from-prod.sh.j2 index c91297b..40b5124 100755 --- a/roles/backup/templates/beta/restore-from-prod.sh.j2 +++ b/roles/backup/templates/beta/restore-from-prod.sh.j2 @@ -273,6 +273,8 @@ then $ZDS_WRAPPER collectstatic print_info "search_engine_manager index_all..." $ZDS_WRAPPER search_engine_manager index_all + print_info "restart memcached..." + service memcached restart # required since sessions are stored in both db and cache, so wipe the cache data from prod server fi From 58d165ca62ef8fb4bc437895fe230695682c77e2 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 27 Oct 2024 20:26:51 +0100 Subject: [PATCH 10/11] =?UTF-8?q?Ajoute=20les=20d=C3=A9pendances=20au=20se?= =?UTF-8?q?rvice=20zds-watchdog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up of 2f06eca --- roles/app/templates/zds-watchdog.service.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/app/templates/zds-watchdog.service.j2 b/roles/app/templates/zds-watchdog.service.j2 index adbd4f6..163d70d 100644 --- a/roles/app/templates/zds-watchdog.service.j2 +++ b/roles/app/templates/zds-watchdog.service.j2 @@ -1,6 +1,7 @@ [Unit] Description=Zeste de Savoir - Watchdog -After=network.target +Requires=mariadb.service zmd.service +After=network.target mariadb.service zmd.service [Service] User={{ appuser }} From 10f13d7f0e0867f811c65720b72ffe96a38b6551 Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 27 Oct 2024 20:27:49 +0100 Subject: [PATCH 11/11] Mise en production de la version 31 --- group_vars/production/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/production/vars.yml b/group_vars/production/vars.yml index 805de8f..b4d7a53 100644 --- a/group_vars/production/vars.yml +++ b/group_vars/production/vars.yml @@ -1,5 +1,5 @@ --- -appversion: v30.6-ostara +appversion: v31 env: prod public: true http_host: zestedesavoir.com