From 911bb5c9c83df9b6911593a79a1ec6e57c3232fa Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 19 May 2024 11:58:03 +0200 Subject: [PATCH] =?UTF-8?q?Remplace=20le=20r=C3=B4le=20ElasticSearch=20par?= =?UTF-8?q?=20Typesense?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- playbook.yml | 2 +- roles/app/tasks/fixtures.yml | 2 +- roles/app/tasks/main.yml | 27 +---- roles/app/templates/config.toml.j2 | 4 +- roles/app/templates/zds-es-index.service.j2 | 8 -- .../zds-search-engine-index.service.j2 | 8 ++ ...er.j2 => zds-search-engine-index.timer.j2} | 2 +- .../templates/beta/restore-from-prod.sh.j2 | 4 +- roles/elasticsearch/files/jvm.options | 94 ---------------- roles/elasticsearch/handlers/main.yml | 5 - roles/elasticsearch/tasks/main.yml | 31 ------ roles/munin/files/plugin-conf.d/elasticsearch | 3 - roles/munin/files/plugins/typesense | 105 ++++++++++++++++++ .../files/systemd/munin-node-override.conf | 2 +- roles/munin/tasks/main.yml | 35 ++---- .../templates/plugin-conf.d/typesense.j2 | 2 + roles/munin/vars/main.yml | 11 +- roles/typesense/tasks/main.yml | 32 ++++++ .../templates/logrotate/typesense.j2 | 8 ++ 19 files changed, 179 insertions(+), 206 deletions(-) delete mode 100644 roles/app/templates/zds-es-index.service.j2 create mode 100644 roles/app/templates/zds-search-engine-index.service.j2 rename roles/app/templates/{zds-es-index.timer.j2 => zds-search-engine-index.timer.j2} (59%) delete mode 100644 roles/elasticsearch/files/jvm.options delete mode 100644 roles/elasticsearch/handlers/main.yml delete mode 100644 roles/elasticsearch/tasks/main.yml delete mode 100644 roles/munin/files/plugin-conf.d/elasticsearch create mode 100755 roles/munin/files/plugins/typesense create mode 100644 roles/munin/templates/plugin-conf.d/typesense.j2 create mode 100644 roles/typesense/tasks/main.yml create mode 100644 roles/typesense/templates/logrotate/typesense.j2 diff --git a/playbook.yml b/playbook.yml index 9ea7d3d..bf0a7aa 100644 --- a/playbook.yml +++ b/playbook.yml @@ -7,7 +7,7 @@ tags: bootstrap - role: backup tags: bootstrap - - role: elasticsearch + - role: typesense tags: bootstrap - role: mysql tags: bootstrap diff --git a/roles/app/tasks/fixtures.yml b/roles/app/tasks/fixtures.yml index 3267d53..5fed8cc 100644 --- a/roles/app/tasks/fixtures.yml +++ b/roles/app/tasks/fixtures.yml @@ -16,5 +16,5 @@ {{ workdir }}/wrapper load_factory_data {{ appdir }}/fixtures/advanced/aide_tuto_media.yaml && {{ workdir }}/wrapper load_fixtures --size=low --all && touch {{ appdir }}/.loaded_fixtures && - {{ workdir }}/wrapper es_manager index_all; \ + {{ workdir }}/wrapper search_engine_manager index_all; \ fi diff --git a/roles/app/tasks/main.yml b/roles/app/tasks/main.yml index 68c5544..41e6798 100644 --- a/roles/app/tasks/main.yml +++ b/roles/app/tasks/main.yml @@ -210,27 +210,6 @@ - bootstrap - upgrade -- name: patch elasticsearch-dsl for Python 3.11 - become: true - become_user: "{{ appuser }}" - ansible.builtin.lineinfile: - path: "{{ virtualenv }}/lib/python3.11/site-packages/elasticsearch_dsl/{{ item }}" - regexp: ^import collections$ - line: import collections.abc as collections - firstmatch: true - with_items: - - search.py - - utils.py - - mapping.py - - field.py - - aggs.py - - document.py - - function.py - - query.py - tags: - - bootstrap - - upgrade - - name: include nodejs installation ansible.builtin.include_role: name: common @@ -352,8 +331,8 @@ - zmd.service - zds.service - zds.socket - - zds-es-index.service - - zds-es-index.timer + - zds-search-engine-index.service + - zds-search-engine-index.timer - zds-watchdog.service tags: - bootstrap @@ -368,7 +347,7 @@ - zmd.service - zds.service - zds.socket - - zds-es-index.timer + - zds-search-engine-index.timer - zds-watchdog.service tags: - bootstrap diff --git a/roles/app/templates/config.toml.j2 b/roles/app/templates/config.toml.j2 index 1a7a3f3..e685ac2 100644 --- a/roles/app/templates/config.toml.j2 +++ b/roles/app/templates/config.toml.j2 @@ -60,8 +60,8 @@ dsn = "{{ sentry_dsn }}" environment = "{{ env }}" {% endif %} -[elasticsearch] -shards = 3 +[typesense] +api_key = "{{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }}" {% if recaptcha is defined %} [recaptcha] diff --git a/roles/app/templates/zds-es-index.service.j2 b/roles/app/templates/zds-es-index.service.j2 deleted file mode 100644 index 6469088..0000000 --- a/roles/app/templates/zds-es-index.service.j2 +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Reindex ES Service - -[Service] -Type=oneshot -User={{ appuser }} -Group={{ appuser }} -ExecStart={{ workdir }}/wrapper es_manager index_flagged diff --git a/roles/app/templates/zds-search-engine-index.service.j2 b/roles/app/templates/zds-search-engine-index.service.j2 new file mode 100644 index 0000000..ce9b2f0 --- /dev/null +++ b/roles/app/templates/zds-search-engine-index.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Reindex new and updated content + +[Service] +Type=oneshot +User={{ appuser }} +Group={{ appuser }} +ExecStart={{ workdir }}/wrapper search_engine_manager index_flagged diff --git a/roles/app/templates/zds-es-index.timer.j2 b/roles/app/templates/zds-search-engine-index.timer.j2 similarity index 59% rename from roles/app/templates/zds-es-index.timer.j2 rename to roles/app/templates/zds-search-engine-index.timer.j2 index c4ed4ed..016185c 100644 --- a/roles/app/templates/zds-es-index.timer.j2 +++ b/roles/app/templates/zds-search-engine-index.timer.j2 @@ -1,5 +1,5 @@ [Unit] -Description=ES reindex flagged contents +Description=Search engine reindex new and updated contents [Timer] OnCalendar=*:30:00 diff --git a/roles/backup/templates/beta/restore-from-prod.sh.j2 b/roles/backup/templates/beta/restore-from-prod.sh.j2 index fa7fc4d..c91297b 100755 --- a/roles/backup/templates/beta/restore-from-prod.sh.j2 +++ b/roles/backup/templates/beta/restore-from-prod.sh.j2 @@ -271,8 +271,8 @@ then $ZDS_WRAPPER migrate print_info "collectstatic..." $ZDS_WRAPPER collectstatic - print_info "es_manager index_all..." - $ZDS_WRAPPER es_manager index_all + print_info "search_engine_manager index_all..." + $ZDS_WRAPPER search_engine_manager index_all fi diff --git a/roles/elasticsearch/files/jvm.options b/roles/elasticsearch/files/jvm.options deleted file mode 100644 index e39fb31..0000000 --- a/roles/elasticsearch/files/jvm.options +++ /dev/null @@ -1,94 +0,0 @@ -## JVM configuration - -################################################################ -## IMPORTANT: JVM heap size -################################################################ -## -## You should always set the min and max JVM heap -## size to the same value. For example, to set -## the heap to 4 GB, set: -## -## -Xms4g -## -Xmx4g -## -## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html -## for more information -## -################################################################ - -# Xms represents the initial size of total heap space -# Xmx represents the maximum size of total heap space - --Xms512m --Xmx1g - -################################################################ -## Expert settings -################################################################ -## -## All settings below this section are considered -## expert settings. Don't tamper with them unless -## you understand what you are doing -## -################################################################ - -## GC configuration --XX:+UseG1GC -# -XX:CMSInitiatingOccupancyFraction=75 -# -XX:+UseCMSInitiatingOccupancyOnly - -## optimizations - -# disable calls to System#gc --XX:+DisableExplicitGC - -# pre-touch memory pages used by the JVM during initialization --XX:+AlwaysPreTouch - -## basic - -# force the server VM --server - -# set to headless, just in case --Djava.awt.headless=true - -# ensure UTF-8 encoding by default (e.g. filenames) --Dfile.encoding=UTF-8 - -# use our provided JNA always versus the system one --Djna.nosys=true - -# flag to explicitly tell Netty to not use unsafe --Dio.netty.noUnsafe=true - -## heap dumps - -# generate a heap dump when an allocation from the Java heap fails -# heap dumps are created in the working directory of the JVM --XX:+HeapDumpOnOutOfMemoryError - -# specify an alternative path for heap dumps -# ensure the directory exists and has sufficient space -#-XX:HeapDumpPath=${heap.dump.path} - -## GC logging - -#-XX:+PrintGCDetails -#-XX:+PrintGCTimeStamps -#-XX:+PrintGCDateStamps -#-XX:+PrintClassHistogram -#-XX:+PrintTenuringDistribution -#-XX:+PrintGCApplicationStoppedTime - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${loggc} - -# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. -# If documents were already indexed with unquoted fields in a previous version -# of Elasticsearch, some operations may throw errors. -# -# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided -# only for migration purposes. -#-Delasticsearch.json.allow_unquoted_field_names=true diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml deleted file mode 100644 index 2937e19..0000000 --- a/roles/elasticsearch/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart elasticsearch - ansible.builtin.systemd: - name: elasticsearch.service - state: restarted diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml deleted file mode 100644 index ff5eb13..0000000 --- a/roles/elasticsearch/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- name: add elasticsearch repository key - ansible.builtin.apt_key: - id: 46095ACC8548582C1A2699A9D27D666CD88E42B4 - keyserver: hkp://keyserver.ubuntu.com:80 - -- name: add elasticsearch repository - ansible.builtin.apt_repository: - filename: elasticsearch - repo: deb https://artifacts.elastic.co/packages/5.x/apt stable main - state: present - -- name: install openjdk-17-jre-headless and elasticsearch - ansible.builtin.apt: - pkg: - - openjdk-17-jre-headless - - elasticsearch - cache_valid_time: 3600 - -- name: copy elasticsearch config files - ansible.builtin.copy: - src: jvm.options - dest: /etc/elasticsearch/jvm.options - mode: u=rw,g=rw - notify: restart elasticsearch - -- name: start elasticsearch - ansible.builtin.systemd: - name: elasticsearch.service - state: started - enabled: true diff --git a/roles/munin/files/plugin-conf.d/elasticsearch b/roles/munin/files/plugin-conf.d/elasticsearch deleted file mode 100644 index 92b0cdd..0000000 --- a/roles/munin/files/plugin-conf.d/elasticsearch +++ /dev/null @@ -1,3 +0,0 @@ -[elasticsearch_*] -env.host localhost -env.port 9200 diff --git a/roles/munin/files/plugins/typesense b/roles/munin/files/plugins/typesense new file mode 100755 index 0000000..226babc --- /dev/null +++ b/roles/munin/files/plugins/typesense @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import urllib.request + +address = os.environ.get("address", "127.0.0.1") +port = os.environ.get("port", "8108") +api_key = os.environ.get("api_key", "") + +show_config = (len(sys.argv) == 2 and sys.argv[1] == "config") + +headers = { + "X-TYPESENSE-API-KEY": api_key, +} +url = f"http://{address}:{port}" + +# https://typesense.org/docs/26.0/api/cluster-operations.html#cluster-metrics +req_metrics = urllib.request.Request(f"{url}/metrics.json", headers=headers) +with urllib.request.urlopen(req_metrics) as f: + metrics = json.loads(f.read().decode("utf-8")) +# Keep only metrics about Typesense, other metrics are collected through other Munin plugins: +metrics = {k: v for k, v in metrics.items() if k.startswith("typesense")} + +# https://typesense.org/docs/26.0/api/cluster-operations.html#api-stats +req_stats = urllib.request.Request(f"{url}/stats.json", headers=headers) +with urllib.request.urlopen(req_stats) as f: + stats = json.loads(f.read().decode("utf-8")) + + + +print("multigraph typesense_memory_bytes") +if show_config: + print("graph_title Memory") + print("graph_args --base 1000") + print("graph_vlabel Memory (MB)") + print("graph_category typesense") + print("graph_scale no") + print() +for k, v in metrics.items(): + if k.startswith("typesense_memory") and k.endswith("bytes"): + name = k[len("typensense_memory"):-len("_bytes")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {int(v)/1024/1024}") + +print() + +print("multigraph typesense_memory_fragmentation_ratio") +if show_config: + print("graph_title Memory fragmentation") + print("graph_vlabel Ratio") + print("graph_category typesense") + print("graph_scale no") + print() + print(f"fragmentation.label Memory fragmentation ratio") + print(f"fragmentation.min 0") +print(f"fragmentation.value {metrics['typesense_memory_fragmentation_ratio']}") + +print() + +print("multigraph typesense_latency") +if show_config: + print("graph_title Latency") + print("graph_args --base 1000") + print("graph_vlabel Latency (ms)") + print("graph_category typesense") + print() +for k, v in stats.items(): + if k.endswith("_latency_ms"): + name = k[:-len("_latency_ms")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {v}") + +print() + +print("multigraph typesense_throughput") +if show_config: + print("graph_title Throughput") + print("graph_args --base 1000") + print("graph_vlabel Requests per second") + print("graph_category typesense") + print() +for k, v in stats.items(): + if k.endswith("_requests_per_second"): + name = k[:-len("_requests_per_second")] + if show_config: + print(f"{name}.label {name}") + print(f"{name}.min 0") + print(f"{name}.value {v}") + +print() + +print("multigraph typesense_pending_write_batches") +if show_config: + print("graph_title Pending write batches") + print("graph_vlabel Number of pending write batches") + print("graph_category typesense") + print() + print(f"pending_write_batches.label Pending write batches") + print(f"pending_write_batches.min 0") +print(f"pending_write_batches.value {stats['pending_write_batches']}") diff --git a/roles/munin/files/systemd/munin-node-override.conf b/roles/munin/files/systemd/munin-node-override.conf index 8aff50c..b5a29e3 100644 --- a/roles/munin/files/systemd/munin-node-override.conf +++ b/roles/munin/files/systemd/munin-node-override.conf @@ -1,2 +1,2 @@ [Unit] -After=network-online.target mariadb.service elasticsearch.service +After=network-online.target mariadb.service typesense-server diff --git a/roles/munin/tasks/main.yml b/roles/munin/tasks/main.yml index b1eae80..c367e91 100644 --- a/roles/munin/tasks/main.yml +++ b/roles/munin/tasks/main.yml @@ -47,23 +47,6 @@ group: root mode: "0644" -- name: get elasticsearch munin plugins - ansible.builtin.get_url: - url: https://raw.githubusercontent.com/y-ken/munin-plugin-elasticsearch/master/elasticsearch_{{ item }} - dest: "{{ munin_available_plugins_dir }}/elasticsearch_{{ item }}" - mode: u=rwx,g=rx,o=rx - with_items: - - cache - - cluster_shards - - docs - - gc_time - - index_size - - index_total - - jvm_memory - - jvm_pools_size - - jvm_threads - - open_files - - name: get memcached_multi munin plugin ansible.builtin.get_url: url: https://raw.githubusercontent.com/mhwest13/Memcached-Munin-Plugin/master/memcached_multi_ @@ -90,6 +73,12 @@ dest: "{{ munin_available_plugins_dir }}/zmd" mode: u=rwx,g=rx,o=rx +- name: copy typesense munin plugin + ansible.builtin.copy: + src: plugins/typesense + dest: "{{ munin_available_plugins_dir }}/typesense" + mode: u=rwx,g=rx,o=rx + - name: create symlinks for munin plugins ansible.builtin.file: src: "{{ munin_available_plugins_dir }}/{{ item.src }}" @@ -115,18 +104,18 @@ - { src: postfix_mailvolume, dest: postfix_mailvolume } when: installed_postfix_check is succeeded -- name: copy configuration file of ElasticSearch munin plugin - ansible.builtin.copy: - src: plugin-conf.d/elasticsearch - dest: /etc/munin/plugin-conf.d/elasticsearch - mode: u=rw,g=r,o=r - - name: copy configuration file of wget_page munin plugin ansible.builtin.template: src: plugin-conf.d/wget_page.j2 dest: /etc/munin/plugin-conf.d/wget_page mode: u=rw,g=r,o=r +- name: copy configuration file of typesense munin plugin + ansible.builtin.template: + src: plugin-conf.d/typesense.j2 + dest: /etc/munin/plugin-conf.d/typesense + mode: '640' # the file contains the Typesense API key + - name: use correct MySQL user for Munin plugin ansible.builtin.lineinfile: path: /etc/munin/plugin-conf.d/munin-node diff --git a/roles/munin/templates/plugin-conf.d/typesense.j2 b/roles/munin/templates/plugin-conf.d/typesense.j2 new file mode 100644 index 0000000..7ce7e47 --- /dev/null +++ b/roles/munin/templates/plugin-conf.d/typesense.j2 @@ -0,0 +1,2 @@ +[typesense] +env.api_key {{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }} diff --git a/roles/munin/vars/main.yml b/roles/munin/vars/main.yml index dbbaafc..217b874 100644 --- a/roles/munin/vars/main.yml +++ b/roles/munin/vars/main.yml @@ -8,16 +8,6 @@ munin_enabled_plugins: - { src: df, dest: df } - { src: df_inode, dest: df_inode } - { src: diskstats, dest: diskstats } - - { src: elasticsearch_cache, dest: elasticsearch_cache } - - { src: elasticsearch_cluster_shards, dest: elasticsearch_cluster_shards } - - { src: elasticsearch_docs, dest: elasticsearch_docs } - - { src: elasticsearch_gc_time, dest: elasticsearch_gc_time } - - { src: elasticsearch_index_size, dest: elasticsearch_index_size } - - { src: elasticsearch_index_total, dest: elasticsearch_index_total } - - { src: elasticsearch_jvm_memory, dest: elasticsearch_jvm_memory } - - { src: elasticsearch_jvm_pools_size, dest: elasticsearch_jvm_pools_size } - - { src: elasticsearch_jvm_threads, dest: elasticsearch_jvm_threads } - - { src: elasticsearch_open_files, dest: elasticsearch_open_files } - { src: entropy, dest: entropy } - { src: forks, dest: forks } - { src: fw_conntrack, dest: fw_conntrack } @@ -88,3 +78,4 @@ munin_enabled_plugins: - { src: zmd, dest: zmd_event_loop_lag } - { src: zmd, dest: zmd_memory } - { src: zmd, dest: zmd_status } + - { src: typesense, dest: typesense } diff --git a/roles/typesense/tasks/main.yml b/roles/typesense/tasks/main.yml new file mode 100644 index 0000000..562050b --- /dev/null +++ b/roles/typesense/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: install Typesense + ansible.builtin.apt: + deb: https://dl.typesense.org/releases/26.0/typesense-server-26.0-amd64.deb + +- name: fix permissions for Typesense configuration file + ansible.builtin.file: + path: /etc/typesense/typesense-server.ini + mode: '640' + +- name: make Typesense listen only on localhost + ansible.builtin.lineinfile: + path: /etc/typesense/typesense-server.ini + search_string: "api-address = 0.0.0.0" + line: "api-address = 127.0.0.1" + +- name: generate logrotate config file + ansible.builtin.template: + src: logrotate/typesense.j2 + dest: /etc/logrotate.d/typesense + mode: u=rw,g=r,o=r + +- name: start Typesense + ansible.builtin.systemd: + name: typesense-server.service + state: started + enabled: true + +- name: get Typesense config file + ansible.builtin.slurp: + src: /etc/typesense/typesense-server.ini + register: typesense_file diff --git a/roles/typesense/templates/logrotate/typesense.j2 b/roles/typesense/templates/logrotate/typesense.j2 new file mode 100644 index 0000000..e7d4bd9 --- /dev/null +++ b/roles/typesense/templates/logrotate/typesense.j2 @@ -0,0 +1,8 @@ +{{ logdir }}/typesense*.log { + rotate 52 + compress + size 2M + missingok + notifempty + delaycompress +}