Remplace le rôle ElasticSearch par Typesense

playbook.yml

@@ -7,7 +7,7 @@
       tags: bootstrap
     - role: backup
       tags: bootstrap
-    - role: elasticsearch
+    - role: typesense
       tags: bootstrap
     - role: mysql
       tags: bootstrap

roles/app/tasks/fixtures.yml

@@ -16,5 +16,5 @@
       {{ workdir }}/wrapper load_factory_data {{ appdir }}/fixtures/advanced/aide_tuto_media.yaml &&
       {{ workdir }}/wrapper load_fixtures --size=low --all &&
       touch {{ appdir }}/.loaded_fixtures &&
-      {{ workdir }}/wrapper es_manager index_all; \
+      {{ workdir }}/wrapper search_engine_manager index_all; \
     fi

roles/app/tasks/main.yml

@@ -210,27 +210,6 @@
     - bootstrap
     - upgrade
 
-- name: patch elasticsearch-dsl for Python 3.11
-  become: true
-  become_user: "{{ appuser }}"
-  ansible.builtin.lineinfile:
-    path: "{{ virtualenv }}/lib/python3.11/site-packages/elasticsearch_dsl/{{ item }}"
-    regexp: ^import collections$
-    line: import collections.abc as collections
-    firstmatch: true
-  with_items:
-    - search.py
-    - utils.py
-    - mapping.py
-    - field.py
-    - aggs.py
-    - document.py
-    - function.py
-    - query.py
-  tags:
-    - bootstrap
-    - upgrade
-
 - name: include nodejs installation
   ansible.builtin.include_role:
     name: common
@@ -352,8 +331,8 @@
     - zmd.service
     - zds.service
     - zds.socket
-    - zds-es-index.service
-    - zds-es-index.timer
+    - zds-search-engine-index.service
+    - zds-search-engine-index.timer
     - zds-watchdog.service
   tags:
     - bootstrap
@@ -368,7 +347,7 @@
     - zmd.service
     - zds.service
     - zds.socket
-    - zds-es-index.timer
+    - zds-search-engine-index.timer
     - zds-watchdog.service
   tags:
     - bootstrap

roles/app/templates/config.toml.j2

@@ -60,8 +60,8 @@ dsn = "{{ sentry_dsn }}"
 environment = "{{ env }}"
 {% endif %}
 
-[elasticsearch]
-shards = 3
+[typesense]
+api_key = "{{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }}"
 
 {% if recaptcha is defined %}
 [recaptcha]

roles/app/templates/zds-search-engine-index.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Reindex new and updated content
+
+[Service]
+Type=oneshot
+User={{ appuser }}
+Group={{ appuser }}
+ExecStart={{ workdir }}/wrapper search_engine_manager index_flagged

roles/app/templates/zds-es-index.timer.j2 → roles/app/templates/zds-search-engine-index.timer.j2

@@ -1,5 +1,5 @@
 [Unit]
-Description=ES reindex flagged contents
+Description=Search engine reindex new and updated contents
 
 [Timer]
 OnCalendar=*:30:00

roles/backup/templates/beta/restore-from-prod.sh.j2

@@ -271,8 +271,8 @@ then
 	$ZDS_WRAPPER migrate
 	print_info "collectstatic..."
 	$ZDS_WRAPPER collectstatic
-	print_info "es_manager index_all..."
-	$ZDS_WRAPPER es_manager index_all
+	print_info "search_engine_manager index_all..."
+	$ZDS_WRAPPER search_engine_manager index_all
 fi
 
 

roles/munin/files/plugins/typesense

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+
+address = os.environ.get("address", "127.0.0.1")
+port = os.environ.get("port", "8108")
+api_key = os.environ.get("api_key", "")
+
+show_config = (len(sys.argv) == 2 and sys.argv[1] == "config")
+
+headers = {
+    "X-TYPESENSE-API-KEY": api_key,
+}
+url = f"http://{address}:{port}"
+
+# https://typesense.org/docs/26.0/api/cluster-operations.html#cluster-metrics
+req_metrics = urllib.request.Request(f"{url}/metrics.json", headers=headers)
+with urllib.request.urlopen(req_metrics) as f:
+    metrics = json.loads(f.read().decode("utf-8"))
+# Keep only metrics about Typesense, other metrics are collected through other Munin plugins:
+metrics  = {k: v for k, v in metrics.items() if k.startswith("typesense")}
+
+# https://typesense.org/docs/26.0/api/cluster-operations.html#api-stats
+req_stats = urllib.request.Request(f"{url}/stats.json", headers=headers)
+with urllib.request.urlopen(req_stats) as f:
+    stats = json.loads(f.read().decode("utf-8"))
+
+
+
+print("multigraph typesense_memory_bytes")
+if show_config:
+    print("graph_title Memory")
+    print("graph_args --base 1000")
+    print("graph_vlabel Memory (MB)")
+    print("graph_category typesense")
+    print("graph_scale no")
+    print()
+for k, v in metrics.items():
+    if k.startswith("typesense_memory") and k.endswith("bytes"):
+        name = k[len("typensense_memory"):-len("_bytes")]
+        if show_config:
+            print(f"{name}.label {name}")
+            print(f"{name}.min 0")
+        print(f"{name}.value {int(v)/1024/1024}")
+
+print()
+
+print("multigraph typesense_memory_fragmentation_ratio")
+if show_config:
+    print("graph_title Memory fragmentation")
+    print("graph_vlabel Ratio")
+    print("graph_category typesense")
+    print("graph_scale no")
+    print()
+    print(f"fragmentation.label Memory fragmentation ratio")
+    print(f"fragmentation.min 0")
+print(f"fragmentation.value {metrics['typesense_memory_fragmentation_ratio']}")
+
+print()
+
+print("multigraph typesense_latency")
+if show_config:
+    print("graph_title Latency")
+    print("graph_args --base 1000")
+    print("graph_vlabel Latency (ms)")
+    print("graph_category typesense")
+    print()
+for k, v in stats.items():
+    if k.endswith("_latency_ms"):
+        name = k[:-len("_latency_ms")]
+        if show_config:
+            print(f"{name}.label {name}")
+            print(f"{name}.min 0")
+        print(f"{name}.value {v}")
+
+print()
+
+print("multigraph typesense_throughput")
+if show_config:
+    print("graph_title Throughput")
+    print("graph_args --base 1000")
+    print("graph_vlabel Requests per second")
+    print("graph_category typesense")
+    print()
+for k, v in stats.items():
+    if k.endswith("_requests_per_second"):
+        name = k[:-len("_requests_per_second")]
+        if show_config:
+            print(f"{name}.label {name}")
+            print(f"{name}.min 0")
+        print(f"{name}.value {v}")
+
+print()
+
+print("multigraph typesense_pending_write_batches")
+if show_config:
+    print("graph_title Pending write batches")
+    print("graph_vlabel Number of pending write batches")
+    print("graph_category typesense")
+    print()
+    print(f"pending_write_batches.label Pending write batches")
+    print(f"pending_write_batches.min 0")
+print(f"pending_write_batches.value {stats['pending_write_batches']}")

roles/munin/files/systemd/munin-node-override.conf

@@ -1,2 +1,2 @@
 [Unit]
-After=network-online.target mariadb.service elasticsearch.service
+After=network-online.target mariadb.service typesense-server