Merge branch 'main' into live

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/ansible-community/ansible-lint.git
-    rev: v24.2.1 # doit aussi être mis à jour dans requirements.txt
+    rev: v24.9.2 # doit aussi être mis à jour dans requirements.txt
     hooks:
       - id: ansible-lint
         files: \.(yaml|yml)$

Gemfile.lock

@@ -1,15 +1,16 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    bcrypt_pbkdf (1.1.0)
+    base64 (0.2.0)
+    bcrypt_pbkdf (1.1.1)
     bigdecimal (3.1.8)
-    builder (3.2.4)
-    chef-utils (18.4.12)
+    builder (3.3.0)
+    chef-utils (18.5.0)
       concurrent-ruby
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.4)
     ed25519 (1.3.0)
-    erubi (1.12.0)
-    ffi (1.16.3)
+    erubi (1.13.0)
+    ffi (1.17.0)
     gssapi (1.3.1)
       ffi (>= 1.0.1)
     gyoku (1.4.0)
@@ -27,37 +28,36 @@ GEM
       tty-box (~> 0.6)
       tty-prompt (~> 0.20)
     little-plugger (1.1.4)
-    logging (2.3.1)
+    logging (2.4.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
     mixlib-install (3.12.30)
       mixlib-shellout
       mixlib-versioning
       thor
-    mixlib-shellout (3.2.7)
+    mixlib-shellout (3.2.8)
       chef-utils
     mixlib-versioning (1.2.12)
     multi_json (1.15.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.2.3)
+    net-ssh (7.3.0)
     net-ssh-gateway (2.0.0)
       net-ssh (>= 4.0.0)
-    nori (2.7.0)
+    nori (2.7.1)
       bigdecimal
     pastel (0.8.0)
       tty-color (~> 0.5)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
-    rubyntlm (0.6.3)
+    rexml (3.3.8)
+    rubyntlm (0.6.5)
+      base64
     rubyzip (2.3.2)
     strings (0.2.1)
       strings-ansi (~> 0.2)
       unicode-display_width (>= 1.5, < 3.0)
       unicode_utils (~> 1.4)
     strings-ansi (0.2.0)
-    strscan (3.1.0)
-    test-kitchen (3.6.0)
+    test-kitchen (3.7.0)
       bcrypt_pbkdf (~> 1.0)
       chef-utils (>= 16.4.35)
       ed25519 (~> 1.2)
@@ -71,7 +71,7 @@ GEM
       winrm (~> 2.0)
       winrm-elevated (~> 1.0)
       winrm-fs (~> 1.1)
-    thor (1.3.1)
+    thor (1.3.2)
     tomlrb (2.0.3)
     tty-box (0.7.0)
       pastel (~> 0.8)
@@ -87,16 +87,17 @@ GEM
       tty-screen (~> 0.8)
       wisper (~> 2.0)
     tty-screen (0.8.2)
-    unicode-display_width (2.5.0)
+    unicode-display_width (2.6.0)
     unicode_utils (1.4.0)
-    winrm (2.3.6)
+    winrm (2.3.9)
       builder (>= 2.1.2)
       erubi (~> 1.8)
       gssapi (~> 1.2)
       gyoku (~> 1.0)
       httpclient (~> 2.2, >= 2.2.0.2)
       logging (>= 1.6.1, < 3.0)
-      nori (~> 2.0)
+      nori (~> 2.0, >= 2.7.1)
+      rexml (~> 3.0)
       rubyntlm (~> 0.6.0, >= 0.6.3)
     winrm-elevated (1.2.3)
       erubi (~> 1.8)

group_vars/production/vars.yml

@@ -1,5 +1,5 @@
 ---
-appversion: v30.6-ostara
+appversion: v31
 env: prod
 public: true
 http_host: zestedesavoir.com

playbook.yml

@@ -7,7 +7,7 @@
       tags: bootstrap
     - role: backup
       tags: bootstrap
-    - role: elasticsearch
+    - role: typesense
       tags: bootstrap
     - role: mysql
       tags: bootstrap

requirements.txt

@@ -1,3 +1,3 @@
-ansible==9.3.0
-ansible-lint==24.2.1  # doit aussi être mis à jour dans .pre-commit-config.yaml
-pre-commit==3.6.2
+ansible==10.4.0
+ansible-lint==24.9.2  # doit aussi être mis à jour dans .pre-commit-config.yaml
+pre-commit==3.8.0

roles/app/tasks/fixtures.yml

@@ -16,5 +16,5 @@
       {{ workdir }}/wrapper load_factory_data {{ appdir }}/fixtures/advanced/aide_tuto_media.yaml &&
       {{ workdir }}/wrapper load_fixtures --size=low --all &&
       touch {{ appdir }}/.loaded_fixtures &&
-      {{ workdir }}/wrapper es_manager index_all; \
+      {{ workdir }}/wrapper search_engine_manager index_all; \
     fi

roles/app/tasks/main.yml

@@ -112,6 +112,14 @@
   tags:
     - bootstrap
 
+- name: get Typesense config file
+  ansible.builtin.slurp:
+    src: /etc/typesense/typesense-server.ini
+  register: typesense_file
+  tags:
+    - bootstrap
+    - upgrade
+
 - name: create app config file
   ansible.builtin.template:
     src: templates/config.toml.j2
@@ -210,27 +218,6 @@
     - bootstrap
     - upgrade
 
-- name: patch elasticsearch-dsl for Python 3.11
-  become: true
-  become_user: "{{ appuser }}"
-  ansible.builtin.lineinfile:
-    path: "{{ virtualenv }}/lib/python3.11/site-packages/elasticsearch_dsl/{{ item }}"
-    regexp: ^import collections$
-    line: import collections.abc as collections
-    firstmatch: true
-  with_items:
-    - search.py
-    - utils.py
-    - mapping.py
-    - field.py
-    - aggs.py
-    - document.py
-    - function.py
-    - query.py
-  tags:
-    - bootstrap
-    - upgrade
-
 - name: include nodejs installation
   ansible.builtin.include_role:
     name: common
@@ -352,8 +339,12 @@
     - zmd.service
     - zds.service
     - zds.socket
-    - zds-es-index.service
-    - zds-es-index.timer
+    - zds-clear-sessions.service
+    - zds-clear-sessions.timer
+    - zds-remove-old-ips.service
+    - zds-remove-old-ips.timer
+    - zds-search-engine-index.service
+    - zds-search-engine-index.timer
     - zds-watchdog.service
   tags:
     - bootstrap
@@ -368,7 +359,9 @@
     - zmd.service
     - zds.service
     - zds.socket
-    - zds-es-index.timer
+    - zds-clear-sessions.timer
+    - zds-remove-old-ips.timer
+    - zds-search-engine-index.timer
     - zds-watchdog.service
   tags:
     - bootstrap

roles/app/templates/config.toml.j2

@@ -60,8 +60,8 @@ dsn = "{{ sentry_dsn }}"
 environment = "{{ env }}"
 {% endif %}
 
-[elasticsearch]
-shards = 3
+[typesense]
+api_key = "{{ typesense_file['content'] | b64decode | regex_findall('api-key = (.+)') | first }}"
 
 {% if recaptcha is defined %}
 [recaptcha]

roles/app/templates/zds-clear-sessions.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clear old sessions from database
+Requires=mariadb.service
+After=mariadb.service
+
+[Service]
+Type=oneshot
+User={{ appuser }}
+Group={{ appuser }}
+ExecStart={{ workdir }}/wrapper clearsessions

roles/app/templates/zds-clear-sessions.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clear old sessions from database
+
+[Timer]
+# between two database backups at 2:00 and 3:15
+OnCalendar=2:30:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/app/templates/zds-remove-old-ips.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Remove IP informations of old messages
+Requires=mariadb.service
+After=mariadb.service
+
+[Service]
+Type=oneshot
+User={{ appuser }}
+Group={{ appuser }}
+ExecStart={{ workdir }}/wrapper remove_one_year_old_ip_addresses

roles/app/templates/zds-remove-old-ips.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Remove IP informations of old messages
+
+[Timer]
+# between two database backups at 2:00 and 3:15 and after clearsessions at 2:30
+OnCalendar=3:05:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/app/templates/zds-search-engine-index.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Reindex new and updated content
+Requires=typesense-server.service mariadb.service
+After=typesense-server.service mariadb.service
+
+[Service]
+Type=oneshot
+User={{ appuser }}
+Group={{ appuser }}
+ExecStart={{ workdir }}/wrapper search_engine_manager index_flagged

roles/app/templates/zds-es-index.timer.j2 → roles/app/templates/zds-search-engine-index.timer.j2

@@ -1,5 +1,5 @@
 [Unit]
-Description=ES reindex flagged contents
+Description=Search engine reindex new and updated contents
 
 [Timer]
 OnCalendar=*:30:00

roles/app/templates/zds-watchdog.service.j2

@@ -1,6 +1,7 @@
 [Unit]
 Description=Zeste de Savoir - Watchdog
-After=network.target
+Requires=mariadb.service zmd.service
+After=network.target mariadb.service zmd.service
 
 [Service]
 User={{ appuser }}

roles/app/templates/zds.service.j2

@@ -1,7 +1,7 @@
 [Unit]
 Description=Zeste de Savoir
-Requires=zds.socket
-After=network.target
+Requires=zds.socket mariadb.service zmd.service
+After=network.target mariadb.service zmd.service
 
 [Service]
 PIDFile={{ workdir }}/run/gunicorn.pid

roles/backup/files/prod/backups.sh

@@ -102,8 +102,10 @@ fi
 # Exception handling: if the first backup fails, we don't want it to stop the others.
 set +e
 backup2beta2023; err1=$?
+echo
 # Ajouter ici les autres appels aux fonctions de sauvegarde
 # backup2toto; err2=?
+# echo
 err=$((err1+err2))
 set -e
 

roles/backup/templates/beta/restore-from-prod.sh.j2

@@ -271,8 +271,10 @@ then
 	$ZDS_WRAPPER migrate
 	print_info "collectstatic..."
 	$ZDS_WRAPPER collectstatic
-	print_info "es_manager index_all..."
-	$ZDS_WRAPPER es_manager index_all
+	print_info "search_engine_manager index_all..."
+	$ZDS_WRAPPER search_engine_manager index_all
+	print_info "restart memcached..."
+	service memcached restart  # required since sessions are stored in both db and cache, so wipe the cache data from prod server
 fi