squid: Handle the start of the service gracefully (#1526)

Signed-off-by: Gondermann <[email protected]>

roles/squid/defaults/main.yml

@@ -12,6 +12,12 @@ docker_network_mtu: 1500
 
 docker_registry_squid: index.docker.io
 
+##########################
+# packages
+
+squid_required_packages:
+  - jq
+
 ##########################
 # squid
 
@@ -27,3 +33,5 @@ squid_port: 3128
 squid_tag: 5.7-23.04_beta  # don't get fooled by "beta". this is actually production ready
 squid_image: "{{ docker_registry_squid }}/ubuntu/squid:{{ squid_tag }}"
 squid_container_name: squid
+
+squid_service_restart: true

roles/squid/handlers/main.yml

@@ -1,7 +1,44 @@
 ---
-# handlers file for squid
 - name: Restart squid service
   become: true
   ansible.builtin.service:
     name: "{{ squid_service_name }}"
     state: restarted
+  register: result
+  until: result.status.ActiveState == "active"
+  retries: 10
+  delay: 20
+  when: squid_service_restart|bool
+  notify:
+    - Wait for squid service to start
+    - Register that squid service was restarted
+
+- name: Wait for squid service to start
+  ansible.builtin.pause:
+    minutes: 1
+  changed_when: true
+  notify:
+    - Wait for an healthy squid service
+
+# NOTE: This handler prevents a squid restart when the service
+# was already started via ansible.builtin.service.
+- name: Register that squid service was restarted
+  ansible.builtin.set_fact:
+    squid_service_restart: false
+
+# NOTE: The command returns a list of IDs of containers from the service
+#       that are currently starting or unhealthy. As long as the list is not empty
+#       the service is not in a good state.
+- name: Wait for an healthy squid service
+  ansible.builtin.shell: |
+    set -o pipefail
+    docker compose --project-directory /opt/squid \
+        ps --all --format json | \
+        jq '. | select(.State=="created" or .State=="exited" or .Health=="starting" or .Health=="unhealthy") | .Name'
+  args:
+    executable: /bin/bash
+  register: result
+  until: "result.stdout | length == 0"
+  retries: 60
+  delay: 10
+  changed_when: true

roles/squid/tasks/install-Debian-family.yml

@@ -0,0 +1,7 @@
+---
+- name: Install required packages
+  become: true
+  ansible.builtin.apt:
+    name: "{{ squid_required_packages }}"
+    state: present
+    lock_timeout: "{{ apt_lock_timeout | default(300) }}"

roles/squid/tasks/install-RedHat-family.yml

@@ -0,0 +1,7 @@
+---
+- name: Install required packages
+  become: true
+  ansible.builtin.dnf:
+    name: "{{ squid_required_packages }}"
+    state: present
+    lock_timeout: "{{ dnf_lock_timeout | default(300) }}"

roles/squid/tasks/main.yml

@@ -1,6 +1,10 @@
 ---
 # tasks file for squid
 
+- name: Include install tasks
+  ansible.builtin.include_tasks: "install-{{ ansible_os_family }}-family.yml"
+  tags: install
+
 - name: Create required directories
   become: true
   ansible.builtin.file:
@@ -39,9 +43,43 @@
     mode: 0640
   notify: Restart squid service
 
-- name: Manage squid service
-  become: true
-  ansible.builtin.service:
-    name: "{{ squid_service_name }}"
-    state: started
-    enabled: true
+- name: Ensure that the squid service is up and running
+  block:  # noqa osism-fqcn
+    - name: Manage squid service
+      become: true
+      ansible.builtin.service:
+        name: "{{ squid_service_name }}"
+        state: started
+        enabled: true
+      register: result
+      until: result.status.ActiveState == "active"
+      retries: 10
+      delay: 20
+      notify: Wait for squid service to start
+  rescue:
+    # Compose is not always reliable when starting services. Therefore,
+    # in case of an error at startup, another stop and start of the
+    # service is tried here.
+    - name: Stop squid service
+      become: true
+      ansible.builtin.service:
+        name: "{{ squid_service_name }}"
+        state: stopped
+
+    - name: Do a manual start of the squid service
+      ansible.builtin.command: "/usr/bin/docker compose --project-directory {{ squid_docker_compose_directory }} up -d --remove-orphans"
+      changed_when: true
+      failed_when: false
+
+    # This does not change anything on the service side, but the unit is
+    # then in the expected state.
+    - name: Start squid service again
+      become: true
+      ansible.builtin.service:
+        name: "{{ squid_service_name }}"
+        state: started
+      register: result
+      until: result.status.ActiveState == "active"
+      retries: 10
+      delay: 20
+      notify: Register that squid service was started