From 1185a5beec49aaf0c068f603273a01811b3e0623 Mon Sep 17 00:00:00 2001
From: Slawek Kaplonski <skaplons@redhat.com>
Date: Wed, 11 Oct 2023 12:42:30 +0200
Subject: [PATCH] Add support for the sidecar containers in neutron_dhcp role

Similarly to what was done in Tripleo, neutron agents which are spawning
some long running processes, like e.g. haproxy or dnsmasq should run them
in the sidecar containers to avoid breakege in the dataplane when
e.g. agent's container is restarted.

This patch adds wrappers for haproxy and dnsmasq sidecars in the
neutron_dhcp role.
Those wrappers script are taken almost directly from the TripleO's
puppet module [1] and are just converted to jinja2 format and to support
only Podman as container_cli.

[1] https://opendev.org/openstack/puppet-tripleo/src/branch/master/templates/neutron
---
 roles/edpm_neutron_dhcp/defaults/main.yml     |  8 +++
 .../edpm_neutron_dhcp/meta/argument_specs.yml | 20 +++++++
 .../molecule/default/verify.yml               | 28 +++++++++
 roles/edpm_neutron_dhcp/tasks/configure.yml   | 10 ++++
 .../templates/wrappers/dnsmasq.j2             | 56 ++++++++++++++++++
 .../templates/wrappers/haproxy.j2             | 57 +++++++++++++++++++
 6 files changed, 179 insertions(+)
 create mode 100644 roles/edpm_neutron_dhcp/templates/wrappers/dnsmasq.j2
 create mode 100644 roles/edpm_neutron_dhcp/templates/wrappers/haproxy.j2

diff --git a/roles/edpm_neutron_dhcp/defaults/main.yml b/roles/edpm_neutron_dhcp/defaults/main.yml
index cf646ad17..19dfd7e8f 100644
--- a/roles/edpm_neutron_dhcp/defaults/main.yml
+++ b/roles/edpm_neutron_dhcp/defaults/main.yml
@@ -26,6 +26,7 @@ edpm_neutron_dhcp_images_download_retries: 5
 
 edpm_neutron_dhcp_agent_config_src: "/var/lib/openstack/configs/neutron-dhcp"
 edpm_neutron_dhcp_agent_config_dir: "/var/lib/config-data/ansible-generated/neutron-dhcp-agent"
+edpm_neutron_dhcp_agent_lib_dir: "/var/lib/neutron"
 edpm_neutron_dhcp_image: "quay.io/podified-antelope-centos9/openstack-neutron-dhcp-agent:current-podified"
 
 edpm_neutron_dhcp_common_volumes:
@@ -35,6 +36,13 @@ edpm_neutron_dhcp_common_volumes:
   - /var/lib/kolla/config_files/neutron_dhcp_agent.json:/var/lib/kolla/config_files/config.json:ro
   - /var/log/containers/neutron:/var/log/neutron:z
   - /run/openvswitch:/run/openvswitch:shared,z
+  - "{{ edpm_neutron_dhcp_agent_lib_dir }}/dhcp_agent_haproxy_wrapper:/usr/local/bin/haproxy:ro"
+  - "{{ edpm_neutron_dhcp_agent_lib_dir }}/dhcp_agent_dnsmasq_wrapper:/usr/local/bin/dnsmasq:ro"
+
+# Sidecar containers settings
+edpm_neutron_dhcp_sidecar_debug: false
+edpm_neutron_dhcp_sidecar_haproxy_image_name: "{{ edpm_neutron_dhcp_image }}"
+edpm_neutron_dhcp_sidecar_dnsmasq_image_name: "{{ edpm_neutron_dhcp_image }}"
 
 # neutron.conf
 # DEFAULT
diff --git a/roles/edpm_neutron_dhcp/meta/argument_specs.yml b/roles/edpm_neutron_dhcp/meta/argument_specs.yml
index 47a7ebac8..061c8dafe 100644
--- a/roles/edpm_neutron_dhcp/meta/argument_specs.yml
+++ b/roles/edpm_neutron_dhcp/meta/argument_specs.yml
@@ -24,6 +24,12 @@ argument_specs:
           The path to the directory containing Neutron DHCP agent config
           files.
         type: str
+      edpm_neutron_dhcp_agent_lib_dir:
+        default: "/var/lib/neutron"
+        description: |
+          The path to the directory containing files required by the Neutron DHCP
+          agent, like e.g. sidecar container wrappers.
+        type: str
       edpm_neutron_dhcp_image:
         default: "quay.io/podified-antelope-centos9/openstack-neutron-dhcp-agent:current-podified"
         description: Neutron DHCP agent container image.
@@ -36,8 +42,22 @@ argument_specs:
           - /var/lib/kolla/config_files/neutron_dhcp_agent.json:/var/lib/kolla/config_files/config.json:ro
           - /var/log/containers/neutron:/var/log/neutron:z
           - /run/openvswitch:/run/openvswitch:shared,z
+          - "{{ edpm_neutron_dhcp_agent_lib_dir }}/dhcp_agent_haproxy_wrapper:/usr/local/bin/haproxy:ro"
+          - "{{ edpm_neutron_dhcp_agent_lib_dir }}/dhcp_agent_dnsmasq_wrapper:/usr/local/bin/dnsmasq:ro"
         description: List of volumes in a mount point form.
         type: list
+      edpm_neutron_dhcp_sidecar_debug:
+        default: false
+        description: ''
+        type: bool
+      edpm_neutron_dhcp_sidecar_haproxy_image_name:
+        default: "{{ edpm_neutron_dhcp_image }}"
+        description: Image used to spawn haproxy sidecar containers.
+        type: str
+      edpm_neutron_dhcp_sidecar_dnsmasq_image_name:
+        default: "{{ edpm_neutron_dhcp_image }}"
+        description: Image used to spawn dnsmasq sidecar containers.
+        type: str
       edpm_neutron_dhcp_DEFAULT_debug:
         default: false
         description: "Enable or disable DEBUG mode in the Neutron DHCP agent"
diff --git a/roles/edpm_neutron_dhcp/molecule/default/verify.yml b/roles/edpm_neutron_dhcp/molecule/default/verify.yml
index 253555228..e4abecb63 100644
--- a/roles/edpm_neutron_dhcp/molecule/default/verify.yml
+++ b/roles/edpm_neutron_dhcp/molecule/default/verify.yml
@@ -51,3 +51,31 @@
             that:
               - dhcp_config.stat.exists
             fail_msg: "dhcp agent config file does not exist"
+
+    - name: Ensure that haproxy wrapper script was created
+      block:
+        - name: haproxy wrapper exists
+          become: true
+          ansible.builtin.stat:
+            path: "/var/lib/neutron/dhcp_agent_haproxy_wrapper"
+          register: haproxy_wrapper
+
+        - name: assert that haproxy wrapper exists
+          ansible.builtin.assert:
+            that:
+              - haproxy_wrapper.stat.exists
+            fail_msg: "haproxy wrapper script does not exist"
+
+    - name: Ensure that dnsmasq wrapper script was created
+      block:
+        - name: dnsmasq wrapper exists
+          become: true
+          ansible.builtin.stat:
+            path: "/var/lib/neutron/dhcp_agent_dnsmasq_wrapper"
+          register: dnsmasq_wrapper
+
+        - name: assert that dnsmasq wrapper exists
+          ansible.builtin.assert:
+            that:
+              - dnsmasq_wrapper.stat.exists
+            fail_msg: "dnsmasq wrapper script does not exist"
diff --git a/roles/edpm_neutron_dhcp/tasks/configure.yml b/roles/edpm_neutron_dhcp/tasks/configure.yml
index a500c9a33..8c291bf03 100644
--- a/roles/edpm_neutron_dhcp/tasks/configure.yml
+++ b/roles/edpm_neutron_dhcp/tasks/configure.yml
@@ -50,3 +50,13 @@
         setype: "container_file_t"
         mode: "0644"
       loop: "{{ edpm_neutron_dhcp_secrets.files }}"
+
+- name: Configure sidecar containers scripts
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ edpm_neutron_dhcp_agent_lib_dir }}/{{ item.dest }}"
+    setype: "container_file_t"
+    mode: "0755"
+  loop:
+    - {"src": "wrappers/haproxy.j2", "dest": "dhcp_agent_haproxy_wrapper"}
+    - {"src": "wrappers/dnsmasq.j2", "dest": "dhcp_agent_dnsmasq_wrapper"}
diff --git a/roles/edpm_neutron_dhcp/templates/wrappers/dnsmasq.j2 b/roles/edpm_neutron_dhcp/templates/wrappers/dnsmasq.j2
new file mode 100644
index 000000000..f9f5fa63c
--- /dev/null
+++ b/roles/edpm_neutron_dhcp/templates/wrappers/dnsmasq.j2
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+{% if edpm_neutron_dhcp_sidecar_debug -%}
+set -x
+{%- endif -%}
+
+ARGS="$@"
+
+IMAGE_NAME={{ edpm_neutron_dhcp_sidecar_dnsmasq_image_name }}
+
+{% raw -%}
+# Extract the network namespace UUID from the command line args provided by
+# neutron. Typically of the form (with dnsmasq as an example):
+#
+# dnsmasq --no-hosts --no-resolv --except-interface=lo \
+#   --pid-file=/var/lib/neutron/dhcp/317716b8-919a-4a6f-8db1-78128ec3b100/pid  \
+#   --dhcp-hostsfile=/var/lib/neutron/dhcp/317716b8-919a-4a6f-8db1-78128ec3b100/host ...
+NETNS=$(ip netns identify)
+NAME=neutron-dnsmasq-${NETNS}
+
+CLI="nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman"
+LOGGING="--log-driver k8s-file --log-opt path=/var/log/containers/stdouts/${NAME}.log"
+CMD='/usr/sbin/dnsmasq -k'
+
+LIST=$($CLI ps -a --filter name=neutron-dnsmasq- --format '{{.ID}}:{{.Names}}:{{.Status}}' | awk '{print $1}')
+
+# Find orphaned containers left for dead after its main process terminated by neutron parent process
+# FIXME(cjeanner): https://github.com/containers/libpod/issues/1703
+ORPHANS=$(printf "%s\n" "${LIST}" | grep -E ":(Exited|Created)")
+if [ -n "${ORPHANS}" ]; then
+    for orphant in $(printf "%s\n" "${ORPHANS}" | awk -F':' '{print $1}'); do
+        echo "Removing orphaned container ${orphant}"
+        # TODO(slaweq): script should at least log what error
+        # prevented to stop or rm orphaned container if there will be any error
+        $CLI stop ${orphant} || true
+        $CLI rm -f ${orphant} || true
+    done
+fi
+
+# If the NAME is already taken by a container, give it an unique name
+printf "%s\n" "${LIST}" | grep -q "${NAME}$" && NAME="${NAME}-$(date +%Y-%m-%d-%H%M%S-%N)"
+echo "Starting a new child container ${NAME}"
+$CLI run --detach ${LOGGING} \
+     -v /var/lib/config-data/puppet-generated/neutron/etc/neutron:/etc/neutron:ro \
+     -v /run/netns:/run/netns:shared \
+     -v /var/lib/neutron:/var/lib/neutron:shared \
+     -v /dev/log:/dev/log \
+     --net host \
+     --pid host \
+     --cgroupns host \
+     --privileged \
+     -u root \
+     --name $NAME \
+    ${IMAGE_NAME} \
+    $CMD $ARGS
+{%- endraw %}
diff --git a/roles/edpm_neutron_dhcp/templates/wrappers/haproxy.j2 b/roles/edpm_neutron_dhcp/templates/wrappers/haproxy.j2
new file mode 100644
index 000000000..66711cb62
--- /dev/null
+++ b/roles/edpm_neutron_dhcp/templates/wrappers/haproxy.j2
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+{% if edpm_neutron_dhcp_sidecar_debug -%}
+set -x
+{%- endif -%}
+
+ARGS="$@"
+
+IMAGE_NAME={{ edpm_neutron_dhcp_sidecar_haproxy_image_name }}
+
+{% raw -%}
+# Extract the network namespace UUID from the command line args provided by
+# neutron. Typically of the form (with dnsmasq as an example):
+#
+# dnsmasq --no-hosts --no-resolv --except-interface=lo \
+#   --pid-file=/var/lib/neutron/dhcp/317716b8-919a-4a6f-8db1-78128ec3b100/pid  \
+#   --dhcp-hostsfile=/var/lib/neutron/dhcp/317716b8-919a-4a6f-8db1-78128ec3b100/host ...
+NETNS=$(ip netns identify)
+NAME=neutron-haproxy-${NETNS}
+HAPROXY_CMD='$(if [ -f /usr/sbin/haproxy-systemd-wrapper ]; then echo "/usr/sbin/haproxy -Ds"; else echo "/usr/sbin/haproxy -Ws"; fi)'
+
+CLI="nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman"
+LOGGING="--log-driver k8s-file --log-opt path=/var/log/containers/stdouts/${NAME}.log"
+CMD='$HAPROXY'
+
+LIST=$($CLI ps -a --filter name=neutron-haproxy- --format '{{.ID}}:{{.Names}}:{{.Status}}' | awk '{print $1}')
+
+# Find orphaned containers left for dead after its main process terminated by neutron parent process
+# FIXME(cjeanner): https://github.com/containers/libpod/issues/1703
+ORPHANS=$(printf "%s\n" "${LIST}" | grep -E ":(Exited|Created)")
+if [ -n "${ORPHANS}" ]; then
+    for orphant in $(printf "%s\n" "${ORPHANS}" | awk -F':' '{print $1}'); do
+        echo "Removing orphaned container ${orphant}"
+        # TODO(slaweq): script should at least log what error
+        # prevented to stop or rm orphaned container if there will be any error
+        $CLI stop ${orphant} || true
+        $CLI rm -f ${orphant} || true
+    done
+fi
+
+# If the NAME is already taken by a container, give it an unique name
+printf "%s\n" "${LIST}" | grep -q "${NAME}$" && NAME="${NAME}-$(date +%Y-%m-%d-%H%M%S-%N)"
+echo "Starting a new child container ${NAME}"
+$CLI run --detach ${LOGGING} \
+     -v /var/lib/config-data/puppet-generated/neutron/etc/neutron:/etc/neutron:ro \
+     -v /run/netns:/run/netns:shared \
+     -v /var/lib/neutron:/var/lib/neutron:shared \
+     -v /dev/log:/dev/log \
+     --net host \
+     --pid host \
+     --cgroupns host \
+     --privileged \
+     -u root \
+     --name $NAME \
+     ${IMAGE_NAME} \
+     /bin/bash -c "HAPROXY=\"$HAPROXY_CMD\"; exec $CMD $ARGS"
+{%- endraw %}