Integrate Masakari checks into rpc-maas (#754)

playbooks/files/rax-maas/plugins/masakari_api_local_check.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import collections
+import ipaddr
+
+from maas_common import generate_local_endpoint
+from maas_common import get_openstack_client
+from maas_common import metric
+from maas_common import metric_bool
+from maas_common import print_output
+from maas_common import status_err
+from maas_common import status_ok
+from requests import exceptions as exc
+
+
+
+def check(args):
+    masakari = get_openstack_client('ha')
+
+    try:
+        masakari_local_endpoint = generate_local_endpoint(
+            str(masakari.get_endpoint()), args.ip, args.port,
+            args.protocol, '/segments'
+        )
+        resp = masakari.session.get(masakari_local_endpoint, timeout=180)
+
+    except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
+        is_up = False
+        metric_bool('client_success', False, m_name='maas_masakari')
+    # Any other exception presumably isn't an API error
+    except Exception as e:
+        metric_bool('client_success', False, m_name='maas_masakari')
+        status_err(str(e), m_name='maas_masakari')
+    else:
+        is_up = resp.ok
+        milliseconds = resp.elapsed.total_seconds() * 1000
+        metric_bool('client_success', True, m_name='maas_masakari')
+
+    status_ok(m_name='maas_masakari')
+    metric_bool('masakari_api_local_status', is_up, m_name='maas_masakari')
+
+    # only want to send other metrics if api is up
+    if is_up:
+        metric('masakari_api_local_response_time',
+               'double',
+               '%.3f' % milliseconds,
+               'ms')
+
+
+def main(args):
+    check(args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='Check Masakari API against local or remote address')
+    parser.add_argument('ip', nargs='?',
+                        type=ipaddr.IPv4Address,
+                        help='Optional Masakari API server address')
+    parser.add_argument('--telegraf-output',
+                        action='store_true',
+                        default=False,
+                        help='Set the output format to telegraf')
+    parser.add_argument('--port',
+                        action='store',
+                        default='15868',
+                        help='Port to use for Masakari API check')
+    parser.add_argument('--protocol',
+                        action='store',
+                        default='http',
+                        help='Protocol to use for local Masakari API')
+    args = parser.parse_args()
+    with print_output(print_telegraf=args.telegraf_output):
+        main(args)

playbooks/maas-openstack-all.yml

@@ -68,3 +68,7 @@
 - import_playbook: maas-openstack-barbican.yml
   tags:
     - maas-openstack
+
+- import_playbook: maas-openstack-masakari.yml
+  tags:
+    - maas-openstack

playbooks/maas-openstack-masakari.yml

@@ -0,0 +1,102 @@
+---
+# Copyright 2023, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install checks for openstack masakari-api
+  hosts: masakari_api
+  gather_facts: true
+  user: "{{ ansible_ssh_user | default('root') }}"
+  become: true
+  pre_tasks:
+    - include_tasks: "common-tasks/maas_excluded_regex.yml"
+
+  tasks:
+    - name: Install masakari api checks
+      template:
+        src: "templates/rax-maas/masakari_api_local_check.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/masakari_api_local_check--{{ inventory_hostname }}.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      delegate_to: "{{ physical_host | default(ansible_host) }}"
+      when: 'physical_host != ansible_host'
+
+    - name: Install masakari api checks
+      template:
+        src: "templates/rax-maas/masakari_api_local_check.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/masakari_api_local_check--{{ inventory_hostname }}.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      when: 'physical_host == ansible_host'
+
+    - name: Install masakari lb checks
+      template:
+        src: "templates/rax-maas/lb_api_check_masakari.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/lb_api_check_masakari.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      delegate_to: "{{ physical_host | default(ansible_host) }}"
+      when:
+        - 'physical_host != ansible_host'
+        - maas_remote_check | bool
+        - not maas_private_monitoring_enabled
+
+    - name: Install masakari lb checks
+      template:
+        src: "templates/rax-maas/lb_api_check_masakari.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/lb_api_check_masakari.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      when:
+        - 'physical_host == ansible_host'
+        - maas_remote_check | bool
+        - not maas_private_monitoring_enabled
+
+    - name: Install masakari private lb checks
+      template:
+        src: "templates/rax-maas/private_lb_api_check_masakari.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/private_lb_api_check_masakari.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      delegate_to: "{{ physical_host | default(ansible_host) }}"
+      when:
+        - 'physical_host != ansible_host'
+        - maas_private_monitoring_enabled
+        - maas_private_monitoring_zone is defined
+
+    - name: Install masakari private lb checks
+      template:
+        src: "templates/rax-maas/private_lb_api_check_masakari.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/private_lb_api_check_masakari.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      when:
+        - 'physical_host == ansible_host'
+        - maas_private_monitoring_enabled
+        - maas_private_monitoring_zone is defined
+
+  vars_files:
+    - vars/main.yml
+    - vars/maas.yml
+    - vars/maas-openstack.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  tags:
+    - maas-openstack-masakari

playbooks/templates/rax-maas/lb_api_check_masakari.yaml.j2

@@ -0,0 +1,31 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "lb_api_check_masakari" %}
+{% set check_name = label %}
+{% set masakari_api_url = ansible_local.maas.api.masakari_public_url | default(ansible_local.maas.api.masakari_internal_url) %}
+{% set masakari_api_ip = ansible_local.maas.api.masakari_public_ip | default(ansible_local.maas.api.masakari_internal_ip) %}
+{% set masakari_fallback_url = (maas_masakari_scheme | default(maas_scheme)) + '://' + maas_external_hostname + ':15868/' %}
+type              : remote.http
+label             : "{{ check_name }}"
+period            : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout           : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled          : "{{ (inventory_hostname != groups['masakari_all'][0] or check_name | regex_search(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+target_resolver   : "IPv4"
+target_hostname   : "{{ masakari_api_ip | default(maas_external_ip_address) }}"
+details           :
+    url           : "{{ masakari_api_url | default(masakari_fallback_url) }}"
+monitoring_zones_poll:
+{% for zone in maas_monitoring_zones %}
+  - {{ zone }}
+{% endfor %}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms            :
+    lb_api_alarm_masakari   :
+        label               : lb_api_alarm_masakari
+        notification_plan_id: "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+        disabled            : {{ ('lb_api_alarm_masakari' | regex_search(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+        criteria            : |
+            :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+            if (metric['code'] != '300') {
+                return new AlarmStatus(CRITICAL, 'API unavailable.');
+            }

playbooks/templates/rax-maas/masakari_api_local_check.yaml.j2

@@ -0,0 +1,24 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "masakari_api_local_check" %}
+{% set check_name = label+'--'+inventory_hostname %}
+type        : agent.plugin
+label       : "{{ check_name }}"
+period      : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout     : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled    : "{{ (check_name | regex_search(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+details     :
+    file    : run_plugin_in_venv.sh
+    args    : ["{{ maas_plugin_dir }}/masakari_api_local_check.py", "{{ container_address | default(ansible_host) }}", "--protocol", "{{ masakari_local_api_protocol }}", "--port", "{{ masakari_local_api_port }}"]
+    timeout : {{ (maas_check_timeout_override[label] | default(maas_check_timeout) * 1000) }}
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms      :
+    masakari_api_local_status :
+        label                   : masakari_api_local_status--{{ inventory_hostname }}
+        notification_plan_id    : "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+        disabled                : {{ (('masakari_api_local_status--'+inventory_hostname) | regex_search(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+        criteria                : |
+            :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+            if (metric["masakari_api_local_status"] != 1) {
+                return new AlarmStatus(CRITICAL, "API unavailable");
+            }

playbooks/templates/rax-maas/private_lb_api_check_masakari.yaml.j2

@@ -0,0 +1,29 @@
+{% from "templates/common/macros.jinja" import get_metadata with context %}
+{% set label = "private_lb_api_check_masakari" %}
+{% set check_name = label %}
+{% set masakari_api_url = ansible_local.maas.api.masakari_public_url | default(ansible_local.maas.api.masakari_internal_url) %}
+{% set masakari_api_ip = ansible_local.maas.api.masakari_public_ip | default(ansible_local.maas.api.masakari_internal_ip) %}
+{% set masakari_fallback_url = (maas_masakari_scheme | default(maas_scheme)) + '://' + maas_external_hostname + ':15868/' %}
+type              : remote.http
+label             : "{{ check_name }}"
+period            : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout           : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled          : "{{ (inventory_hostname != groups['masakari_all'][0] or check_name | regex_search(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+target_resolver   : "IPv4"
+target_hostname   : "{{ masakari_api_ip | default(maas_external_ip_address) }}"
+details           :
+    url           : "{{ masakari_api_url | default(masakari_fallback_url) }}"
+monitoring_zones_poll:
+  - "{{ maas_private_monitoring_zone }}"
+{{ get_metadata(label).strip() }}
+{# Add extra metadata options with two leading white spaces #}
+alarms            :
+    private_lb_api_alarm_masakari   :
+        label               : private_lb_api_alarm_masakari
+        notification_plan_id: "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+        disabled            : {{ ('private_lb_api_alarm_masakari' | regex_search(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+        criteria            : |
+            :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+            if (metric['code'] != '300') {
+                return new AlarmStatus(CRITICAL, 'API unavailable.');
+            }

playbooks/vars/main.yml

@@ -156,6 +156,16 @@ ironic_local_api_port: '6385'
 #
 ironic_local_api_protocol: 'http'
 
+#
+# masakari_local_api_port: Port number for the local masakri api service
+#
+masakari_local_api_port:  "{{ masakari_service_port | default('15868') }}"
+
+#
+# masakari_local_api_protocol: Protocol for the local masakari API checks
+#
+masakari_local_api_protocol: "{{ masakari_service_proto | default('http') }}"
+
 #
 # horizon service
 #