From 5b875e3748a5fa0fb1f097768657d94b15bb1514 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Tue, 3 Sep 2024 12:21:03 -0500 Subject: [PATCH] feat(workflows): sensor and trigger to sync keystone to nautobot Added an EventSource listener for keystone notifications. Added a Sensor to trigger a workflow to be run when projects are created, updated or deleted in keystone. Added a workflow to execute the keystone sync script on each of those triggers. # new file: apps/understack-workflows/workflowtemplates/kustomization.yaml --- .github/workflows/yamllint.yaml | 1 + .../eventsource-openstack/argo-rabbitmq.yaml | 18 +++++- .../eventsource-openstack/kustomization.yaml | 1 + .../openstack-event-source.yaml | 39 +++++++++++- .../sensor-keystone-event-project.yaml | 62 +++++++++++++++++++ apps/understack-workflows/kustomization.yaml | 1 + .../workflowtemplates/kustomization.yaml | 9 +++ .../workflowtemplates/openstack-svc-acct.yaml | 28 +++++++++ .../wf-keystone-event-project.yaml | 38 ++++++++++++ 9 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 apps/understack-workflows/eventsource-openstack/sensor-keystone-event-project.yaml create mode 100644 apps/understack-workflows/workflowtemplates/kustomization.yaml create mode 100644 apps/understack-workflows/workflowtemplates/openstack-svc-acct.yaml create mode 100644 apps/understack-workflows/workflowtemplates/wf-keystone-event-project.yaml diff --git a/.github/workflows/yamllint.yaml b/.github/workflows/yamllint.yaml index cb6827d74..0e3fe02fd 100644 --- a/.github/workflows/yamllint.yaml +++ b/.github/workflows/yamllint.yaml @@ -46,6 +46,7 @@ jobs: argo-workflows/**/workflowtemplates/*.y*ml argo-workflows/**/sensors/*.y*ml argo-workflows/**/workflows/*.y*ml + apps/understack-workflows/workflowtemplates/wf-*.y*ml shellcheck: runs-on: ubuntu-latest diff --git a/apps/understack-workflows/eventsource-openstack/argo-rabbitmq.yaml b/apps/understack-workflows/eventsource-openstack/argo-rabbitmq.yaml index 6cf4d3e4f..25331fd48 100644 --- a/apps/understack-workflows/eventsource-openstack/argo-rabbitmq.yaml +++ b/apps/understack-workflows/eventsource-openstack/argo-rabbitmq.yaml @@ -18,8 +18,24 @@ spec: name: "argo" # name of a user.rabbitmq.com in the same namespace; must specify either spec.userReference or spec.user permissions: write: "^$" - configure: "^$" + configure: ".*" read: "^ironic_versioned_notifications.info$" rabbitmqClusterReference: name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource namespace: openstack +--- +apiVersion: rabbitmq.com/v1beta1 +kind: Permission +metadata: + name: argo-to-keystone-permission +spec: + vhost: "keystone" + userReference: + name: "argo" # name of a user.rabbitmq.com in the same namespace; must specify either spec.userReference or spec.user + permissions: + write: "^$" + configure: ".*" + read: "^notifications.info$" + rabbitmqClusterReference: + name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource + namespace: openstack diff --git a/apps/understack-workflows/eventsource-openstack/kustomization.yaml b/apps/understack-workflows/eventsource-openstack/kustomization.yaml index 5f308fc53..6828ca22d 100644 --- a/apps/understack-workflows/eventsource-openstack/kustomization.yaml +++ b/apps/understack-workflows/eventsource-openstack/kustomization.yaml @@ -7,3 +7,4 @@ resources: - argo-rabbitmq.yaml - eventbus-default.yaml - openstack-event-source.yaml + - sensor-keystone-event-project.yaml diff --git a/apps/understack-workflows/eventsource-openstack/openstack-event-source.yaml b/apps/understack-workflows/eventsource-openstack/openstack-event-source.yaml index 3c0c22f4c..4e71dfe5e 100644 --- a/apps/understack-workflows/eventsource-openstack/openstack-event-source.yaml +++ b/apps/understack-workflows/eventsource-openstack/openstack-event-source.yaml @@ -7,7 +7,6 @@ spec: openstack: # amqp server url url: amqp://rabbitmq-server-0.rabbitmq-nodes.openstack.svc.cluster.local:5672/ironic - routingKey: 'ironic_versioned_notifications.info' # jsonBody specifies that all event body payload coming from this # source will be JSON jsonBody: true @@ -16,6 +15,44 @@ spec: exchangeType: topic exchangeDeclare: durable: false + # routing key for messages within the exchange + routingKey: 'ironic_versioned_notifications.info' + # optional consume settings + # if not provided, default values will be used + consume: + consumerTag: "argo-events" + autoAck: true + exclusive: false + noLocal: false + # username and password for authentication + # use secret selectors + auth: + username: + name: argo-user-credentials + key: username + password: + name: argo-user-credentials + key: password +--- +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: openstack-keystone +spec: + amqp: + notifications: + # amqp server url + url: amqp://rabbitmq-server-0.rabbitmq-nodes.openstack.svc.cluster.local:5672/keystone + # jsonBody specifies that all event body payload coming from this + # source will be JSON + jsonBody: true + # name of the exchange. + exchangeName: keystone + exchangeType: topic + exchangeDeclare: + durable: false + # routing key for messages within the exchange + routingKey: 'notifications.info' # optional consume settings # if not provided, default values will be used consume: diff --git a/apps/understack-workflows/eventsource-openstack/sensor-keystone-event-project.yaml b/apps/understack-workflows/eventsource-openstack/sensor-keystone-event-project.yaml new file mode 100644 index 000000000..f7ed12fdd --- /dev/null +++ b/apps/understack-workflows/eventsource-openstack/sensor-keystone-event-project.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + finalizers: + - sensor-controller + labels: + argocd.argoproj.io/instance: argo-events + name: keystone-event-project + namespace: argo-events + annotations: + workflows.argoproj.io/description: | + Defined in `apps/understack-workflows/sensors/sensor-keystone-event-project.yaml` +spec: + dependencies: + - eventName: notifications + eventSourceName: openstack-keystone + name: keystone-msg + transform: + jq: ".body[\"oslo.message\"] | fromjson" + filters: + dataLogicalOperator: "and" + data: + - path: "event_type" + type: "string" + value: + - "identity.project.created" + - "identity.project.updated" + - "identity.project.deleted" + template: + serviceAccountName: operate-workflow-sa + triggers: + - template: + name: keystone-event-project + argoWorkflow: + operation: submit + parameters: + - dest: spec.arguments.parameters.0.value + src: + dataKey: event_type + dependencyName: keystone-msg + - dest: spec.arguments.parameters.1.value + src: + dataKey: payload.target.id + dependencyName: keystone-msg + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: keystone-event-project- + namespace: argo-events + spec: + arguments: + parameters: + - name: event_type + value: "replace by parameters section" + - name: project_uuid + value: "replaced by parameters section" + serviceAccountName: workflow + workflowTemplateRef: + name: keystone-event-project diff --git a/apps/understack-workflows/kustomization.yaml b/apps/understack-workflows/kustomization.yaml index bc3323f17..d8a7641f5 100644 --- a/apps/understack-workflows/kustomization.yaml +++ b/apps/understack-workflows/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - eventsource-openstack + - workflowtemplates diff --git a/apps/understack-workflows/workflowtemplates/kustomization.yaml b/apps/understack-workflows/workflowtemplates/kustomization.yaml new file mode 100644 index 000000000..a156e0b94 --- /dev/null +++ b/apps/understack-workflows/workflowtemplates/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# this is where our workflows currently run +namespace: argo-events + +resources: + - openstack-svc-acct.yaml + - wf-keystone-event-project.yaml diff --git a/apps/understack-workflows/workflowtemplates/openstack-svc-acct.yaml b/apps/understack-workflows/workflowtemplates/openstack-svc-acct.yaml new file mode 100644 index 000000000..0793273ce --- /dev/null +++ b/apps/understack-workflows/workflowtemplates/openstack-svc-acct.yaml @@ -0,0 +1,28 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: openstack-svc-acct +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openstack + target: + name: openstack-svc-acct + template: + engineVersion: v2 + data: + clouds.yaml: | + clouds: + understack: + auth_url: http://keystone-api.openstack.svc.cluster.local:5000/v3 + user_domain_name: {{ .user_domain }} + username: {{ .username }} + password: {{ .password }} + # this should switch to where we will be creating the ironic nodes + # in the future + project_domain_name: default + project_name: undercloud + dataFrom: + - extract: + key: svc-acct-argoworkflow diff --git a/apps/understack-workflows/workflowtemplates/wf-keystone-event-project.yaml b/apps/understack-workflows/workflowtemplates/wf-keystone-event-project.yaml new file mode 100644 index 000000000..7a7ad5747 --- /dev/null +++ b/apps/understack-workflows/workflowtemplates/wf-keystone-event-project.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +metadata: + name: keystone-event-project + annotations: + workflows.argoproj.io/description: | + Defined in `apps/understack-workflows/workflowtemplates/wf-keystone-event-project.yaml` +kind: WorkflowTemplate +spec: + serviceAccountName: workflow + entrypoint: sync-keystone + templates: + - name: sync-keystone + container: + image: ghcr.io/rackerlabs/understack/ironic-nautobot-client:latest + command: + - sync-keystone + args: + - "--only-domain" + - "default" + - "{{workflow.parameters.event_type}}" + - "{{workflow.parameters.project_uuid}}" + volumeMounts: + - mountPath: /etc/nb-token/ + name: nb-token + readOnly: true + - mountPath: /etc/openstack + name: 'clouds.yaml' + readOnly: true + inputs: + parameters: + - name: project_uuid + volumes: + - name: nb-token + secret: + secretName: nautobot-token + - name: openstack-svc-acct + secret: + secretName: openstack-svc-acct