From ba95f8d87cad302d2cca1796c316611a60b71844 Mon Sep 17 00:00:00 2001 From: lemaitre-aneo <91961298+lemaitre-aneo@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:45:27 +0100 Subject: [PATCH 1/9] Draft revamp infra --- AEP/aep-xxxxx.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 AEP/aep-xxxxx.md diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md new file mode 100644 index 0000000..540471b --- /dev/null +++ b/AEP/aep-xxxxx.md @@ -0,0 +1,126 @@ +ArmoniK inputs (from storage module outputs) + +```tf +variable "control_plane_env" { + description = "Environment variables to pass to the control plane" + type = map(string) + default = {} + // Example: + // { + // "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/amqp/ArmoniK.Core.Adapters.Amqp.dll" + // } +} + + +variable "control_plane_env_cm" { + description = "Environment variables from existing config map to pass to the control plane" + type = set(string) + default = [] + // Example: + // [ + // "amqp-cm" + // ] + // + // amqp-cm: + // Amqp__MaxRetries: 5 + // Amqp__LinkCredit: 2 +} + +variable "control_plane_env_secret" { + description = "Environment variables from existing secret to pass to the control plane" + type = set(string) + default = [] + // Example: + // [ + // "amqp-secret" + // ] + // + // amqp-secret: + // Amqp__Host: amqp.svc + // Amqp__Scheme: AMQPS + // Amqp__AllowHostMismatch: true +} + +variable "control_plane_env_from_secret" { + description = "Environment variables from existing secret to pass to the control plane" + type = map(object({ + secret = string + field = string + })) + default = {} + // Example: + // { + // "Amqp__User" = { + // secret = "amqp-credentials" + // field = "user" + // }, + // "Amqp__Password" = { + // secret = "amqp-credentials" + // field = "password" + // } + // } + // + // amqp-credentials: + // user: dG90bw== + // password: dG90bw== +} + +variable "control_plane_mount_cm" { + description = "Volume mount from ConfigMap to pass to the control plane" + type = map(object({ + configmap = string + path = string + subpath = optional(string) + mode = optional(string, "644") + items = map(object({ + mode = optional(string, "644") + field = string + })) + })) + default = {} +} + +variable "control_plane_mount_secret" { + description = "Volume mount from Secret to pass to the control plane" + type = map(object({ + secret = string + path = string + subpath = optional(string) + mode = optional(string, "644") + items = map(object({ + mode = optional(string, "644") + field = string + })) + })) + default = {} + // Example: + // { + // "amqp-certificates" = { + // secret = "amqp-certs" + // path = "/amqp" + // subpath = null + // mode = "600" + // items = { + // "amqp.pem" = { + // mode = "600" + // field = "ca.crt" + // } + // } + // } + // } + // + // amqp-certificates: + // ca.crt: BASE64== + // + // /amqp + // └── amqp.pem +} + + +variable "control_plane_mount_volume" { + type = ToBeDefined +} +variable "control_plane_mount_volume_claim" { + type = ToBeDefined +} +``` \ No newline at end of file From 68b6a6384e3a6bb0b716290946b09544eff3dfb0 Mon Sep 17 00:00:00 2001 From: lemaitre-aneo <91961298+lemaitre-aneo@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:57:27 +0100 Subject: [PATCH 2/9] optionals --- AEP/aep-xxxxx.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index 540471b..8043524 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -72,10 +72,10 @@ variable "control_plane_mount_cm" { path = string subpath = optional(string) mode = optional(string, "644") - items = map(object({ + items = optional(map(object({ mode = optional(string, "644") field = string - })) + })), {}) })) default = {} } @@ -87,10 +87,10 @@ variable "control_plane_mount_secret" { path = string subpath = optional(string) mode = optional(string, "644") - items = map(object({ + items = optional(map(object({ mode = optional(string, "644") field = string - })) + })), {}) })) default = {} // Example: From 0614c6775268aacedbaea42c918d15754971d3f9 Mon Sep 17 00:00:00 2001 From: jdongo-aneo <158571650+jdongo-aneo@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:52:00 +0000 Subject: [PATCH 3/9] update infra aep --- AEP/aep-xxxxx.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index 8043524..c808d63 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -1,4 +1,33 @@ -ArmoniK inputs (from storage module outputs) +# AEP X: ArmoniK Infrastructure redesign + +| |ArmoniK Enhancement Proposal| +---: |:--- +**AEP** | +**Title** | ArmoniK Infrastructure redesign +**Author** | +**Status** | Draft +**Type** | Standard +**Creation Date** | 2024-03-04 + + +# Abstract + +This AEP describes the choice and give a global view to redesign ArmoniK's infrastructure. + +# Motivation + +The motivation behind the redesign of the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, the switch between one technology vendor to another one will be made easy. + + +# Rationale + +Updating or proposing alternatives solution for various component with the current infrastructure is challenging at time. + +# Specifications + +First we define the input variables that are expected from Armonik. This is the output provided by the modules. These variables are environment variables. Kubernetes defines 3 ways to poupulate these viarbles: using environment variable methods, config maps, secret, and mount from envir + +## ArmoniK inputs (from storage module outputs) ```tf variable "control_plane_env" { From 09998eb2521d556e6cd0a6f877006e5035619c8c Mon Sep 17 00:00:00 2001 From: Junior Dongo Date: Wed, 17 Apr 2024 17:03:16 +0200 Subject: [PATCH 4/9] adding specifications --- AEP/aep-xxxxx.md | 149 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 3 deletions(-) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index c808d63..3c13acf 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -16,7 +16,7 @@ This AEP describes the choice and give a global view to redesign ArmoniK's infra # Motivation -The motivation behind the redesign of the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, the switch between one technology vendor to another one will be made easy. +The motivation behind the redesign of the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, the switch between one technology vendor to another one will be made easy. # Rationale @@ -25,7 +25,42 @@ Updating or proposing alternatives solution for various component with the curre # Specifications -First we define the input variables that are expected from Armonik. This is the output provided by the modules. These variables are environment variables. Kubernetes defines 3 ways to poupulate these viarbles: using environment variable methods, config maps, secret, and mount from envir +First, we define the input variables that are expected from Armonik. This is the output provided by the modules. These variables are referred to as environment variables. Kubernetes defines three ways to populate these variables: using environment variable methods, config maps, secret, and mount from environment. + +## Approach + +Our approach is to have an infrastructure built on a modular basis. We will first define all the modules needed by Armonik. +We distinguish the components Armonik needs to work (for example a mongoDB database) lets call them external and the components that are part of ArmoniK (such as compute plane) internal. + +The internal components modules depend on external components ones to work or to be built. + + +Modules outputs ---> ArmoniK + +```mermaid + graph LR + A[External module 1] -- outputs --> B(Armonik) + C[External module ...] -- outputs --> B + D[External module n] -- outputs --> B + +``` + +It is important to note that internal components require a specific format for the information they need. Therefore, it is essential to define and specify these requirements for each external module. As an example, the compute plane requires information from the activeMQ module, as outlined below: + +```tf + "Components__QueueAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Amqp.QueueBuilder" + "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/amqp/ArmoniK.Core.Adapters.Amqp.dll" + "Amqp__User" = "" + "Amqp__Password" = "" + "Amqp__Host" = "" + "Amqp__Port" = "" + "Amqp__Scheme" = "AMQP" + "Amqp__MaxPriority" = "" + "Amqp__MaxRetries" = "" + "Amqp__LinkCredit" = "" + +``` +The internal module will then acts as a consumer of the various outputs from external modules. ## ArmoniK inputs (from storage module outputs) @@ -152,4 +187,112 @@ variable "control_plane_mount_volume" { variable "control_plane_mount_volume_claim" { type = ToBeDefined } -``` \ No newline at end of file +``` + + +Three components are part of the Armonik Core and are those consuming the outputs. + +control-plane + +compute-plane +- polling-agent +- worker + +metrics-exporter + +## Component outputs + +Component outputs will be something like that: + +```tf +# optional +output "env" { + value = { + VARIABLE_NAME = "value" + } +} + +# optional +output "env_configmap" { + value = [ + "component-cm", + ] +} + +# env_secret like env_configmap + +# optional +output "env_from_configmap" { + value = { + VARIABLE_NAME = { + configmap = "component-cm" + field = "field-within-cm" + } + } +} + +# env_from_secret like env_from_configmap + +# optional +output "mount_configmap" { + value = { + "mount-name" = { + configmap = "component-cm" + path = "/path/within/pod" + subpath = "subpath/in/cm" # optional + mode = "600" # optional + items = { # optional + "file" = { + mode = "600" # optional + field = "field-within-cm" + } + } + } + } +} + +# mount_secret like mount_configmap + +# optional +output "mount_volume" { + value = ToBeDefined +} + +# optional +output "mount_volume_claim" { + value = ToBeDefined +} +``` + +## Modules passing to the ArmoniK module + + +```tf +module "armonik" { + source = "path/to/armonik" + + control_plane = { + ... + conf = [ module.activemq, module.s3, module.mongodb ] + } + + metrics_exporter = { + ... + conf = [ module.mongodb ] + } + + ... +} +``` +# Backwards Compatibility + +This infrastrure redesign should be compatible with previous version of ArmoniK (Core, API, ...). + +# How to Teach This + +A document will be deliver on how to integrate a new module in the infrastructure so that it can be used. + + +# Copyright + +This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. \ No newline at end of file From db7f7618f5998de8bac39c65f408237e31679219 Mon Sep 17 00:00:00 2001 From: lemaitre-aneo <91961298+lemaitre-aneo@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:55:32 +0200 Subject: [PATCH 5/9] [WIP] consistent inputs/outputs --- AEP/aep-xxxxx.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index 3c13acf..cbf4ac7 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -65,6 +65,19 @@ The internal module will then acts as a consumer of the various outputs from ext ## ArmoniK inputs (from storage module outputs) ```tf +variable "control_plane" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} variable "control_plane_env" { description = "Environment variables to pass to the control plane" type = map(string) From c7a2d5db20a8fb73360d20df937e914ac4e6674d Mon Sep 17 00:00:00 2001 From: Junior Dongo Date: Fri, 19 Apr 2024 16:14:58 +0200 Subject: [PATCH 6/9] update inputs --- AEP/aep-xxxxx.md | 158 ++++++++++++++--------------------------------- 1 file changed, 46 insertions(+), 112 deletions(-) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index cbf4ac7..ed11ab8 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -78,127 +78,61 @@ variable "control_plane" { })), []) }) } -variable "control_plane_env" { - description = "Environment variables to pass to the control plane" - type = map(string) - default = {} - // Example: - // { - // "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/amqp/ArmoniK.Core.Adapters.Amqp.dll" - // } -} - - -variable "control_plane_env_cm" { - description = "Environment variables from existing config map to pass to the control plane" - type = set(string) - default = [] - // Example: - // [ - // "amqp-cm" - // ] - // - // amqp-cm: - // Amqp__MaxRetries: 5 - // Amqp__LinkCredit: 2 -} -variable "control_plane_env_secret" { - description = "Environment variables from existing secret to pass to the control plane" - type = set(string) - default = [] - // Example: - // [ - // "amqp-secret" - // ] - // - // amqp-secret: - // Amqp__Host: amqp.svc - // Amqp__Scheme: AMQPS - // Amqp__AllowHostMismatch: true -} - -variable "control_plane_env_from_secret" { - description = "Environment variables from existing secret to pass to the control plane" - type = map(object({ - secret = string +variable "compute_plane" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string field = string - })) - default = {} - // Example: - // { - // "Amqp__User" = { - // secret = "amqp-credentials" - // field = "user" - // }, - // "Amqp__Password" = { - // secret = "amqp-credentials" - // field = "password" - // } - // } - // - // amqp-credentials: - // user: dG90bw== - // password: dG90bw== + })), {}) + })), []) + }) } -variable "control_plane_mount_cm" { - description = "Volume mount from ConfigMap to pass to the control plane" - type = map(object({ +variable "polling_agent" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ configmap = string - path = string - subpath = optional(string) - mode = optional(string, "644") - items = optional(map(object({ - mode = optional(string, "644") - field = string - })), {}) - })) - default = {} + field = string + })), {}) + })), []) + }) } -variable "control_plane_mount_secret" { - description = "Volume mount from Secret to pass to the control plane" - type = map(object({ - secret = string - path = string - subpath = optional(string) - mode = optional(string, "644") - items = optional(map(object({ - mode = optional(string, "644") - field = string - })), {}) - })) - default = {} - // Example: - // { - // "amqp-certificates" = { - // secret = "amqp-certs" - // path = "/amqp" - // subpath = null - // mode = "600" - // items = { - // "amqp.pem" = { - // mode = "600" - // field = "ca.crt" - // } - // } - // } - // } - // - // amqp-certificates: - // ca.crt: BASE64== - // - // /amqp - // └── amqp.pem +variable "worker" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) } - -variable "control_plane_mount_volume" { - type = ToBeDefined -} -variable "control_plane_mount_volume_claim" { - type = ToBeDefined +variable "metric_exporter" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) } ``` From cacf2080c87bb118008de6d2cd901d58dd900bab Mon Sep 17 00:00:00 2001 From: Junior Dongo Date: Tue, 7 May 2024 15:47:07 +0200 Subject: [PATCH 7/9] update --- AEP/aep-xxxxx.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-xxxxx.md index ed11ab8..dd632c7 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-xxxxx.md @@ -12,27 +12,29 @@ # Abstract -This AEP describes the choice and give a global view to redesign ArmoniK's infrastructure. +This AEP describes the choice and gives a global view to redesigning ArmoniK's infrastructure. # Motivation -The motivation behind the redesign of the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, the switch between one technology vendor to another one will be made easy. +The motivation behind redesigning the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, switching from one technology vendor to another will be easy. # Rationale -Updating or proposing alternatives solution for various component with the current infrastructure is challenging at time. +Updating or proposing alternative solutions for various components within the current infrastructure can be a challenging task. # Specifications -First, we define the input variables that are expected from Armonik. This is the output provided by the modules. These variables are referred to as environment variables. Kubernetes defines three ways to populate these variables: using environment variable methods, config maps, secret, and mount from environment. +First, we define the input variables that are expected from Armonik. These variables are essentially the outputs that are generated by the modules. We commonly refer to these variables as environment variables. Kubernetes defines three different approaches to populate these variables: using environment variable methods, configmaps, secrets, and volume mounts. ## Approach -Our approach is to have an infrastructure built on a modular basis. We will first define all the modules needed by Armonik. -We distinguish the components Armonik needs to work (for example a mongoDB database) lets call them external and the components that are part of ArmoniK (such as compute plane) internal. -The internal components modules depend on external components ones to work or to be built. +Our approach is to have an infrastructure built on a modular basis.  We will first define all the modules Armonik needs. + +We classify these modules into two categories: internal and external components. The internal components are a part of Armonik such as the compute plane, while the external components like MongoDB database are required for Armonik to function. + +The internal components modules depend on external components either to function or to be created. Modules outputs ---> ArmoniK @@ -45,7 +47,7 @@ Modules outputs ---> ArmoniK ``` -It is important to note that internal components require a specific format for the information they need. Therefore, it is essential to define and specify these requirements for each external module. As an example, the compute plane requires information from the activeMQ module, as outlined below: +It is important to note that internal components require a specific format for the information they need. Therefore, it is essential to define and specify these requirements for every external module . As an example, the compute plane requires specific information from the activeMQ module to operate correctly. The necessary information from activeMQ is outlined below: ```tf "Components__QueueAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Amqp.QueueBuilder" @@ -60,7 +62,7 @@ It is important to note that internal components require a specific format for t "Amqp__LinkCredit" = "" ``` -The internal module will then acts as a consumer of the various outputs from external modules. +The internal module acts as a consumer of the various outputs from external modules. ## ArmoniK inputs (from storage module outputs) @@ -149,7 +151,7 @@ metrics-exporter ## Component outputs -Component outputs will be something like that: +Component outputs should be like that: ```tf # optional @@ -233,11 +235,11 @@ module "armonik" { ``` # Backwards Compatibility -This infrastrure redesign should be compatible with previous version of ArmoniK (Core, API, ...). +This infrastructure redesign should be compatible with previous version of ArmoniK (Core, API, ...). # How to Teach This -A document will be deliver on how to integrate a new module in the infrastructure so that it can be used. +A document will be delivered on how to integrate a new module into the infrastructure so that it can be used. # Copyright From affee735e5d8d197d3b7dc7ff09b3b19eff6c48a Mon Sep 17 00:00:00 2001 From: Junior Dongo Date: Wed, 2 Oct 2024 15:09:08 +0200 Subject: [PATCH 8/9] fixing typos --- AEP/{aep-xxxxx.md => aep-00005.md} | 489 ++++++++++++++--------------- 1 file changed, 242 insertions(+), 247 deletions(-) rename AEP/{aep-xxxxx.md => aep-00005.md} (89%) diff --git a/AEP/aep-xxxxx.md b/AEP/aep-00005.md similarity index 89% rename from AEP/aep-xxxxx.md rename to AEP/aep-00005.md index dd632c7..4a52b90 100644 --- a/AEP/aep-xxxxx.md +++ b/AEP/aep-00005.md @@ -1,247 +1,242 @@ -# AEP X: ArmoniK Infrastructure redesign - -| |ArmoniK Enhancement Proposal| ----: |:--- -**AEP** | -**Title** | ArmoniK Infrastructure redesign -**Author** | -**Status** | Draft -**Type** | Standard -**Creation Date** | 2024-03-04 - - -# Abstract - -This AEP describes the choice and gives a global view to redesigning ArmoniK's infrastructure. - -# Motivation - -The motivation behind redesigning the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, switching from one technology vendor to another will be easy. - - -# Rationale - -Updating or proposing alternative solutions for various components within the current infrastructure can be a challenging task. - -# Specifications - -First, we define the input variables that are expected from Armonik. These variables are essentially the outputs that are generated by the modules. We commonly refer to these variables as environment variables. Kubernetes defines three different approaches to populate these variables: using environment variable methods, configmaps, secrets, and volume mounts. - -## Approach - - -Our approach is to have an infrastructure built on a modular basis.  We will first define all the modules Armonik needs. - -We classify these modules into two categories: internal and external components. The internal components are a part of Armonik such as the compute plane, while the external components like MongoDB database are required for Armonik to function. - -The internal components modules depend on external components either to function or to be created. - - -Modules outputs ---> ArmoniK - -```mermaid - graph LR - A[External module 1] -- outputs --> B(Armonik) - C[External module ...] -- outputs --> B - D[External module n] -- outputs --> B - -``` - -It is important to note that internal components require a specific format for the information they need. Therefore, it is essential to define and specify these requirements for every external module . As an example, the compute plane requires specific information from the activeMQ module to operate correctly. The necessary information from activeMQ is outlined below: - -```tf - "Components__QueueAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Amqp.QueueBuilder" - "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/amqp/ArmoniK.Core.Adapters.Amqp.dll" - "Amqp__User" = "" - "Amqp__Password" = "" - "Amqp__Host" = "" - "Amqp__Port" = "" - "Amqp__Scheme" = "AMQP" - "Amqp__MaxPriority" = "" - "Amqp__MaxRetries" = "" - "Amqp__LinkCredit" = "" - -``` -The internal module acts as a consumer of the various outputs from external modules. - -## ArmoniK inputs (from storage module outputs) - -```tf -variable "control_plane" { - type = object({ - // ... - conf = optional(list(object({ - env = optional(map(string), {}) - env_configmap = optional(set(string), []) - env_from_configmap = optional(map(object({ - configmap = string - field = string - })), {}) - })), []) - }) -} - -variable "compute_plane" { - type = object({ - // ... - conf = optional(list(object({ - env = optional(map(string), {}) - env_configmap = optional(set(string), []) - env_from_configmap = optional(map(object({ - configmap = string - field = string - })), {}) - })), []) - }) -} - -variable "polling_agent" { - type = object({ - // ... - conf = optional(list(object({ - env = optional(map(string), {}) - env_configmap = optional(set(string), []) - env_from_configmap = optional(map(object({ - configmap = string - field = string - })), {}) - })), []) - }) -} - -variable "worker" { - type = object({ - // ... - conf = optional(list(object({ - env = optional(map(string), {}) - env_configmap = optional(set(string), []) - env_from_configmap = optional(map(object({ - configmap = string - field = string - })), {}) - })), []) - }) -} - -variable "metric_exporter" { - type = object({ - // ... - conf = optional(list(object({ - env = optional(map(string), {}) - env_configmap = optional(set(string), []) - env_from_configmap = optional(map(object({ - configmap = string - field = string - })), {}) - })), []) - }) -} -``` - - -Three components are part of the Armonik Core and are those consuming the outputs. - -control-plane - -compute-plane -- polling-agent -- worker - -metrics-exporter - -## Component outputs - -Component outputs should be like that: - -```tf -# optional -output "env" { - value = { - VARIABLE_NAME = "value" - } -} - -# optional -output "env_configmap" { - value = [ - "component-cm", - ] -} - -# env_secret like env_configmap - -# optional -output "env_from_configmap" { - value = { - VARIABLE_NAME = { - configmap = "component-cm" - field = "field-within-cm" - } - } -} - -# env_from_secret like env_from_configmap - -# optional -output "mount_configmap" { - value = { - "mount-name" = { - configmap = "component-cm" - path = "/path/within/pod" - subpath = "subpath/in/cm" # optional - mode = "600" # optional - items = { # optional - "file" = { - mode = "600" # optional - field = "field-within-cm" - } - } - } - } -} - -# mount_secret like mount_configmap - -# optional -output "mount_volume" { - value = ToBeDefined -} - -# optional -output "mount_volume_claim" { - value = ToBeDefined -} -``` - -## Modules passing to the ArmoniK module - - -```tf -module "armonik" { - source = "path/to/armonik" - - control_plane = { - ... - conf = [ module.activemq, module.s3, module.mongodb ] - } - - metrics_exporter = { - ... - conf = [ module.mongodb ] - } - - ... -} -``` -# Backwards Compatibility - -This infrastructure redesign should be compatible with previous version of ArmoniK (Core, API, ...). - -# How to Teach This - -A document will be delivered on how to integrate a new module into the infrastructure so that it can be used. - - -# Copyright - -This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. \ No newline at end of file +# AEP 5: ArmoniK Infrastructure redesign + +| |ArmoniK Enhancement Proposal| +---: |:--- +**AEP** | 5 +**Title** | ArmoniK Infrastructure redesign +**Author** | Junior Dongo <>, Florian Lemaitre <> +**Status** | Draft +**Type** | Standard +**Creation Date** | 2024-03-04 + +# Abstract + +This AEP describes the choices and gives a global view to redesigning ArmoniK's infrastructure. + +# Motivation + +The motivation behind redesigning the infrastructure is to have an infrastructure that is light, modular and easy to maintain. By standardizing the expected outputs for the various modules, switching from one technology vendor to another will be easier. + +# Rationale + +Updating or proposing alternative solutions for various components within the current infrastructure can be a challenging task. + +# Specifications + +First, we define the input variables that are expected from Armonik. These variables are essentially the outputs that are generated by the modules. We commonly refer to these variables as environment variables. Kubernetes defines three different approaches to populate these variables: using environment variable methods, configmaps, secrets, and volume mounts. + +## Approach + +Our approach is to have an infrastructure built on a modular basis.  We will first define all the modules Armonik needs. + +We classify these modules into two categories: internal and external components. The internal components are a part of Armonik such as the compute plane, while the external components like MongoDB database are required for Armonik to function. + +The internal components modules depend on external components either to function or to be created. + +Modules outputs ---> ArmoniK + +```mermaid + graph LR + A[External module 1] -- outputs --> B(Armonik) + C[External module ...] -- outputs --> B + D[External module n] -- outputs --> B + +``` + +It is important to note that internal components require a specific format for the information they need. Therefore, it is essential to define and specify these requirements for every external module . As an example, the compute plane requires specific information from the activeMQ module to operate correctly. The necessary information from activeMQ is outlined below: + +```tf + "Components__QueueAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Amqp.QueueBuilder" + "Components__QueueAdaptorSettings__AdapterAbsolutePath" = "/adapters/queue/amqp/ArmoniK.Core.Adapters.Amqp.dll" + "Amqp__User" = "" + "Amqp__Password" = "" + "Amqp__Host" = "" + "Amqp__Port" = "" + "Amqp__Scheme" = "AMQP" + "Amqp__MaxPriority" = "" + "Amqp__MaxRetries" = "" + "Amqp__LinkCredit" = "" + +``` + +The internal module acts as a consumer of the various outputs from external modules. + +## ArmoniK inputs (from storage module outputs) + +```tf +variable "control_plane" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} + +variable "compute_plane" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} + +variable "polling_agent" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} + +variable "worker" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} + +variable "metric_exporter" { + type = object({ + // ... + conf = optional(list(object({ + env = optional(map(string), {}) + env_configmap = optional(set(string), []) + env_from_configmap = optional(map(object({ + configmap = string + field = string + })), {}) + })), []) + }) +} +``` + +Three components are part of the Armonik Core and are those consuming the outputs. + +control-plane + +compute-plane +- polling-agent +- worker + +metrics-exporter + +## Component outputs + +The components output format should be the following: + +```tf +# optional +output "env" { + value = { + VARIABLE_NAME = "value" + } +} + +# optional +output "env_configmap" { + value = [ + "component-cm", + ] +} + +# env_secret like env_configmap + +# optional +output "env_from_configmap" { + value = { + VARIABLE_NAME = { + configmap = "component-cm" + field = "field-within-cm" + } + } +} + +# env_from_secret like env_from_configmap + +# optional +output "mount_configmap" { + value = { + "mount-name" = { + configmap = "component-cm" + path = "/path/within/pod" + subpath = "subpath/in/cm" # optional + mode = "600" # optional + items = { # optional + "file" = { + mode = "600" # optional + field = "field-within-cm" + } + } + } + } +} + +# mount_secret like mount_configmap + +# optional +output "mount_volume" { + value = ToBeDefined +} + +# optional +output "mount_volume_claim" { + value = ToBeDefined +} +``` + +## Modules passing to the ArmoniK module + +```tf +module "armonik" { + source = "path/to/armonik" + + control_plane = { + ... + conf = [ module.activemq, module.s3, module.mongodb ] + } + + metrics_exporter = { + ... + conf = [ module.mongodb ] + } + + ... +} +``` + +# Backwards Compatibility + +This infrastructure redesign should be compatible with previous version of ArmoniK (Core, API, ...). + +# How to Teach This + +A document will be delivered on how to integrate a new module into the infrastructure so that it can be used. + +# Copyright + +This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. From 9a9df5de0d0826287a3fbeaa3eddcdaa3990f9c8 Mon Sep 17 00:00:00 2001 From: Junior Dongo Date: Wed, 2 Oct 2024 15:37:51 +0200 Subject: [PATCH 9/9] fix AmoniK spelling --- AEP/aep-00005.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AEP/aep-00005.md b/AEP/aep-00005.md index 4a52b90..8f71907 100644 --- a/AEP/aep-00005.md +++ b/AEP/aep-00005.md @@ -23,13 +23,13 @@ Updating or proposing alternative solutions for various components within the cu # Specifications -First, we define the input variables that are expected from Armonik. These variables are essentially the outputs that are generated by the modules. We commonly refer to these variables as environment variables. Kubernetes defines three different approaches to populate these variables: using environment variable methods, configmaps, secrets, and volume mounts. +First, we define the input variables that are expected from ArmoniK. These variables are essentially the outputs that are generated by the modules. We commonly refer to these variables as environment variables. Kubernetes defines three different approaches to populate these variables: using environment variable methods, configmaps, secrets, and volume mounts. ## Approach -Our approach is to have an infrastructure built on a modular basis.  We will first define all the modules Armonik needs. +Our approach is to have an infrastructure built on a modular basis.  We will first define all the modules ArmoniK needs. -We classify these modules into two categories: internal and external components. The internal components are a part of Armonik such as the compute plane, while the external components like MongoDB database are required for Armonik to function. +We classify these modules into two categories: internal and external components. The internal components are a part of ArmoniK such as the compute plane, while the external components like MongoDB database are required for ArmoniK to function. The internal components modules depend on external components either to function or to be created. @@ -37,7 +37,7 @@ Modules outputs ---> ArmoniK ```mermaid graph LR - A[External module 1] -- outputs --> B(Armonik) + A[External module 1] -- outputs --> B(ArmoniK) C[External module ...] -- outputs --> B D[External module n] -- outputs --> B