refactoring the configuration

linux_service/moneo_prestart.sh

@@ -21,3 +21,4 @@ fi
 mkdir -p /tmp/moneo-worker
 
 cp -rf $MONEO_PATH/src/worker/* /tmp/moneo-worker/
+cp -f $MONEO_PATH/moneo_config.json /tmp/moneo-worker/

linux_service/moneo_service_deploy.sh

@@ -8,11 +8,10 @@
 
 MONEO_VERSION=v0.3.4 # Release tag
 MONITOR_DIR=/opt/azurehpc/tools # install directory
-IDENTITY_CLIENT_ID="38b84eb5-8aec-4971-aaeb-ddd7e9bfef98" # This is the client ID of the Managed Identity for the Azure Prometheus Monitor Workspace
-INGESTION_ENDPOINT="https://moneo-amw-q14z.southcentralus-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-c0192b4cd2c748f88ffd422e7a0d77ac/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24" # This is the ingestion endpoint for the Azure Prometheus Monitor Workspace
-MONEO_PATH=$MONITOR_DIR/Moneo
+IDENTITY_CLIENT_ID="" # This is the client ID of the Managed Identity for the Azure Prometheus Monitor Workspace
+INGESTION_ENDPOINT=""
 PublisherMethod="" # This is the publisher method for Moneo. Options are azure_monitor, geneva (Msft internal Use), or leave blank for Azure Managed Prometheus
-
+MONEO_PATH=$MONITOR_DIR/Moneo
 # clone source to specified directory
 if [[ -d "$MONEO_PATH" ]]; then
     pushd $MONEO_PATH 
@@ -36,9 +35,9 @@ fi
 sudo chmod -R 777 $MONEO_PATH
 
 # Configure step
-echo "{
-    \"IDENTITY_CLIENT_ID\": \"$IDENTITY_CLIENT_ID\",
-    \"INGESTION_ENDPOINT\":  \"$INGESTION_ENDPOINT\" }" > $MONEO_PATH/src/worker/publisher/config/managed_prom_config.json
+jq '(.prom_config.IDENTITY_CLIENT_ID |= "'"$IDENTITY_CLIENT_ID"'")' "$MONEO_PATH/moneo_config.json" > "$MONEO_PATH/temp.json" && mv "$MONEO_PATH/temp.json" "$MONEO_PATH/moneo_config.json"
+jq '(.prom_config.INGESTION_ENDPOINT |= "'"$INGESTION_ENDPOINT"'")' "$MONEO_PATH/moneo_config.json" > "$MONEO_PATH/temp.json" && mv "$MONEO_PATH/temp.json" "$MONEO_PATH/moneo_config.json"
+rm -f "$MONEO_PATH/temp.json"
 
 pushd $MONEO_PATH/linux_service
     sudo ./configure_service.sh >> moneoServiceInstall.log

moneo.py

@@ -123,6 +123,10 @@ def deploy_worker(self, hosts_file, max_threads=16):    # noqa: C901
         logging.info('Copying files to workers')
         out = pscp(copy_path, destination_dir, hosts_file, user=self.args.user)
         logging.info(out)
+        copy_path = './moneo_config.json'
+        destination_dir = '/tmp/moneo-worker'
+        out = pscp(copy_path, destination_dir, hosts_file, user=self.args.user)
+        logging.info(out)
         print('--------------------------')
         if self.args.skip_install:
             pass

moneo_config.json

@@ -0,0 +1,25 @@
+
+{
+    "prom_config":{
+        "IDENTITY_CLIENT_ID": "<identity client id>",
+        "INGESTION_ENDPOINT": "<metrics ingestion endpoint>"
+    },
+    "geneva_config":{
+        "AccountName": "<account name>",
+        "MDMEndPoint": "<endpoint>",
+        "UmiObjectId": "<object ID>"
+    },
+    "publisher_config":{
+        "common_config": {
+            "metrics_ports": "8000,8001,8002",
+            "metrics_namespace": "<metrics_namespace>",
+            "interval": "20"
+        },
+        "geneva_agent_config": {
+            "metrics_account": "<metrics_account>"
+        },
+        "azure_monitor_agent_config": {
+            "connection_string": "<connectionString>"
+        }
+    }
+}

src/worker/publisher/metrics_publisher.py

@@ -94,7 +94,7 @@ def get_publisher_metrics_config():
     Returns:
         config(dict): The geneva metrics configuration
     """
-    with open('/tmp/moneo-worker/publisher/config/publisher_config.json') as f:
+    with open('/tmp/moneo-worker/publisher/config/moneor_config.json') as f:
         config = json.load(f)
     return config
 
@@ -272,15 +272,15 @@ def get_tags_from_metric(self, metric_labels):
     agent_config = get_publisher_metrics_config()
 
     # Get common config
-    metrics_ports = agent_config['common_config']['metrics_ports']
-    metrics_namespace = agent_config['common_config']['metrics_namespace']
-    interval = int(agent_config['common_config']['interval'])
+    metrics_ports = agent_config['publisher_config']['common_config']['metrics_ports']
+    metrics_namespace = agent_config['publisher_config']['common_config']['metrics_namespace']
+    interval = int(agent_config['publisher_config']['common_config']['interval'])
 
     # Get publisher agent config
     if publisher_agent == 'geneva':
-        metrics_auth = agent_config['geneva_agent_config']['metrics_account']
+        metrics_auth = agent_config['publisher_config']['geneva_agent_config']['metrics_account']
     elif publisher_agent == 'azure_monitor':
-        metrics_auth = agent_config['azure_monitor_agent_config']['connection_string']
+        metrics_auth = agent_config['publisher_config']['azure_monitor_agent_config']['connection_string']
     else:
         raise Exception('##[ERROR]The publisher agent is not supported')
 

src/worker/start.sh

@@ -56,7 +56,7 @@ then
                 then
                     # Start Geneva Metrics Extension(MA) docker container with UMI
                     echo "Starting Geneva Metrics Extension(MA) docker container with $PUBLISHER_AUTH"
-                    $WORK_DIR/start_geneva.sh $PUBLISHER_AUTH $WORK_DIR/publisher/config
+                    $WORK_DIR/start_geneva.sh $PUBLISHER_AUTH $WORK_DIR
                 else
                     # Unsupported auth type
                     echo "Publisher auth not supported"

src/worker/start_geneva.sh

@@ -11,8 +11,8 @@ if sudo docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
     exit 0
 fi
 
-GENEVA_ACCOUNT_NAME=$(jq -r '.AccountName' $GENEVA_CONFIG)
-GENEVA_MDM_ENDPOINT=$(jq -r '.MDMEndPoint' $GENEVA_CONFIG)
+GENEVA_ACCOUNT_NAME=$(jq -r '.geneva_config.AccountName' $GENEVA_CONFIG)
+GENEVA_MDM_ENDPOINT=$(jq -r '.geneva_config.MDMEndPoint' $GENEVA_CONFIG)
 
 # Set Geneva Metrics Extension(MA) endpoint
 if [[ "$GENEVA_MDM_ENDPOINT" == *"ppe"* ]]; then
@@ -32,7 +32,7 @@ echo "GENEVA_DIR: $GENEVA_DIR"
 
 if [ $AUTH == "umi" ];
 then
-    GENEVA_UMI_OBJECT_ID=$(jq -r '.UmiObjectId' $GENEVA_CONFIG)
+    GENEVA_UMI_OBJECT_ID=$(jq -r '.geneva_config.UmiObjectId' $GENEVA_CONFIG)
     GENEVA_UMI_DIR=$GENEVA_DIR'/auth_umi.json'
 
 cat > $GENEVA_UMI_DIR << EOF

src/worker/start_managed_prometheus.sh

@@ -3,8 +3,8 @@
 INSTANCE_NAME=$(hostname)
 WORK_DIR="${1:-/tmp/moneo-worker}"
 PROM_CONFIG=$WORK_DIR/prometheus.yml
-CONFIG_DIR=$WORK_DIR/publisher/config
-MANAGED_PROM_CONFIG=$CONFIG_DIR/managed_prom_config.json
+CONFIG_DIR=$WORK_DIR
+MANAGED_PROM_CONFIG=$CONFIG_DIR/moneo_config.json
 
 get_subscription(){
     subscription_name=$(curl -s -H Metadata:true "http://169.254.169.254/metadata/instance/compute/subscriptionId?api-version=2021-02-01&format=text")
@@ -32,8 +32,8 @@ SUBSCRIPTION_NAME=$(get_subscription)
 CLUSTER_NAME=$(get_cluster_name)
 PHYS_HOST_NAME=$(get_physical_host_name)
 
-IDENTITY_CLIENT_ID=$(jq -r '.IDENTITY_CLIENT_ID' $MANAGED_PROM_CONFIG)
-INGESTION_ENDPOINT=$(jq -r '.INGESTION_ENDPOINT' $MANAGED_PROM_CONFIG)
+IDENTITY_CLIENT_ID=$(jq -r '.prom_config.IDENTITY_CLIENT_ID' $MANAGED_PROM_CONFIG)
+INGESTION_ENDPOINT=$(jq -r '.prom_config.INGESTION_ENDPOINT' $MANAGED_PROM_CONFIG)
 
 generate_prom(){
     DCMG_TARGET="        - $INSTANCE_NAME:8000\n"