From be75f06cb59c5b7a64b5a008fc3e0bf21f02e446 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 23 Apr 2024 22:24:07 +0000 Subject: [PATCH] refactoring the configuration --- linux_service/moneo_prestart.sh | 1 + linux_service/moneo_service_deploy.sh | 13 ++++++------ moneo.py | 4 ++++ moneo_config.json | 25 +++++++++++++++++++++++ src/worker/publisher/metrics_publisher.py | 12 +++++------ src/worker/start.sh | 2 +- src/worker/start_geneva.sh | 6 +++--- src/worker/start_managed_prometheus.sh | 8 ++++---- 8 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 moneo_config.json diff --git a/linux_service/moneo_prestart.sh b/linux_service/moneo_prestart.sh index 2954943..d1e3eed 100755 --- a/linux_service/moneo_prestart.sh +++ b/linux_service/moneo_prestart.sh @@ -21,3 +21,4 @@ fi mkdir -p /tmp/moneo-worker cp -rf $MONEO_PATH/src/worker/* /tmp/moneo-worker/ +cp -f $MONEO_PATH/moneo_config.json /tmp/moneo-worker/ diff --git a/linux_service/moneo_service_deploy.sh b/linux_service/moneo_service_deploy.sh index 65f6439..c30dea4 100755 --- a/linux_service/moneo_service_deploy.sh +++ b/linux_service/moneo_service_deploy.sh @@ -8,11 +8,10 @@ MONEO_VERSION=v0.3.4 # Release tag MONITOR_DIR=/opt/azurehpc/tools # install directory -IDENTITY_CLIENT_ID="38b84eb5-8aec-4971-aaeb-ddd7e9bfef98" # This is the client ID of the Managed Identity for the Azure Prometheus Monitor Workspace -INGESTION_ENDPOINT="https://moneo-amw-q14z.southcentralus-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-c0192b4cd2c748f88ffd422e7a0d77ac/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24" # This is the ingestion endpoint for the Azure Prometheus Monitor Workspace -MONEO_PATH=$MONITOR_DIR/Moneo +IDENTITY_CLIENT_ID="" # This is the client ID of the Managed Identity for the Azure Prometheus Monitor Workspace +INGESTION_ENDPOINT="" PublisherMethod="" # This is the publisher method for Moneo. Options are azure_monitor, geneva (Msft internal Use), or leave blank for Azure Managed Prometheus - +MONEO_PATH=$MONITOR_DIR/Moneo # clone source to specified directory if [[ -d "$MONEO_PATH" ]]; then pushd $MONEO_PATH @@ -36,9 +35,9 @@ fi sudo chmod -R 777 $MONEO_PATH # Configure step -echo "{ - \"IDENTITY_CLIENT_ID\": \"$IDENTITY_CLIENT_ID\", - \"INGESTION_ENDPOINT\": \"$INGESTION_ENDPOINT\" }" > $MONEO_PATH/src/worker/publisher/config/managed_prom_config.json +jq '(.prom_config.IDENTITY_CLIENT_ID |= "'"$IDENTITY_CLIENT_ID"'")' "$MONEO_PATH/moneo_config.json" > "$MONEO_PATH/temp.json" && mv "$MONEO_PATH/temp.json" "$MONEO_PATH/moneo_config.json" +jq '(.prom_config.INGESTION_ENDPOINT |= "'"$INGESTION_ENDPOINT"'")' "$MONEO_PATH/moneo_config.json" > "$MONEO_PATH/temp.json" && mv "$MONEO_PATH/temp.json" "$MONEO_PATH/moneo_config.json" +rm -f "$MONEO_PATH/temp.json" pushd $MONEO_PATH/linux_service sudo ./configure_service.sh >> moneoServiceInstall.log diff --git a/moneo.py b/moneo.py index 3116d59..2c4aac3 100644 --- a/moneo.py +++ b/moneo.py @@ -123,6 +123,10 @@ def deploy_worker(self, hosts_file, max_threads=16): # noqa: C901 logging.info('Copying files to workers') out = pscp(copy_path, destination_dir, hosts_file, user=self.args.user) logging.info(out) + copy_path = './moneo_config.json' + destination_dir = '/tmp/moneo-worker' + out = pscp(copy_path, destination_dir, hosts_file, user=self.args.user) + logging.info(out) print('--------------------------') if self.args.skip_install: pass diff --git a/moneo_config.json b/moneo_config.json new file mode 100644 index 0000000..04537b8 --- /dev/null +++ b/moneo_config.json @@ -0,0 +1,25 @@ + +{ + "prom_config":{ + "IDENTITY_CLIENT_ID": "", + "INGESTION_ENDPOINT": "" + }, + "geneva_config":{ + "AccountName": "", + "MDMEndPoint": "", + "UmiObjectId": "" + }, + "publisher_config":{ + "common_config": { + "metrics_ports": "8000,8001,8002", + "metrics_namespace": "", + "interval": "20" + }, + "geneva_agent_config": { + "metrics_account": "" + }, + "azure_monitor_agent_config": { + "connection_string": "" + } + } +} diff --git a/src/worker/publisher/metrics_publisher.py b/src/worker/publisher/metrics_publisher.py index 23b77e8..919adb3 100644 --- a/src/worker/publisher/metrics_publisher.py +++ b/src/worker/publisher/metrics_publisher.py @@ -94,7 +94,7 @@ def get_publisher_metrics_config(): Returns: config(dict): The geneva metrics configuration """ - with open('/tmp/moneo-worker/publisher/config/publisher_config.json') as f: + with open('/tmp/moneo-worker/publisher/config/moneor_config.json') as f: config = json.load(f) return config @@ -272,15 +272,15 @@ def get_tags_from_metric(self, metric_labels): agent_config = get_publisher_metrics_config() # Get common config - metrics_ports = agent_config['common_config']['metrics_ports'] - metrics_namespace = agent_config['common_config']['metrics_namespace'] - interval = int(agent_config['common_config']['interval']) + metrics_ports = agent_config['publisher_config']['common_config']['metrics_ports'] + metrics_namespace = agent_config['publisher_config']['common_config']['metrics_namespace'] + interval = int(agent_config['publisher_config']['common_config']['interval']) # Get publisher agent config if publisher_agent == 'geneva': - metrics_auth = agent_config['geneva_agent_config']['metrics_account'] + metrics_auth = agent_config['publisher_config']['geneva_agent_config']['metrics_account'] elif publisher_agent == 'azure_monitor': - metrics_auth = agent_config['azure_monitor_agent_config']['connection_string'] + metrics_auth = agent_config['publisher_config']['azure_monitor_agent_config']['connection_string'] else: raise Exception('##[ERROR]The publisher agent is not supported') diff --git a/src/worker/start.sh b/src/worker/start.sh index 555781d..1335f7a 100755 --- a/src/worker/start.sh +++ b/src/worker/start.sh @@ -56,7 +56,7 @@ then then # Start Geneva Metrics Extension(MA) docker container with UMI echo "Starting Geneva Metrics Extension(MA) docker container with $PUBLISHER_AUTH" - $WORK_DIR/start_geneva.sh $PUBLISHER_AUTH $WORK_DIR/publisher/config + $WORK_DIR/start_geneva.sh $PUBLISHER_AUTH $WORK_DIR else # Unsupported auth type echo "Publisher auth not supported" diff --git a/src/worker/start_geneva.sh b/src/worker/start_geneva.sh index 084523a..a5cec95 100755 --- a/src/worker/start_geneva.sh +++ b/src/worker/start_geneva.sh @@ -11,8 +11,8 @@ if sudo docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then exit 0 fi -GENEVA_ACCOUNT_NAME=$(jq -r '.AccountName' $GENEVA_CONFIG) -GENEVA_MDM_ENDPOINT=$(jq -r '.MDMEndPoint' $GENEVA_CONFIG) +GENEVA_ACCOUNT_NAME=$(jq -r '.geneva_config.AccountName' $GENEVA_CONFIG) +GENEVA_MDM_ENDPOINT=$(jq -r '.geneva_config.MDMEndPoint' $GENEVA_CONFIG) # Set Geneva Metrics Extension(MA) endpoint if [[ "$GENEVA_MDM_ENDPOINT" == *"ppe"* ]]; then @@ -32,7 +32,7 @@ echo "GENEVA_DIR: $GENEVA_DIR" if [ $AUTH == "umi" ]; then - GENEVA_UMI_OBJECT_ID=$(jq -r '.UmiObjectId' $GENEVA_CONFIG) + GENEVA_UMI_OBJECT_ID=$(jq -r '.geneva_config.UmiObjectId' $GENEVA_CONFIG) GENEVA_UMI_DIR=$GENEVA_DIR'/auth_umi.json' cat > $GENEVA_UMI_DIR << EOF diff --git a/src/worker/start_managed_prometheus.sh b/src/worker/start_managed_prometheus.sh index 4e62435..42ffd8f 100755 --- a/src/worker/start_managed_prometheus.sh +++ b/src/worker/start_managed_prometheus.sh @@ -3,8 +3,8 @@ INSTANCE_NAME=$(hostname) WORK_DIR="${1:-/tmp/moneo-worker}" PROM_CONFIG=$WORK_DIR/prometheus.yml -CONFIG_DIR=$WORK_DIR/publisher/config -MANAGED_PROM_CONFIG=$CONFIG_DIR/managed_prom_config.json +CONFIG_DIR=$WORK_DIR +MANAGED_PROM_CONFIG=$CONFIG_DIR/moneo_config.json get_subscription(){ subscription_name=$(curl -s -H Metadata:true "http://169.254.169.254/metadata/instance/compute/subscriptionId?api-version=2021-02-01&format=text") @@ -32,8 +32,8 @@ SUBSCRIPTION_NAME=$(get_subscription) CLUSTER_NAME=$(get_cluster_name) PHYS_HOST_NAME=$(get_physical_host_name) -IDENTITY_CLIENT_ID=$(jq -r '.IDENTITY_CLIENT_ID' $MANAGED_PROM_CONFIG) -INGESTION_ENDPOINT=$(jq -r '.INGESTION_ENDPOINT' $MANAGED_PROM_CONFIG) +IDENTITY_CLIENT_ID=$(jq -r '.prom_config.IDENTITY_CLIENT_ID' $MANAGED_PROM_CONFIG) +INGESTION_ENDPOINT=$(jq -r '.prom_config.INGESTION_ENDPOINT' $MANAGED_PROM_CONFIG) generate_prom(){ DCMG_TARGET=" - $INSTANCE_NAME:8000\n"