Features/momalka/add stdi stream support (#1010)

[comment]: # (Note that your PR title should follow the conventional
commit format: https://conventionalcommits.org/en/v1.0.0/#summary)
# PR Description

[comment]: # (The below checklist is for PRs adding new features. If a
box is not checked, add a reason why it's not needed.)
# New Feature Checklist

- [ ] List telemetry added about the feature.
- [ ] Link to the one-pager about the feature.
- [ ] List any tasks necessary for release (3P docs, AKS RP chart
changes, etc.) after merging the PR.
- [ ] Attach results of scale and perf testing.

[comment]: # (The below checklist is for code changes. Not all boxes
necessarily need to be checked. Build, doc, and template changes do not
need to fill out the checklist.)
# Tests Checklist

- [ ] Have end-to-end Ginkgo tests been run on your cluster and passed?
To bootstrap your cluster to run the tests, follow [these
instructions](/otelcollector/test/README.md#bootstrap-a-dev-cluster-to-run-ginkgo-tests).
  - Labels used when running the tests on your cluster:
    - [ ] `operator`
    - [ ] `windows`
    - [ ] `arm64`
    - [ ] `arc-extension`
    - [ ] `fips`
- [ ] Have new tests been added? For features, have tests been added for
this feature? For fixes, is there a test that could have caught this
issue and could validate that the fix works?
  - [ ] Is a new scrape job needed?
- [ ] The scrape job was added to the folder
[test-cluster-yamls](/otelcollector/test/test-cluster-yamls/) in the
correct configmap or as a CR.
  - [ ] Was a new test label added?
- [ ] A string constant for the label was added to
[constants.go](/otelcollector/test/utils/constants.go).
- [ ] The label and description was added to the [test
README](/otelcollector/test/README.md).
- [ ] The label was added to this [PR
checklist](/.github/pull_request_template).
- [ ] The label was added as needed to
[testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml).
  - [ ] Are additional API server permissions needed for the new tests?
- [ ] These permissions have been added to
[api-server-permissions.yaml](/otelcollector/test/testkube/api-server-permissions.yaml).
  - [ ] Was a new test suite (a new folder under `/tests`) added?
- [ ] The new test suite is included in
[testkube-test-crs.yaml](/otelcollector/test/testkube/testkube-test-crs.yaml).

---------

Co-authored-by: Moshe Malka <[email protected]>
Co-authored-by: Vishwanath <[email protected]>
Co-authored-by: Renato Monteiro <[email protected]>

tools/az-prom-rules-converter/README.md

@@ -36,3 +36,8 @@ Options:
   -l, --location <string>                   Rule group location.
   -h, --help                                display help for command
 ```
+
+## usage with pipeline
+```bash
+Get-Content .\examples\yaml-example1.yml | node az-prom-rules-converter
+```

tools/az-prom-rules-converter/dist/cli.js

@@ -24,9 +24,10 @@ program.description('Azure Prometheus rule groups tool');
 function yaml2arm(inputPath, options, command) {
     var _a, _b, _c;
     return __awaiter(this, void 0, void 0, function* () {
-        const inputAbsolutePath = path_1.default.resolve(inputPath);
-        const str = (_a = (yield promises_1.default.readFile(inputAbsolutePath))) === null || _a === void 0 ? void 0 : _a.toString();
-        const flowResult = (0, _1.default)(str, options);
+        const inputStr = inputPath ?
+            (_a = (yield promises_1.default.readFile(path_1.default.resolve(inputPath)))) === null || _a === void 0 ? void 0 : _a.toString() :
+            yield readStdin();
+        const flowResult = (0, _1.default)(inputStr, options);
         if (flowResult.success == false) {
             console.error((_b = flowResult.error) === null || _b === void 0 ? void 0 : _b.title);
             console.error(JSON.stringify((_c = flowResult.error) === null || _c === void 0 ? void 0 : _c.details, null, 2));
@@ -44,7 +45,7 @@ function yaml2arm(inputPath, options, command) {
 }
 program //.command('yaml2arm')
     .description('Convert Prometheus rules Yaml file to ARM template')
-    .argument('<input>', 'Input Prometheus rule groups Yaml file path.')
+    .argument('[input]', 'Input Prometheus rule groups Yaml (or Json) file path.')
     .option('-amw, --azure-monitor-workspace <string>', 'Azure monitor workspace id\'s that this rule group is scoped to.')
     .option('-c, --cluster-name <string>', 'The cluster name of the rule group evaluation.')
     .option('-a, --action-group-id <string>', 'The resource id of the action group to use for alerting rules.')
@@ -53,3 +54,19 @@ program //.command('yaml2arm')
     .option('-l, --location <string>', 'Rule group location.')
     .action(yaml2arm);
 program.parse();
+function readStdin() {
+    return __awaiter(this, void 0, void 0, function* () {
+        return new Promise((resolve, reject) => {
+            let input = '';
+            process.stdin.on('data', chunk => {
+                input += chunk;
+            });
+            process.stdin.on('end', () => {
+                resolve(input);
+            });
+            process.stdin.on('error', err => {
+                reject(err);
+            });
+        });
+    });
+}

tools/az-prom-rules-converter/dist/steps/to-arm-template.js

@@ -68,7 +68,6 @@ const getArmTemplateFormat = (params) => {
         resources: []
     };
     ['clusterName', 'actionGroupId', 'azureMonitorWorkspace', 'location'].forEach((paramName) => {
-        console.log(paramName, params[paramName]);
         if (params[paramName]) {
             result.parameters[paramName].defaultValue = params[paramName];
         }

tools/az-prom-rules-converter/examples/json-example1.json

@@ -0,0 +1,18 @@
+{
+  "groups": [
+    {
+      "name": "example",
+      "interval": "2m",
+      "rules": [
+        {
+          "record": "job:http_inprogress_requests:sum",
+          "expr": "sum by (job) (http_inprogress_requests)"
+        },
+        {
+          "alert": "alert name",
+          "expr": "sum by (job) (http_inprogress_requests)"
+        }
+      ]
+    }
+  ]
+}

tools/az-prom-rules-converter/examples/result1.json

@@ -0,0 +1,62 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "location": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]"
+    },
+    "clusterName": {
+      "type": "string",
+      "metadata": {
+        "description": "Cluster name"
+      }
+    },
+    "actionGroupId": {
+      "type": "string",
+      "metadata": {
+        "description": "Action Group ResourceId"
+      }
+    },
+    "azureMonitorWorkspace": {
+      "type": "string",
+      "metadata": {
+        "description": "ResourceId of Azure monitor workspace to associate to"
+      }
+    }
+  },
+  "variables": {},
+  "resources": [
+    {
+      "name": "example",
+      "type": "Microsoft.AlertsManagement/prometheusRuleGroups",
+      "apiVersion": "2023-03-01",
+      "location": "[parameters('location')]",
+      "properties": {
+        "interval": "PT2M",
+        "scopes": ["[parameters('azureMonitorWorkspace')]"],
+        "clusterName": "[parameters('clusterName')]",
+        "rules": [
+          {
+            "record": "job:http_inprogress_requests:sum",
+            "expression": "sum by (job) (http_inprogress_requests)"
+          },
+          {
+            "severity": 3,
+            "resolveConfiguration": {
+              "autoResolved": true,
+              "timeToResolve": "PT10M"
+            },
+            "actions": [
+              {
+                "actionGroupId": "[parameters('actionGroupId')]"
+              }
+            ],
+            "alert": "alert name",
+            "expression": "sum by (job) (http_inprogress_requests)"
+          }
+        ]
+      }
+    }
+  ]
+}

tools/az-prom-rules-converter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "az-prom-rules-converter",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "",
   "main": "dist/index.js",
   "bin": "dist/cli.js",

tools/az-prom-rules-converter/src/cli.ts

@@ -12,9 +12,10 @@ program.description('Azure Prometheus rule groups tool');
 // program.version(pack.version);
 
 async function yaml2arm(inputPath: string, options: any, command: Command) {
-  const inputAbsolutePath = path.resolve(inputPath);
-  const str = (await fs.readFile(inputAbsolutePath))?.toString();
-  const flowResult: StepResult = toArmTemplateFlow(str, options);
+  const inputStr = inputPath ? 
+                  (await fs.readFile(path.resolve(inputPath)))?.toString() :
+                  await readStdin();
+  const flowResult: StepResult = toArmTemplateFlow(inputStr, options);
 
   if (flowResult.success == false){
     console.error(flowResult.error?.title);
@@ -33,7 +34,7 @@ async function yaml2arm(inputPath: string, options: any, command: Command) {
 
 program//.command('yaml2arm')
   .description('Convert Prometheus rules Yaml file to ARM template')
-  .argument('<input>', 'Input Prometheus rule groups Yaml file path.')
+  .argument('[input]', 'Input Prometheus rule groups Yaml (or Json) file path.')
   .option('-amw, --azure-monitor-workspace <string>', 'Azure monitor workspace id\'s that this rule group is scoped to.')
   .option('-c, --cluster-name <string>', 'The cluster name of the rule group evaluation.')
   .option('-a, --action-group-id <string>', 'The resource id of the action group to use for alerting rules.')
@@ -42,4 +43,23 @@ program//.command('yaml2arm')
   .option('-l, --location <string>', 'Rule group location.')
   .action(yaml2arm);
 
-program.parse();
+program.parse();
+
+
+async function readStdin(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let input = '';
+
+    process.stdin.on('data', chunk => {
+      input += chunk;
+    });
+
+    process.stdin.on('end', () => {
+      resolve(input);
+    });
+
+    process.stdin.on('error', err => {
+      reject(err);
+    });
+  });
+}

tools/az-prom-rules-converter/src/index.test.ts

@@ -0,0 +1,32 @@
+import yamlToArmTemplateFlow from './index';
+import StepResult from './types/step-result';
+import fs from "fs/promises";
+
+describe('toArmTemplateFlow example 1', () => {
+  let expectedResult: any;
+
+  beforeAll(async () => {
+    expectedResult = JSON.parse(
+      await fs.readFile("./examples/result1.json", { encoding: "utf-8" })
+    );
+  });
+
+  test('Successful flow with valid YAML input', async () => {
+    await readFileAndRunFlow("./examples/yaml-example1.yml");
+
+  });
+
+  test("Successful flow with valid JSON input", async () => {
+    await readFileAndRunFlow("./examples/json-example1.json");
+  });
+
+  async function readFileAndRunFlow(examplePath : string) {
+    const options = {};
+    const str = (await fs.readFile(examplePath))?.toString();
+    const result: StepResult = yamlToArmTemplateFlow(str, options);
+
+    expect(result.success).toBe(true);
+    expect(result.output).toEqual(expectedResult);
+  }
+
+});

tools/az-prom-rules-converter/src/index.ts

@@ -1,6 +1,6 @@
 import yaml2json from './steps/yaml2json'
 import StepResult from './types/step-result'
-import   validatePromSchema  from './steps/validate-prom-schemas'
+import validatePromSchema  from './steps/validate-prom-schemas'
 import validateArmTemplate from './steps/validate-arm-template'
 import toArmTemplate from './steps/to-arm-template'
 import validateInputNotEmpty from './steps/validate-input-not-empty'

tools/az-prom-rules-converter/src/steps/to-arm-template.ts

@@ -66,7 +66,6 @@ const getArmTemplateFormat = (params: any) : any => {
     resources: [  ]
   };
   ['clusterName', 'actionGroupId', 'azureMonitorWorkspace', 'location'].forEach( (paramName) => {
-    console.log(paramName, params[paramName]);
     if (params[paramName]) {
       result.parameters[paramName].defaultValue = params[paramName];
     }