Add new source type for shell command execution

Implements [0].
Added UT and CT for the new source.

Also,

 - remove 'const' for variables map in sources processing, in
   order to allow to create the internal 'rc' variable from
   command execution.
 - Fix some markdown links within README.md file, to respect
   capital letters.

[0] #66

ct/src/conftest.py

@@ -709,6 +709,28 @@ def send(content, responseBodyRef = VALID_GLOBAL_VARIABLES__RESPONSE_BODY, respo
 }}
 '''
 
+TRANSFORM_FOO_BAR_COMMAND_PROVISION_TEMPLATE='''
+{{
+  "requestMethod":"POST",
+  "requestUri":"/app/v1/foo/bar",
+  "responseCode":200,
+  "responseHeaders": {{
+    "content-type":"{ct}",
+    "x-version":"1.0.0"
+  }},
+  "transform": [
+    {{
+      "source": "command.{command}",
+      "target": "response.body.json.string./output"
+    }} ,
+    {{
+      "source": "var.rc",
+      "target": "response.body.json.integer./rc"
+    }}
+  ]
+}}
+'''
+
 NESTED_VAR1_VAR2_REQUEST='''
 {
   "var1value": {

ct/src/transform/no_filter_test.py

@@ -1,7 +1,7 @@
 import pytest
 import json
 from conftest import BASIC_FOO_BAR_PROVISION_TEMPLATE, string2dict, ADMIN_SERVER_PROVISION_URI, VALID_PROVISIONS__RESPONSE_BODY
-from conftest import NESTED_NODE1_NODE2_REQUEST, NESTED_VAR1_VAR2_REQUEST, TRANSFORM_FOO_BAR_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_AND_VAR1_VAR2_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_TWO_TRANSFERS_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_RESPONSE_BODY_DATA_PROVISION_TEMPLATE
+from conftest import NESTED_NODE1_NODE2_REQUEST, NESTED_VAR1_VAR2_REQUEST, TRANSFORM_FOO_BAR_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_AND_VAR1_VAR2_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_TWO_TRANSFERS_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_RESPONSE_BODY_DATA_PROVISION_TEMPLATE, TRANSFORM_FOO_BAR_COMMAND_PROVISION_TEMPLATE
 
 
 @pytest.mark.transform
@@ -651,3 +651,14 @@ def test_050_valueToResponseBodyBadHexStringOddNumber(admin_server_provision, h2
 #  response = h2ac_traffic.postDict("/app/v1/foo/bar")
 #  h2ac_traffic.assert_response__status_body_headers(response, 200, "")
 
+
+def test_051_commandExecutionToResponseBodyString(admin_server_provision, h2ac_traffic):
+
+  # Provision
+  admin_server_provision(string2dict(TRANSFORM_FOO_BAR_COMMAND_PROVISION_TEMPLATE, ct="text/html", command="echo -n foo"))
+
+  # Traffic
+  response = h2ac_traffic.postDict("/app/v1/foo/bar")
+  responseBodyRef = { "output":"foo", "rc":0 }
+  h2ac_traffic.assert_response__status_body_headers(response, 200, responseBodyRef)
+

src/jsonSchema/AdminSchemas.hpp

@@ -225,7 +225,7 @@ const nlohmann::json server_provision = R"(
         "properties": {
           "source": {
             "type": "string",
-            "pattern": "^request\\.(uri(\\.(path$|param\\..+))?|body(\\..+)?|header\\..+)$|^response\\.body(\\..+)?$|^eraser$|^math\\..*|^random\\.[-+]{0,1}[0-9]+\\.[-+]{0,1}[0-9]+$|^randomset\\..+|^timestamp\\.[m|u|n]{0,1}s$|^strftime\\..+|^recvseq$|^(var|globalVar|event)\\..+|^(value)\\..*|^inState$|^txtFile\\..+|^binFile\\..+"
+            "pattern": "^request\\.(uri(\\.(path$|param\\..+))?|body(\\..+)?|header\\..+)$|^response\\.body(\\..+)?$|^eraser$|^math\\..*|^random\\.[-+]{0,1}[0-9]+\\.[-+]{0,1}[0-9]+$|^randomset\\..+|^timestamp\\.[m|u|n]{0,1}s$|^strftime\\..+|^recvseq$|^(var|globalVar|event)\\..+|^(value)\\..*|^inState$|^txtFile\\..+|^binFile\\..+|^command\\..+"
           },
           "target": {
             "type": "string",

src/model/AdminServerProvision.cpp

@@ -40,6 +40,7 @@ SOFTWARE.
 #include <time.h>       /* time_t, struct tm, time, localtime, strftime */
 #include <string>
 #include <algorithm>
+//#include <fcntl.h> // non-blocking fgets call
 
 #include <nlohmann/json.hpp>
 #include <arashpartow/exprtk.hpp>
@@ -93,7 +94,7 @@ AdminServerProvision::AdminServerProvision() : in_state_(DEFAULT_ADMIN_SERVER_PR
 
 bool AdminServerProvision::processSources(std::shared_ptr<Transformation> transformation,
         TypeConverter& sourceVault,
-        const std::map<std::string, std::string>& variables,
+        std::map<std::string, std::string>& variables,
         const std::string &requestUri,
         const std::string &requestUriPath,
         const std::map<std::string, std::string> &requestQueryParametersMap,
@@ -306,6 +307,35 @@ bool AdminServerProvision::processSources(std::shared_ptr<Transformation> transf
         file_manager_->read(path, content, false/*binary*/);
         sourceVault.setString(std::move(content));
     }
+    else if (transformation->getSourceType() == Transformation::SourceType::Command) {
+        std::string command = transformation->getSource();
+        replaceVariables(command, transformation->getSourcePatterns(), variables, global_variable_->get());
+
+        static char buffer[256];
+        std::string output;
+
+        FILE *fp = popen(command.c_str(), "r");
+        if (fp) {
+            /* This makes asyncronous the command execution, but we will have broken pipe and cannot capture anything.
+            // fgets is blocking (https://stackoverflow.com/questions/6055702/using-fgets-as-non-blocking-function-c/6055774#6055774)
+            int fd = fileno(fp);
+            int flags = fcntl(fd, F_GETFL, 0);
+            flags |= O_NONBLOCK;
+            fcntl(fd, F_SETFL, flags);
+            */
+
+            while(fgets(buffer, sizeof(buffer), fp))
+            {
+                output += buffer;
+            }
+            variables["rc"] = std::to_string(WEXITSTATUS(/* status = */pclose(fp))); // rc = status >>= 8; // divide by 256
+        }
+        else {
+            variables["rc"] = "-1";
+        }
+
+        sourceVault.setString(std::move(output));
+    }
 
 
     return true;

src/model/AdminServerProvision.hpp

@@ -119,7 +119,7 @@ class AdminServerProvision
     // Three processing stages: get sources, apply filters and store targets:
     bool processSources(std::shared_ptr<Transformation> transformation,
                         TypeConverter& sourceVault,
-                        const std::map<std::string, std::string>& variables,
+                        std::map<std::string, std::string>& variables,
                         const std::string &requestUri,
                         const std::string &requestUriPath,
                         const std::map<std::string, std::string> &requestQueryParametersMap,

src/model/Transformation.cpp

@@ -189,6 +189,7 @@ bool Transformation::load(const nlohmann::json &j) {
     // - inState: current processing state.
     // + txtFile.`<path>`: reads text content from file with the path provided.
     // + binFile.`<path>`: reads binary content from file with the path provided.
+    // + command.`<command>`: executes command on process shell and captures the standard output.
 
     // Regex needed:
     static std::regex requestUriParam("^request.uri.param.(.*)", std::regex::optimize); // no need to escape dots as this is validated in schema
@@ -206,6 +207,7 @@ bool Transformation::load(const nlohmann::json &j) {
     static std::regex event("^event.(.*)", std::regex::optimize);
     static std::regex txtFile("^txtFile.(.*)", std::regex::optimize);
     static std::regex binFile("^binFile.(.*)", std::regex::optimize);
+    static std::regex command("^command.(.*)", std::regex::optimize);
 
     std::smatch matches; // to capture regex group(s)
     // BE CAREFUL!: https://stackoverflow.com/a/51709911/2576671
@@ -299,6 +301,10 @@ bool Transformation::load(const nlohmann::json &j) {
             source_ = matches.str(1);
             source_type_ = SourceType::SBinFile;
         }
+        else if (std::regex_match(sourceSpec, matches, command)) { // command string
+            source_ = matches.str(1);
+            source_type_ = SourceType::Command;
+        }
         else { // some things could reach this (strange characters within value.* for example):
             ert::tracing::Logger::error(ert::tracing::Logger::asString("Cannot identify source type for: %s", sourceSpec.c_str()), ERT_FILE_LOCATION);
             return false;
@@ -483,6 +489,9 @@ std::string Transformation::asString() const {
         else if (source_type_ == SourceType::STxtFile || source_type_ == SourceType::SBinFile) {
             ss << " (path file)";
         }
+        else if (source_type_ == SourceType::STxtFile || source_type_ == SourceType::Command) {
+            ss << " (shell command expression)";
+        }
 
         if (!source_patterns_.empty()) {
             ss << " | source variables:";

src/model/Transformation.hpp

@@ -59,10 +59,10 @@ class Transformation
         has_filter_(false), filter_(""), filter_number_type_(0), filter_i_(0), filter_u_(0), filter_f_(0) {;}
 
     // Source type
-    enum SourceType { RequestUri = 0, RequestUriPath, RequestUriParam, RequestBody, ResponseBody, RequestHeader, Eraser, Math, Random, RandomSet, Timestamp, Strftime, Recvseq, SVar, SGVar, Value, Event, InState, STxtFile, SBinFile };
+    enum SourceType { RequestUri = 0, RequestUriPath, RequestUriParam, RequestBody, ResponseBody, RequestHeader, Eraser, Math, Random, RandomSet, Timestamp, Strftime, Recvseq, SVar, SGVar, Value, Event, InState, STxtFile, SBinFile, Command };
     const char* SourceTypeAsText(const SourceType & type) const
     {
-        static const char* text [] = { "RequestUri", "RequestUriPath", "RequestUriParam", "RequestBody", "ResponseBody", "RequestHeader", "Eraser", "Math", "Random", "RandomSet", "Timestamp", "Strftime", "Recvseq", "SVar", "SGVar", "Value", "Event", "InState", "STxtFile", "SBinFile" };
+        static const char* text [] = { "RequestUri", "RequestUriPath", "RequestUriParam", "RequestBody", "ResponseBody", "RequestHeader", "Eraser", "Math", "Random", "RandomSet", "Timestamp", "Strftime", "Recvseq", "SVar", "SGVar", "Value", "Event", "InState", "STxtFile", "SBinFile", "Command" };
         return text [type];
     }
 
@@ -106,7 +106,7 @@ class Transformation
 
     SourceType source_type_{};
     std::string source_{}; // RequestUriParam, RequestBody(empty: whole, path: node), ResponseBody(empty: whole, path: node),
-    // RequestHeader, Math, Timestamp, Strftime, SVar, SGVar, Value, Event, STxtFile(path), SBinFile (path)
+    // RequestHeader, Math, Timestamp, Strftime, SVar, SGVar, Value, Event, STxtFile(path), SBinFile (path), Command(expression)
     std::vector<std::string> source_tokenized_{}; // RandomSet
     int source_i1_{}, source_i2_{}; // Random
 

ut/model/Transformation/transform.cpp

@@ -175,6 +175,14 @@ const nlohmann::json ProvisionConfiguration_Sources = R"delim(
     {
       "source": "eraser",
       "target": "txtFile./tmp/h2agent.ut.@{myvar}.txt"
+    },
+    {
+      "source": "command.echo -n foo",
+      "target": "response.body.json.string./command-output"
+    },
+    {
+      "source": "var.rc",
+      "target": "response.body.json.string./command-rc"
     }
   ]
 }
@@ -444,7 +452,9 @@ TEST_F(Transform_test, TransformWithSources) // test different sources
       "unix_ns": "1653872192363705636",
       "unix_s": "1653872192",
       "math-calculation": 19,
-      "file-content": "file content"
+      "file-content": "file content",
+      "command-output": "foo",
+      "command-rc": "0"
     }
     )"_json;
     for(auto i: {
@@ -534,13 +544,13 @@ TEST_F(Transform_test, TransformationAsString) // test different sources
 {
     int transformationItems = ProvisionConfiguration_Sources["transform"].size();
 
-    EXPECT_EQ(transformationItems, 37);
+    EXPECT_EQ(transformationItems, 39);
     for (int k = 0; k < transformationItems; k++) {
         EXPECT_TRUE(Transform_test::transformation_.load(ProvisionConfiguration_Sources["transform"][k]));
     }
 
     // Last one:
-    EXPECT_EQ(transformation_.asString(), "SourceType: Eraser | TargetType: TTxtFile | target_: /tmp/h2agent.ut.@{myvar}.txt (path file) | target variables: myvar");
+    EXPECT_EQ(transformation_.asString(), "SourceType: SVar | source_: rc | TargetType: ResponseBodyJson_String | target_: /command-rc (empty: whole, path: node)");
 }
 
 TEST_F(Transform_test, TransformationWithFilterAsString) // test different sources