Allow Users to Run Multiple Scripts

Until now it was only possible to run a single script within one
Prometheus job, because the `script` parameter could only be provided
one time.

With the changes in this commit it is now possible to run multiple
scripts within one Prometheus job, for that the `script` parameter can
be specified multiple times. For each specified script we are running
the same logic as for a single script and then we merge the output of
each script execution into one result.

Limitations:
- The specified parameters are used for all scripts, it is not possible
  to specifiy different parameters for the scripts.
- The timeout specified via the `X-Prometheus-Scrape-Timeout-Seconds`
  might not work correctly, if the offset is not large enough, since the
  timout logic is applied for all scripts.
- If there is an error in the logic for one script, e.g. one of the
  defined scripts can not be found, the other scripts will also return
  not output.

pkg/exporter/metrics.go

@@ -15,16 +15,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
-	// Get script from url parameter
+func (e *Exporter) metricsHandler(r *http.Request, scriptName string) (string, error) {
 	params := r.URL.Query()
-	scriptName := params.Get("script")
-	if scriptName == "" {
-		errorStr := "Script parameter is missing"
-		level.Error(e.Logger).Log("err", errorStr)
-		http.Error(w, errorStr, http.StatusBadRequest)
-		return
-	}
 
 	// Get prefix from url parameter
 	prefix := params.Get("prefix")
@@ -45,7 +37,6 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	w.Header().Set("Content-Type", "text/plain")
 	scriptStartTime := time.Now()
 
 	// When the script configuration contains a cache duration we use the result from the cache when the entry is not
@@ -55,8 +46,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 		formattedOutput, successStatus, exitCode := getCacheResult(scriptName, paramValues, *cacheDuration)
 		if formattedOutput != nil && successStatus != nil && exitCode != nil {
 			level.Debug(e.Logger).Log("msg", "Returning cached result", "script", scriptName)
-			fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, *successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, *exitCode, *formattedOutput)
-			return
+			return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, *successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, *exitCode, *formattedOutput), nil
 		}
 	}
 
@@ -65,8 +55,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		errorStr := fmt.Sprintf("Script '%s' not found", scriptName)
 		level.Error(e.Logger).Log("err", errorStr, "script", scriptName)
-		http.Error(w, errorStr, http.StatusBadRequest)
-		return
+		return "", fmt.Errorf(errorStr)
 	}
 	// Append args passed via scrape query parameters
 	runArgs = append(runArgs, paramValues...)
@@ -92,8 +81,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 	// true.
 	outputParam := params.Get("output")
 	if outputParam == "ignore" || (successStatus == 0 && e.Config.GetIgnoreOutputOnFail(scriptName)) {
-		fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode)
-		return
+		return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode), nil
 	}
 
 	// Format output
@@ -133,7 +121,30 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 		setCacheResult(scriptName, paramValues, formattedOutput, successStatus, exitCode)
 	}
 
-	fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode, formattedOutput)
+	return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode, formattedOutput), nil
+}
+
+func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+
+	// Get scripts from url parameter
+	scriptNames := r.URL.Query()["script"]
+	if len(scriptNames) == 0 {
+		errorStr := "Script parameter is missing"
+		level.Error(e.Logger).Log("err", errorStr)
+		http.Error(w, errorStr, http.StatusBadRequest)
+		return
+	}
+
+	// Run each script and return the output
+	for _, scriptName := range scriptNames {
+		output, err := e.metricsHandler(r, scriptName)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		fmt.Fprintf(w, output)
+	}
 }
 
 // SetupMetrics creates and registers our internal Prometheus metrics,