Skip to content

Commit

Permalink
Allow Users to Run Multiple Scripts
Browse files Browse the repository at this point in the history
Until now it was only possible to run a single script within one
Prometheus job, because the `script` parameter could only be provided
one time.

With the changes in this commit it is now possible to run multiple
scripts within one Prometheus job, for that the `script` parameter can
be specified multiple times. For each specified script we are running
the same logic as for a single script and then we merge the output of
each script execution into one result.

Limitations:
- The specified parameters are used for all scripts, it is not possible
  to specifiy different parameters for the scripts.
- The timeout specified via the `X-Prometheus-Scrape-Timeout-Seconds`
  might not work correctly, if the offset is not large enough, since the
  timout logic is applied for all scripts.
- If there is an error in the logic for one script, e.g. one of the
  defined scripts can not be found, the other scripts will also return
  not output.
  • Loading branch information
ricoberger committed Dec 1, 2023
1 parent 6a34b64 commit 0753bfc
Showing 1 changed file with 28 additions and 17 deletions.
45 changes: 28 additions & 17 deletions pkg/exporter/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,8 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
)

func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
// Get script from url parameter
func (e *Exporter) metricsHandler(r *http.Request, scriptName string) (string, error) {
params := r.URL.Query()
scriptName := params.Get("script")
if scriptName == "" {
errorStr := "Script parameter is missing"
level.Error(e.Logger).Log("err", errorStr)
http.Error(w, errorStr, http.StatusBadRequest)
return
}

// Get prefix from url parameter
prefix := params.Get("prefix")
Expand All @@ -45,7 +37,6 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
}
}

w.Header().Set("Content-Type", "text/plain")
scriptStartTime := time.Now()

// When the script configuration contains a cache duration we use the result from the cache when the entry is not
Expand All @@ -55,8 +46,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
formattedOutput, successStatus, exitCode := getCacheResult(scriptName, paramValues, *cacheDuration)
if formattedOutput != nil && successStatus != nil && exitCode != nil {
level.Debug(e.Logger).Log("msg", "Returning cached result", "script", scriptName)
fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, *successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, *exitCode, *formattedOutput)
return
return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, *successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, *exitCode, *formattedOutput), nil
}
}

Expand All @@ -65,8 +55,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
errorStr := fmt.Sprintf("Script '%s' not found", scriptName)
level.Error(e.Logger).Log("err", errorStr, "script", scriptName)
http.Error(w, errorStr, http.StatusBadRequest)
return
return "", fmt.Errorf(errorStr)
}
// Append args passed via scrape query parameters
runArgs = append(runArgs, paramValues...)
Expand All @@ -92,8 +81,7 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
// true.
outputParam := params.Get("output")
if outputParam == "ignore" || (successStatus == 0 && e.Config.GetIgnoreOutputOnFail(scriptName)) {
fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode)
return
return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode), nil
}

// Format output
Expand Down Expand Up @@ -133,7 +121,30 @@ func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
setCacheResult(scriptName, paramValues, formattedOutput, successStatus, exitCode)
}

fmt.Fprintf(w, "%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode, formattedOutput)
return fmt.Sprintf("%s\n%s\n%s_success{script=\"%s\"} %d\n%s\n%s\n%s_duration_seconds{script=\"%s\"} %f\n%s\n%s\n%s_exit_code{script=\"%s\"} %d\n%s\n", scriptSuccessHelp, scriptSuccessType, namespace, scriptName, successStatus, scriptDurationSecondsHelp, scriptDurationSecondsType, namespace, scriptName, time.Since(scriptStartTime).Seconds(), scriptExitCodeHelp, scriptExitCodeType, namespace, scriptName, exitCode, formattedOutput), nil
}

func (e *Exporter) MetricsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")

// Get scripts from url parameter
scriptNames := r.URL.Query()["script"]
if len(scriptNames) == 0 {
errorStr := "Script parameter is missing"
level.Error(e.Logger).Log("err", errorStr)
http.Error(w, errorStr, http.StatusBadRequest)
return
}

// Run each script and return the output
for _, scriptName := range scriptNames {
output, err := e.metricsHandler(r, scriptName)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
fmt.Fprint(w, output)
}
}

// SetupMetrics creates and registers our internal Prometheus metrics,
Expand Down

0 comments on commit 0753bfc

Please sign in to comment.