jenkins: fix: triggering jobs with parameter does not work

Triggering jenkins jobs that have parameter does not work.
This is fixed by:
- posting the parameters as x-www-urlencoded payload in a "json" field,
- changing the JSON data format to the one described in
  https://wiki.jenkins.io/display/JENKINS/Remote+access+API

The data also must be posted to the "build" endpoint, posting it to the
buildWithParameters endpoint does not work.

config.example.toml

@@ -55,7 +55,7 @@ queue_pr_head_label = "autoupdater-first"
     endpoint = "job/build/{{ queryescape .PullRequestNr }}/build"
 
   [[ci.job]]
-    endpoint = "job/test/buildWithParameters"
+    endpoint = "job/test/build"
 
     [ci.job.parameters]
       branch = "{{ .Branch }}"

internal/jenkins/client.go

@@ -1,13 +1,13 @@
 package jenkins
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/simplesurance/directorius/internal/goorderr"
@@ -50,11 +50,12 @@ func (s *Client) Build(ctx context.Context, j *Job) error {
 		return fmt.Errorf("creating http-request failed: %w", err)
 	}
 
-	req.Header.Add("User-Agent", userAgent)
 	if req.Body != nil {
-		req.Header.Add("Content-Type", userAgent)
+		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 	}
 
+	req.Header.Add("User-Agent", userAgent)
+
 	req.SetBasicAuth(s.auth.user, s.auth.password)
 
 	resp, err := s.clt.Do(req)
@@ -76,6 +77,9 @@ func (s *Client) Build(ctx context.Context, j *Job) error {
 		- 404 because the multibranch job was not created yet but is soonish,
 		- 502, 504 jenkins temporarily down,
 		- 401: temporary issues with jenkins auth backend,
+		- 403: because of the bug that we encounter, probably related
+		       to github auth, where Jenkins from now and then fails with 403
+		       in the UI and APIs and then works after some retries
 		etc
 		*/
 		return goorderr.NewRetryableAnytimeError(fmt.Errorf("server returned status code: %d", resp.StatusCode))
@@ -98,7 +102,11 @@ func toRequestBody(j *Job) io.Reader {
 		return nil
 	}
 
-	return bytes.NewReader(j.parametersJSON)
+	formData := url.Values{
+		"json": []string{string(j.parametersJSON)},
+	}
+
+	return strings.NewReader(formData.Encode())
 }
 
 func (s *Client) String() string {

internal/jenkins/client_test.go

@@ -0,0 +1,108 @@
+package jenkins
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunJobWithParameters(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+
+		assert.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type"))
+		assert.Positive(t, r.ContentLength)
+
+		if !assert.Equal(t, "/build/mybranch", r.URL.Path) {
+			w.WriteHeader(http.StatusBadRequest)
+			return // required because we can't use require in go-routines
+		}
+
+		body, err := io.ReadAll(r.Body)
+		if !assert.NoError(t, err) {
+			w.WriteHeader(http.StatusBadRequest)
+			return // required because we can't use require in go-routines
+		}
+
+		urlVals, err := url.ParseQuery(string(body))
+		if !assert.NoError(t, err) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		paramsJSON := urlVals.Get("json")
+		if !assert.NotEmpty(t, paramsJSON) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		var params jenkinsParameters
+
+		err = json.Unmarshal([]byte(paramsJSON), &params)
+		if !assert.NoError(t, err) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		if !assert.Len(t, params.Parameter, 2) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		p1 := params.Parameter[0]
+		var pVerFound, pBranchFound bool
+		for _, p := range params.Parameter {
+			switch p.Name {
+			case "version":
+				if !assert.Equal(t, "123", p.Value) {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				pVerFound = true
+			case "branch":
+				if !assert.Equal(t, "mybranch", p.Value) {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				pBranchFound = true
+			default:
+				t.Error("unexpected parameter", p1.Name)
+				w.WriteHeader(http.StatusBadRequest)
+				return
+
+			}
+		}
+
+		if !assert.True(t, pVerFound) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		if !assert.True(t, pBranchFound) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		w.WriteHeader(http.StatusCreated)
+	}))
+	t.Cleanup(srv.Close)
+
+	clt := NewClient(srv.URL, "", "")
+	jt := JobTemplate{
+		RelURL:     "build/{{ .Branch }}",
+		Parameters: map[string]string{"version": "123", "branch": "{{ .Branch }}"},
+	}
+	job, err := jt.Template(TemplateData{PullRequestNumber: "123", Branch: "mybranch"})
+	require.NoError(t, err)
+
+	err = clt.Build(context.Background(), job)
+	require.NoError(t, err)
+
+	srv.Close()
+}

internal/jenkins/jobtemplate.go

@@ -24,6 +24,17 @@ type TemplateData struct {
 	Branch            string
 }
 
+type jenkinsParameter struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// jenkinsParameters represent the data format that Jenkins expects for passing
+// parameters as JSON.
+type jenkinsParameters struct {
+	Parameter []*jenkinsParameter `json:"parameter"`
+}
+
 // Template creates a concrete [Job] from j by templating it with
 // [templateData] and [templateFuncs.
 func (j *JobTemplate) Template(data TemplateData) (*Job, error) {
@@ -46,16 +57,11 @@ func (j *JobTemplate) Template(data TemplateData) (*Job, error) {
 		}, nil
 	}
 
-	templatedParams, err := j.templateParameters(data, templ)
+	jsonParams, err := j.paramsToJSON(templ, data)
 	if err != nil {
 		return nil, err
 	}
 
-	jsonParams, err := json.Marshal(templatedParams)
-	if err != nil {
-		return nil, fmt.Errorf("converting templated job parameters to json failed: %w", err)
-	}
-
 	return &Job{
 		relURL:         relURLTemplated.String(),
 		parametersJSON: jsonParams,
@@ -91,3 +97,24 @@ func (j *JobTemplate) templateParameters(data TemplateData, templ *template.Temp
 
 	return templatedParams, nil
 }
+
+func (j *JobTemplate) paramsToJSON(templ *template.Template, data TemplateData) ([]byte, error) {
+	var jenkinsParams jenkinsParameters
+
+	templatedParams, err := j.templateParameters(data, templ)
+	if err != nil {
+		return nil, err
+	}
+
+	jenkinsParams.Parameter = make([]*jenkinsParameter, 0, len(templatedParams))
+	for k, v := range templatedParams {
+		jenkinsParams.Parameter = append(jenkinsParams.Parameter, &jenkinsParameter{Name: k, Value: v})
+	}
+
+	jsonParams, err := json.Marshal(jenkinsParams)
+	if err != nil {
+		return nil, fmt.Errorf("converting parameter struct to json failed: %w", err)
+	}
+
+	return jsonParams, err
+}

internal/jenkins/jobtemplate_test.go

@@ -1,43 +1,11 @@
 package jenkins
 
 import (
-	"encoding/json"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestTemplate(t *testing.T) {
-	jt := JobTemplate{
-		RelURL: "job/{{ pathescape .Branch }}/{{ .PullRequestNumber }}/build?branch={{ queryescape .Branch }}",
-		Parameters: map[string]string{
-			"Branch": "{{ .Branch }}",
-			"PRNr":   "{{ .PullRequestNumber }}",
-		},
-	}
-
-	d := TemplateData{
-		PullRequestNumber: "456",
-		Branch:            "ma/i-n br",
-	}
-
-	j, err := jt.Template(d)
-	require.NoError(t, err)
-
-	var params map[string]string
-
-	err = json.Unmarshal(j.parametersJSON, &params)
-	require.NoError(t, err)
-
-	assert.Contains(t, params, "Branch")
-	assert.Equal(t, d.Branch, params["Branch"])
-	assert.Contains(t, params, "PRNr")
-	assert.Equal(t, d.PullRequestNumber, params["PRNr"])
-
-	assert.Equal(t, "job/ma%2Fi-n%20br/456/build?branch=ma%2Fi-n+br", j.relURL)
-}
-
 func TestTemplateFailsOnUndefinedKey(t *testing.T) {
 	jt := JobTemplate{
 		RelURL: "abc",