From 0250c55078ad07bfb04a6059e60e4b1d68360f0e Mon Sep 17 00:00:00 2001 From: Andrea Berlingieri Date: Tue, 2 Jul 2024 16:22:47 +0200 Subject: [PATCH 1/4] Fix support for job plugins Change JobCommand struct to correctly parse the plugins block from the XML job definition Add JobPlugins struct to parse LogFilterPlugins Change the job resource schema to include the log_filter_plugin attribute Change parsing of resource data into a plugin object to instantiate the command's plugins Add documentation about the plugins block Add example to the docs for the new log_filter_plugin attribute --- rundeck/job.go | 10 ++++-- rundeck/resource_job.go | 65 +++++++++++++++++++++++++++++------- website/docs/r/job.html.md | 67 ++++++++++++++++++++++++++++++-------- 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/rundeck/job.go b/rundeck/job.go index 90aa56f08..164ca96a4 100644 --- a/rundeck/job.go +++ b/rundeck/job.go @@ -276,7 +276,7 @@ type JobCommand struct { Job *JobCommandJobRef `xml:"jobref"` // Configuration for a step plugin to run as this command. - StepPlugin *JobPlugin `xml:"step-plugin"` + Plugins *JobPlugins `xml:"plugins"` // Configuration for a node step plugin to run as this command. NodeStepPlugin *JobPlugin `xml:"node-step-plugin"` @@ -307,7 +307,13 @@ type JobCommandJobRefArguments string type JobPlugin struct { XMLName xml.Name Type string `xml:"type,attr"` - Config JobPluginConfig `xml:"configuration"` + Config JobPluginConfig `xml:"config"` +} + +// Plugins is a configuration for a list of LogFilter plugins to run within a job. +type JobPlugins struct { + XMLName xml.Name + LogFilterPlugins []JobLogFilter `xml:"LogFilter"` } // JobPluginConfig is a specialization of map[string]string for job plugin configuration. diff --git a/rundeck/resource_job.go b/rundeck/resource_job.go index 8c2a362e7..4d65b73ca 100644 --- a/rundeck/resource_job.go +++ b/rundeck/resource_job.go @@ -376,10 +376,10 @@ func resourceRundeckJobCommand() *schema.Resource { Elem: resourceRundeckJobCommandJob(), }, - "step_plugin": { + "plugins": { Type: schema.TypeList, Optional: true, - Elem: resourceRundeckJobPluginResource(), + Elem: resourceRundeckJobPluginsResource(), }, "node_step_plugin": { Type: schema.TypeList, @@ -441,10 +441,10 @@ func resourceRundeckJobCommandErrorHandler() *schema.Resource { Elem: resourceRundeckJobCommandJob(), }, - "step_plugin": { + "plugins": { Type: schema.TypeList, Optional: true, - Elem: resourceRundeckJobPluginResource(), + Elem: resourceRundeckJobPluginsResource(), }, "node_step_plugin": { @@ -534,6 +534,18 @@ func resourceRundeckJobPluginResource() *schema.Resource { } } +func resourceRundeckJobPluginsResource() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "log_filter_plugin": { + Type: schema.TypeList, + Required: true, + Elem: resourceRundeckJobPluginResource(), + }, + }, + } +} + func resourceRundeckJobFilter() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -1187,7 +1199,7 @@ func commandFromResourceData(commandI interface{}) (*JobCommand, error) { if command.Job, err = jobCommandJobRefFromResourceData("job", commandMap); err != nil { return nil, err } - if command.StepPlugin, err = singlePluginFromResourceData("step_plugin", commandMap); err != nil { + if command.Plugins, err = pluginsFromResourceData("plugins", commandMap); err != nil { return nil, err } if command.NodeStepPlugin, err = singlePluginFromResourceData("node_step_plugin", commandMap); err != nil { @@ -1227,6 +1239,32 @@ func jobCommandJobRefFromResourceData(key string, commandMap map[string]interfac return jobRef, nil } +func pluginsFromResourceData(key string, commandMap map[string]interface{}) (*JobPlugins, error) { + pluginsList := commandMap[key].([]interface{}) + if len(pluginsList) == 0 { + return nil, nil + } + if len(pluginsList) > 1 { + return nil, fmt.Errorf("rundeck command job reference may have no more than one plugins section") + } + pluginsI := pluginsList[0].(map[string]interface{}) + var plugins = new(JobPlugins) + for _, pluginI := range pluginsI["log_filter_plugin"].([]interface{}) { + pluginMap := pluginI.(map[string]interface{}) + configI := pluginMap["config"].(map[string]interface{}) + var config = JobLogFilterConfig{} + for key, value := range configI { + config[key] = value.(string) + } + plugin := &JobLogFilter{ + Type: pluginMap["type"].(string), + Config: &config, + } + plugins.LogFilterPlugins = append(plugins.LogFilterPlugins, *plugin) + } + return plugins, nil +} + func singlePluginFromResourceData(key string, commandMap map[string]interface{}) (*JobPlugin, error) { stepPluginsI := commandMap[key].([]interface{}) if len(stepPluginsI) > 1 { @@ -1295,13 +1333,18 @@ func commandToResourceData(command *JobCommand) (map[string]interface{}, error) commandConfigI["job"] = append([]interface{}{}, jobRefConfigI) } - if command.StepPlugin != nil { - commandConfigI["step_plugin"] = []interface{}{ - map[string]interface{}{ - "type": command.StepPlugin.Type, - "config": map[string]string(command.StepPlugin.Config), - }, + if command.Plugins != nil { + logFilterPluginI := []map[string]interface{}{} + for _, plugin := range command.Plugins.LogFilterPlugins { + pluginI := map[string]interface{}{ + "type": plugin.Type, + "config": (map[string]string)(*plugin.Config), + } + logFilterPluginI = append(logFilterPluginI, pluginI) } + commandConfigI["plugins"] = append([]interface{}{}, map[string]interface{}{ + "log_filter_plugin": logFilterPluginI, + }) } if command.NodeStepPlugin != nil { diff --git a/website/docs/r/job.html.md b/website/docs/r/job.html.md index 9219b8af6..1b2e9e9e9 100644 --- a/website/docs/r/job.html.md +++ b/website/docs/r/job.html.md @@ -18,21 +18,58 @@ Each job belongs to a project. A project can be created with the `rundeck_projec ```hcl resource "rundeck_job" "bounceweb" { - name = "Bounce All Web Servers" - project_name = "${rundeck_project.terraform.name}" - node_filter_query = "tags: web" - description = "Restart the service daemons on all the web servers" + name = "Bounce All Web Servers" + project_name = "${rundeck_project.terraform.name}" + node_filter_query = "tags: web" + description = "Restart the service daemons on all the web servers" + + command { + shell_command = "sudo service anvils restart" + } + notification { + type = "on_success" + email { + recipients = ["example@foo.bar"] + } + } + } +``` + +## Example Usage (Key-Value Data Log filter to pass data between jobs) +```hcl + resource "rundeck_job" "update_review_environments" { + name = "Update review environments" + project_name = "${rundeck_project.terraform.name}" + node_filter_query = "tags: dev_server" + description = "Update the code in review environments checking out the given branch" + command { + description = null + inline_script = "#!/bin/sh\nenvironment_numbers=$(find /var/review_environments -mindepth 3 -maxdepth 4 -name '$1' | awk -F/ -vORS=, '{ print $3 }' | sed 's/.$//')\necho \"RUNDECK:DATA:environment_numbers=\"$environment_numbers\"" + script_file_args = "$${option.git_branch}" + plugins { + log_filter_plugin { + config = { + invalidKeyPattern = "\\s|\\$|\\{|\\}|\\\\" + logData = "true" + regex = "^RUNDECK:DATA:\\s*([^\\s]+?)\\s*=\\s*(.+)$" + } + type = "key-value-data" + } + } + } command { - shell_command = "sudo service anvils restart" + job { + args = "-environment_numbers $${data.environment_numbers}" + name = "git_pull_review_environments" + } } - notification { - type = "on_success" - email { - recipients = ["example@foo.bar"] - } + option { + name = "git_branch" + required = true } -} + } + ``` ## Argument Reference @@ -208,7 +245,7 @@ The following arguments are supported: * A `job` block, described below, causes another job within the same project to be executed as a command. -* A `step_plugin` block, described below, causes a step plugin to be executed as a command. +* A `plugins` block, described below, contains a list of plugins to add to the command. At the moment, only [Log Filters](https://docs.rundeck.com/docs/manual/log-filters/) are supported * A `node_step_plugin` block, described below, causes a node step plugin to be executed once for each node. @@ -244,7 +281,11 @@ A command's `node_filters` block has the following structure: * `exclude_filter`: (Optional) The query string for nodes ***not to use***. -A command's `step_plugin` or `node_step_plugin` block both have the following structure, as does the job's +A command's `plugins` block has the following structure: + +* `log_filter_plugin`: A log filter plugin to add to the command. Can be repeated to add multiple log filters. See below for the structure. + +A command's `log_filter_plugin` or `node_step_plugin` block both have the following structure, as does the job's `global_log_filter` blocks: * `type`: (Required) The name of the plugin to execute. From f1acae43b84775567e1a4e506e6ec9e8668504d3 Mon Sep 17 00:00:00 2001 From: Andrea Berlingieri Date: Wed, 3 Jul 2024 10:23:57 +0200 Subject: [PATCH 2/4] Add acceptance tests for the plugins field --- rundeck/resource_job_test.go | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/rundeck/resource_job_test.go b/rundeck/resource_job_test.go index 20fec0458..9acd29d73 100644 --- a/rundeck/resource_job_test.go +++ b/rundeck/resource_job_test.go @@ -268,6 +268,44 @@ func TestAccJobOptions_secure_choice(t *testing.T) { }) } +func TestAccJob_plugins(t *testing.T) { + var job JobDetail + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccJobCheckDestroy(&job), + Steps: []resource.TestStep{ + { + Config: testAccJobConfig_plugins, + Check: resource.ComposeTestCheckFunc( + testAccJobCheckExists("rundeck_job.test", &job), + func(s *terraform.State) error { + jobCommand := job.CommandSequence.Commands[0] + if jobCommand.Plugins == nil { + return fmt.Errorf("JobCommands[0].plugins shouldn't be nil") + } + keyValuePlugin := jobCommand.Plugins.LogFilterPlugins[0] + if expected := "key-value-data"; keyValuePlugin.Type != expected { + return fmt.Errorf("wrong plugin type; expected %v, got %v", expected, keyValuePlugin.Type) + } + if expected := "\\s|\\$|\\{|\\}|\\\\"; (*keyValuePlugin.Config)["invalidKeyPattern"] != expected { + return fmt.Errorf("failed to set plugin config; expected %v for \"invalidKeyPattern\", got %v", expected, (*keyValuePlugin.Config)["invalidKeyPattern"]) + } + if expected := "true"; (*keyValuePlugin.Config)["logData"] != expected { + return fmt.Errorf("failed to set plugin config; expected %v for \"logData\", got %v", expected, (*keyValuePlugin.Config)["logData"]) + } + if expected := "^RUNDECK:DATA:\\s*([^\\s]+?)\\s*=\\s*(.+)$"; (*keyValuePlugin.Config)["regex"] != expected { + return fmt.Errorf("failed to set plugin config; expected %v for \"regex\", got %v", expected, (*keyValuePlugin.Config)["regex"]) + } + return nil + }, + ), + }, + }, + }) +} + const testAccJobConfig_basic = ` resource "rundeck_project" "test" { name = "terraform-acc-test-job" @@ -708,3 +746,57 @@ resource "rundeck_project" "test" { } } ` + +const testAccJobConfig_plugins = ` +resource "rundeck_project" "test" { + name = "terraform-acc-test-job" + description = "parent project for job acceptance tests" + + resource_model_source { + type = "file" + config = { + format = "resourcexml" + file = "/tmp/terraform-acc-tests.xml" + } + } +} +resource "rundeck_job" "test" { + project_name = "${rundeck_project.test.name}" + name = "basic-job" + description = "A basic job" + execution_enabled = true + node_filter_query = "example" + allow_concurrent_executions = true + nodes_selected_by_default = true + success_on_empty_node_filter = true + max_thread_count = 1 + rank_order = "ascending" + timeout = "42m" + schedule = "0 0 12 * * * *" + schedule_enabled = true + option { + name = "foo" + default_value = "bar" + } + command { + description = "Prints Hello World" + shell_command = "echo Hello World" + plugins { + log_filter_plugin { + config = { + invalidKeyPattern = "\\s|\\$|\\{|\\}|\\\\" + logData = "true" + regex = "^RUNDECK:DATA:\\s*([^\\s]+?)\\s*=\\s*(.+)$" + } + type = "key-value-data" + } + } + } + notification { + type = "on_success" + email { + recipients = ["foo@foo.bar"] + } + } +} +` From ad8b9782245ca80115b55769aea72590c62f0303 Mon Sep 17 00:00:00 2001 From: Andrea Berlingieri Date: Wed, 3 Jul 2024 11:20:33 +0200 Subject: [PATCH 3/4] Restore step-plugin (it's actually used) --- rundeck/job.go | 7 +++++-- rundeck/resource_job.go | 22 +++++++++++++++++++++- website/docs/r/job.html.md | 3 ++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/rundeck/job.go b/rundeck/job.go index 164ca96a4..e7b979e47 100644 --- a/rundeck/job.go +++ b/rundeck/job.go @@ -276,6 +276,9 @@ type JobCommand struct { Job *JobCommandJobRef `xml:"jobref"` // Configuration for a step plugin to run as this command. + StepPlugin *JobPlugin `xml:"step-plugin"` + + // Configuration for log filter plugins to run for this command. Plugins *JobPlugins `xml:"plugins"` // Configuration for a node step plugin to run as this command. @@ -307,10 +310,10 @@ type JobCommandJobRefArguments string type JobPlugin struct { XMLName xml.Name Type string `xml:"type,attr"` - Config JobPluginConfig `xml:"config"` + Config JobPluginConfig `xml:"configuration"` } -// Plugins is a configuration for a list of LogFilter plugins to run within a job. +// Plugin is a configuration for a filter plugin to run for a step type JobPlugins struct { XMLName xml.Name LogFilterPlugins []JobLogFilter `xml:"LogFilter"` diff --git a/rundeck/resource_job.go b/rundeck/resource_job.go index 4d65b73ca..70c8d4096 100644 --- a/rundeck/resource_job.go +++ b/rundeck/resource_job.go @@ -375,7 +375,11 @@ func resourceRundeckJobCommand() *schema.Resource { Optional: true, Elem: resourceRundeckJobCommandJob(), }, - + "step_plugin": { + Type: schema.TypeList, + Optional: true, + Elem: resourceRundeckJobPluginResource(), + }, "plugins": { Type: schema.TypeList, Optional: true, @@ -440,6 +444,11 @@ func resourceRundeckJobCommandErrorHandler() *schema.Resource { Optional: true, Elem: resourceRundeckJobCommandJob(), }, + "step_plugin": { + Type: schema.TypeList, + Optional: true, + Elem: resourceRundeckJobPluginResource(), + }, "plugins": { Type: schema.TypeList, @@ -1199,6 +1208,9 @@ func commandFromResourceData(commandI interface{}) (*JobCommand, error) { if command.Job, err = jobCommandJobRefFromResourceData("job", commandMap); err != nil { return nil, err } + if command.StepPlugin, err = singlePluginFromResourceData("step_plugin", commandMap); err != nil { + return nil, err + } if command.Plugins, err = pluginsFromResourceData("plugins", commandMap); err != nil { return nil, err } @@ -1332,6 +1344,14 @@ func commandToResourceData(command *JobCommand) (map[string]interface{}, error) } commandConfigI["job"] = append([]interface{}{}, jobRefConfigI) } + if command.StepPlugin != nil { + commandConfigI["step_plugin"] = []interface{}{ + map[string]interface{}{ + "type": command.StepPlugin.Type, + "config": map[string]string(command.StepPlugin.Config), + }, + } + } if command.Plugins != nil { logFilterPluginI := []map[string]interface{}{} diff --git a/website/docs/r/job.html.md b/website/docs/r/job.html.md index 1b2e9e9e9..3c41ea011 100644 --- a/website/docs/r/job.html.md +++ b/website/docs/r/job.html.md @@ -244,6 +244,7 @@ The following arguments are supported: * A `job` block, described below, causes another job within the same project to be executed as a command. +* A `step_plugin` block, described below, causes a step plugin to be executed as a command. * A `plugins` block, described below, contains a list of plugins to add to the command. At the moment, only [Log Filters](https://docs.rundeck.com/docs/manual/log-filters/) are supported @@ -285,7 +286,7 @@ A command's `plugins` block has the following structure: * `log_filter_plugin`: A log filter plugin to add to the command. Can be repeated to add multiple log filters. See below for the structure. -A command's `log_filter_plugin` or `node_step_plugin` block both have the following structure, as does the job's +A command's `log_filter_plugin`, `step_plugin` or `node_step_plugin` block both have the following structure, as does the job's `global_log_filter` blocks: * `type`: (Required) The name of the plugin to execute. From ddc4161eadd97d3135b1889c05e77ee5d8a066a6 Mon Sep 17 00:00:00 2001 From: Forrest Evans Date: Fri, 20 Sep 2024 16:03:28 -0700 Subject: [PATCH 4/4] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b80176c7..094caf2e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.4.8 - Added Job step option for `script_url` - Added ability to reference jobs in other projects through the use of `project_name` in `job` command type. +- Added support for Log Filters on individual Job Steps ## 0.4.7 - Added Password Resource