diff --git a/step-templates/octopus-serialize-project-to-terraform.json b/step-templates/octopus-serialize-project-to-terraform.json index 3c002a85..96ea7281 100644 --- a/step-templates/octopus-serialize-project-to-terraform.json +++ b/step-templates/octopus-serialize-project-to-terraform.json @@ -3,12 +3,12 @@ "Name": "Octopus - Serialize Project to Terraform", "Description": "Serialize an Octopus project as a Terraform module and upload the resulting package to the Octopus built in feed.\n\nThis step uses naming conventions to exclude resources from the generated module:\n\n* Variables starting with `Private.` are excluded\n* Runbooks starting with `__ ` are excluded\n* The environment called `Sync` is removed from any variable scopes\n\nBecause serializing Terraform modules is done via the API, the values of any secret variables are not available, and are not included in the module generated by this step.\n\nHowever, by following a variable naming and scoping convention, it is possible to export and then apply a project in a Terraform module recreating secret variables, without ever including the secrets in the exported module.\n\nThe project to be exported must define all secret variables with a unique name and a single value. For example, the secret variable `Test.Database.Password` can be scoped to the `Test` environment and the secret variable `Production.Database.Password` can be scoped to the `Production` environment. You can not have a single secret variable called `Database.Password` with two values for the different environments though.\n\nTo collapse the unique secret variables into a single variable used by steps, it is possible to create a non-secret variable called `Database.Password` with two values `#{Test.Database.Password}` and `#{Production.Database.Password}` scoped to appropriate environments.\n\nIn this way steps can still reference a single variable called `Database.Password`, but all secret variables have unique names and only one value.\n\nAll secret variables are then scoped to an additional environment called `Sync`, which means all secret variables are exposed to runbooks run in the `Step` environment. The `Sync` environment is used to apply the Terraform module exported by this step, `Apply a Terraform template` step to perform variable replacements with secret variables.\n\nThe secret values in the Terraform module then have default values set to the Octostache template referencing the secret variable. For example, the Octopus variables in the Terraform module have default values like `#{Test.Database.Password}` and `#{Production.Database.Password}`. These templates are replaced at runtime by the `Apply a Terraform template` step, run in the `Sync` environment, effectively injecting the secret values back into the newly created project.\n\nThis allows secret variables to be recreated with their original values, without ever exporting the secret values. ", "ActionType": "Octopus.Script", - "Version": 13, + "Version": 14, "CommunityActionTemplateId": null, "Packages": [], "Properties": { "Octopus.Action.RunOnServer": "true", - "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nimport urllib.request\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--ignore-all-changes',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreAllChanges') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreAllChanges') or 'false',\n help='Set to true to set the \"lifecycle.ignore_changes\" ' +\n 'setting on each exported resource to \"all\"')\n parser.add_argument('--ignore-variable-changes',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreVariableChanges') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreVariableChanges') or 'false',\n help='Set to true to set the \"lifecycle.ignore_changes\" ' +\n 'setting on each exported octopus variable to \"all\"')\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--project-name',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.Name') or get_octopusvariable_quiet(\n 'Exported.Project.Name') or get_octopusvariable_quiet(\n 'Octopus.Project.Name'),\n help='Set this to the name of the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignore-cac-managed-values',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreCacValues') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreCacValues') or 'false',\n help='Set this to true to exclude cac managed values like non-secret variables, ' +\n 'deployment processes, and project versioning into the Terraform module. ' +\n 'Set to false to have these values embedded into the module.')\n parser.add_argument('--exclude-cac-project-settings',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.ExcludeCacProjectValues') or get_octopusvariable_quiet(\n 'Exported.Project.ExcludeCacProjectValues') or 'false',\n help='Set this to true to exclude CaC settings like git connections from the exported module.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredTenants'),\n help='A comma separated list of tenants to ignore.')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Project.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--lookup-project-link-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.LookupProjectLinkTenants') or get_octopusvariable_quiet(\n 'Exported.Project.LookupProjectLinkTenants') or 'false',\n help='Set this option to link tenants and create tenant project variables.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeProject.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSet', x] for x in ignores_library_variable_sets]\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccounts', x] for x in ignored_accounts]\n\n# Build the arguments to ignore tenants\nignored_tenants = parser.ignored_tenants.split(',')\nignored_tenants_args = [['-excludeTenants', x] for x in ignored_tenants]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # the name of the project to serialize\n '-projectName', parser.project_name,\n # ignoreProjectChanges can be set to ignore all changes to the project, variables, runbooks etc\n '-ignoreProjectChanges=' + parser.ignore_all_changes,\n # use data sources to lookup external dependencies (like environments, accounts etc) rather\n # than serialize those external resources\n '-lookupProjectDependencies',\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues',\n # Any value that can't be replaced with an Octostache template, add a dummy value\n '-dummySecretVariableValues',\n # detach any step templates, allowing the exported project to be used in a new space\n '-detachProjectTemplates=' + str(not parser.include_step_templates),\n # allow the downstream project to move between project groups\n '-ignoreProjectGroupChanges',\n # allow the downstream project to change names\n '-ignoreProjectNameChanges',\n # CaC enabled projects will not export the deployment process, non-secret variables, and other\n # CaC managed project settings if ignoreCacManagedValues is true. It is usually desirable to\n # set this value to true, but it is false here because CaC projects created by Terraform today\n # save some variables in the database rather than writing them to the Git repo.\n '-ignoreCacManagedValues=' + parser.ignore_cac_managed_values,\n # Excluding CaC values means the resulting module does not include things like git credentials.\n # Setting excludeCaCProjectSettings to true and ignoreCacManagedValues to false essentially\n # converts a CaC project back to a database project.\n '-excludeCaCProjectSettings=' + parser.exclude_cac_project_settings,\n # This value is always true. Either this is an unmanaged project, in which case we are never\n # reapplying it; or it is a variable configured project, in which case we need to ignore\n # variable changes, or it is a shared CaC project, in which case we don't use Terraform to\n # manage variables.\n '-ignoreProjectVariableChanges=' + parser.ignore_variable_changes,\n # To have secret variables available when applying a downstream project, they must be scoped\n # to the Sync environment. But we do not need this scoping in the downstream project, so the\n # Sync environment is removed from any variable scopes when serializing it to Terraform.\n '-excludeVariableEnvironmentScopes', 'Sync',\n # Exclude any variables starting with \"Private.\"\n '-excludeProjectVariableRegex', 'Private\\\\..*',\n # Capture the octopus endpoint, space ID, and space name as output vars. This is useful when\n # querying th Terraform state file to know which space and instance the resources were\n # created in. The scripts used to update downstream projects in bulk work by querying the\n # Terraform state, finding all the downstream projects, and using the space name to only process\n # resources that match the current tenant (because space names and tenant names are the same).\n # The output variables added by this option are octopus_server, octopus_space_id, and\n # octopus_space_name.\n '-includeOctopusOutputVars',\n # Where steps do not explicitly define a worker pool and reference the default one, this\n # option explicitly exports the default worker pool by name. This means if two spaces have\n # different default pools, the exported project still uses the pool that the original project\n # used.\n '-lookUpDefaultWorkerPools',\n # Link any tenants that were originally link to the project and create project tenant variables\n '-lookupProjectLinkTenants=' + parser.lookup_project_link_tenants,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export',\n # This is a management runbook that we do not wish to export\n '-excludeRunbookRegex', '__ .*'] + list(chain(*ignores_library_variable_sets_args)) + list(\n chain(*ignored_accounts)) + list(chain(*ignored_tenants_args))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', parser.project_name),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', parser.project_name),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', parser.project_name) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', parser.project_name) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", + "Octopus.Action.Script.ScriptBody": "import argparse\nimport os\nimport stat\nimport re\nimport socket\nimport subprocess\nimport sys\nfrom datetime import datetime\nfrom urllib.parse import urlparse\nimport urllib.request\nfrom itertools import chain\nimport platform\nfrom urllib.request import urlretrieve\nimport zipfile\nimport json\nimport tarfile\nimport random, time\n\n# If this script is not being run as part of an Octopus step, return variables from environment variables.\n# Periods are replaced with underscores, and the variable name is converted to uppercase\nif \"get_octopusvariable\" not in globals():\n def get_octopusvariable(variable):\n return os.environ[re.sub('\\\\.', '_', variable.upper())]\n\n# If this script is not being run as part of an Octopus step, print directly to std out.\nif \"printverbose\" not in globals():\n def printverbose(msg):\n print(msg)\n\n\ndef printverbose_noansi(output):\n \"\"\"\n Strip ANSI color codes and print the output as verbose\n :param output: The output to print\n \"\"\"\n if not output:\n return\n\n # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python\n output_no_ansi = re.sub(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])', '', output)\n printverbose(output_no_ansi)\n\n\ndef get_octopusvariable_quiet(variable):\n \"\"\"\n Gets an octopus variable, or an empty string if it does not exist.\n :param variable: The variable name\n :return: The variable value, or an empty string if the variable does not exist\n \"\"\"\n try:\n return get_octopusvariable(variable)\n except:\n return ''\n\n\ndef retry_with_backoff(fn, retries=5, backoff_in_seconds=1):\n x = 0\n while True:\n try:\n return fn()\n except Exception as e:\n\n print(e)\n\n if x == retries:\n raise\n\n sleep = (backoff_in_seconds * 2 ** x +\n random.uniform(0, 1))\n time.sleep(sleep)\n x += 1\n\n\ndef execute(args, cwd=None, env=None, print_args=None, print_output=printverbose_noansi):\n \"\"\"\n The execute method provides the ability to execute external processes while capturing and returning the\n output to std err and std out and exit code.\n \"\"\"\n process = subprocess.Popen(args,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n cwd=cwd,\n env=env)\n stdout, stderr = process.communicate()\n retcode = process.returncode\n\n if print_args is not None:\n print_output(' '.join(args))\n\n if print_output is not None:\n print_output(stdout)\n print_output(stderr)\n\n return stdout, stderr, retcode\n\n\ndef is_windows():\n return platform.system() == 'Windows'\n\n\ndef init_argparse():\n parser = argparse.ArgumentParser(\n usage='%(prog)s [OPTION] [FILE]...',\n description='Serialize an Octopus project to a Terraform module'\n )\n parser.add_argument('--ignore-all-changes',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreAllChanges') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreAllChanges') or 'false',\n help='Set to true to set the \"lifecycle.ignore_changes\" ' +\n 'setting on each exported resource to \"all\"')\n parser.add_argument('--ignore-variable-changes',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreVariableChanges') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreVariableChanges') or 'false',\n help='Set to true to set the \"lifecycle.ignore_changes\" ' +\n 'setting on each exported octopus variable to \"all\"')\n parser.add_argument('--terraform-backend',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Terraform.Backend') or get_octopusvariable_quiet(\n 'ThisInstance.Terraform.Backend') or 'pg',\n help='Set this to the name of the Terraform backend to be included in the generated module.')\n parser.add_argument('--server-url',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Server.Url') or get_octopusvariable_quiet(\n 'ThisInstance.Server.Url'),\n help='Sets the server URL that holds the project to be serialized.')\n parser.add_argument('--api-key',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.ThisInstance.Api.Key') or get_octopusvariable_quiet(\n 'ThisInstance.Api.Key'),\n help='Sets the Octopus API key.')\n parser.add_argument('--space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Space.Id') or get_octopusvariable_quiet(\n 'Exported.Space.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID containing the project to be serialized.')\n parser.add_argument('--project-name',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.Name') or get_octopusvariable_quiet(\n 'Exported.Project.Name') or get_octopusvariable_quiet(\n 'Octopus.Project.Name'),\n help='Set this to the name of the project to be serialized.')\n parser.add_argument('--upload-space-id',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Octopus.UploadSpace.Id') or get_octopusvariable_quiet(\n 'Octopus.UploadSpace.Id') or get_octopusvariable_quiet('Octopus.Space.Id'),\n help='Set this to the space ID of the Octopus space where ' +\n 'the resulting package will be uploaded to.')\n parser.add_argument('--ignore-cac-managed-values',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoreCacValues') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoreCacValues') or 'false',\n help='Set this to true to exclude cac managed values like non-secret variables, ' +\n 'deployment processes, and project versioning into the Terraform module. ' +\n 'Set to false to have these values embedded into the module.')\n parser.add_argument('--exclude-cac-project-settings',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.ExcludeCacProjectValues') or get_octopusvariable_quiet(\n 'Exported.Project.ExcludeCacProjectValues') or 'false',\n help='Set this to true to exclude CaC settings like git connections from the exported module.')\n parser.add_argument('--ignored-library-variable-sets',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredLibraryVariableSet') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredLibraryVariableSet'),\n help='A comma separated list of library variable sets to ignore.')\n parser.add_argument('--ignored-accounts',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredAccounts') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredAccounts'),\n help='A comma separated list of accounts to ignore.')\n parser.add_argument('--ignored-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredTenants') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredTenants'),\n help='A comma separated list of tenants to ignore.')\n parser.add_argument('--ignored-channels',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IgnoredChannels') or get_octopusvariable_quiet(\n 'Exported.Project.IgnoredChannels'),\n help='A comma separated list of channels to ignore.')\n parser.add_argument('--include-step-templates',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.IncludeStepTemplates') or get_octopusvariable_quiet(\n 'Exported.Project.IncludeStepTemplates') or 'false',\n help='Set this to true to include step templates in the exported module. ' +\n 'This disables the default behaviour of detaching step templates.')\n parser.add_argument('--lookup-project-link-tenants',\n action='store',\n default=get_octopusvariable_quiet(\n 'SerializeProject.Exported.Project.LookupProjectLinkTenants') or get_octopusvariable_quiet(\n 'Exported.Project.LookupProjectLinkTenants') or 'false',\n help='Set this option to link tenants and create tenant project variables.')\n\n return parser.parse_known_args()\n\n\ndef get_latest_github_release(owner, repo, filename):\n url = f\"https://api.github.com/repos/{owner}/{repo}/releases/latest\"\n releases = urllib.request.urlopen(url).read()\n contents = json.loads(releases)\n\n download = [asset for asset in contents.get('assets') if asset.get('name') == filename]\n\n if len(download) != 0:\n return download[0].get('browser_download_url')\n\n return None\n\n\ndef ensure_octo_cli_exists():\n if is_windows():\n print(\"Checking for the Octopus CLI\")\n try:\n stdout, _, exit_code = execute(['octo.exe', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.win-x64.zip',\n 'OctopusTools.zip')\n with zipfile.ZipFile('OctopusTools.zip', 'r') as zip_ref:\n zip_ref.extractall(os.getcwd())\n return os.getcwd()\n else:\n print(\"Checking for the Octopus CLI for Linux\")\n try:\n stdout, _, exit_code = execute(['octo', 'help'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octo CLI not found\"\n return \"\"\n except:\n print(\"Downloading the Octopus CLI for Linux\")\n urlretrieve('https://download.octopusdeploy.com/octopus-tools/9.0.0/OctopusTools.9.0.0.linux-x64.tar.gz',\n 'OctopusTools.tar.gz')\n with tarfile.open('OctopusTools.tar.gz') as file:\n file.extractall(os.getcwd())\n os.chmod(os.path.join(os.getcwd(), 'octo'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\ndef ensure_octoterra_exists():\n if is_windows():\n print(\"Checking for the Octoterra tool for Windows\")\n try:\n stdout, _, exit_code = execute(['octoterra.exe', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Windows\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_windows_amd64.exe\",\n 'octoterra.exe'), 10, 30)\n return os.getcwd()\n else:\n print(\"Checking for the Octoterra tool for Linux\")\n try:\n stdout, _, exit_code = execute(['octoterra', '-version'])\n printverbose(stdout)\n if not exit_code == 0:\n raise \"Octoterra not found\"\n return \"\"\n except:\n print(\"Downloading Octoterra CLI for Linux\")\n retry_with_backoff(lambda: urlretrieve(\n \"https://github.com/OctopusSolutionsEngineering/OctopusTerraformExport/releases/latest/download/octoterra_linux_amd64\",\n 'octoterra'), 10, 30)\n os.chmod(os.path.join(os.getcwd(), 'octoterra'), stat.S_IRWXO | stat.S_IRWXU | stat.S_IRWXG)\n return os.getcwd()\n\n\noctocli_path = ensure_octo_cli_exists()\noctoterra_path = ensure_octoterra_exists()\nparser, _ = init_argparse()\n\n# Variable precondition checks\nif len(parser.server_url) == 0:\n print(\"--server-url, ThisInstance.Server.Url, or SerializeProject.ThisInstance.Server.Url must be defined\")\n sys.exit(1)\n\nif len(parser.api_key) == 0:\n print(\"--api-key, ThisInstance.Api.Key, or ThisInstance.Api.Key must be defined\")\n sys.exit(1)\n\nprint(\"Octopus URL: \" + parser.server_url)\nprint(\"Octopus Space ID: \" + parser.space_id)\n\n# Build the arguments to ignore library variable sets\nignores_library_variable_sets = parser.ignored_library_variable_sets.split(',')\nignores_library_variable_sets_args = [['-excludeLibraryVariableSet', x] for x in ignores_library_variable_sets]\n\n# Build the arguments to ignore accounts\nignored_accounts = parser.ignored_accounts.split(',')\nignored_accounts = [['-excludeAccounts', x] for x in ignored_accounts]\n\n# Build the arguments to ignore tenants\nignored_tenants = parser.ignored_tenants.split(',')\nignored_tenants_args = [['-excludeTenants', x] for x in ignored_tenants]\n\n# Build the arguments to ignore channels\nignored_channels = parser.ignored_channels.split(',')\nignored_channels_args = [['-excludeChannels', x] for x in ignored_channels]\n\nos.mkdir(os.getcwd() + '/export')\n\nexport_args = [os.path.join(octoterra_path, 'octoterra'),\n # the url of the instance\n '-url', parser.server_url,\n # the api key used to access the instance\n '-apiKey', parser.api_key,\n # add a postgres backend to the generated modules\n '-terraformBackend', parser.terraform_backend,\n # dump the generated HCL to the console\n '-console',\n # dump the project from the current space\n '-space', parser.space_id,\n # the name of the project to serialize\n '-projectName', parser.project_name,\n # ignoreProjectChanges can be set to ignore all changes to the project, variables, runbooks etc\n '-ignoreProjectChanges=' + parser.ignore_all_changes,\n # use data sources to lookup external dependencies (like environments, accounts etc) rather\n # than serialize those external resources\n '-lookupProjectDependencies',\n # for any secret variables, add a default value set to the octostache value of the variable\n # e.g. a secret variable called \"database\" has a default value of \"#{database}\"\n '-defaultSecretVariableValues',\n # Any value that can't be replaced with an Octostache template, add a dummy value\n '-dummySecretVariableValues',\n # detach any step templates, allowing the exported project to be used in a new space\n '-detachProjectTemplates=' + str(not parser.include_step_templates),\n # allow the downstream project to move between project groups\n '-ignoreProjectGroupChanges',\n # allow the downstream project to change names\n '-ignoreProjectNameChanges',\n # CaC enabled projects will not export the deployment process, non-secret variables, and other\n # CaC managed project settings if ignoreCacManagedValues is true. It is usually desirable to\n # set this value to true, but it is false here because CaC projects created by Terraform today\n # save some variables in the database rather than writing them to the Git repo.\n '-ignoreCacManagedValues=' + parser.ignore_cac_managed_values,\n # Excluding CaC values means the resulting module does not include things like git credentials.\n # Setting excludeCaCProjectSettings to true and ignoreCacManagedValues to false essentially\n # converts a CaC project back to a database project.\n '-excludeCaCProjectSettings=' + parser.exclude_cac_project_settings,\n # This value is always true. Either this is an unmanaged project, in which case we are never\n # reapplying it; or it is a variable configured project, in which case we need to ignore\n # variable changes, or it is a shared CaC project, in which case we don't use Terraform to\n # manage variables.\n '-ignoreProjectVariableChanges=' + parser.ignore_variable_changes,\n # To have secret variables available when applying a downstream project, they must be scoped\n # to the Sync environment. But we do not need this scoping in the downstream project, so the\n # Sync environment is removed from any variable scopes when serializing it to Terraform.\n '-excludeVariableEnvironmentScopes', 'Sync',\n # Exclude any variables starting with \"Private.\"\n '-excludeProjectVariableRegex', 'Private\\\\..*',\n # Capture the octopus endpoint, space ID, and space name as output vars. This is useful when\n # querying th Terraform state file to know which space and instance the resources were\n # created in. The scripts used to update downstream projects in bulk work by querying the\n # Terraform state, finding all the downstream projects, and using the space name to only process\n # resources that match the current tenant (because space names and tenant names are the same).\n # The output variables added by this option are octopus_server, octopus_space_id, and\n # octopus_space_name.\n '-includeOctopusOutputVars',\n # Where steps do not explicitly define a worker pool and reference the default one, this\n # option explicitly exports the default worker pool by name. This means if two spaces have\n # different default pools, the exported project still uses the pool that the original project\n # used.\n '-lookUpDefaultWorkerPools',\n # Link any tenants that were originally link to the project and create project tenant variables\n '-lookupProjectLinkTenants=' + parser.lookup_project_link_tenants,\n # Add support for experimental step templates\n '-experimentalEnableStepTemplates=' + parser.include_step_templates,\n # The directory where the exported files will be saved\n '-dest', os.getcwd() + '/export',\n # This is a management runbook that we do not wish to export\n '-excludeRunbookRegex', '__ .*'] + list(chain(*ignores_library_variable_sets_args)) + list(\n chain(*ignored_accounts)) + list(chain(*ignored_tenants_args)) + list(chain(*ignored_channels_args))\n\nprint(\"Exporting Terraform module\")\n_, _, octoterra_exit = execute(export_args)\n\nif not octoterra_exit == 0:\n print(\"Octoterra failed. Please check the logs for more information.\")\n sys.exit(1)\n\ndate = datetime.now().strftime('%Y.%m.%d.%H%M%S')\n\nprint(\"Creating Terraform module package\")\nif is_windows():\n execute([os.path.join(octocli_path, 'octo.exe'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', parser.project_name),\n '--version', date,\n '--basePath', os.getcwd() + '\\\\export',\n '--outFolder', os.getcwd()])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'pack',\n '--format', 'zip',\n '--id', re.sub('[^0-9a-zA-Z]', '_', parser.project_name),\n '--version', date,\n '--basePath', os.getcwd() + '/export',\n '--outFolder', os.getcwd()])\n\nprint(\"Uploading Terraform module package\")\nif is_windows():\n _, _, _ = execute([os.path.join(octocli_path, 'octo.exe'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"\\\\\" +\n re.sub('[^0-9a-zA-Z]', '_', parser.project_name) + '.' + date + '.zip',\n '--replace-existing'])\nelse:\n _, _, _ = execute([os.path.join(octocli_path, 'octo'),\n 'push',\n '--apiKey', parser.api_key,\n '--server', parser.server_url,\n '--space', parser.upload_space_id,\n '--package', os.getcwd() + \"/\" +\n re.sub('[^0-9a-zA-Z]', '_', parser.project_name) + '.' + date + '.zip',\n '--replace-existing'])\n\nprint(\"##octopus[stdout-default]\")\n\nprint(\"Done\")\n", "Octopus.Action.Script.ScriptSource": "Inline", "Octopus.Action.Script.Syntax": "Python" }, @@ -143,6 +143,16 @@ "Octopus.ControlType": "SingleLineText" } }, + { + "Id": "ab67746c-b66b-4983-ac17-9a278b711037", + "Name": "SerializeProject.Exported.Project.IgnoredChannels", + "Label": "Ignored Channels", + "HelpText": "A comma separated list of channels that will not be included in the Terraform module.", + "DefaultValue": "", + "DisplaySettings": { + "Octopus.ControlType": "SingleLineText" + } + }, { "Id": "e456bc3f-a537-4982-8963-a091d3f31cf0", "Name": "SerializeProject.Exported.Project.IncludeStepTemplates",