feat: Orb Tools Version 12 (#181)

* feat: add injection to continue

* feat: optional workspaces

* refactor!: snake_case

* feat: print modified config

* fix: modify config

* chore: silent curl

* docs: update usage examples for v12

* feat: RC010 snake_case

* docs: v12 migration guide

* feat: include empty orb in test-deploy template

* docs: change message

* feat: add custom orb name

* docs: remove double negatives

* feat: make checkout optional

* chore: update python in lint job

* chore: pin base images to current-22.04

* chore: pin the circlecicli image for stability

* feat: executor parameters for jobs

* feat: base img -> python for less executors

* fix: executors

* chore: quotes

Co-authored-by: Eric Ribeiro <[email protected]>

* chore: quotes

Co-authored-by: Eric Ribeiro <[email protected]>

* chore: wording

* refactor: improve readability

* feat: exit with error message if cant continue

* chore: comments

* refactor: replace echo with printf

* refactor: replace echo with printf

* refactor: replace echo with printf

* feat: add support for private orbs

* feat: publish multiple dev tags

* refactor: remove slashes from paths

* refactor: use anchors for filters

* refactor!: main config does not require filters

* fix: macos compatible

Co-authored-by: Eric Ribeiro <[email protected]>

* refactor: git mv

Co-authored-by: Eric Ribeiro <[email protected]>

* chore: remove shellcheck exclusions

---------

Co-authored-by: Eric Ribeiro <[email protected]>

.circleci/config.yml

@@ -5,47 +5,23 @@ orbs:
   #   meaning this dev version will need to be published every 90 days.
   # No other orbs will need to import themselves here.
   orb-tools-alpha: circleci/orb-tools@dev:alpha
-  shellcheck: circleci/shellcheck@3.0
+  shellcheck: circleci/shellcheck@3.1
 
 workflows:
   lint-pack:
     jobs:
-      - orb-tools-alpha/lint:
-          filters:
-            tags:
-              only: /.*/
-      - orb-tools-alpha/pack:
-          filters:
-            tags:
-              only: /.*/
-      - orb-tools-alpha/review:
-          filters:
-            tags:
-              only: /.*/
-      - shellcheck/check:
-          exclude: SC2148,SC2038,SC2086,SC2002,SC2016
-          filters:
-            tags:
-              only: /.*/
-      - orb-tools-alpha/publish:
-          orb-name: circleci/orb-tools
-          vcs-type: << pipeline.project.type >>
+      - orb-tools-alpha/lint
+      - orb-tools-alpha/pack
+      - orb-tools-alpha/review
+      - shellcheck/check
+      - orb-tools-alpha/continue:
+          pipeline_number: << pipeline.number >>
+          vcs_type: << pipeline.project.type >>
+          orb_name: orb-tools-alpha
           requires:
             [
               orb-tools-alpha/lint,
               orb-tools-alpha/review,
               orb-tools-alpha/pack,
               shellcheck/check,
             ]
-          github-token: GHI_TOKEN
-          context: orb-publisher
-          filters:
-            tags:
-              only: /.*/
-      - orb-tools-alpha/continue:
-          pipeline-number: << pipeline.number >>
-          vcs-type: << pipeline.project.type >>
-          requires: [orb-tools-alpha/publish]
-          filters:
-            tags:
-              only: /.*/

.circleci/test-deploy.yml

@@ -1,30 +1,36 @@
 version: 2.1
 orbs:
-  orb-tools-alpha: circleci/orb-tools@dev:<<pipeline.git.revision>>
+  orb-tools-alpha: {}
+
+filters: &filters
+  tags:
+    only: /.*/
 
 workflows:
   test-deploy:
     jobs:
       - orb-tools-alpha/lint:
-          filters:
-            tags:
-              only: /.*/
+          filters: *filters
       - orb-tools-alpha/pack:
-          filters:
-            tags:
-              only: /.*/
+          filters: *filters
       - orb-tools-alpha/review:
-          filters:
-            tags:
-              only: /.*/
+          filters: *filters
       - orb-tools-alpha/publish:
-          orb-name: circleci/orb-tools
-          vcs-type: <<pipeline.project.type>>
-          pub-type: production
+          name: publish_dev_test
+          orb_name: circleci/orb-tools
+          vcs_type: <<pipeline.project.type>>
+          pub_type: dev
           requires:
             [orb-tools-alpha/lint, orb-tools-alpha/review, orb-tools-alpha/pack]
           context: orb-publisher
-          github-token: GHI_TOKEN
+      - orb-tools-alpha/publish:
+          orb_name: circleci/orb-tools
+          vcs_type: <<pipeline.project.type>>
+          pub_type: production
+          requires:
+            [orb-tools-alpha/lint, orb-tools-alpha/review, orb-tools-alpha/pack, publish_dev_test]
+          context: orb-publisher
+          github_token: GHI_TOKEN
           filters:
             branches:
               ignore: /.*/

MIGRATION.md

@@ -1,5 +1,36 @@
 # Migration Guide
 
+## v12.0.0
+
+Version 12 of orb-tools enhances the contributor experience by removing the roadblock of needing to publish a development orb for testing. Orbs are now _injected_ dynamically into the pipeline, allowing for orb testing without the need for access to a publishing token.
+
+_Note: If you are upgrading from 11.x to 12.x proceed here. If you are upgrading from an earlier version, please see the [migration guide for v11.x](https://github.com/CircleCI-Public/orb-tools-orb/blob/v11.6.1/MIGRATION.md)._
+
+### Notable Changes:
+
+- **Removed the need for a dev publishing token**
+  - Previously, after the orb was built, it would be published to a development tag and referenced in the test pipeline. This required a publishing token to be present in the dev pipeline.
+   - The dynamic configuration system allows us to inject the orb directly into the pipeline as an "in-line" orb, allowing for orb testing without the need for access to a publishing token.
+- **Snake_Case Renamed Components**
+  - To keep consistency with CircleCI's _native_ configuration schema, all components and parameters have been renamed to use snake_case.
+  (Example: `parameter-name` -> `parameter_name`)
+- **RC010 - Check for Snake_Case**
+  - The `review` job will automatically check for snake_case naming conventions and provide an error if it detects a violation. This (like all RC checks) can be skipped.
+
+### How to Migrate
+
+1. Clone your orb project locally.
+1. Create a new branch for upgrading your pipeline
+   ```sh
+   git checkout -b orb-tools-12-migration
+   ````
+1. Copy the `migrate.sh` script from this repository into your project's root directory.
+1. Run the script
+   ```sh
+   chmod +x migrate.sh && ./migrate.sh
+   ```
+
+After executing the script, your orb's component names and parameters will be converted to snake_case, and references to them within your config files will be updated. The reference to your orb in the `test-deploy` config will be removed, as the orb will now be dynamically injected into the pipeline.
 ## v11.0.0
 
 Version 11.0.0 of orb-tools is composed of a major re-write that greatly changes and improves the way orbs are deployed, makes use of CircleCI's [dynamic configuration](https://circleci.com/docs/2.0/dynamic-config/), and can even automatically test for best practices.
@@ -19,7 +50,7 @@ Version 11.0.0 of orb-tools is composed of a major re-write that greatly changes
   - Automatically comment on the PR associated with a commit when each new orb version is published (dev or production.)
   - The comment will include a link to the Orb Registry to preview dev versions of the orb, and a live link to the production version of the orb.
 
-## How to Migrate
+### How to Migrate
 
 1. Enable "[dynamic configuration](https://circleci.com/docs/2.0/dynamic-config/#getting-started-with-dynamic-config-in-circleci)" for your project on CircleCI.
    - Visit https://app.circleci.com/ and navigate to your project.

migrate.sh

@@ -1,19 +1,27 @@
 #!/bin/bash
-# Use this tool to assist in migrating your Orb Development Kit pipeline.
+# Use this tool to assist in migrating your Orb Development Kit pipeline from v11+ to v12+.
+
+orb_config_files=(
+  ".circleci/config.yml"
+  ".circleci/test-deploy.yml"
+)
+component_types=("commands" "executors" "jobs" "examples")
+orb_name=""
+declare -a token_map
 
 verify_run() {
   CHECK_FAILED=false
   # Ensure .circleci/config.yml exists
-  if [ ! -f .circleci/config.yml ]; then
+  if [ ! -f "${orb_config_files[0]}" ]; then
     echo "No .circleci/config.yml found"
     echo "This does not appear to be the root of a CircleCI project"
     CHECK_FAILED=true
   fi
 
-  # Ensure jq is installed
-  if ! command -v jq >/dev/null 2>&1; then
-    echo "Looks like you don't have \"jq\" installed"
-    echo "Please install it and run the script again: https://stedolan.github.io/jq/download/."
+  # Ensure .circleci/test-deploy.yml exists
+  if [ ! -f "${orb_config_files[1]}" ]; then
+    echo "No .circleci/test-deploy.yml found"
+    echo "This version of the migration tool expects your orb to have been built with Orb-Tools v11 or higher. If you are upgrading from an earlier version of orb tools, you must first upgrade to v11+ by using the v11 version of this tool from https://github.com/CircleCI-Public/orb-tools-orb/tree/v11.6.1"
     CHECK_FAILED=true
   fi
 
@@ -24,89 +32,130 @@ verify_run() {
     CHECK_FAILED=true
   fi
 
-  if [ "$CHECK_FAILED" == true ]; then 
+  # Warn the user to backup their files
+  if [ "$CHECK_FAILED" == false ]; then
+    echo "This script will rename all your orb components and parameters to use snake_case."
+    echo "Yaml files will be standardized to .yml."
+    echo "Please backup your files or create a new branch before running this script."
+    echo
+    read -rp 'Continue? [y/N] ' CONTINUE
+    if [ "$CONTINUE" != "y" ]; then
+      echo "Exiting..."
+      exit 1
+    fi
+  fi
+
+  if [ "$CHECK_FAILED" == true ]; then
     exit 1
   fi
 }
 
-backup_contents() {
-  # Backup the existing files in .circleci/
-  for file in .circleci/*; do
+# Gets all the YAML files in the src/<component> directory
+get_components() {
+  # Return an array of command names as given by the filename in `src/<component>/`
+  local -a component_names
+  component_dir="./src/${1}"
+  for file in "${component_dir}"/*.yaml "${component_dir}"/*.yml; do
     if [ -f "$file" ]; then
-      mv "$file" "$file.bak"
+      component_names+=("$file")
     fi
   done
+  echo "${component_names[@]}"
 }
 
-download_template() {
-  ORB_TEMPLATE_TAG_LIST=$(curl --request GET \
-    --url https://api.github.com/repos/CircleCI-Public/Orb-Template/tags \
-    --header 'Accept: application/vnd.github.v3+json' | jq -r '.[].name' | grep -v '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -Vr)
-  ORB_TEMPLATE_VERSION=$(echo "$ORB_TEMPLATE_TAG_LIST" | head -n 1)
-  ORB_TEMPLATE_DOWNLOAD_URL="https://github.com/CircleCI-Public/Orb-Template/archive/refs/tags/${ORB_TEMPLATE_VERSION}.tar.gz"
-  ORB_TEMP_DIR=$(mktemp -d)
-
-  curl -Ls "$ORB_TEMPLATE_DOWNLOAD_URL" -o "$ORB_TEMP_DIR/orb-template.tar.gz"
-  tar -xzf "$ORB_TEMP_DIR/orb-template.tar.gz" -C "$ORB_TEMP_DIR" --strip-components 1
-  cp -r "${ORB_TEMP_DIR}/.circleci/." .circleci/
+convert_to_snake_case() {
+  # Convert the string to snake_case
+  echo "$1" | sed -E 's/-/_/g' | tr '[:upper:]' '[:lower:]'
+}
 
-  if [[ ! -e .yamllint ]]; then
-    cp "${ORB_TEMP_DIR}/.yamllint" .yamllint
-  fi
+# Takes in a list of files and renames them to snake_case and standardizes the file extension to .yml
+rename_components() {
+  for file in ${1}; do
+    # Get the filename without the extension
+    filename=$(basename -- "$file")
+    filename="${filename%.*}"
+    token_map+=("$filename")
+    # Convert the filename to snake_case
+    filename=$(convert_to_snake_case "$filename")
+    filepath=$(dirname "$file")
+    # Rename the file
+    if [ -d ".git" ]; then
+      git mv --no-commit "$file" "${filepath}/${filename}.yml" 2>/dev/null
+    else
+      mv "$file" "${filepath}/${filename}.yml" 2>/dev/null
+    fi
+  done
 }
 
-copy_custom_components() {
-  ORIGINAL_EXECUTORS=$(yq '.executors' .circleci/config.yml.bak)
-  ORIGINAL_JOBS=$(yq '.jobs' .circleci/config.yml.bak)
-  ORIGINAL_COMMANDS=$(yq '.commands' .circleci/config.yml.bak)
-  export ORIGINAL_EXECUTORS
-  export ORIGINAL_JOBS
-  export ORIGINAL_COMMANDS
-  if [[ -n "$ORIGINAL_EXECUTORS" && ! "$ORIGINAL_EXECUTORS" == "null" ]]; then
-    yq -i '. += {"executors": env(ORIGINAL_EXECUTORS)}' .circleci/test-deploy.yml
-  fi
-  if [[ -n "$ORIGINAL_JOBS" && ! "$ORIGINAL_JOBS" == "null" ]]; then
-    yq -i '. += {"jobs": env(ORIGINAL_JOBS)}' .circleci/test-deploy.yml
-  fi
-  if [[ -n "$ORIGINAL_COMMANDS" && ! "$ORIGINAL_COMMANDS" == "null" ]]; then
-    yq -i '. += {"commands": env(ORIGINAL_COMMANDS)}' .circleci/test-deploy.yml
-  fi
+destroy_script() {
+  rm -f "$0"
 }
 
-user_input() {
-  read -rp 'Namespace: ' ORB_NAMESPACE
-  read -rp 'Orb name: ' ORB_NAME
-  read -rp 'Context name: ' ORB_CONTEXT_NAME
+find_and_replace() {
+  #1 the file
+  #2 the string to find
+  #3 the string to replace it with
+  sed -i "" -e "s/$2/$3/g" "$1"
 }
 
-replace_values() {
-  sed -i '' "s/<namespace>/$ORB_NAMESPACE/g" .circleci/config.yml
-  sed -i '' "s/<orb-name>/$ORB_NAME/g" .circleci/config.yml
-  sed -i '' "s/<publishing-context>/$ORB_CONTEXT_NAME/g" .circleci/config.yml
+main() {
+  # Set the orb name
+  read -rp 'Enter the name of your orb: ' orb_name
+  # Get and rename all orb components and add them to the token map
+  for component_type in "${component_types[@]}"; do
+    components=$(get_components "${component_type}")
+    echo "Renaming ${component_type}..."
+    for component in ${components}; do
+      # The component is renamed and added to the token map
+      rename_components "${component}"
+    done
+  done
 
-  sed -i '' "s/<namespace>/$ORB_NAMESPACE/g" .circleci/test-deploy.yml
-  sed -i '' "s/<orb-name>/$ORB_NAME/g" .circleci/test-deploy.yml
-  sed -i '' "s/<publishing-context>/$ORB_CONTEXT_NAME/g" .circleci/test-deploy.yml
-}
+  # Get all orb component parameter keys and add them to the token map
+  for component_type in "${component_types[@]}"; do
+    components=$(get_components "${component_type}")
+    echo "Fetching ${component_type} parameter keys..."
+    for component in ${components}; do
+      keys=$(yq -e '.parameters | keys | .[]' "$component" 2>/dev/null)
+      for key in ${keys}; do
+        token_map+=("$key")
+        # # find_and_replace "$component" "$key" "$(convert_to_snake_case "$key")"
+      done
+    done
+  done
 
-msg_success() {
-  echo "Successfully upgraded config files."
-  echo "You must now open \"test-deploy.yml\" and add your integrations tests."
-  echo "Docs: https://circleci.com/docs/2.0/testing-orbs/#integration-testing"
-  echo
-  echo "When complete, delete the '.bak' files in the .circleci directory."
-  echo 'Commit your changes and the next version of your orb will be published when a tag is created.'
-}
+  # Find and replace all tokens in the orb components
+  for component_type in "${component_types[@]}"; do
+    components=$(get_components "${component_type}")
+    echo "Replacing tokens in ${component_type}..."
+    for component in ${components}; do
+      for token in "${token_map[@]}"; do
+        find_and_replace "$component" "$token" "$(convert_to_snake_case "$token")"
+      done
+    done
+  done
+
+  # Find and replace all tokens in the orb config files
+  for orb_config_file in "${orb_config_files[@]}"; do
+    echo "Replacing tokens in ${orb_config_file}..."
+    for token in "${token_map[@]}"; do
+      find_and_replace "$orb_config_file" "$token" "$(convert_to_snake_case "$token")"
+    done
+  done
+
+  # Remove the orb import from the test-deploy.yml file
+  echo "Removing orb import from ${orb_config_files[1]}..."
+  ORB_IMPORT_KEY=$(yq eval ".orbs | to_entries | map(select(.value == \"*${orb_name}*\")) | .[0].key" "${orb_config_files[1]}" 2>/dev/null)
+  if [ -n "$ORB_IMPORT_KEY" ]; then
+    yq eval "del(.orbs.${ORB_IMPORT_KEY})" -i "${orb_config_files[1]}"
+  else
+    echo "Could not find orb import in ${orb_config_files[1]}"
+    echo "Please remove the import manually."
+  fi
 
-destroy_script() {
-  rm -f "$0"
 }
 
 verify_run
-user_input
-backup_contents
-download_template
-replace_values
-copy_custom_components
-msg_success
+main
+echo "Migration complete!"
 destroy_script