integ-test.yml

name: integ-test
on:
  schedule:
    - cron: "0 4 * * *"
  workflow_dispatch:
    inputs:
      integ_tests_include:
        type: string
        description: |-
          List integration tests to include.
          Use "*" to run all tests.
          Use regex like 'node|^git_issue' to run only a subset.
        default: ""
      # dns_config|time_server - NTP cannot be reconfigured if DNS is invalid
      # git_issues - slow, do not run on each push. TODO - run them only once a day
      # oidc_config - during reconfiguration API returns 500/502 errors for other requests
      # utils_login - it uses OIDC user to login
      # smtp - email_alert test requires a configured SMTP
      # role_cluster_config - role cluster_config reconfigures DNS, SMTP, OIDC. And it is slow.
      # version_update_single_node - role would change version, VSNS system cannot be updated.
      integ_tests_exclude:
        type: string
        description: |-
          List integration tests to exclude.
          Use "*" to exclude all tests.
          Use regex like 'node|^git_issue|^dns_config$' to exclude only a subset.
        default: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^utils_login$|^certificate$"
      examples_tests_include:
        type: string
        description: |-
          List examples to include into testing.
          Use "*" to test all examples.
        default: "iso_info"
env:
  INTEG_TESTS_INCLUDE_SCHEDULE: "*"
  INTEG_TESTS_EXCLUDE_SCHEDULE: "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^utils_login$|^certificate$"
  EXAMPLES_TESTS_INCLUDE_SCHEDULE: "*"
  # ansible-test needs special directory structure.
  # WORKDIR is a subdir of GITHUB_WORKSPACE
  WORKDIR: work-dir/ansible_collections/scale_computing/hypercore
  LANG: C.UTF-8
# Run only one workflow for specific branch.
concurrency:
  group: ${{ github.ref_name }}
  cancel-in-progress: true

jobs:
  # This should be run at least once before first integration tests run.
  # As long as prepared HC3 host remains prepared, we do not need (except on very first run)
  # to delay integ-test until integration-prepare-env finishes.
  integration-prepare-env:
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:8
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      matrix:
        sc_host:
          - https://10.5.11.200
          - https://10.5.11.201
          - https://10.5.11.202 # HC 9.3.x
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: ansible-galaxy collection install community.crypto community.general
      - run: ansible-galaxy collection list
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: ${{ matrix.sc_host }}
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-playbook tests/integration/prepare/prepare_iso.yml
      - run: ansible-playbook tests/integration/prepare/prepare_vm.yml
      - run: ansible-playbook tests/integration/prepare/prepare_examples.yml
    if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}"

  integ-matrix:
    runs-on: [ubuntu-latest]
    container: quay.io/justinc1_github/scale_ci_integ:8
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      # We want to run all integ tests periodically - "integ_tests_include || '*'", the '*' is used.
      # When running with workflow-dispatch, user is required to put some non-empty string into integ_tests_include.
      - run: echo 'INTEG_TESTS_INCLUDE=${{ github.event.inputs.integ_tests_include || env.INTEG_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV
      - run: echo 'INTEG_TESTS_EXCLUDE=${{ github.event.inputs.integ_tests_exclude || env.INTEG_TESTS_EXCLUDE_SCHEDULE }}' >> $GITHUB_ENV
#      - run: echo 'INTEG_TESTS_TO_RUN_SEQUENTIALLY=${{ github.event.inputs.integ_tests_to_run_sequentially || env.INTEG_TESTS_TO_RUN_SEQUENTIALLY }}' >> $GITHUB_ENV

      - id: set-matrix
        shell: bash
        run: |-
          echo "matrix=$(
            ls -r tests/integration/targets |
            grep -v -E "${{ env.INTEG_TESTS_EXCLUDE }}" |
            grep -E "${{ env.INTEG_TESTS_INCLUDE }}" |
            jq -R -s -c 'split("\n")[:-1]'
          )" >> $GITHUB_OUTPUT
    if: "${{ github.event.inputs.integ_tests_include || github.event.schedule }}"

  examples-matrix:
    runs-on: [ ubuntu-latest ]
    container: quay.io/justinc1_github/scale_ci_integ:8
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: |
          if [ "" != "${{ github.event.schedule }}" ]
          then
            echo 'EXAMPLES_TESTS_INCLUDE=${{ env.EXAMPLES_TESTS_INCLUDE_SCHEDULE }}' >> $GITHUB_ENV
          else
            echo 'EXAMPLES_TESTS_INCLUDE=${{ github.event.inputs.examples_tests_include }}' >> $GITHUB_ENV
          fi
      # A few files are not an example playbook:
      #   hypercore_inventory.yml - inventory example
      #   cloud-init-user-data-example.yml - cloud-init user-data example
      - id: set-matrix
        shell: bash
        run: |-
          echo "matrix=$(
            ls -r examples |
            grep -v -E "^README.md$|^hypercore_inventory.yml$|^cloud-init-user-data-example.yml$" |
            grep -E "${{ env.EXAMPLES_TESTS_INCLUDE }}" |
            jq -R -s -c 'split("\n")[:-1]'
          )" >> $GITHUB_OUTPUT
    if: "${{ github.event.inputs || github.event.schedule }}"

  examples-run:
    needs:
      - examples-matrix
      # run examples after integ - they will both try to modify say syslog_server etc
      - integ
      - integ-seq
    if: "always() && (needs.examples-matrix.result=='success')"
    runs-on: [ self-hosted2 ]
    container: quay.io/justinc1_github/scale_ci_integ:8
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
      DEBIAN_FRONTEND: noninteractive
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      matrix:
        ansible: [ 2.15.0 ]
        example_name: ${{ fromJson(needs.examples-matrix.outputs.matrix) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: pip install ansible-core~=${{ matrix.ansible }}
      - run: ansible-galaxy collection install community.crypto community.general
      - run: ansible-galaxy collection list

      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.202
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: |
          pwd
          ls -al tests/integration/integration_config.yml
          head tests/integration/integration_config.yml

      - run: |
          eval export SC_HOST=$(cat tests/integration/integration_config.yml | yq '.sc_host')
          eval export SC_USERNAME=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_username')
          eval export SC_PASSWORD=$(cat tests/integration/integration_config.yml | yq '.sc_config."https://10.5.11.200".sc_password')
          eval export SC_TIMEOUT=$(cat tests/integration/integration_config.yml | yq '.sc_timeout')
          ansible-playbook -i localhost, -e hypercore_example_check_mode=true examples/${{ matrix.example_name }}

  integ:
    needs:
      - integ-matrix
      # - units-test
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:8
    env:
      DEBIAN_FRONTEND: noninteractive
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      matrix:
        # ansible: [2.15.0]
        # python: [3.11]
        # test_name: [user_info]
        test_name: ${{ fromJson(needs.integ-matrix.outputs.matrix) }}
        sc_host:
          - https://10.5.11.200
          - https://10.5.11.201
          - https://10.5.11.202
        include:
          - sc_host: https://10.5.11.50
            test_name: vm_replication
          - sc_host: https://10.5.11.50
            test_name: vm_replication_info
          - sc_host: https://10.5.11.50
            test_name: support_tunnel
          - sc_host: https://10.5.11.50
            test_name: vm_clone__replicated
        exclude:
          # The VSNS were not configured with remote replication cluster.
          - sc_host: https://10.5.11.200
            test_name: vm_replication
          - sc_host: https://10.5.11.201
            test_name: vm_replication
          - sc_host: https://10.5.11.202
            test_name: vm_replication
          - sc_host: https://10.5.11.200
            test_name: vm_clone__replicated
          - sc_host: https://10.5.11.201
            test_name: vm_clone__replicated
          - sc_host: https://10.5.11.202
            test_name: vm_clone__replicated
          # Seem as VSNS nodes cannot open remote tunnel.
          # Code/port that worked (was opened, then closed) on real HyperCode
          # did not work with VSNS.
          - sc_host: https://10.5.11.200
            test_name: support_tunnel
          - sc_host: https://10.5.11.201
            test_name: support_tunnel
          - sc_host: https://10.5.11.202
            test_name: support_tunnel
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: pip install ansible-core~=2.15.0
      # We have ansible.cfg "for testing" in git repo
      # (it is excluded in galaxy.yml, so it is not part of collection artifact)
      # But it does affect ansible-galaxy and ansible-test commands.
      - run: ansible-galaxy collection install community.crypto
      - run: ansible-galaxy collection list
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: ${{ matrix.sc_host }}
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-test integration --local ${{ matrix.test_name }}

  integ-seq:
    needs:
      - integ-matrix
      - integ
    if: "always() && (needs.integ-matrix.result=='success')"
    runs-on: [ self-hosted2 ]
    container: quay.io/justinc1_github/scale_ci_integ:3
    env:
      DEBIAN_FRONTEND: noninteractive
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    strategy:
      fail-fast: false
      # The max number of concurent jobs
      max-parallel: 1
      matrix:
        # "^dns_config$|^cluster_shutdown$|^version_update$|^oidc_config$|^smtp$|^role_cluster_config$|^role_version_update_single_node$|^utils_login$|^certificate$"
        # Some tests can be tested on VSNS - those are run on VSNS.
        # Some cannot, those are run on test cluster 10.5.11.50.
        test_name:
          - dns_config
          # - cluster_shutdown
          # - version_update
          - oidc_config
          - smtp
          - role_cluster_config
          # - role_version_update_single_node
          # after oidc_config configures OIDC login, the utils_login can be tested too
          - utils_login
          - certificate
        sc_host:
          - https://10.5.11.200
          - https://10.5.11.201
          - https://10.5.11.202
        exclude:
          # role cluster_config can be used with HC3 9.1, but you need to omit cluster_name setting.
          # CI job role_cluster_config does try to set cluster_name, so it would fail.
          - sc_host: https://10.5.11.200
            test_name: role_cluster_config
        # include:  # change later to use env.INTEG_TESTS_TO_RUN_SEQUENTIALLY
        #   - sc_host: https://10.5.11.50
        #     test_name: dns_config
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: pip install ansible-core~=2.13.0
      # We have ansible.cfg "for testing" in git repo
      # (it is excluded in galaxy.yml, so it is not part of collection artifact)
      # But it does affect ansible-galaxy and ansible-test commands.
      - run: ansible-galaxy collection install community.crypto
      - run: ansible-galaxy collection list
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: ${{ matrix.sc_host }}
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-test integration --local ${{ matrix.test_name }}

  replica_cleanup:
    needs:
      - integ
      - integ-seq
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:8
    env:
      ANSIBLE_COLLECTIONS_PATH: $GITHUB_WORKSPACE/work-dir
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      - run: pip install ansible-core~=2.15.0
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.50
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: ansible-galaxy collection install community.general
      - run: ansible-playbook tests/integration/cleanup/ci_replica_cleanup.yml

  smb_cleanup:
    needs:
      - integ
      - integ-seq
    runs-on: [self-hosted2]
    container: quay.io/justinc1_github/scale_ci_integ:8
    defaults:
      run:
        working-directory: ${{ env.WORKDIR }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: ${{ env.WORKDIR }}
      # ${{ env.WORKDIR }} cannot be used
      - uses: ./work-dir/ansible_collections/scale_computing/hypercore/.github/actions/make-integ-config
        with:
          sc_host: https://10.5.11.200
          sc_password_50: ${{ secrets.CI_CONFIG_HC_IP50_SC_PASSWORD }}
          smb_password: ${{ secrets.CI_CONFIG_HC_IP50_SMB_PASSWORD }}
          oidc_client_secret: ${{ secrets.OIDC_CLIENT_SECRET }}
          oidc_users_0_password: ${{ secrets.OIDC_USERS_0_PASSWORD }}
          working_directory: ${{ env.WORKDIR }}
      - run: |
          cd tests/integration/cleanup && ./smb_cleanup.sh \
            "$(cat ../integration_config.yml | yq '.smb_server' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_share' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_username' | sed -e 's/^"//' -e 's/"$//')" \
            "$(cat ../integration_config.yml | yq '.smb_password' | sed -e 's/^"//' -e 's/"$//')"