testing: add more dryruns to github ci

.github/workflows/node.js.yml

@@ -2,22 +2,41 @@ name: CI
 
 on: [push]
 
+env:
+  UNFURL_BRANCH: standalone-gui-naive-reload
+
+  GCP_DNS_ZONE: deploythis.io
+  AWS_DNS_ZONE: opencloudservices.net
+
+  MAIL_USERNAME: [email protected]
+  MAIL_PASSWORD: ocPC2cnGkQsz35I
+  SMTP_HOST: mailu.untrusted.me
+
+  AWS_ACCESS_KEY_ID: foobarbaz
+  AWS_SECRET_ACCESS_KEY: asldfjkasldkjalqweioptuqpeortiu
+  AWS_DEFAULT_REGION: eu-central-1
+
+  ARM_CLIENT_ID: 2b03ea39-628d-4443-96ec-535bc8b374eb
+  ARM_TENANT_ID: 1211e508-10ae-48e9-9591-74b82d108762
+  ARM_SUBSCRIPTION_ID: 2a4d88e0-165e-422b-bb41-aa4f1461e581
+  ARM_CLIENT_SECRET: asldfjkasldkjalqweioptuqpeortiu
+  AZURE_RESOURCE_GROUP: example-resources
+  AZURE_SUBNET: default
+  AZURE_VIRTUAL_NETWORK: example-network
+
+  GOOGLE_APPLICATION_CREDENTIALS: testing-shared/fixtures/service-account.json
+
+  DIGITALOCEAN_TOKEN:  asldfjkasldkjalqweioptuqpeortiu
+
+  FAIL_FAST_ENABLED: true
+  CY_COMMAND_TIMEOUT: 30000
+  TEST_VERSIONS: v2
+  UNFURL_TEST_TMPDIR: tmp
+
 jobs:
   build:
     name: Test & Build
     runs-on: ubuntu-latest
-    env:
-      UNFURL_BRANCH: standalone-gui
-      AWS_DNS_ZONE: opencloudservices.net 
-      AWS_ACCESS_KEY_ID: foobarbaz
-      AWS_SECRET_ACCESS_KEY: asldfjkasldkjalqweioptuqpeortiu
-      AWS_DEFAULT_REGION: eu-central-1
-      MAIL_USERNAME: [email protected]
-      MAIL_PASSWORD: ocPC2cnGkQsz35I
-      SMTP_HOST: mailu.untrusted.me
-      CY_COMMAND_TIMEOUT: 30000
-      TEST_VERSIONS: v2
-      UNFURL_TEST_TMPDIR: tmp
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -55,8 +74,7 @@ jobs:
       - name: Create release artifacts
         if: startsWith(github.ref, 'refs/tags/')
         run: tar -czvf unfurl-gui-dist.tar.gz dist
-      - name: Upload release artifact
-        if: startsWith(github.ref, 'refs/tags/')
+      - name: Upload dist
         uses: actions/upload-artifact@v2
         with:
           name: unfurl-gui-dist
@@ -77,3 +95,83 @@ jobs:
       uses: softprops/action-gh-release@v2
       with:
         files: unfurl-gui-dist.tar.gz
+
+# TODO some yaml merging or smth (GitHub doesn't support yaml anchors)
+# This could leverage .gitlab/ci-tests.yaml and have automated documentation
+  dryrun_a:
+    name: Dryrun A
+    needs: build
+    env:
+      SPEC_GLOBS: aws* az*
+      SPEC_SKIP_GLOBS: '*container-webapp* *nestedcloud* *draft aws__baserow*' # we already ran baserow
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install yarn dependencies
+        run: npm i -g yarn && YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install
+      - name: Compile # TODO just extract the artifact from previous step
+        run: yarn build
+      - name: Install unfurl
+        run: git clone --recurse-submodules --branch $UNFURL_BRANCH --single-branch --depth 1 https://github.com/onecommons/unfurl $HOME/unfurl; pip install -e $HOME/unfurl
+      - name: Integration test
+        run: yarn integration-test run --namespace onecommons/blueprints -- -e GENERATE_SUBDOMAINS=true --browser chrome
+      - name: Upload Cypress screenshots
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunA-cypress-screenshots
+          path: cypress/screenshots
+      - name: Upload dashboard
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunA-dashboard
+          path: ${{ env.UNFURL_TEST_TMPDIR }}/ufsv
+      - name: Upload unfurl server log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunA-unfurl-server-log
+          path: ${{ env.UNFURL_TEST_TMPDIR }}/unfurl.log
+
+  dryrun_b:
+    name: Dryrun B
+    needs: build
+    env:
+      SPEC_GLOBS: gcp* do*
+      SPEC_SKIP_GLOBS: '*container-webapp* *nestedcloud* *draft aws__baserow*' # we already ran baserow
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install yarn dependencies
+        run: npm i -g yarn && YARN_ENABLE_IMMUTABLE_INSTALLS=false yarn install
+      - name: Compile # TODO just extract the artifact from previous step
+        run: yarn build
+      - name: Install unfurl
+        run: git clone --recurse-submodules --branch $UNFURL_BRANCH --single-branch --depth 1 https://github.com/onecommons/unfurl $HOME/unfurl; pip install -e $HOME/unfurl
+      - name: Integration test
+        run: yarn integration-test run --namespace onecommons/blueprints -- -e GENERATE_SUBDOMAINS=true --browser chrome
+      - name: Upload Cypress screenshots
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunB-cypress-screenshots
+          path: cypress/screenshots
+      - name: Upload dashboard
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunB-dashboard
+          path: ${{ env.UNFURL_TEST_TMPDIR }}/ufsv
+      - name: Upload unfurl server log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: dryrunB-unfurl-server-log
+          path: ${{ env.UNFURL_TEST_TMPDIR }}/unfurl.log

cypress/plugins/index.js

@@ -31,6 +31,8 @@ module.exports = (on, config) => {
   //
   config.env.ELECTRON_EXTRA_LAUNCH_ARGS = '--js-flags="--expose-gc"'
 
+  require('cypress-fail-fast/plugin')(on, config)
+
   const OC_URL = config.env.OC_URL = config.env.OC_URL || 'http://localhost:8080'
   config.env.OC_GRAPHQL_ENDPOINT = (function(baseUrl){
     const url = new URL(baseUrl)
@@ -84,7 +86,6 @@ module.exports = (on, config) => {
   if (typeof TEARDOWN == 'string') TEARDOWN = TEARDOWN.toLowerCase()
   if (typeof GENERATE_SUBDOMAINS == 'string') GENERATE_SUBDOMAINS = GENERATE_SUBDOMAINS.toLowerCase()
 
-  console.log({TEARDOWN, GENERATE_SUBDOMAINS, INTEGRATION_TEST_ARGS})
   // default true
   config.env.TEARDOWN = !['0', 'false', 'no', false].includes(TEARDOWN)
   // default false

cypress/support/create-aws-environment.js

@@ -69,6 +69,8 @@ Cypress.Commands.add('createAWSEnvironment', (options) => {
     options
   )
 
+  let environmentCreated
+
   cy.whenEnvironmentAbsent(environmentName, () => {
     cy.visit(`/${DASHBOARD_DEST}/-/environments`)
     createEnvironmentButton().click()
@@ -77,12 +79,13 @@ Cypress.Commands.add('createAWSEnvironment', (options) => {
     cy.awsAuthenticateEnvironment({region: AWS_DEFAULT_REGION})
     cy.contains(environmentName).should('exist')
     cy.contains('Amazon Web Services').should('exist')
+    environmentCreated = true
   })
 
   // create external resource
   if (shouldCreateExternalResource) {
     cy.whenInstancesAbsent(environmentName, () => {
-      cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
+      environmentCreated || cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
       if(shouldCreateDNS) {
         cy.uncheckedCreateDNS(AWS_DNS_TYPE, AWS_DNS_ZONE)
       }

cypress/support/create-az-environment.js

@@ -49,6 +49,8 @@ Cypress.Commands.add('createAzEnvironment', (options) => {
     options
   )
 
+  let environmentCreated
+
   cy.whenEnvironmentAbsent(environmentName, () => {
     cy.visit(`/${DASHBOARD_DEST}/-/environments`)
     createEnvironmentButton().click()
@@ -74,14 +76,16 @@ Cypress.Commands.add('createAzEnvironment', (options) => {
       cy.contains('button', 'Save Changes').click()
     })
 
+    environmentCreated = true
     cy.wait(5000)
   })
 
-  cy.contains('a', 'Resources').click()
 
   // create external resource
   if (shouldCreateExternalResource) {
     cy.whenInstancesAbsent(environmentName, () => {
+      environmentCreated || cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
+      cy.contains('a', 'Resources').click()
       if(shouldCreateDNS) {
         cy.uncheckedCreateDNS(AWS_DNS_TYPE, AWS_DNS_ZONE)
       }

cypress/support/create-digitalocean-environment.js

@@ -42,6 +42,8 @@ Cypress.Commands.add('createDigitalOceanEnvironment', (options) => {
     options
   )
 
+  let environmentCreated
+
   cy.whenEnvironmentAbsent(environmentName, () => {
     cy.visit(`/${DASHBOARD_DEST}/-/environments`)
     createEnvironmentButton().click()
@@ -64,14 +66,16 @@ Cypress.Commands.add('createDigitalOceanEnvironment', (options) => {
       cy.contains('button', 'Save Changes').click()
     })
 
+    environmentCreated = true
     cy.wait(5000)
   })
 
-  cy.contains('a', 'Resources').click()
 
   // create external resource
   if (shouldCreateExternalResource) {
     cy.whenInstancesAbsent(environmentName, () => {
+      environmentCreated || cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
+      cy.contains('a', 'Resources').click()
       if(shouldCreateDNS) {
         cy.uncheckedCreateDNS(AWS_DNS_TYPE, AWS_DNS_ZONE)
       }

cypress/support/create-gcp-environment.js

@@ -7,6 +7,8 @@ const USERNAME = Cypress.env('OC_IMPERSONATE')
 const DASHBOARD_DEST = Cypress.env('DASHBOARD_DEST')
 
 function createGCPEnvironment({environmentName, shouldCreateExternalResource, shouldCreateDNS}) {
+  let environmentCreated
+
   cy.whenEnvironmentAbsent(environmentName, () => {
     cy.visit(`/${DASHBOARD_DEST}/-/environments`)
     cy.clickCreateEnvironmentButton()
@@ -15,11 +17,13 @@ function createGCPEnvironment({environmentName, shouldCreateExternalResource, sh
     authenticateGCP()
 
     validateGCPEnvironment()
+    environmentCreated = true
   })
 
   // create external resource
   if (shouldCreateExternalResource) {
     cy.whenInstancesAbsent(environmentName, () => {
+      environmentCreated || cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
       if(shouldCreateDNS) {
         cy.uncheckedCreateDNS(GCP_DNS_TYPE, GCP_DNS_ZONE)
       }

cypress/support/create-kubernetes-environment.js

@@ -94,6 +94,8 @@ Cypress.Commands.add('createK8SEnvironment', (options) => {
     options
   )
 
+  let environmentCreated
+
   cy.whenEnvironmentAbsent(environmentName, () => {
     cy.visit(`/${DASHBOARD_DEST}/-/environments`)
     createEnvironmentButton().click()
@@ -127,14 +129,16 @@ Cypress.Commands.add('createK8SEnvironment', (options) => {
       cy.get('#ci-variable-type').select('File')
       cy.get('[data-qa-selector="ci_variable_save_button"]').click()
     }
-    cy.contains('a', 'Resources').click()
-
     addK8sAnnotations()
+
+    environmentCreated = true
   })
 
   // create external resource
   if (shouldCreateExternalResource) {
     cy.whenInstancesAbsent(environmentName, () => {
+      environmentCreated || cy.visit(`/${DASHBOARD_DEST}/-/environments/${environmentName}`)
+      cy.contains('a', 'Resources').click()
       if(shouldCreateDNS) {
         cy.uncheckedCreateDNS(AWS_DNS_TYPE, AWS_DNS_ZONE)
       }

cypress/support/e2e.js

@@ -14,6 +14,7 @@
 // ***********************************************************
 
 // Import commands.js using ES2015 syntax:
+import 'cypress-fail-fast'
 import './commands'
 
 const USERNAME = Cypress.env('OC_USERNAME')

cypress/support/run-recreate-deployment.js

@@ -345,10 +345,10 @@ Cypress.Commands.add('recreateDeployment', options => {
           const deploymentDir = store.getters.lookupDeployPath(deployment.name, deployment._environment)?.name
           expect(deploymentDir).to.exist
 
-          cy.execLoud(
-            `./testing-shared/run-unfurl.sh deploy --dryrun --use-environment ${deployment._environment} --approve --jobexitcode error ${deploymentDir}`,
-            { timeout: BASE_TIMEOUT * 10 }
-          )
+          cy.execLoud(`./testing-shared/run-unfurl.sh deploy --dryrun --use-environment ${deployment._environment} --approve --jobexitcode error ${deploymentDir}`, {
+            timeout: BASE_TIMEOUT * 100,  // unfurl isn't known to hang, so we can wait a long time
+            failOnNonZeroExit: false // we'll fail later when we check the status - this is better for the log
+          })
         })
 
         cy.reload()

package.json

@@ -84,6 +84,7 @@
     "babel-jest": "^29.6.0",
     "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2",
     "cypress": "11.2.0",
+    "cypress-fail-fast": "^7.1.0",
     "cypress-file-upload": "^5.0.8",
     "cypress-wait-until": "^1.7.2",
     "eslint": "^7.2.0",

scripts/src/forwarded-variables.json

@@ -67,5 +67,6 @@
   "KUBECONFIG",
   "SKIP_NESTEDCLOUD",
   "DASHBOARD_DEST",
-  "UNFURL_CMD"
+  "UNFURL_CMD",
+  "FAIL_FAST_ENABLED"
 ]

scripts/src/integration-test.js

@@ -21,6 +21,14 @@ if(STANDALONE_UNFURL && ! PORT) {
   PORT = new URL(OC_URL).port
 }
 
+process.env.FAIL_FAST_ENABLED = process.env.FAIL_FAST_ENABLED || 'false'
+
+if(STANDALONE_UNFURL) {
+  // allow some specs to self-filter
+  process.env.NO_FLAKY = '1'
+  process.env.DRYRUN = '1'
+}
+
 const GENERATED_PASSWORD = !STANDALONE_UNFURL && btoa(Number.MAX_SAFE_INTEGER * Math.random())
 const FIXTURES_TMP = path.join(unfurlGuiRoot, 'cypress/fixtures/tmp')
 
@@ -172,7 +180,14 @@ function createDashboardCommand(username, dashboardRepo) {
   }
 }
 
-function invokeCypressCommand(baseArgs, forwardedEnv) {
+async function invokeCypressCommand(baseArgs, forwardedEnv) {
+  if(!baseArgs.includes('-s') && baseArgs[0] == 'run') {
+    process.env.SPEC_GLOBS = process.env.SPEC_GLOBS || '*'
+    const {getBlueprintSpecs} = (await import('../../testing-shared/fixture-specs.mjs'))
+    baseArgs.push('-s')
+    baseArgs.push(getBlueprintSpecs())
+  }
+
   const args = ['run', 'cypress', ...baseArgs]
   const options = {stdio: 'inherit', env: {...process.env, ...forwardedEnv}}
   return spawnSync.bind(null, 'yarn', args, options)
@@ -224,7 +239,7 @@ async function main() {
     delete forwardedEnv['CYPRESS_OC_IMPERSONATE']
   }
 
-  const cypressCommand = invokeCypressCommand(args._, forwardedEnv)
+  const cypressCommand = await invokeCypressCommand(args._, forwardedEnv)
 
   if(prepareUserCommand) prepareUserCommand()
 
@@ -271,14 +286,17 @@ async function main() {
       outfile: `${UNFURL_TEST_TMPDIR}/unfurl.log`
     })
 
+    unfurlServer.process.on('exit', () => {
+      console.error('Unfurl server exited early')
+      process.exit(1)
+    })
+
     await unfurlServer.waitUntilReady()
   }
   const cypressResult = cypressCommand()
   console.log(cypressResult)
   const status = cypressResult.status
-  if(status) {
-    process.exit(status)
-  }
+  process.exit(status)
 }
 
 async function tryMain() {