Add QIT Custom E2E support and migrate tests

.github/workflows/e2e-tests.yml

@@ -42,6 +42,9 @@ jobs:
         shell: bash
         run: composer install --no-progress
 
+      - name: Add partner for QIT
+        run: ./vendor/bin/qit partner:add --user='${{ secrets.PARTNER_USER }}' --application_password='${{ secrets.PARTNER_SECRET }}'
+
       # Node
 
       - name: Setup Node
@@ -71,27 +74,30 @@ jobs:
           npm run build
 
       # E2E test environment
-
-      - name: Prepare test environment
-        shell: bash
-        env:
-          GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }}
-          STRIPE_PUB_KEY: ${{ secrets.E2E_STRIPE_PUBLISHABLE_KEY }}
-          STRIPE_SECRET_KEY: ${{ secrets.E2E_STRIPE_SECRET_KEY }}
-        run: npm run test:e2e-setup
+      - name: Fill in .env
+        run: |
+          echo 'STRIPE_PUB_KEY="${{ secrets.E2E_STRIPE_PUBLISHABLE_KEY }}"' >> .env
+          echo 'STRIPE_SECRET_KEY="${{ secrets.E2E_STRIPE_SECRET_KEY }}"' >> .env
+          echo 'PAYPAL_MERCHANT_ID="${{ secrets.PAYPAL_MERCHANT_ID }}"' >> .env
+          echo 'PAYPAL_MERCHANT_EMAIL="${{ secrets.PAYPAL_MERCHANT_EMAIL }}"' >> .env
+          echo 'PAYPAL_CLIENT_ID="${{ secrets.PAYPAL_CLIENT_ID }}"' >> .env
+          echo 'PAYPAL_CLIENT_SECRET="${{ secrets.PAYPAL_CLIENT_SECRET }}"' >> .env
+          echo 'PAYPAL_CUSTOMER_EMAIL="${{ secrets.PAYPAL_CUSTOMER_EMAIL }}"' >> .env
+          echo 'PAYPAL_CUSTOMER_PASSWORD="${{ secrets.PAYPAL_CUSTOMER_PASSWORD }}"' >> .env
 
       - name: Run ${{ matrix.checkout }} E2E tests
         shell: bash
-        env:
-          STRIPE_PUB_KEY: ${{ secrets.E2E_STRIPE_PUBLISHABLE_KEY }}
-          STRIPE_SECRET_KEY: ${{ secrets.E2E_STRIPE_SECRET_KEY }}
-        run: npm run test:e2e${{ matrix.checkout == 'Legacy' && '-legacy' || '' }}
+        run: npm run test:e2e${{ matrix.checkout == 'Legacy' && '-legacy' || '' }} -- --env_file .env --plugin woocommerce-paypal-payments:bootstrap
 
+      - name: Set the path in an env var
+        if: ${{ failure() }}
+        run: echo "E2E_REPORT_PATH=$(./vendor/bin/qit e2e-report --dir_only --local)/playwright" >> $GITHUB_ENV
+
       - name: Upload ${{ matrix.checkout }} E2E test results
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.checkout }}-WP_latest-WC_latest-results
-          path: tests/e2e/test-results
+          name: ${{ matrix.checkout }}-test-results
+          path: ${{ env.E2E_REPORT_PATH }}
           if-no-files-found: ignore
           retention-days: 14

.github/workflows/qit-e2e-tests-upload.yml

@@ -0,0 +1,30 @@
+name: Upload QIT E2E tests
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    name: New release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set PHP version
+      uses: shivammathur/setup-php@v2
+      with:
+        php-version: 7.4
+        tools: composer:v2
+        coverage: none
+
+    - name: Install QIT via composer
+      run: composer require woocommerce/qit-cli
+
+    - name: Add partner
+      run: ./vendor/bin/qit partner:add --user='${{ secrets.PARTNER_USER }}' --application_password='${{ secrets.PARTNER_SECRET }}'
+
+    - name: Upload E2E tests tags to QIT
+      run: npm run test:upload-tags

composer.json

@@ -17,7 +17,8 @@
 				"phpunit/phpunit": "7.5.20",
 				"yoast/phpunit-polyfills": "^2.0",
 				"woocommerce/woocommerce-sniffs": "0.1.0",
-				"wp-cli/wp-cli-bundle": "2.5.0"
+				"wp-cli/wp-cli-bundle": "2.5.0",
+        "woocommerce/qit-cli": "^0.7.1"
     },
     "extra": {
       "installer-disable": true

package.json

@@ -151,14 +151,12 @@
     "jt:setup": "npm run up && ./bin/jurassic-tube-setup.sh",
     "jt:start": "./docker/bin/jt/tunnel.sh",
     "jt:stop": "./docker/bin/jt/tunnel.sh break",
-    "test:e2e-setup": "./tests/e2e/bin/setup.sh",
-    "test:e2e-up": "./tests/e2e/bin/up.sh",
-    "test:e2e-down": "./tests/e2e/bin/down.sh",
-    "test:e2e-cleanup": "npm run test:e2e-down && ./tests/e2e/bin/cleanup.sh",
-    "test:e2e": "./tests/e2e/bin/run-tests.sh --project=default",
-    "test:e2e-debug": "npm run test:e2e -- --debug",
-    "test:e2e-legacy": "./tests/e2e/bin/run-tests.sh --project=legacy",
-    "test:e2e-legacy-debug": "npm run test:e2e-legacy -- --debug"
+		"test:e2e": "./vendor/bin/qit run:e2e woocommerce-gateway-stripe ./tests/e2e --source ./ --pw_options=\"--grep-invert @legacy\"",
+		"test:e2e-legacy": "./vendor/bin/qit run:e2e woocommerce-gateway-stripe ./tests/e2e --source ./ --pw_options=\"--grep @legacy\"",
+		"test:e2e-local": "./vendor/bin/qit run:e2e woocommerce-gateway-stripe ./tests/e2e --source ./ --pw_options=\"--grep-invert @legacy\" --env_file ./tests/e2e/config/local.env",
+		"test:e2e-legacy-local": "./vendor/bin/qit run:e2e woocommerce-gateway-stripe ./tests/e2e --source ./ --pw_options=\"--grep @legacy\" --env_file ./tests/e2e/config/local.env",
+		"test:e2e-all-local": "./vendor/bin/qit run:e2e woocommerce-gateway-stripe ./tests/e2e --source ./ --env_file ./tests/e2e/config/local.env",
+		"test:upload-tags": "sh ./tests/e2e/bin/upload-tags.sh"
   },
   "engines": {
     "node": ">=16.17.0",

qit-env.json

@@ -0,0 +1,14 @@
+{
+	"php_version": "7.4",
+	"plugins": {
+		"woocommerce": {
+			"action": "bootstrap"
+		},
+		"woocommerce-subscriptions": {
+			"action": "bootstrap"
+		}
+	},
+	"themes": [
+		"storefront"
+	]
+}

tests/e2e/README.md

@@ -1,6 +1,6 @@
 # WooCommerce Gateway Stripe End to End Tests
 
-We use [Playwright](https://playwright.dev/) as our test runner. 
+We use [QIT Custom E2E tests](https://qit.woo.com/docs/custom-tests/introduction) to run our end to end tests. 
 
 ## Table of contents
 
@@ -10,7 +10,7 @@ We use [Playwright](https://playwright.dev/) as our test runner.
     - [Pre-requisites](#pre-requisites)
     - [Environment Setup](#environment-setup)
     - [Running tests](#running-tests)
-    - [Debugging tests](#debugging-tests)
+    - [Compatibility Tests](#compatibility-tests)
     - [Running only selected test suites](#running-only-selected-test-suites)
   - [Guide for writing e2e tests](#guide-for-writing-e2e-tests)
     - [Creating the test structure](#creating-the-test-structure)
@@ -22,13 +22,8 @@ We use [Playwright](https://playwright.dev/) as our test runner.
 
 - Node.js ([Installation instructions](https://nodejs.org/en/download/))
 - NVM ([Installation instructions](https://github.com/nvm-sh/nvm))
-- A test site to run the tests on. **Jurassic Ninja sites are recommended**.
-- Admin credentials (`wp-admin`) to the test site.
-
-**For the automated setup (optional)**
-
-- SSH access to the test site.
-- WP CLI available on the test site server.
+- QIT CLI
+- Docker
 - Test keys for a Stripe account.
 
 ### Environment Setup
@@ -37,68 +32,37 @@ We use [Playwright](https://playwright.dev/) as our test runner.
 - Edit the variables on the `local.env` file.
 
 ### Running tests
+To set up the test environment and run test, run following commands:
 
-**Test Setup**
-
-To set up the test environment, run the command:
-
-`npm run test:e2e-setup -- --base_url=SOME_URL_HERE`
-
-This command will perform the following actions:
-
-- Connect to the test server using SSH and the credentials in the `/tests/e2e/config/local.env` file.
-- Install the latest version of WooCommerce from the official WordPress repository.
-- Install the latest version of the WooCommerce Gateway Stripe plugin from the official WordPress repository. 
-  **Note:** you can specify a different version to test by using the `--version` flag. In this case, the plugin will be downloaded from GitHub instead.
-- Install and activate the StoreFront theme.
-- Configure WooCommerce on the test site (e.g. store address, currency, shipping methods).
-- Import test products into WooCommerce.
-- Create pages for the Cart blocks and Checkout blocks from WooCommerce Blocks.
-- Set up the Stripe gateway using the keys from the `/tests/e2e/config/local.env` file and create a webhook endpoint on Stripe.
-
-**Note:** To run this command, SSH and admin credentials are required. 
-
-The SSH and admin credentials are mandatory (view the parameters `--with_woo_setup` and `--with_stripe_setup` below for more info).
-
-**Test execution**
-
-`npm run test:e2e -- --base_url=SOME_URL_HERE`
-
-The default command to run the tests. It'll run the tests in the URL indicated by the `--base_url` parameter.
-
-**Optional Parameters**
-
-`--version`: Allows you to specify a specific plugin version to test. This will download the specified version from GitHub and upload it to the test site before running the tests. If no version is specified, the tests will use the version currently installed on the test site.
-
-`--with_woo_setup`: Use this option when setting up a test site for the first time. It will use the SSH credentials from `tests/e2e/config/local.env` to set up the WooCommerce plugin with test products, store address, currency, and shipping methods, as well as installing and activating the StoreFront theme.
-
-`--with_stripe_setup`: Use this option when setting up a test site for the first time. It will use the Stripe keys from `tests/e2e/config/local.env` to set up the plugin, create a webhook endpoint on Stripe, and set up the webhook secret in the Stripe plugin.
-
-**⚠️ All the other parameters are passed to the Playwright CLI**
-
-[Playwright CLI Docs](https://playwright.dev/docs/test-cli)
+To run tests locally:
+- **Default tests:** `npm run test:e2e-local`
+- **Legacy tests:** `npm run test:e2e-legacy-local`
+- **All tests:** `npm run test:e2e-all-local`
 
-### Debugging tests
+To run tests in CI:
+- **Default tests:** `npm run test:e2e -- --env_file ./PATH_TO_ENV_FILE`
+- **Legacy tests:** `npm run test:e2e-legacy -- --env_file ./PATH_TO_ENV_FILE`
 
-`npm run test:e2e-debug`
+### Compatibility tests
+QIT Custom E2E tests provide a way to run tests for other plugins. To test plugin compatibility with other Woo extensions you have access to, you can use the `--plugin` flag. You can find more information on this [here](https://qit.woo.com/docs/custom-tests/running-other-plugins-tests):
 
-[Documentation](https://playwright.dev/docs/debug)
+`npm run test:e2e -- --plugin woocommerce-paypal-payments:test`
 
 ### Running only selected test suites
 
 **Running Tests by Annotation**
 
 Certain tests are annotated to indicate their specific focus, such as subscriptions, blocks, or smoke tests. These annotations are indicated in the test name with the `@` symbol in front of them, for example `Test XYZ @subscriptions`.
 
-To only run tests with a specific annotation, use the `--grep @annotation` parameter when running the tests. For example:
+To only run tests with a specific annotation, use the `--grep @annotation` in the `pw_options` parameter when running the tests. For example:
 
-`npm run test:e2e -- --base_url=SOME_URL_HERE --grep @subscriptions`
+`npm run test:e2e-all-local -- --pw_options="--grep @subscriptions"`
 
 **Running Tests by File Name**
 
 You can also run tests by specifying the file name containing the test you want to run. Keep in mind that there may be duplicate file names, especially between tests run in the regular checkout and in the blocks checkout.
 
- `npm run test:e2e -- --base_url=SOME_URL_HERE normal-card`
+`npm run test:e2e-all-local -- --pw_options="normal-card"`
 
  In the above example, the command would run the tests with a file name containing `normal-card`.