From c887de90a36948653e180ad7084487e4ad1708aa Mon Sep 17 00:00:00 2001 From: Momodu Afegbua Date: Wed, 28 Aug 2024 02:28:18 -0500 Subject: [PATCH] [WORKFLOW] Subgraph deploy matrix (#2003) --- .github/workflows/call.deploy-subgraph.yml | 89 ++++++------- .../handler.deploy-production-subgraphs.yml | 12 +- .../previous-versions-testing/package.json | 2 +- .../previous-versions-testing/queryTests.ts | 5 + packages/subgraph/tasks/deploy.sh | 124 +++++++++--------- packages/subgraph/tasks/vendorNetworkMap.json | 6 + 6 files changed, 121 insertions(+), 117 deletions(-) create mode 100644 packages/subgraph/tasks/vendorNetworkMap.json diff --git a/.github/workflows/call.deploy-subgraph.yml b/.github/workflows/call.deploy-subgraph.yml index a4503a6725..36656db577 100644 --- a/.github/workflows/call.deploy-subgraph.yml +++ b/.github/workflows/call.deploy-subgraph.yml @@ -5,7 +5,7 @@ on: inputs: vendor: required: true - description: "Where to deploy subgraph to; superfluid, goldsky, graph or airstack" + description: "Where to deploy subgraph to; superfluid, goldsky, or graph" type: string deployment_env: required: true @@ -24,7 +24,7 @@ on: description: "Satsuma deploy key" required: false THE_GRAPH_ACCESS_TOKEN: - description: "The Graph (hosted service) access token for deploying subgraphs" + description: "The Graph access token for deploying subgraphs" required: false GOLDSKY_API_KEY: description: "Goldsky API key for deploying subgraphs" @@ -55,12 +55,38 @@ jobs: echo github.head_ref: "$HEAD_REF" echo github.base_ref: ${{ github.base_ref }} + determine-networks: + name: Determine Networks + runs-on: ubuntu-latest + outputs: + networks: ${{ steps.determine.outputs.networks }} + steps: + - uses: actions/checkout@v4 + + - name: "Determine vendors for the network" + id: determine + run: | + networks=$(jq -r --arg vendor "${{ inputs.vendor }}" --arg network "${{ inputs.network }}" ' + if $network == "all" then + .[$vendor] | @json + else + [$network] | @json + end' packages/subgraph/tasks/vendorNetworkMap.json) + if [ -z "$networks" ] || [ "$networks" == "null" ]; then + echo "Unsupported vendor or network: ${{ inputs.vendor }} / ${{ inputs.network }}" + exit 1 + fi + echo "networks=$networks" >> "$GITHUB_OUTPUT" + deploy-subgraph: name: Build and Deploy Subgraph runs-on: ubuntu-latest - env: - subgraph-working-directory: ./packages/subgraph + needs: determine-networks + strategy: + matrix: + network: ${{ fromJson(needs.determine-networks.outputs.networks) }} + fail-fast: false defaults: run: @@ -69,7 +95,9 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@v13 + - uses: cachix/install-nix-action@v19 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} - name: "Install dependencies" run: yarn install --frozen-lockfile @@ -78,53 +106,14 @@ jobs: run: yarn build working-directory: ./packages/ethereum-contracts - - name: "Get ABI" - run: node scripts/getAbi.js - working-directory: ${{ env.subgraph-working-directory }} - - - name: "Prepare subgraph manifest" - # This step is required for generating a preliminary subgraph.yaml - # and addresses.ts file for the yarn codegen step below - # using a mock.json networks file. - run: ./tasks/prepare-manifest.sh mock - working-directory: ${{ env.subgraph-working-directory }} - - - name: "Generate meta.ignore.ts file" - # Collects metadata like the git branch and commit hash - run: "yarn generate-sf-meta" - working-directory: ${{ env.subgraph-working-directory }} - env: - COMMIT_HASH: ${{ github.sha }} - CONFIGURATION: ${{ inputs.deployment_env }} - - - name: "Generate AssemblyScript types" - run: "yarn codegen" - working-directory: ${{ env.subgraph-working-directory }} - - - name: Get Hosted Service Networks from metadata - run: npx ts-node ./scripts/getHostedServiceNetworks.ts - working-directory: ${{ env.subgraph-working-directory }} - - # - name: Ensure deployment_env property is passed - # run: | - # if [ "${{ inputs.deployment_env }}" == "" ];then - # echo "You must pass in the deployment_env at a minimum." - # exit 1 - # fi - - # If tagged, get subgraph tag name that starts with subgraph@. Outputs $GIT_TAG_NAME - - uses: olegtarasov/get-tag@v2.1.2 - id: "getSubgraphTag" - with: - tagRegex: "subgraph@(.*)" - - # Version is the git tag or revision + # Version is from subgraph@ git tag if exists, else short git commit revision - name: "Deploy to a node" run: | shortRev=$(git rev-parse --short ${{ github.sha }}) - versionLabel="${GIT_TAG_NAME:-$shortRev}" - ./tasks/deploy.sh -o ${{ inputs.vendor }} -n ${{ inputs.network }} -r ${{ inputs.deployment_env }} -v $versionLabel - working-directory: ${{ env.subgraph-working-directory }} + versionTag=$(git tag --contains ${{ github.sha }} | grep "^subgraph@" | sed 's/^subgraph@//') + versionLabel="${versionTag:-$shortRev}" + ./tasks/deploy.sh -o ${{ inputs.vendor }} -n ${{ matrix.network }} -r ${{ inputs.deployment_env }} -v $versionLabel + working-directory: ./packages/subgraph env: SUBGRAPH_URL_TEMPLATE: ${{ secrets.SUBGRAPH_URL_TEMPLATE }} SUPERFLUID_IPFS_API: ${{ secrets.SUPERFLUID_IPFS_API }} @@ -132,3 +121,5 @@ jobs: SATSUMA_DEPLOY_KEY: ${{ secrets.SATSUMA_DEPLOY_KEY }} GOLDSKY_API_KEY: ${{ secrets.GOLDSKY_API_KEY }} AIRSTACK_API_KEY: ${{ secrets.AIRSTACK_API_KEY }} + COMMIT_HASH: ${{ github.sha }} + CONFIGURATION: ${{ inputs.deployment_env }} diff --git a/.github/workflows/handler.deploy-production-subgraphs.yml b/.github/workflows/handler.deploy-production-subgraphs.yml index 19d9aaa702..0d9561e995 100644 --- a/.github/workflows/handler.deploy-production-subgraphs.yml +++ b/.github/workflows/handler.deploy-production-subgraphs.yml @@ -13,13 +13,12 @@ on: vendor: required: true description: "Where to deploy subgraph to; one of `superfluid`, `goldsky`, `graph` or `airstack`" - default: "superfluid" + default: "superfluid" type: choice options: - superfluid - goldsky - graph - - airstack deployment_env: required: true description: 'The subgraph deployment env: v1, dev, feature.' @@ -46,12 +45,11 @@ jobs: subgraph-release: '' subgraph-endpoint: http://localhost:8000/subgraphs/name/superfluid-test - deploy-subgraph: + deploy-subgraphs: + if: ${{ success() }} uses: ./.github/workflows/call.deploy-subgraph.yml - needs: [ - build-and-test-local-subgraph - , build-and-test-local-subgraph-against-previous-sdk-core-releases - ] + # build-and-test-local-subgraph-against-previous-sdk-core-releases is currently not "needed" here because it fails. + needs: [build-and-test-local-subgraph] name: Deploy graph to ${{ github.event.inputs.vendor }} vendor with: deployment_env: ${{ github.event.inputs.deployment_env }} diff --git a/packages/sdk-core/previous-versions-testing/package.json b/packages/sdk-core/previous-versions-testing/package.json index 9cf28c6ae9..7bb012d58e 100644 --- a/packages/sdk-core/previous-versions-testing/package.json +++ b/packages/sdk-core/previous-versions-testing/package.json @@ -6,7 +6,7 @@ "run-query-tests": "npx hardhat test ./queryTests.ts" }, "devDependencies": { - "@superfluid-finance/sdk-core": "0.4.3" + "@superfluid-finance/sdk-core": "0.8.0" }, "resolutions": { "@superfluid-finance/metadata": ">1.2.0" diff --git a/packages/sdk-core/previous-versions-testing/queryTests.ts b/packages/sdk-core/previous-versions-testing/queryTests.ts index fe1d1e07a2..ce69cedae7 100644 --- a/packages/sdk-core/previous-versions-testing/queryTests.ts +++ b/packages/sdk-core/previous-versions-testing/queryTests.ts @@ -16,10 +16,15 @@ export const getChainId = () => { }; export const testQueryClassFunctions = async (query: Query) => { + console.log("query listAllSuperTokens..."); await query.listAllSuperTokens({}, { take: 10 }); + console.log("query listIndexes..."); await query.listIndexes({}, { take: 10 }); + console.log("query listIndexSubscriptions..."); await query.listIndexSubscriptions({}, { take: 10 }); + console.log("query listStreams..."); await query.listStreams({}, { take: 10 }); + console.log("query listUserInteractedSuperTokens..."); await query.listUserInteractedSuperTokens({}, { take: 10 }); }; diff --git a/packages/subgraph/tasks/deploy.sh b/packages/subgraph/tasks/deploy.sh index 638e6fee4b..6072bf1582 100755 --- a/packages/subgraph/tasks/deploy.sh +++ b/packages/subgraph/tasks/deploy.sh @@ -8,26 +8,6 @@ GRAPH_CLI="npx --package=@graphprotocol/graph-cli --yes -- graph" GOLDSKY_CLI="npx --package=@goldskycom/cli --yes -- goldsky" SUPPORTED_VENDORS=( "graph" "satsuma" "superfluid" "goldsky" "airstack" ) -# list of supported networks by vendor - -# shellcheck disable=SC2034,SC2207 -GRAPH_NETWORKS=( "polygon-mainnet" "eth-mainnet" ) -# shellcheck disable=SC2034 -SATSUMA_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "eth-mainnet" "eth-sepolia" "optimism-mainnet" "base-mainnet") -# shellcheck disable=SC2034 -SUPERFLUID_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "base-mainnet" "optimism-mainnet" "arbitrum-one" "celo-mainnet" "bsc-mainnet" "avalanche-c" "optimism-sepolia" "scroll-sepolia" "scroll-mainnet" "degenchain" "base-sepolia") -# shellcheck disable=SC2034 -GOLDSKY_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "eth-mainnet" "base-mainnet" "optimism-mainnet" "arbitrum-one" "bsc-mainnet" "avalanche-c" "optimism-sepolia" "scroll-sepolia" "scroll-mainnet" "eth-sepolia" "avalanche-fuji" "base-sepolia") -# shellcheck disable=SC2034 -AIRSTACK_NETWORKS=( "degenchain") - -declare -A VENDOR_NETWORKS=( - ["graph"]="${GRAPH_NETWORKS[@]}" - ["satsuma"]="${SATSUMA_NETWORKS[@]}" - ["superfluid"]="${SUPERFLUID_NETWORKS[@]}" - ["goldsky"]="${GOLDSKY_NETWORKS[@]}" - ["airstack"]="${AIRSTACK_NETWORKS[@]}" -) VENDOR="" NETWORK="" @@ -65,6 +45,32 @@ while [[ $# -gt 0 ]]; do esac done +prepare_deployment() { + # Read environment variables directly, with a fallback Git command for commit_hash + local commit_hash="${GITHUB_SHA:-$(git rev-parse HEAD)}" + local configuration="${CONFIGURATION:-v1}" + + # Get ABI + echo "Getting ABI..." + node "./scripts/getAbi.js" + + # Prepare subgraph manifest + echo "Preparing subgraph manifest..." + ./tasks/prepare-manifest.sh mock + + # Generate meta.ignore.ts file + echo "Generating meta.ignore.ts file..." + COMMIT_HASH="$commit_hash" CONFIGURATION="$configuration" yarn generate-sf-meta + + # Generate AssemblyScript types + echo "Generating AssemblyScript types..." + yarn codegen + + # Get Hosted Service Networks from metadata + echo "Getting Hosted Service Networks from metadata..." + npx ts-node "./scripts/getHostedServiceNetworks.ts" +} + deploy_to_graph() { local network="$1" @@ -79,10 +85,10 @@ deploy_to_graph() { echo "********* Deploying $network subgraph $subgraphName to The Graph (hosted service). **********" - $GRAPH_CLI deploy \ - --studio "$subgraphName" \ - --deploy-key "$THE_GRAPH_ACCESS_TOKEN" \ - --version-label "$VERSION_LABEL" + if ! $GRAPH_CLI deploy --studio "$subgraphName" --deploy-key "$THE_GRAPH_ACCESS_TOKEN" --version-label "$VERSION_LABEL"; then + echo "Error: Deployment to The Graph (hosted service) failed for $network" + exit 1 + fi } deploy_to_satsuma() { @@ -112,31 +118,37 @@ deploy_to_superfluid() { local subgraphName="protocol-$DEPLOYMENT_ENV" echo "node url: $nodeUrl, subgraph name: $subgraphName" - $GRAPH_CLI create "$subgraphName" --node "$nodeUrl" - $GRAPH_CLI deploy "$subgraphName" \ - --version-label "$VERSION_LABEL" \ - --node "$nodeUrl" \ - --ipfs "$SUPERFLUID_IPFS_API" + if ! $GRAPH_CLI create "$subgraphName" --node "$nodeUrl"; then + echo "Error: Creation of subgraph $subgraphName on Superfluid (self hosted) failed for $network" + exit 1 + fi + if ! $GRAPH_CLI deploy "$subgraphName" --version-label "$VERSION_LABEL" --node "$nodeUrl" --ipfs "$SUPERFLUID_IPFS_API"; then + echo "Error: Deployment to Superfluid (self hosted) failed for $network" + exit 1 + fi } deploy_to_goldsky() { local network="$1" # TODO: use tagging? - #Get subgraph version from package.json + # Get subgraph version from package.json PACKAGE_JSON_PATH="package.json" SUBGRAPH_VERSION=$(jq -r '.version' $PACKAGE_JSON_PATH) local subgraphName="protocol-$DEPLOYMENT_ENV-$network/$SUBGRAPH_VERSION" - # Note: when using Graph CLI to deploy, it implicitly triggers build too, but Goldsky CLI doesn't, so we do it explicitly. - $GRAPH_CLI build + # Note: when using Graph CLI to deploy, it implicitly triggers build too, but Goldsky CLI doesn't, so we do it explicitly. + if ! $GRAPH_CLI build; then + echo "Error: Build for Goldsky failed" + exit 1 + fi echo "********* Deploying $network subgraph $subgraphName to Goldsky. **********" - $GOLDSKY_CLI subgraph deploy \ - "$subgraphName" \ - --path . \ - --token "$GOLDSKY_API_KEY" + if ! $GOLDSKY_CLI subgraph deploy "$subgraphName" --path . --token "$GOLDSKY_API_KEY"; then + echo "Error: Deployment to Goldsky failed for $network" + exit 1 + fi } deploy_to_airstack() { @@ -145,14 +157,14 @@ deploy_to_airstack() { local subgraphName="protocol-$DEPLOYMENT_ENV-$network" echo "********* Deploying $network subgraph $subgraphName to Airstack. **********" - $GRAPH_CLI create "$subgraphName" --node "$nodeUrl" --access-token "$AIRSTACK_API_KEY" - $GRAPH_CLI deploy \ - --version-label "$VERSION_LABEL" \ - --node "$nodeUrl" \ - --deploy-key "$AIRSTACK_API_KEY" \ - --ipfs https://ipfs.airstack.xyz/ipfs/api/v0 \ - --headers '{"Authorization": "'"$AIRSTACK_API_KEY"'"}' \ - "$subgraphName" + if ! $GRAPH_CLI create "$subgraphName" --node "$nodeUrl" --access-token "$AIRSTACK_API_KEY"; then + echo "Error: Creation of subgraph $subgraphName on Airstack failed for $network" + exit 1 + fi + if ! $GRAPH_CLI deploy --version-label "$VERSION_LABEL" --node "$nodeUrl" --deploy-key "$AIRSTACK_API_KEY" --ipfs https://ipfs.airstack.xyz/ipfs/api/v0 --headers '{"Authorization": "'"$AIRSTACK_API_KEY"'"}' "$subgraphName"; then + echo "Error: Deployment to Airstack failed for $network" + exit 1 + fi } # Vendor specific function dispatcher @@ -163,15 +175,6 @@ deploy_to() { local vendor="$1" local network="$2" - # check if network is supported by vendor - local -n networksRef="${vendor^^}_NETWORKS" - # We can safely ignore this warning, becasue the value in network won't contain whitespaces - # shellcheck disable=SC2199,SC2076 - if [[ ! " ${networksRef[@]} " =~ " $network " ]]; then - echo "The network, $network, is currently not on the list of networks supported by $vendor." - exit 1 - fi - npx ts-node ./scripts/buildNetworkConfig.ts "$network" "$vendor" # prepare the manifest prior to deployment @@ -212,11 +215,12 @@ if [[ ! " ${SUPPORTED_VENDORS[@]} " =~ " $VENDOR " ]]; then print_usage_and_exit fi -# Handle all vs specific network -if [ "$NETWORK" == "all" ]; then - for network in ${VENDOR_NETWORKS[$VENDOR]}; do - deploy_to "$VENDOR" "$network" - done -else - deploy_to "$VENDOR" "$NETWORK" +#Prepare deployment +# Prepare deployment +if ! prepare_deployment; then + echo "Error: Failed to prepare deployment" + exit 1 fi + +# Deploy the specified network +deploy_to "$VENDOR" "$NETWORK" diff --git a/packages/subgraph/tasks/vendorNetworkMap.json b/packages/subgraph/tasks/vendorNetworkMap.json new file mode 100644 index 0000000000..41e953aa25 --- /dev/null +++ b/packages/subgraph/tasks/vendorNetworkMap.json @@ -0,0 +1,6 @@ +{ + "superfluid": ["polygon-mainnet", "xdai-mainnet", "base-mainnet", "optimism-mainnet", "arbitrum-one", "celo-mainnet", "bsc-mainnet", "avalanche-c", "optimism-sepolia", "scroll-sepolia", "scroll-mainnet", "degenchain", "base-sepolia"], + "goldsky": ["polygon-mainnet", "xdai-mainnet", "eth-mainnet", "base-mainnet", "optimism-mainnet", "arbitrum-one", "bsc-mainnet", "avalanche-c", "optimism-sepolia", "scroll-sepolia", "scroll-mainnet", "eth-sepolia", "avalanche-fuji", "base-sepolia", "degenchain"], + "graph": ["polygon-mainnet", "eth-mainnet", "celo-mainnet"] +} +