Skip to content

Commit

Permalink
[WORKFLOW] Subgraph deploy matrix (#2003)
Browse files Browse the repository at this point in the history
  • Loading branch information
mmd-afegbua authored Aug 28, 2024
1 parent 349f760 commit c887de9
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 117 deletions.
89 changes: 40 additions & 49 deletions .github/workflows/call.deploy-subgraph.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
inputs:
vendor:
required: true
description: "Where to deploy subgraph to; superfluid, goldsky, graph or airstack"
description: "Where to deploy subgraph to; superfluid, goldsky, or graph"
type: string
deployment_env:
required: true
Expand All @@ -24,7 +24,7 @@ on:
description: "Satsuma deploy key"
required: false
THE_GRAPH_ACCESS_TOKEN:
description: "The Graph (hosted service) access token for deploying subgraphs"
description: "The Graph access token for deploying subgraphs"
required: false
GOLDSKY_API_KEY:
description: "Goldsky API key for deploying subgraphs"
Expand Down Expand Up @@ -55,12 +55,38 @@ jobs:
echo github.head_ref: "$HEAD_REF"
echo github.base_ref: ${{ github.base_ref }}
determine-networks:
name: Determine Networks
runs-on: ubuntu-latest
outputs:
networks: ${{ steps.determine.outputs.networks }}
steps:
- uses: actions/checkout@v4

- name: "Determine vendors for the network"
id: determine
run: |
networks=$(jq -r --arg vendor "${{ inputs.vendor }}" --arg network "${{ inputs.network }}" '
if $network == "all" then
.[$vendor] | @json
else
[$network] | @json
end' packages/subgraph/tasks/vendorNetworkMap.json)
if [ -z "$networks" ] || [ "$networks" == "null" ]; then
echo "Unsupported vendor or network: ${{ inputs.vendor }} / ${{ inputs.network }}"
exit 1
fi
echo "networks=$networks" >> "$GITHUB_OUTPUT"
deploy-subgraph:
name: Build and Deploy Subgraph

runs-on: ubuntu-latest
env:
subgraph-working-directory: ./packages/subgraph
needs: determine-networks
strategy:
matrix:
network: ${{ fromJson(needs.determine-networks.outputs.networks) }}
fail-fast: false

defaults:
run:
Expand All @@ -69,7 +95,9 @@ jobs:
steps:
- uses: actions/checkout@v4

- uses: DeterminateSystems/nix-installer-action@v13
- uses: cachix/install-nix-action@v19
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}

- name: "Install dependencies"
run: yarn install --frozen-lockfile
Expand All @@ -78,57 +106,20 @@ jobs:
run: yarn build
working-directory: ./packages/ethereum-contracts

- name: "Get ABI"
run: node scripts/getAbi.js
working-directory: ${{ env.subgraph-working-directory }}

- name: "Prepare subgraph manifest"
# This step is required for generating a preliminary subgraph.yaml
# and addresses.ts file for the yarn codegen step below
# using a mock.json networks file.
run: ./tasks/prepare-manifest.sh mock
working-directory: ${{ env.subgraph-working-directory }}

- name: "Generate meta.ignore.ts file"
# Collects metadata like the git branch and commit hash
run: "yarn generate-sf-meta"
working-directory: ${{ env.subgraph-working-directory }}
env:
COMMIT_HASH: ${{ github.sha }}
CONFIGURATION: ${{ inputs.deployment_env }}

- name: "Generate AssemblyScript types"
run: "yarn codegen"
working-directory: ${{ env.subgraph-working-directory }}

- name: Get Hosted Service Networks from metadata
run: npx ts-node ./scripts/getHostedServiceNetworks.ts
working-directory: ${{ env.subgraph-working-directory }}

# - name: Ensure deployment_env property is passed
# run: |
# if [ "${{ inputs.deployment_env }}" == "" ];then
# echo "You must pass in the deployment_env at a minimum."
# exit 1
# fi

# If tagged, get subgraph tag name that starts with subgraph@. Outputs $GIT_TAG_NAME
- uses: olegtarasov/[email protected]
id: "getSubgraphTag"
with:
tagRegex: "subgraph@(.*)"

# Version is the git tag or revision
# Version is from subgraph@<version> git tag if exists, else short git commit revision
- name: "Deploy to a node"
run: |
shortRev=$(git rev-parse --short ${{ github.sha }})
versionLabel="${GIT_TAG_NAME:-$shortRev}"
./tasks/deploy.sh -o ${{ inputs.vendor }} -n ${{ inputs.network }} -r ${{ inputs.deployment_env }} -v $versionLabel
working-directory: ${{ env.subgraph-working-directory }}
versionTag=$(git tag --contains ${{ github.sha }} | grep "^subgraph@" | sed 's/^subgraph@//')
versionLabel="${versionTag:-$shortRev}"
./tasks/deploy.sh -o ${{ inputs.vendor }} -n ${{ matrix.network }} -r ${{ inputs.deployment_env }} -v $versionLabel
working-directory: ./packages/subgraph
env:
SUBGRAPH_URL_TEMPLATE: ${{ secrets.SUBGRAPH_URL_TEMPLATE }}
SUPERFLUID_IPFS_API: ${{ secrets.SUPERFLUID_IPFS_API }}
THE_GRAPH_ACCESS_TOKEN: ${{ secrets.THE_GRAPH_ACCESS_TOKEN }}
SATSUMA_DEPLOY_KEY: ${{ secrets.SATSUMA_DEPLOY_KEY }}
GOLDSKY_API_KEY: ${{ secrets.GOLDSKY_API_KEY }}
AIRSTACK_API_KEY: ${{ secrets.AIRSTACK_API_KEY }}
COMMIT_HASH: ${{ github.sha }}
CONFIGURATION: ${{ inputs.deployment_env }}
12 changes: 5 additions & 7 deletions .github/workflows/handler.deploy-production-subgraphs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,12 @@ on:
vendor:
required: true
description: "Where to deploy subgraph to; one of `superfluid`, `goldsky`, `graph` or `airstack`"
default: "superfluid"
default: "superfluid"
type: choice
options:
- superfluid
- goldsky
- graph
- airstack
deployment_env:
required: true
description: 'The subgraph deployment env: v1, dev, feature.'
Expand All @@ -46,12 +45,11 @@ jobs:
subgraph-release: ''
subgraph-endpoint: http://localhost:8000/subgraphs/name/superfluid-test

deploy-subgraph:
deploy-subgraphs:
if: ${{ success() }}
uses: ./.github/workflows/call.deploy-subgraph.yml
needs: [
build-and-test-local-subgraph
, build-and-test-local-subgraph-against-previous-sdk-core-releases
]
# build-and-test-local-subgraph-against-previous-sdk-core-releases is currently not "needed" here because it fails.
needs: [build-and-test-local-subgraph]
name: Deploy graph to ${{ github.event.inputs.vendor }} vendor
with:
deployment_env: ${{ github.event.inputs.deployment_env }}
Expand Down
2 changes: 1 addition & 1 deletion packages/sdk-core/previous-versions-testing/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"run-query-tests": "npx hardhat test ./queryTests.ts"
},
"devDependencies": {
"@superfluid-finance/sdk-core": "0.4.3"
"@superfluid-finance/sdk-core": "0.8.0"
},
"resolutions": {
"@superfluid-finance/metadata": ">1.2.0"
Expand Down
5 changes: 5 additions & 0 deletions packages/sdk-core/previous-versions-testing/queryTests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,15 @@ export const getChainId = () => {
};

export const testQueryClassFunctions = async (query: Query) => {
console.log("query listAllSuperTokens...");
await query.listAllSuperTokens({}, { take: 10 });
console.log("query listIndexes...");
await query.listIndexes({}, { take: 10 });
console.log("query listIndexSubscriptions...");
await query.listIndexSubscriptions({}, { take: 10 });
console.log("query listStreams...");
await query.listStreams({}, { take: 10 });
console.log("query listUserInteractedSuperTokens...");
await query.listUserInteractedSuperTokens({}, { take: 10 });
};

Expand Down
124 changes: 64 additions & 60 deletions packages/subgraph/tasks/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,6 @@ GRAPH_CLI="npx --package=@graphprotocol/graph-cli --yes -- graph"
GOLDSKY_CLI="npx --package=@goldskycom/cli --yes -- goldsky"
SUPPORTED_VENDORS=( "graph" "satsuma" "superfluid" "goldsky" "airstack" )

# list of supported networks by vendor

# shellcheck disable=SC2034,SC2207
GRAPH_NETWORKS=( "polygon-mainnet" "eth-mainnet" )
# shellcheck disable=SC2034
SATSUMA_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "eth-mainnet" "eth-sepolia" "optimism-mainnet" "base-mainnet")
# shellcheck disable=SC2034
SUPERFLUID_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "base-mainnet" "optimism-mainnet" "arbitrum-one" "celo-mainnet" "bsc-mainnet" "avalanche-c" "optimism-sepolia" "scroll-sepolia" "scroll-mainnet" "degenchain" "base-sepolia")
# shellcheck disable=SC2034
GOLDSKY_NETWORKS=( "polygon-mainnet" "xdai-mainnet" "eth-mainnet" "base-mainnet" "optimism-mainnet" "arbitrum-one" "bsc-mainnet" "avalanche-c" "optimism-sepolia" "scroll-sepolia" "scroll-mainnet" "eth-sepolia" "avalanche-fuji" "base-sepolia")
# shellcheck disable=SC2034
AIRSTACK_NETWORKS=( "degenchain")

declare -A VENDOR_NETWORKS=(
["graph"]="${GRAPH_NETWORKS[@]}"
["satsuma"]="${SATSUMA_NETWORKS[@]}"
["superfluid"]="${SUPERFLUID_NETWORKS[@]}"
["goldsky"]="${GOLDSKY_NETWORKS[@]}"
["airstack"]="${AIRSTACK_NETWORKS[@]}"
)

VENDOR=""
NETWORK=""
Expand Down Expand Up @@ -65,6 +45,32 @@ while [[ $# -gt 0 ]]; do
esac
done

prepare_deployment() {
# Read environment variables directly, with a fallback Git command for commit_hash
local commit_hash="${GITHUB_SHA:-$(git rev-parse HEAD)}"
local configuration="${CONFIGURATION:-v1}"

# Get ABI
echo "Getting ABI..."
node "./scripts/getAbi.js"

# Prepare subgraph manifest
echo "Preparing subgraph manifest..."
./tasks/prepare-manifest.sh mock

# Generate meta.ignore.ts file
echo "Generating meta.ignore.ts file..."
COMMIT_HASH="$commit_hash" CONFIGURATION="$configuration" yarn generate-sf-meta

# Generate AssemblyScript types
echo "Generating AssemblyScript types..."
yarn codegen

# Get Hosted Service Networks from metadata
echo "Getting Hosted Service Networks from metadata..."
npx ts-node "./scripts/getHostedServiceNetworks.ts"
}

deploy_to_graph() {
local network="$1"

Expand All @@ -79,10 +85,10 @@ deploy_to_graph() {

echo "********* Deploying $network subgraph $subgraphName to The Graph (hosted service). **********"

$GRAPH_CLI deploy \
--studio "$subgraphName" \
--deploy-key "$THE_GRAPH_ACCESS_TOKEN" \
--version-label "$VERSION_LABEL"
if ! $GRAPH_CLI deploy --studio "$subgraphName" --deploy-key "$THE_GRAPH_ACCESS_TOKEN" --version-label "$VERSION_LABEL"; then
echo "Error: Deployment to The Graph (hosted service) failed for $network"
exit 1
fi
}

deploy_to_satsuma() {
Expand Down Expand Up @@ -112,31 +118,37 @@ deploy_to_superfluid() {
local subgraphName="protocol-$DEPLOYMENT_ENV"
echo "node url: $nodeUrl, subgraph name: $subgraphName"

$GRAPH_CLI create "$subgraphName" --node "$nodeUrl"
$GRAPH_CLI deploy "$subgraphName" \
--version-label "$VERSION_LABEL" \
--node "$nodeUrl" \
--ipfs "$SUPERFLUID_IPFS_API"
if ! $GRAPH_CLI create "$subgraphName" --node "$nodeUrl"; then
echo "Error: Creation of subgraph $subgraphName on Superfluid (self hosted) failed for $network"
exit 1
fi
if ! $GRAPH_CLI deploy "$subgraphName" --version-label "$VERSION_LABEL" --node "$nodeUrl" --ipfs "$SUPERFLUID_IPFS_API"; then
echo "Error: Deployment to Superfluid (self hosted) failed for $network"
exit 1
fi
}

deploy_to_goldsky() {
local network="$1"
# TODO: use tagging?

#Get subgraph version from package.json
# Get subgraph version from package.json
PACKAGE_JSON_PATH="package.json"
SUBGRAPH_VERSION=$(jq -r '.version' $PACKAGE_JSON_PATH)

local subgraphName="protocol-$DEPLOYMENT_ENV-$network/$SUBGRAPH_VERSION"

# Note: when using Graph CLI to deploy, it implicitly triggers build too, but Goldsky CLI doesn't, so we do it explicitly.
$GRAPH_CLI build
# Note: when using Graph CLI to deploy, it implicitly triggers build too, but Goldsky CLI doesn't, so we do it explicitly.
if ! $GRAPH_CLI build; then
echo "Error: Build for Goldsky failed"
exit 1
fi

echo "********* Deploying $network subgraph $subgraphName to Goldsky. **********"
$GOLDSKY_CLI subgraph deploy \
"$subgraphName" \
--path . \
--token "$GOLDSKY_API_KEY"
if ! $GOLDSKY_CLI subgraph deploy "$subgraphName" --path . --token "$GOLDSKY_API_KEY"; then
echo "Error: Deployment to Goldsky failed for $network"
exit 1
fi
}

deploy_to_airstack() {
Expand All @@ -145,14 +157,14 @@ deploy_to_airstack() {
local subgraphName="protocol-$DEPLOYMENT_ENV-$network"

echo "********* Deploying $network subgraph $subgraphName to Airstack. **********"
$GRAPH_CLI create "$subgraphName" --node "$nodeUrl" --access-token "$AIRSTACK_API_KEY"
$GRAPH_CLI deploy \
--version-label "$VERSION_LABEL" \
--node "$nodeUrl" \
--deploy-key "$AIRSTACK_API_KEY" \
--ipfs https://ipfs.airstack.xyz/ipfs/api/v0 \
--headers '{"Authorization": "'"$AIRSTACK_API_KEY"'"}' \
"$subgraphName"
if ! $GRAPH_CLI create "$subgraphName" --node "$nodeUrl" --access-token "$AIRSTACK_API_KEY"; then
echo "Error: Creation of subgraph $subgraphName on Airstack failed for $network"
exit 1
fi
if ! $GRAPH_CLI deploy --version-label "$VERSION_LABEL" --node "$nodeUrl" --deploy-key "$AIRSTACK_API_KEY" --ipfs https://ipfs.airstack.xyz/ipfs/api/v0 --headers '{"Authorization": "'"$AIRSTACK_API_KEY"'"}' "$subgraphName"; then
echo "Error: Deployment to Airstack failed for $network"
exit 1
fi
}

# Vendor specific function dispatcher
Expand All @@ -163,15 +175,6 @@ deploy_to() {
local vendor="$1"
local network="$2"

# check if network is supported by vendor
local -n networksRef="${vendor^^}_NETWORKS"
# We can safely ignore this warning, becasue the value in network won't contain whitespaces
# shellcheck disable=SC2199,SC2076
if [[ ! " ${networksRef[@]} " =~ " $network " ]]; then
echo "The network, $network, is currently not on the list of networks supported by $vendor."
exit 1
fi

npx ts-node ./scripts/buildNetworkConfig.ts "$network" "$vendor"

# prepare the manifest prior to deployment
Expand Down Expand Up @@ -212,11 +215,12 @@ if [[ ! " ${SUPPORTED_VENDORS[@]} " =~ " $VENDOR " ]]; then
print_usage_and_exit
fi

# Handle all vs specific network
if [ "$NETWORK" == "all" ]; then
for network in ${VENDOR_NETWORKS[$VENDOR]}; do
deploy_to "$VENDOR" "$network"
done
else
deploy_to "$VENDOR" "$NETWORK"
#Prepare deployment
# Prepare deployment
if ! prepare_deployment; then
echo "Error: Failed to prepare deployment"
exit 1
fi

# Deploy the specified network
deploy_to "$VENDOR" "$NETWORK"
6 changes: 6 additions & 0 deletions packages/subgraph/tasks/vendorNetworkMap.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"superfluid": ["polygon-mainnet", "xdai-mainnet", "base-mainnet", "optimism-mainnet", "arbitrum-one", "celo-mainnet", "bsc-mainnet", "avalanche-c", "optimism-sepolia", "scroll-sepolia", "scroll-mainnet", "degenchain", "base-sepolia"],
"goldsky": ["polygon-mainnet", "xdai-mainnet", "eth-mainnet", "base-mainnet", "optimism-mainnet", "arbitrum-one", "bsc-mainnet", "avalanche-c", "optimism-sepolia", "scroll-sepolia", "scroll-mainnet", "eth-sepolia", "avalanche-fuji", "base-sepolia", "degenchain"],
"graph": ["polygon-mainnet", "eth-mainnet", "celo-mainnet"]
}

0 comments on commit c887de9

Please sign in to comment.