Merge branch 'snowbridge' into ron/ark-bls-integration

.config/lychee.toml

@@ -2,9 +2,9 @@
 # Run with `lychee -c .config/lychee.toml ./**/*.rs ./**/*.prdoc`
 
 cache = true
-max_cache_age = "1d"
+max_cache_age = "10d"
 max_redirects = 10
-max_retries = 6
+max_retries = 3
 
 # Exclude localhost et.al.
 exclude_all_private = true
@@ -32,12 +32,10 @@ exclude = [
 	"https://github.com/paritytech/polkadot-sdk/substrate/frame/timestamp",
 	"https://github.com/paritytech/substrate/frame/fast-unstake",
 	"https://github.com/zkcrypto/bls12_381/blob/e224ad4ea1babfc582ccd751c2bf128611d10936/src/test-data/mod.rs",
+	"https://polkadot-try-runtime-node.parity-chains.parity.io/",
 	"https://polkadot.network/the-path-of-a-parachain-block/",
-	"https://research.web3.foundation/en/latest/polkadot/BABE/Babe/#6-practical-results",
 	"https://research.web3.foundation/en/latest/polkadot/NPoS/3.%20Balancing.html",
 	"https://research.web3.foundation/en/latest/polkadot/Token%20Economics.html#inflation-model",
-	"https://research.web3.foundation/en/latest/polkadot/block-production/Babe.html",
-	"https://research.web3.foundation/en/latest/polkadot/block-production/Babe.html#-6.-practical-results",
 	"https://research.web3.foundation/en/latest/polkadot/networking/3-avail-valid.html#topology",
 	"https://research.web3.foundation/en/latest/polkadot/overview/2-token-economics.html",
 	"https://research.web3.foundation/en/latest/polkadot/overview/2-token-economics.html#inflation-model",
@@ -51,4 +49,8 @@ exclude = [
 	# Behind a captcha (code 403):
 	"https://iohk.io/en/blog/posts/2023/11/03/partner-chains-are-coming-to-cardano/",
 	"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
+	# 403 rate limited:
+	"https://etherscan.io/block/11090290",
+	"https://subscan.io/",
+	"https://substrate.stackexchange.com/.*",
 ]

.config/zepter.yaml

@@ -25,9 +25,13 @@ workflows:
         '--show-path',
         '--quiet',
       ]
-  # Same as `check`, but with the `--fix` flag.
+  # The umbrella crate uses more features, so we to check those too:
+  check_umbrella:
+    - [ $check.0, '--features=serde,experimental,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
+  # Same as `check_*`, but with the `--fix` flag.
   default:
     - [ $check.0, '--fix' ]
+    - [ $check_umbrella.0, '--fix' ]
 
 # Will be displayed when any workflow fails:
 help:

.forklift/config.toml

@@ -0,0 +1,33 @@
+[compression]
+type = "zstd"
+
+[compression.zstd]
+compressionLevel = 3
+
+[general]
+jobNameVariable = "CI_JOB_NAME"
+jobsBlackList = []
+logLevel = "warn"
+threadsCount = 6
+
+[cache]
+extraEnv = ["RUNTIME_METADATA_HASH"]
+
+[metrics]
+enabled = true
+pushEndpoint = "placeholder"
+
+[metrics.extraLabels]
+environment = "production"
+job_name = "$CI_JOB_NAME"
+project_name = "$CI_PROJECT_PATH"
+
+[storage]
+type = "s3"
+
+[storage.s3]
+accessKeyId = "placeholder"
+bucketName = "placeholder"
+concurrency = 10
+endpointUrl = "placeholder"
+secretAccessKey = "placeholder"

.github/codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  precision: 2
+  round: down
+  range: "1...100"
+  status:
+    project:
+      default:
+        target: 1.0
+        threshold: 2.0

.github/env

@@ -0,0 +1 @@
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408"

.github/scripts/check-prdoc.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+'''
+Ensure that the prdoc files are valid.
+
+# Example
+
+```sh
+python3 -m pip install cargo-workspace
+python3 .github/scripts/check-prdoc.py Cargo.toml prdoc/*.prdoc
+```
+
+Produces example output:
+```pre
+🔎 Reading workspace polkadot-sdk/Cargo.toml
+📦 Checking 32 prdocs against 493 crates.
+✅ All prdocs are valid
+```
+'''
+
+import os
+import yaml
+import argparse
+import cargo_workspace
+
+def check_prdoc_crate_names(root, paths):
+	'''
+	Check that all crates of the `crates` section of each prdoc is present in the workspace.
+	'''
+
+	print(f'🔎 Reading workspace {root}.')
+	workspace = cargo_workspace.Workspace.from_path(root)
+	crate_names = [crate.name for crate in workspace.crates]
+
+	print(f'📦 Checking {len(paths)} prdocs against {len(crate_names)} crates.')
+	faulty = {}
+
+	for path in paths:
+		with open(path, 'r') as f:
+			prdoc = yaml.safe_load(f)
+
+		for crate in prdoc.get('crates', []):
+			crate = crate['name']
+			if crate in crate_names:
+				continue
+
+			faulty.setdefault(path, []).append(crate)
+
+	if len(faulty) == 0:
+		print('✅ All prdocs are valid.')
+	else:
+		print('❌ Some prdocs are invalid.')
+		for path, crates in faulty.items():
+			print(f'💥 {path} lists invalid crate: {", ".join(crates)}')
+		exit(1)
+
+def parse_args():
+	parser = argparse.ArgumentParser(description='Check prdoc files')
+	parser.add_argument('root', help='The cargo workspace manifest', metavar='root', type=str, nargs=1)
+	parser.add_argument('prdoc', help='The prdoc files', metavar='prdoc', type=str, nargs='*')
+	args = parser.parse_args()
+
+	if len(args.prdoc) == 0:
+		print('❌ Need at least one prdoc file as argument.')
+		exit(1)
+
+	return { 'root': os.path.abspath(args.root[0]), 'prdocs': args.prdoc }
+
+if __name__ == '__main__':
+	args = parse_args()
+	check_prdoc_crate_names(args['root'], args['prdocs'])

.github/scripts/deny-git-deps.py

@@ -0,0 +1,40 @@
+"""
+Script to deny Git dependencies in the Cargo workspace. Can be passed one optional argument for the
+root folder. If not provided, it will use the cwd.
+
+## Usage
+	python3 .github/scripts/deny-git-deps.py polkadot-sdk
+"""
+
+import os
+import sys
+
+from cargo_workspace import Workspace, DependencyLocation
+
+KNOWN_BAD_GIT_DEPS = {
+	'simple-mermaid': ['xcm-docs'],
+	# Fix in <https://github.com/paritytech/polkadot-sdk/issues/2922>
+	'bandersnatch_vrfs': ['sp-core'],
+}
+
+root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
+workspace = Workspace.from_path(root)
+
+def check_dep(dep, used_by):
+	if dep.location != DependencyLocation.GIT:
+		return
+
+	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
+		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
+	else:
+		print(f'🚫 Found git dependency {dep.name} in {used_by}')
+		sys.exit(1)	
+
+# Check the workspace dependencies that can be inherited:
+for dep in workspace.dependencies:
+	check_dep(dep, "workspace")
+
+# And the dependencies of each crate:
+for crate in workspace.crates:
+	for dep in crate.dependencies:
+		check_dep(dep, crate.name)

.github/workflows/check-changed-files.yml

@@ -0,0 +1,57 @@
+# Reusable workflow to perform checks and generate conditions for other workflows. 
+# Currently it checks if any Rust (build-related) file is changed
+# and if the current (caller) workflow file is changed.
+# Example:
+#
+# jobs:
+#   changes:
+#     permissions:
+#       pull-requests: read
+#     uses: ./.github/workflows/check-changed-files.yml
+#   some-job:
+#     needs: changes
+#     if: ${{ needs.changes.outputs.rust }}
+#  ....... 
+
+name: Check changes files
+
+on:
+  workflow_call:
+    # Map the workflow outputs to job outputs
+    outputs:
+      rust: 
+        value: ${{ jobs.changes.outputs.rust }}
+        description: 'true if any of the build-related OR current (caller) workflow files have changed'
+      current-workflow: 
+        value: ${{ jobs.changes.outputs.current-workflow }}
+        description: 'true if current (caller) workflow file has changed'      
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      # true if current workflow (caller) file is changed
+      rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }}
+      current-workflow: ${{ steps.filter.outputs.current-workflow }}
+    steps:
+    - id: current-file
+      run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
+    - run: echo "${{ steps.current-file.outputs.current-workflow-file }}"
+      # For pull requests it's not necessary to checkout the code
+    - id: filter
+      uses: dorny/paths-filter@v3
+      with:
+        predicate-quantifier: 'every'
+        # current-workflow - check if the current (caller) workflow file is changed
+        # rust - check if any Rust (build-related) file is changed
+        filters: |
+          current-workflow:
+            - '${{ steps.current-file.outputs.current-workflow-file }}'
+          rust:
+            - '**/*'
+            - '!.github/**/*'
+            - '!prdoc/**/*'
+            - '!docs/**/*'
+    #

.github/workflows/check-runtime-migration.yml

@@ -0,0 +1,122 @@
+name: check-runtime-migration
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  FORKLIFT_storage_s3_bucketName: ${{ secrets.FORKLIFT_storage_s3_bucketName }}
+  FORKLIFT_storage_s3_accessKeyId: ${{ secrets.FORKLIFT_storage_s3_accessKeyId }}
+  FORKLIFT_storage_s3_secretAccessKey: ${{ secrets.FORKLIFT_storage_s3_secretAccessKey }}
+  FORKLIFT_storage_s3_endpointUrl: ${{ secrets.FORKLIFT_storage_s3_endpointUrl }}
+  FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
+
+jobs:
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+  # rococo and westend are disabled for now (no access to parity-chains.parity.io)
+  check-runtime-migration:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    timeout-minutes: 30
+    needs: [set-image]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    strategy:
+      fail-fast: false
+      matrix:
+        network: [
+            # westend,
+            # rococo,
+            asset-hub-westend,
+            asset-hub-rococo,
+            bridge-hub-westend,
+            bridge-hub-rococo,
+            contracts-rococo,
+            collectives-westend,
+            coretime-rococo,
+          ]
+        include:
+          # - network: westend
+          #   package: westend-runtime
+          #   wasm: westend_runtime.compact.compressed.wasm
+          #   uri: "wss://westend-try-runtime-node.parity-chains.parity.io:443"
+          #   subcommand_extra_args: "--no-weight-warnings"
+          #   command_extra_args: ""
+          # - network: rococo
+          #   package: rococo-runtime
+          #   wasm: rococo_runtime.compact.compressed.wasm
+          #   uri: "wss://rococo-try-runtime-node.parity-chains.parity.io:443"
+          #   subcommand_extra_args: "--no-weight-warnings"
+          #   command_extra_args: ""
+          - network: asset-hub-westend
+            package: asset-hub-westend-runtime
+            wasm: asset_hub_westend_runtime.compact.compressed.wasm
+            uri: "wss://westend-asset-hub-rpc.polkadot.io:443"
+            subcommand_extra_args: ""
+            command_extra_args: ""
+          - network: "asset-hub-rococo"
+            package: "asset-hub-rococo-runtime"
+            wasm: "asset_hub_rococo_runtime.compact.compressed.wasm"
+            uri: "wss://rococo-asset-hub-rpc.polkadot.io:443"
+            subcommand_extra_args: ""
+            command_extra_args: ""
+          - network: "bridge-hub-westend"
+            package: "bridge-hub-westend-runtime"
+            wasm: "bridge_hub_westend_runtime.compact.compressed.wasm"
+            uri: "wss://westend-bridge-hub-rpc.polkadot.io:443"
+          - network: "bridge-hub-rococo"
+            package: "bridge-hub-rococo-runtime"
+            wasm: "bridge_hub_rococo_runtime.compact.compressed.wasm"
+            uri: "wss://rococo-bridge-hub-rpc.polkadot.io:443"
+          - network: "contracts-rococo"
+            package: "contracts-rococo-runtime"
+            wasm: "contracts_rococo_runtime.compact.compressed.wasm"
+            uri: "wss://rococo-contracts-rpc.polkadot.io:443"
+          - network: "collectives-westend"
+            package: "collectives-westend-runtime"
+            wasm: "collectives_westend_runtime.compact.compressed.wasm"
+            uri: "wss://westend-collectives-rpc.polkadot.io:443"
+            command_extra_args: "--disable-spec-name-check"
+          - network: "coretime-rococo"
+            package: "coretime-rococo-runtime"
+            wasm: "coretime_rococo_runtime.compact.compressed.wasm"
+            uri: "wss://rococo-coretime-rpc.polkadot.io:443"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: script
+        run: |
+          echo "Running ${{ matrix.network }} runtime migration check"
+          export RUST_LOG=remote-ext=debug,runtime=debug
+
+          echo "---------- Downloading try-runtime CLI ----------"
+          curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+          chmod +x ./try-runtime
+          echo "Using try-runtime-cli version:"
+          ./try-runtime --version
+
+          echo "---------- Building ${{ matrix.package }} runtime ----------"
+          time forklift cargo build --release --locked -p ${{ matrix.package }} --features try-runtime
+
+          echo "---------- Executing on-runtime-upgrade for ${{ matrix.network }} ----------"
+          time ./try-runtime ${{ matrix.command_extra_args }} \
+              --runtime ./target/release/wbuild/${{ matrix.package }}/${{ matrix.wasm }} \
+              on-runtime-upgrade --disable-spec-version-check --checks=all ${{ matrix.subcommand_extra_args }} live --uri ${{ matrix.uri }}
+          sleep 5