diff --git a/.github/workflows/dependency-updates.yml b/.github/workflows/dependency-updates.yml index e7219013d..5d38d5a0b 100644 --- a/.github/workflows/dependency-updates.yml +++ b/.github/workflows/dependency-updates.yml @@ -10,7 +10,7 @@ jobs: with: reviewers: aalu1418 run: | - make upgrade-e2e-solana-image + make upgrade-solana-image image=$(curl https://api.github.com/repos/solana-labs/solana/releases/latest | jq -r '.tag_name') # outputs diff --git a/.github/workflows/nix-packages-test.yml b/.github/workflows/nix-packages-test.yml new file mode 100644 index 000000000..a1441ec09 --- /dev/null +++ b/.github/workflows/nix-packages-test.yml @@ -0,0 +1,29 @@ +on: + pull_request: + push: + branches: + - develop + +jobs: + nix-packages-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + + - name: Install Nix + uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: "sandbox = false" + + - name: Build and test solana-test-validator + run: nix run .#solana-test-validator + + - name: Build solana-cli-env + run: nix build .#solana-cli-env --print-out-paths + + - name: Test solana-cli-shell + run: nix develop .#solana-cli + + + \ No newline at end of file diff --git a/Makefile b/Makefile index 06e3b0077..22f59f596 100644 --- a/Makefile +++ b/Makefile @@ -120,8 +120,8 @@ lint-go-integration-tests: lint-go-relay: cd ./pkg && golangci-lint --max-issues-per-linter 0 --max-same-issues 0 --color=always --exclude=dot-imports --timeout 10m --out-format checkstyle:golangci-lint-relay-report.xml run -.PHONY: upgrade-e2e-solana-image -upgrade-e2e-solana-image: +.PHONY: upgrade-solana-image +upgrade-solana-image: ./scripts/update-solana.sh .PHONY: update-e2e-core-deps diff --git a/flake.nix b/flake.nix index 339966756..0761c5330 100644 --- a/flake.nix +++ b/flake.nix @@ -11,7 +11,16 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default ]; }; + solanaPkgs = pkgs.callPackage ./solana.nix {}; in rec { - devShell = pkgs.callPackage ./shell.nix {}; - }); + devShells = { + default = pkgs.callPackage ./shell.nix {}; + solana-cli = solanaPkgs.solana-cli-shell; + }; + + packages = { + solana-test-validator = solanaPkgs.solana-test-validator; + solana-cli-env = solanaPkgs.solana-cli-env; + }; + }); } diff --git a/scripts/setup-localnet/localnet.down.sh b/scripts/setup-localnet/localnet.down.sh new file mode 100644 index 000000000..0987e1855 --- /dev/null +++ b/scripts/setup-localnet/localnet.down.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Cleaning up test validator container.." + +echo "Checking for existing 'chainlink-solana.test-validator' docker container..." +dpid=`docker ps -a | grep chainlink-solana.test-validator | awk '{print $1}'`; +if [ -z "$dpid" ] +then + echo "No docker test validator container running."; +else + docker kill $dpid; + docker rm $dpid; +fi + +echo "Cleanup finished." \ No newline at end of file diff --git a/scripts/setup-localnet/localnet.sh b/scripts/setup-localnet/localnet.sh new file mode 100755 index 000000000..a00857d23 --- /dev/null +++ b/scripts/setup-localnet/localnet.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -euo pipefail +cpu_struct="linux"; + +# Clean up first +bash "$(dirname -- "$0";)/localnet.down.sh" + +container_version=v1.18.23 +container_name="chainlink-solana.test-validator" + +echo "Starting $container_name@$container_version" + +# NOTE: solanalabs/solana docker image only supports linux/amd64 +# https://hub.docker.com/r/solanalabs/solana/tags +# If you are running on an ARM machine, Following error will be thrown: +# "Incompatible CPU detected: missing AVX support. Please build from source on the target " +docker run -d \ + --platform linux/amd64 \ + -p 127.0.0.1:8899:8899 \ + -p 127.0.0.1:8900:8900 \ + -p 127.0.0.1:9900:9900 \ + --name "${container_name}" \ + --entrypoint /bin/sh \ + "solanalabs/solana:${container_version}" \ + -c "solana-test-validator && echo 'Validator started successfully'" + # --network-alias "${container_name}" \ + # --network chainlink \ + +echo "Waiting for test validator to become ready.." +start_time=$(date +%s) +prev_output="" +while true +do + output=$(docker logs chainlink-solana.test-validator 2>&1) + if [[ "${output}" != "${prev_output}" ]]; then + echo -n "${output#$prev_output}" + prev_output="${output}" + fi + + if [[ $output == *"Finalized Slot"* ]]; then + echo "" + echo "solana-test-validator is ready." + exit 0 + fi + + if [[ $output == *"Incompatible CPU detected"* || $output == *"Aborted"* ]]; then + echo "" + echo "solanalabs/solana docker image only supports linux/amd64" + exit 1 + fi + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if (( elapsed_time > 10 )); then + echo "Error: Command did not become ready within 10 seconds" + exit 1 + fi + + sleep 3 +done \ No newline at end of file diff --git a/scripts/update-solana-nix-hashes.sh b/scripts/update-solana-nix-hashes.sh new file mode 100755 index 000000000..058274c5a --- /dev/null +++ b/scripts/update-solana-nix-hashes.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Solana Nix Package SHA256 Hash Updater +# +# This script updates the SHA256 hashes for Solana binaries in the solana.nix file. +# +# Usage: ./update_nix_hashes.sh + +set -e + +if [ $# -eq 0 ]; then + echo "Error: No version provided" + echo "Usage: $0 " + exit 1 +fi + +VERSION=$1 + +echo "Updating SHA256 hashes for Nix packages (Solana version: $VERSION)" + +get_nix_hash() { + local url=$1 + nix hash convert --hash-algo sha256 $(nix-prefetch-url --unpack $url) +} + +linux_hash=$(get_nix_hash https://github.com/anza-xyz/agave/releases/download/${VERSION}/solana-release-x86_64-unknown-linux-gnu.tar.bz2) +darwin_hash=$(get_nix_hash https://github.com/anza-xyz/agave/releases/download/${VERSION}/solana-release-aarch64-apple-darwin.tar.bz2) + +echo "Linux Hash: $linux_hash" +echo "Darwin Hash: $darwin_hash" + +LINUX_START_MARKER="### BEGIN_LINUX_SHA256 ###" +LINUX_END_MARKER="### END_LINUX_SHA256 ###" +DARWIN_START_MARKER="### BEGIN_DARWIN_SHA256 ###" +DARWIN_END_MARKER="### END_DARWIN_SHA256 ###" + +if [ "$(uname -s)" = "Darwin" ]; then + sed_in_place=(-i '') +else + sed_in_place=(-i) +fi + +sed "${sed_in_place[@]}" \ + -e "/$LINUX_START_MARKER/,/$LINUX_END_MARKER/ s|sha256 = \"[^\"]*\"|sha256 = \"$linux_hash\"|" \ + solana.nix + +sed "${sed_in_place[@]}" \ + -e "/$DARWIN_START_MARKER/,/$DARWIN_END_MARKER/ s|sha256 = \"[^\"]*\"|sha256 = \"$darwin_hash\"|" \ + solana.nix + +echo "Updated hashes in solana.nix" \ No newline at end of file diff --git a/scripts/update-solana.sh b/scripts/update-solana.sh index a115fbf53..1d433c7ae 100755 --- a/scripts/update-solana.sh +++ b/scripts/update-solana.sh @@ -9,9 +9,17 @@ testVersion=$(grep -oh "solanalabs/solana:v[0-9]*.[0-9]*.[0-9]*" testconfig/defa echo "Current E2E Test Version: $testVersion" cd .. +localnetVersion=$(grep -oh "container_version=v[0-9]*.[0-9]*.[0-9]*" scripts/setup-localnet/localnet.sh) +echo "Current Version in Localnet Container: $localnetVersion" + +nixVersion=$(grep -oh "version = \"v[0-9]*.[0-9]*.[0-9]*\"" solana.nix) +echo "Current Version in Nix packages: $nixVersion" + latestTag=$(curl https://api.github.com/repos/solana-labs/solana/releases/latest | jq -r '.tag_name') latestVersion="solanalabs/solana:$latestTag" latestCLI="release.solana.com/$latestTag" +latestLocalnet="container_version=$latestTag" +latestNix="version = \"$latestTag\"" echo "Latest Solana Mainnet Version: $latestTag" if [ "$testVersion" = "$latestVersion" ] && [ "$cliVersion" = "$latestCLI" ] ; then @@ -25,11 +33,19 @@ if [ "$(uname -s)" = "Darwin" ]; then sed -i '' -e "s~$cliVersion~$latestCLI~" scripts/install-solana-ci.sh cd integration-tests sed -i '' -e "s~$testVersion~$latestVersion~" testconfig/default.toml + cd .. + sed -i '' -e "s~$localnetVersion~$latestLocalnet~" scripts/setup-localnet/localnet.sh + sed -i '' -e "s~$nixVersion~$latestNix~" solana.nix else sed -i -e "s~$cliVersion~$latestCLI~" scripts/install-solana-ci.sh cd integration-tests sed -i -e "s~$testVersion~$latestVersion~" testconfig/default.toml + cd .. + sed -i -e "s~$containerVersion~$latestContainer~" scripts/setup-localnet/localnet.sh + sed -i -e "s~$nixVersion~$latestNix~" solana.nix fi -cd .. + +# Update the Nix hashes +"$(dirname -- "$0")/update-solana-nix-hashes.sh" "$latestTag" echo "Done" diff --git a/solana.nix b/solana.nix new file mode 100644 index 000000000..f60e2289c --- /dev/null +++ b/solana.nix @@ -0,0 +1,97 @@ +{ + system ? builtins.currentSystem, + pkgs ? import { inherit system; }, +}: + +# Solana integration +let + version = "v1.18.23"; + getBinDerivation = + { + name, + filename, + sha256, + }: + pkgs.stdenv.mkDerivation rec { + inherit name; + url = "https://github.com/anza-xyz/agave/releases/download/${version}/${filename}"; + src = pkgs.fetchzip { + inherit url sha256; + }; + + installPhase = '' + mkdir -p $out/bin + ls -lah $src + cp -r $src/bin/* $out/bin + ''; + }; + + # It provides two derivations, one for x86_64-linux and another for aarch64-apple-darwin. + # Each derivation downloads the corresponding Solana release. + + # The SHA256 hashes below are automatically updated by action.(dependency-updates.yml) + # The update script(./scripts/update-solana-nix-hashes.sh) looks for the BEGIN and END markers to locate the lines to modify. + # Do not modify these markers or the lines between them manually. + solanaBinaries = { + x86_64-linux = getBinDerivation { + name = "solana-cli-x86_64-linux"; + filename = "solana-release-x86_64-unknown-linux-gnu.tar.bz2"; + ### BEGIN_LINUX_SHA256 ### + sha256 = "sha256-L7N8z1MjDWkSoOKLAe4y/iuKTRgLpqg2mDpb9h1RXH0="; + ### END_LINUX_SHA256 ### + }; + aarch64-apple-darwin = getBinDerivation { + name = "solana-cli-aarch64-apple-darwin"; + filename = "solana-release-aarch64-apple-darwin.tar.bz2"; + ### BEGIN_DARWIN_SHA256 ### + sha256 = "sha256-D6hJL3yQncHltuWtF4QMAzvp/s7LV/S3NHwHiJG8wQ0="; + ### END_DARWIN_SHA256 ### + }; + }; +in +{ + # Provides environment package for Solana CLI + solana-cli-env = pkgs.buildEnv { + name = "solana-cli-env"; + paths = pkgs.lib.optionals pkgs.stdenv.isLinux [ + solanaBinaries.x86_64-linux + ] + ++ pkgs.lib.optionals (pkgs.stdenv.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) [ + solanaBinaries.aarch64-apple-darwin + ]; + }; + + # Provides interactive dev shell with Solana CLI tool accessibility. + solana-cli-shell = pkgs.mkShell { + buildInputs = pkgs.lib.optionals pkgs.stdenv.isLinux [ + solanaBinaries.x86_64-linux + ] + ++ pkgs.lib.optionals (pkgs.stdenv.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) [ + solanaBinaries.aarch64-apple-darwin + ]; + shellHook = '' + echo "====================================================" + echo "Welcome to the Solana CLI dev shell." + echo "Current environment: $(uname -a)" + echo "You are using the package for ${pkgs.stdenv.hostPlatform.system}." + echo "----------------------------------------------------" + echo "Solana CLI information:" + solana --version + solana config get + echo "====================================================" + ''; + }; + + # Provides dockerized Solana test validator accessibility. https://hub.docker.com/r/solanalabs/solana + # Currently the official docker image only supports x86_64-linux.(https://github.com/anza-xyz/agave/tree/master/sdk/docker-solana) + solana-test-validator = pkgs.stdenv.mkDerivation rec { + name = "solana-test-validator"; + src = ./scripts/setup-localnet; + installPhase = '' + mkdir -p $out/bin + cp $src/localnet.sh $out/bin/${name} + cp $src/localnet.down.sh $out/bin/ + chmod +x $out/bin/${name} + ''; + }; +} \ No newline at end of file