deployment/memory: Solana support (develop) (#15889)

* deployment: memory: Generate more transmitter key types, expose in JD

* deployment: memory: Configure nodes with solana config too

* Use CTF to spin up the solana validator for in-memory tests

* Use autopatchelf on solana binaries to make them usable on NixOS

* memory: solana: Shut down the container when test terminates

* go mod tidy

* Use latest upstream CTF

* Add missing import

* make modgraph

* Use framework.DefaultNetwork()

core/scripts/go.mod

@@ -82,8 +82,8 @@ require (
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytecodealliance/wasmtime-go/v23 v23.0.0 // indirect
-	github.com/bytedance/sonic v1.11.6 // indirect
-	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/bytedance/sonic v1.12.3 // indirect
+	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
@@ -135,7 +135,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.6 // indirect
 	github.com/gagliardetto/binary v0.8.0 // indirect
 	github.com/gagliardetto/solana-go v1.12.0 // indirect
 	github.com/gagliardetto/treeout v0.1.4 // indirect
@@ -164,7 +164,7 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.22.0 // indirect
+	github.com/go-playground/validator/v10 v10.22.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
 	github.com/go-webauthn/webauthn v0.9.4 // indirect
 	github.com/go-webauthn/x v0.1.5 // indirect
@@ -251,7 +251,7 @@ require (
 	github.com/maruel/natural v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -284,7 +284,7 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/prometheus/prometheus v0.54.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/rs/cors v1.10.1 // indirect

deployment/environment/memory/chain.go

@@ -1,8 +1,14 @@
 package memory
 
 import (
+	"encoding/json"
 	"math/big"
+	"os"
+	"path"
+	"strconv"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -11,14 +17,20 @@ import (
 	"github.com/ethereum/go-ethereum/ethclient/simulated"
 	"github.com/gagliardetto/solana-go"
 	solRpc "github.com/gagliardetto/solana-go/rpc"
-
+	"github.com/hashicorp/consul/sdk/freeport"
 	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
 
 	solTestUtil "github.com/smartcontractkit/chainlink-ccip/chains/solana/contracts/tests/testutils"
 
 	chainsel "github.com/smartcontractkit/chain-selectors"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
+
+	chainselectors "github.com/smartcontractkit/chain-selectors"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 )
 
@@ -30,7 +42,9 @@ type EVMChain struct {
 
 type SolanaChain struct {
 	Client      *solRpc.Client
-	DeployerKey *solana.PrivateKey
+	URL         string
+	WSURL       string
+	DeployerKey solana.PrivateKey
 }
 
 func fundAddress(t *testing.T, from *bind.TransactOpts, to common.Address, amount *big.Int, backend *simulated.Backend) {
@@ -80,13 +94,12 @@ func GenerateChainsSol(t *testing.T, numChains int) map[uint64]SolanaChain {
 	chains := make(map[uint64]SolanaChain)
 	for i := 0; i < numChains; i++ {
 		chainID := testSolanaChainSelectors[i]
-		url, _ := solTestUtil.SetupLocalSolNodeWithFlags(t)
-		admin, gerr := solana.NewRandomPrivateKey()
-		solTestUtil.FundTestAccounts(t, []solana.PublicKey{admin.PublicKey()}, url)
-		require.NoError(t, gerr)
+		solChain := solChain(t)
+		admin := solChain.DeployerKey
+		solTestUtil.FundTestAccounts(t, []solana.PublicKey{admin.PublicKey()}, solChain.URL)
 		chains[chainID] = SolanaChain{
-			Client:      solRpc.New(url),
-			DeployerKey: &admin,
+			Client:      solChain.Client,
+			DeployerKey: solChain.DeployerKey,
 		}
 	}
 	return chains
@@ -126,3 +139,66 @@ func evmChain(t *testing.T, numUsers int) EVMChain {
 		Users:       users,
 	}
 }
+
+var once = &sync.Once{}
+
+func solChain(t *testing.T) SolanaChain {
+	t.Helper()
+
+	// initialize the docker network used by CTF
+	err := framework.DefaultNetwork(once)
+	require.NoError(t, err)
+
+	deployerKey, err := solana.NewRandomPrivateKey()
+	require.NoError(t, err)
+
+	t.TempDir()
+	// store the generated keypair somewhere
+	bytes, err := json.Marshal([]byte(deployerKey))
+	require.NoError(t, err)
+	keypairPath := path.Join(t.TempDir(), "solana-keypair.json")
+	err = os.WriteFile(keypairPath, bytes, 0600)
+	require.NoError(t, err)
+
+	port := freeport.GetOne(t)
+
+	bcInput := &blockchain.Input{
+		Type:      "solana",
+		ChainID:   chainselectors.SOLANA_DEVNET.ChainID,
+		PublicKey: deployerKey.PublicKey().String(),
+		Port:      strconv.Itoa(port),
+		// TODO: ContractsDir & SolanaPrograms via env vars
+	}
+	output, err := blockchain.NewBlockchainNetwork(bcInput)
+	require.NoError(t, err)
+	testcontainers.CleanupContainer(t, output.Container)
+
+	url := output.Nodes[0].HostHTTPUrl
+	wsURL := output.Nodes[0].HostWSUrl
+
+	// Wait for api server to boot
+	client := solRpc.New(url)
+	var ready bool
+	for i := 0; i < 30; i++ {
+		time.Sleep(time.Second)
+		out, err := client.GetHealth(tests.Context(t))
+		if err != nil || out != solRpc.HealthOk {
+			t.Logf("API server not ready yet (attempt %d)\n", i+1)
+			continue
+		}
+		ready = true
+		break
+	}
+	if !ready {
+		t.Logf("solana-test-validator is not ready after 30 attempts")
+	}
+	require.True(t, ready)
+	t.Logf("solana-test-validator is ready at %s", url)
+
+	return SolanaChain{
+		Client:      client,
+		URL:         url,
+		WSURL:       wsURL,
+		DeployerKey: deployerKey,
+	}
+}

deployment/environment/memory/environment.go

@@ -128,10 +128,10 @@ func generateMemoryChainSol(t *testing.T, inputs map[uint64]SolanaChain) map[uin
 		chains[cid] = deployment.SolChain{
 			Selector:    cid,
 			Client:      chain.Client,
-			DeployerKey: chain.DeployerKey,
+			DeployerKey: &chain.DeployerKey,
 			Confirm: func(instructions []solana.Instruction, opts ...solCommomUtil.TxModifier) error {
 				_, err := solCommomUtil.SendAndConfirm(
-					context.Background(), chain.Client, instructions, *chain.DeployerKey, solRpc.CommitmentConfirmed, opts...,
+					context.Background(), chain.Client, instructions, chain.DeployerKey, solRpc.CommitmentConfirmed, opts...,
 				)
 				if err != nil {
 					return err
@@ -153,13 +153,13 @@ func NewNodes(t *testing.T, logLevel zapcore.Level, chains map[uint64]deployment
 	// since we won't run a bootstrapper and a plugin oracle on the same
 	// chainlink node in production.
 	for i := 0; i < numBootstraps; i++ {
-		node := NewNode(t, ports[i], chains, logLevel, true /* bootstrap */, registryConfig)
+		node := NewNode(t, ports[i], chains, nil, logLevel, true /* bootstrap */, registryConfig)
 		nodesByPeerID[node.Keys.PeerID.String()] = *node
 		// Note in real env, this ID is allocated by JD.
 	}
 	for i := 0; i < numNodes; i++ {
 		// grab port offset by numBootstraps, since above loop also takes some ports.
-		node := NewNode(t, ports[numBootstraps+i], chains, logLevel, false /* bootstrap */, registryConfig)
+		node := NewNode(t, ports[numBootstraps+i], chains, nil, logLevel, false /* bootstrap */, registryConfig)
 		nodesByPeerID[node.Keys.PeerID.String()] = *node
 		// Note in real env, this ID is allocated by JD.
 	}

deployment/environment/memory/job_client.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"slices"
-	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -153,49 +152,12 @@ func (j JobClient) ListNodeChainConfigs(ctx context.Context, in *nodev1.ListNode
 	if !ok {
 		return nil, fmt.Errorf("node id not found: %s", in.Filter.NodeIds[0])
 	}
-	evmBundle := n.Keys.OCRKeyBundles[chaintype.EVM]
-	offpk := evmBundle.OffchainPublicKey()
-	cpk := evmBundle.ConfigEncryptionPublicKey()
-
-	evmKeyBundle := &nodev1.OCR2Config_OCRKeyBundle{
-		BundleId:              evmBundle.ID(),
-		ConfigPublicKey:       common.Bytes2Hex(cpk[:]),
-		OffchainPublicKey:     common.Bytes2Hex(offpk[:]),
-		OnchainSigningAddress: evmBundle.OnChainPublicKey(),
-	}
-
 	var chainConfigs []*nodev1.ChainConfig
-	for evmChainID, transmitter := range n.Keys.TransmittersByEVMChainID {
-		chainConfigs = append(chainConfigs, &nodev1.ChainConfig{
-			Chain: &nodev1.Chain{
-				Id:   strconv.Itoa(int(evmChainID)),
-				Type: nodev1.ChainType_CHAIN_TYPE_EVM,
-			},
-			AccountAddress: transmitter.String(),
-			AdminAddress:   transmitter.String(), // TODO: custom address
-			Ocr1Config:     nil,
-			Ocr2Config: &nodev1.OCR2Config{
-				Enabled:     true,
-				IsBootstrap: n.IsBoostrap,
-				P2PKeyBundle: &nodev1.OCR2Config_P2PKeyBundle{
-					PeerId: n.Keys.PeerID.String(),
-				},
-				OcrKeyBundle:     evmKeyBundle,
-				Multiaddr:        n.Addr.String(),
-				Plugins:          nil,
-				ForwarderAddress: ptr(""),
-			},
-		})
-	}
 	for _, selector := range n.Chains {
 		family, err := chainsel.GetSelectorFamily(selector)
 		if err != nil {
 			return nil, err
 		}
-		if family == chainsel.FamilyEVM {
-			// already handled above
-			continue
-		}
 
 		// NOTE: this supports non-EVM too
 		chainID, err := chainsel.GetChainIDFromSelector(selector)
@@ -220,7 +182,6 @@ func (j JobClient) ListNodeChainConfigs(ctx context.Context, in *nodev1.ListNode
 		}
 
 		bundle := n.Keys.OCRKeyBundles[ocrtype]
-
 		offpk := bundle.OffchainPublicKey()
 		cpk := bundle.ConfigEncryptionPublicKey()
 
@@ -245,13 +206,15 @@ func (j JobClient) ListNodeChainConfigs(ctx context.Context, in *nodev1.ListNode
 			panic(fmt.Sprintf("Unsupported chain family %v", family))
 		}
 
+		transmitter := n.Keys.Transmitters[selector]
+
 		chainConfigs = append(chainConfigs, &nodev1.ChainConfig{
 			Chain: &nodev1.Chain{
 				Id:   chainID,
 				Type: ctype,
 			},
-			AccountAddress: "", // TODO: support AccountAddress
-			AdminAddress:   "",
+			AccountAddress: transmitter,
+			AdminAddress:   transmitter,
 			Ocr1Config:     nil,
 			Ocr2Config: &nodev1.OCR2Config{
 				Enabled:     true,
@@ -266,7 +229,6 @@ func (j JobClient) ListNodeChainConfigs(ctx context.Context, in *nodev1.ListNode
 			},
 		})
 	}
-	// TODO: I think we can pull it from the feeds manager.
 	return &nodev1.ListNodeChainConfigsResponse{
 		ChainConfigs: chainConfigs,
 	}, nil