merge polymer changes

.github/workflows/docker-op-stack.yml

@@ -0,0 +1,65 @@
+name: docker-peptide
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+env:
+  IMAGE_NAME: ghcr.io/polymerdao/optimism
+
+jobs:
+  docker-build-peptide:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.build-ref }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{raw}}
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.source=https://github.com/polymerdao/optimism-dev
+            org.opencontainers.image.title=polymer
+            org.opencontainers.image.url=https://github.com/polymerdao/optimism-dev
+
+      - name: Authenticate Docker
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Docker images
+        run: |
+          docker buildx bake \
+          		--progress plain \
+          		--load \
+          		-f docker-bake.hcl \
+          		op-node op-batcher op-proposer op-challenger
+        env:
+          REGISTRY: ghcr.io
+          REPOSITORY: polymerdao/optimism
+          IMAGE_TAGS: ${{ github.ref_name }}
+          GIT_COMMIT: ${{ github.github_sha }}
+
+      - name: Push Docker images
+        run: |
+          IMAGES=("op-node" "op-batcher" "op-proposer" "op-challenger")
+          for image in "${IMAGES[@]}"
+          do
+            echo "Pushing $image"
+            docker push ghcr.io/polymerdao/optimism/$image:${{ github.ref_name }}
+          done

op-e2e/bridge_test.go

@@ -23,6 +23,7 @@ import (
 // TestERC20BridgeDeposits tests the the L1StandardBridge bridge ERC20
 // functionality.
 func TestERC20BridgeDeposits(t *testing.T) {
+	t.Skip("the method eth_maxPriorityFeePerGas does not exist/is not available")
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)

op-e2e/deposit_test.go

@@ -16,6 +16,7 @@ import (
 )
 
 func TestMintOnRevertedDeposit(t *testing.T) {
+	t.Skip("fails with: the method eth_getTransactionCount does not exist/is not available")
 	InitParallel(t)
 	cfg := DefaultSystemConfig(t)
 	delete(cfg.Nodes, "verifier")
@@ -76,6 +77,7 @@ func TestMintOnRevertedDeposit(t *testing.T) {
 }
 
 func TestDepositTxCreateContract(t *testing.T) {
+	t.Skip("this test sends a tx that creates a contract on the L2 - we don't support that")
 	InitParallel(t)
 	cfg := DefaultSystemConfig(t)
 	delete(cfg.Nodes, "verifier")

op-e2e/e2eutils/external_polymer/.gitignore

@@ -0,0 +1,3 @@
+op-geth
+polymer-peptide
+shim

op-e2e/e2eutils/external_polymer/Makefile

@@ -0,0 +1,23 @@
+build: shim polymer-peptide
+.PHONY: build
+
+PEPTIDE_DIR := ../../../polymerase/chain/cmd/peptide
+
+polymer-peptide:
+	@test -d $(PEPTIDE_DIR) || { echo "$(PEPTIDE_DIR) does not exist" ; exit 1; }
+	cd $(PEPTIDE_DIR) && go build -o $(realpath .)/polymer-peptide ./
+.PHONY: polymer-peptide
+
+shim: main.go
+	go build -o shim .
+.PHONY: shim
+
+test-shim: build
+	go test -v ./...
+.PHONY: test-shim
+
+clean:
+	rm -f shim polymer-peptide
+.PHONY: clean
+
+.DEFAULT_GOAL := build

op-e2e/e2eutils/external_polymer/README.md

@@ -0,0 +1,35 @@
+# external_polymer shim
+
+See the [external_geth README file](../external_geth/README.md) for context.
+
+## run the shim tests
+
+This test makes sure the shim is able to start the polymer execution engine.
+Simply run
+
+```sh
+cd op-e2e/external_polymer
+make test-shim
+```
+
+## run the e2e tests
+
+This is a starting point while polymer as an L2 worked on. This command runs a single test that will
+check for basic communication between the op-node and the execution engine
+
+```sh
+cd op-e2e
+make -C external_polymer/ build
+go test --externalL2 ./external_polymer/ -v -run TestSystemE2E
+```
+
+To run all e2e tests, we can use the following target
+
+```sh
+cd op-e2e
+make -C external_polymer/ build
+make test-external-polymer
+```
+
+**Note that the e2e tests expect to find the shim and polymer within the `external_polymer` directory
+and there's no internal hook to automatically build them.**

op-e2e/e2eutils/external_polymer/main.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/onsi/gomega/gbytes"
+	"github.com/onsi/gomega/gexec"
+)
+
+func main() {
+	var configPath string
+	flag.StringVar(&configPath, "config", "", "Execute based on the config in this file")
+	flag.Parse()
+	if err := run(configPath); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+// relative to where the shim binary lives
+const opPolymerBin = "polymer-peptide"
+
+func run(configPath string) error {
+	if configPath == "" {
+		return fmt.Errorf("must supply a '--config <path>' flag")
+	}
+
+	configFile, err := os.Open(configPath)
+	if err != nil {
+		return fmt.Errorf("could not open config: %w", err)
+	}
+
+	var config external.Config
+	if err := json.NewDecoder(configFile).Decode(&config); err != nil {
+		return fmt.Errorf("could not decode config file: %w", err)
+	}
+
+	binPath, err := filepath.Abs(opPolymerBin)
+	if err != nil {
+		return fmt.Errorf("could not get absolute path of op-polymer")
+	}
+	if _, err := os.Stat(binPath); err != nil {
+		return fmt.Errorf("could not locate op-polymer in working directory")
+	}
+
+	fmt.Printf("================== op-polymer shim initializing chain config ==========================\n")
+	if err := initialize(binPath, config); err != nil {
+		return fmt.Errorf("could not initialize datadir: %s %w", binPath, err)
+	}
+
+	fmt.Printf("================== op-polymer shim sealing chain ==========================\n")
+	if err := seal(binPath, config); err != nil {
+		return fmt.Errorf("could not seal datadir: %s %w", binPath, err)
+	}
+
+	sess, err := execute(binPath, config)
+	if err != nil {
+		return fmt.Errorf("could not execute polymer: %w", err)
+	}
+	defer sess.Close()
+
+	fmt.Printf("==================    op-polymer shim encoding ready-file  ==========================\n")
+	if err := external.AtomicEncode(config.EndpointsReadyPath, sess.endpoints); err != nil {
+		return fmt.Errorf("could not encode endpoints")
+	}
+
+	fmt.Printf("==================    op-polymer shim awaiting termination  ==========================\n")
+
+	// Listen for kill signals
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
+
+	<-sigCh
+	sess.Close()
+
+	select {
+	case <-sess.session.Exited:
+		return fmt.Errorf("geth exited")
+	case <-time.After(30 * time.Minute):
+		return fmt.Errorf("exiting after 30 minute timeout")
+	}
+}
+
+func initialize(binPath string, config external.Config) error {
+	cmd := exec.Command(
+		binPath,
+		"init",
+		"--home", config.DataDir,
+		"--l1-hash", config.L1Hash,
+		"--l1-height", fmt.Sprintf("%d", config.L1Height),
+	)
+	return cmd.Run()
+}
+
+func seal(binPath string, config external.Config) error {
+	var outb, errb bytes.Buffer
+	cmd := exec.Command(
+		binPath,
+		"seal",
+		"--home", config.DataDir,
+		"--genesis-time", fmt.Sprintf("%d", config.L2Time),
+	)
+	cmd.Stdout = &outb
+	cmd.Stderr = &errb
+	err := cmd.Run()
+	fmt.Printf("%v\n", outb.String())
+	fmt.Printf("%v\n", errb.String())
+	return err
+}
+
+type gethSession struct {
+	session   *gexec.Session
+	endpoints *external.Endpoints
+}
+
+func (es *gethSession) Close() {
+	es.session.Terminate()
+	select {
+	case <-time.After(5 * time.Second):
+		es.session.Kill()
+	case <-es.session.Exited:
+	}
+}
+
+func execute(binPath string, config external.Config) (*gethSession, error) {
+	cmd := exec.Command(binPath, "start",
+		"--app-rpc-address", "localhost:0",
+		"--ee-http-server-address", "localhost:0",
+		"--home", config.DataDir,
+	)
+	sess, err := gexec.Start(cmd, os.Stdout, os.Stderr)
+	if err != nil {
+		return nil, fmt.Errorf("could not start op-polymer session: %w", err)
+	}
+	matcher := gbytes.Say("Execution engine rpc server enabled")
+	var httpUrl, wsUrl string
+	urlRE := regexp.MustCompile(`Execution engine rpc server enabled\s+http=(.+)\sws=(.+)`)
+	for httpUrl == "" && wsUrl == "" {
+		match, err := matcher.Match(sess.Out)
+		if err != nil {
+			return nil, fmt.Errorf("could not execute matcher")
+		}
+		if !match {
+			if sess.Out.Closed() {
+				return nil, fmt.Errorf("op-polymer exited before announcing http ports")
+			}
+			// Wait for a bit more output, then try again
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+
+		for _, line := range strings.Split(string(sess.Out.Contents()), "\n") {
+			found := urlRE.FindStringSubmatch(line)
+			if len(found) == 3 {
+				httpUrl = found[1]
+				wsUrl = found[2]
+				continue
+			}
+		}
+	}
+
+	genesisFile, err := os.Open(filepath.Join(config.DataDir, "config", "genesis.json"))
+	if err != nil {
+		return nil, fmt.Errorf("could not open genesis file: %w", err)
+	}
+
+	var genesis struct {
+		GenesisBlock eth.BlockID `json:"genesis_block"`
+	}
+	if err := json.NewDecoder(genesisFile).Decode(&genesis); err != nil {
+		return nil, fmt.Errorf("could not decode genesis file: %w", err)
+	}
+
+	return &gethSession{
+		session: sess,
+		endpoints: &external.Endpoints{
+			HTTPEndpoint:       httpUrl,
+			WSEndpoint:         wsUrl,
+			HTTPAuthEndpoint:   httpUrl,
+			WSAuthEndpoint:     wsUrl,
+			GenesisBlockHash:   genesis.GenesisBlock.Hash.Hex(),
+			GenesisBlockHeight: strconv.FormatUint(genesis.GenesisBlock.Number, 10),
+		},
+	}, nil
+}