build: dynamically generate flatc rust at compile time

.devcontainer/devcontainer.json

@@ -13,7 +13,8 @@
 			"extensions": [
 				"vadimcn.vscode-lldb",
 				"fill-labs.dependi",
-				"rust-lang.rust-analyzer"
+				"rust-lang.rust-analyzer",
+				"gaborv.flatbuffers"
 			]
 		}
 	},

.github/docker/Dockerfile.build

@@ -1,16 +1,48 @@
-# Stage: Bonsol Test
+# Stage 1: flatc build
+FROM debian:stable-slim AS flatc-build
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y -q --no-install-recommends \
+    build-essential \
+    cmake \
+    ca-certificates \
+    curl \
+    git \
+    gnupg2 \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# flatc
+WORKDIR /flatbuffers
+ARG FLATC_VERSION=24.3.25
+ADD https://github.com/google/flatbuffers/archive/refs/tags/v${FLATC_VERSION}.tar.gz v${FLATC_VERSION}.tar.gz
+RUN tar -zxvf v${FLATC_VERSION}.tar.gz || { echo "Failed to extract tarball"; exit 1; }
+WORKDIR /flatbuffers/flatbuffers-${FLATC_VERSION}
+RUN cmake -G "Unix Makefiles" && make -j && make install
+RUN strip /usr/local/bin/flatc
+
+# Stage 2: Bonsol Test
 FROM ghcr.io/anagrambuild/risczero:latest
 
+# flatbuffers
+COPY --from=flatc-build /usr/local/bin/flatc /usr/local/bin/flatc
+COPY --from=flatc-build /usr/local/include/flatbuffers /usr/local/include/flatbuffers
+COPY --from=flatc-build /usr/local/lib/libflatbuffers.a /usr/local/lib/libflatbuffers.a
+COPY --from=flatc-build /usr/local/lib/cmake/flatbuffers /usr/local/lib/cmake/flatbuffers
+COPY --from=flatc-build /usr/local/lib/pkgconfig/flatbuffers.pc /usr/local/lib/pkgconfig/flatbuffers.pc
+
+
 ENV USER=solana
 ARG SOLANA=1.18.22
 ENV CARGO_HOME=/usr/local/cargo
 ENV RUSTUP_HOME=/usr/local/rustup
-ENV PATH=${PATH}:/usr/local/cargo/bin:/go/bin:/home/solana/.local/share/solana/install/releases/${SOLANA}/bin
-USER solana
+ENV PATH=${PATH}:/usr/local/cargo/bin:/go/bin:/home/${USER}/.local/share/solana/install/releases/${SOLANA}/bin
 
 # Set user and working directory
 ARG PACKAGE=bonsol
 WORKDIR /workspaces/${PACKAGE}
+USER solana
 
 # Install Rust components
 RUN rustup component add \
@@ -19,9 +51,10 @@ RUN rustup component add \
 RUN rustup toolchain install nightly  && \
     rustup component add rustfmt --toolchain nightly
 
-COPY . .
+COPY --chown=${USER}:${USER} . .
 
 RUN /go/bin/yamlfmt -lint .github/workflows/*.yaml .github/workflows/*.yml .github/*.yaml .github/*.yml
 
+RUN cargo check
 RUN cargo +nightly fmt --all -- --check
 RUN cargo test   

.gitignore

@@ -13,4 +13,5 @@ charts/bonsol-node/alloy.yaml
 relay/MyLogFile.log
 NodeDragon.toml
 images/**/manifest.json
-stark
+stark
+*_generated.rs

.rustfmt.toml

@@ -0,0 +1 @@
+ignore = ["**/*_generated.rs"]

.vscode/tasks.json

@@ -6,7 +6,7 @@
         {
             "label": "fmt",
             "type": "shell",
-            "command": "cargo +nightly fmt --check",
+            "command": "cargo +nightly fmt --all -- --check",
             "options": {
                 "cwd": "${workspaceFolder}"
             },

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 * `risc0-groth16-prover` binaries (rapidsnark & stark-verify) are available to the nix store, partially unblocking NixOS support.
+* `flatbuffers` code is now dynamically generated at build time
 
 ## [0.2.1] - 2024-10-13
 

Cargo.toml

@@ -11,7 +11,7 @@ members = [
   "schemas-rust",
   "cli",
   "sdk",
-  "prover",
+  "prover"
 ]
 resolver = "2"
 # Only necessary for nix cargo workspace artifacts to build without warnings.

Dockerfile

@@ -3,12 +3,15 @@
 # Stage 1: Node setup
 FROM debian:stable-slim AS node-slim
 RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt update && \
-    apt install -y -q --no-install-recommends \
-    build-essential git gnupg2 curl \
-    ca-certificates && \
-    apt clean && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get update && \
+    apt-get install -y -q --no-install-recommends \
+    ca-certificates \
+    curl \
+    git \
+    gnupg2 \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ENV NODE_VERSION=v22.3.0
 ENV NVM_DIR=/usr/local/nvm
@@ -17,9 +20,40 @@ RUN mkdir -p ${NVM_DIR}
 ADD https://raw.githubusercontent.com/creationix/nvm/master/install.sh /usr/local/etc/nvm/install.sh
 RUN bash /usr/local/etc/nvm/install.sh
 
+# Stage 2: flatc build
+FROM debian:stable-slim AS flatc-build
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y -q --no-install-recommends \
+    build-essential \
+    cmake \
+    ca-certificates \
+    curl \
+    git \
+    gnupg2 \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# flatc
+WORKDIR /flatbuffers
+ARG FLATC_VERSION=24.3.25
+ADD https://github.com/google/flatbuffers/archive/refs/tags/v${FLATC_VERSION}.tar.gz v${FLATC_VERSION}.tar.gz
+RUN tar -zxvf v${FLATC_VERSION}.tar.gz || { echo "Failed to extract tarball"; exit 1; }
+WORKDIR /flatbuffers/flatbuffers-${FLATC_VERSION}
+RUN cmake -G "Unix Makefiles" && make -j && make install
+RUN strip /usr/local/bin/flatc
+
 # Stage 2: Bonsol Dev
 FROM ghcr.io/anagrambuild/risczero:latest
 
+# flatbuffers
+COPY --from=flatc-build /usr/local/bin/flatc /usr/local/bin/flatc
+COPY --from=flatc-build /usr/local/include/flatbuffers /usr/local/include/flatbuffers
+COPY --from=flatc-build /usr/local/lib/libflatbuffers.a /usr/local/lib/libflatbuffers.a
+COPY --from=flatc-build /usr/local/lib/cmake/flatbuffers /usr/local/lib/cmake/flatbuffers
+COPY --from=flatc-build /usr/local/lib/pkgconfig/flatbuffers.pc /usr/local/lib/pkgconfig/flatbuffers.pc
+
 ENV USER=solana
 ARG SOLANA=1.18.22
 ENV CARGO_HOME=/usr/local/cargo
@@ -48,6 +82,7 @@ ENV PATH      ${NVM_NODE_PATH}/bin:$PATH
 COPY --from=node-slim --chown=${USER}:${USER} /usr/local/nvm /usr/local/nvm
 RUN bash -c ". $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm alias default $NODE_VERSION && nvm use default"
 
+
 RUN npm install npm -g
 RUN npm install yarn -g
 

bin/gen.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
-flatc --rust --gen-object-api --gen-mutable --reflect-names -o schemas-rust/src schemas/*.fbs
+
+
+if [ ! -d "schemas" ]; then
+  echo "No schemas directory found"
+  exit 1
+fi
+
+if [ ! -x $(which flatc) ]; then
+  echo "Flatbuffers compiler is required"
+  exit 1
+fi
+
 flatc --ts --gen-object-api --gen-mutable --reflect-names -o schemas-ts schemas/*.fbs
 sed -i.bak '
   /mutate_writable(value:boolean):boolean {/,/}/ {

schemas-rust/build.rs

@@ -0,0 +1,60 @@
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn main() {
+    // Define schema directory and target directory for generated Rust code.
+    let schema_dir = Path::new("../schemas");
+    let generated_src =
+        PathBuf::from(env::var("GENERATED_CODE_DIR").unwrap_or_else(|_| "src".to_string()));
+
+    // Collect all .fbs files in the schema directory.
+    let file_list: Vec<_> = fs::read_dir(&schema_dir)
+        .expect("Schema directory not found")
+        .filter_map(|entry| {
+            entry.ok().and_then(|e| {
+                let path = e.path();
+                if path.extension().and_then(|ext| ext.to_str()) == Some("fbs") {
+                    Some(path)
+                } else {
+                    None
+                }
+            })
+        })
+        .collect();
+
+    // Build flatc arguments.
+    let mut args = vec![
+        "--gen-mutable",
+        "--gen-object-api",
+        "--reflect-names",
+        "--rust",
+        "-o",
+        generated_src
+            .to_str()
+            .expect("Invalid path for generated source"),
+    ];
+
+    // Add schema file paths to the arguments.
+    args.extend(file_list.iter().map(|path| path.to_str().unwrap()));
+
+    // Execute flatc.
+    let compile_status = Command::new("flatc")
+        .args(&args)
+        .status()
+        .expect("Failed to execute flatc command");
+
+    assert!(
+        compile_status.success(),
+        "flatc failed to compile schema files: {:?}",
+        file_list
+    );
+
+    // Set an environment variable with the generated source path, stripping "src/" if present.
+    let generated_path = generated_src.strip_prefix("src").unwrap_or(&generated_src);
+    println!("cargo:rustc-env=GENERATED_SRC={}", generated_path.display());
+
+    // Instruct Cargo to re-run this script if schema files change.
+    println!("cargo:rerun-if-changed={}", schema_dir.display());
+}