Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revamp Reproducible Builds #56

Merged
merged 41 commits into from
Jul 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
474cdc6
standardize to x86
rauljordan Jul 22, 2024
6fbb593
rem rust stable tag
rauljordan Jul 23, 2024
aa07e14
macro
rauljordan Jul 23, 2024
9dd5721
macro
rauljordan Jul 23, 2024
469abc6
cache updates working
rauljordan Jul 25, 2024
022dcc3
arg comment
rauljordan Jul 25, 2024
f6230b9
cargo update deps
rauljordan Jul 25, 2024
509fd29
include edits
rauljordan Jul 25, 2024
7ba09c8
Update check/src/main.rs
rauljordan Jul 25, 2024
a742881
detect host os
rauljordan Jul 25, 2024
a369208
Merge branch 'docker-checks' of github.com:OffchainLabs/cargo-stylus …
rauljordan Jul 25, 2024
daf8cf0
merge
rauljordan Jul 25, 2024
3da9895
standardize to x86 toolchain
rauljordan Jul 25, 2024
e907744
docker
rauljordan Jul 25, 2024
016d5c5
edits
rauljordan Jul 25, 2024
080bbd0
edits
rauljordan Jul 25, 2024
54da4c1
source
rauljordan Jul 25, 2024
2db8e41
edits
rauljordan Jul 25, 2024
292e6fb
edits
rauljordan Jul 25, 2024
b400d28
edits
rauljordan Jul 25, 2024
4dbb300
edits
rauljordan Jul 25, 2024
4d14d7e
edits
rauljordan Jul 25, 2024
a12c6ea
edits
rauljordan Jul 25, 2024
910b078
edits
rauljordan Jul 25, 2024
0384336
edit
rauljordan Jul 25, 2024
51090f5
edit
rauljordan Jul 25, 2024
ede85a6
edit
rauljordan Jul 26, 2024
01aee88
edits
rauljordan Jul 26, 2024
aac869b
arg
rauljordan Jul 26, 2024
89e8e95
fix
rauljordan Jul 26, 2024
fe82489
use toolchain as source of truth
rauljordan Jul 26, 2024
fe5b4d2
edits to no verify
rauljordan Jul 26, 2024
210a04e
better error
rauljordan Jul 26, 2024
497f3b1
Update check/src/main.rs
rauljordan Jul 26, 2024
46bff53
Update check/src/main.rs
rauljordan Jul 26, 2024
87dfdc2
edit
rauljordan Jul 26, 2024
7951c00
edit
rauljordan Jul 26, 2024
afad358
edits
rauljordan Jul 26, 2024
c3bcaef
Merge branch 'main' into docker-checks
rauljordan Jul 26, 2024
8a8190c
build
rauljordan Jul 26, 2024
18dc6e8
confs
rauljordan Jul 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions check/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,4 @@ wasmparser = "0.213.0"
wasm-encoder = "0.213.0"
wasm-gen = "0.1.4"
toml = "0.8.14"
sys-info = "0.9.1"
9 changes: 6 additions & 3 deletions check/src/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

use crate::{
check::ArbWasm::ArbWasmErrors,
constants::{ARB_WASM_H160, ONE_ETH},
constants::{ARB_WASM_H160, ONE_ETH, TOOLCHAIN_FILE_NAME},
macros::*,
project::{self, BuildConfig},
project::{self, extract_toolchain_channel, BuildConfig},
CheckConfig,
};
use alloy_primitives::{Address, B256, U256};
Expand Down Expand Up @@ -122,7 +122,10 @@ impl CheckConfig {
if let Some(wasm) = self.wasm_file.clone() {
return Ok((wasm, [0u8; 32]));
}
let cfg = BuildConfig::new(self.common_cfg.rust_stable);
let toolchain_file_path = PathBuf::from(".").as_path().join(TOOLCHAIN_FILE_NAME);
let toolchain_channel = extract_toolchain_channel(&toolchain_file_path)?;
let rust_stable = !toolchain_channel.contains("nightly");
let cfg = BuildConfig::new(rust_stable);
let wasm = project::build_dylib(cfg.clone())?;
let project_hash =
project::hash_files(self.common_cfg.source_files_for_project_hash.clone(), cfg)?;
Expand Down
5 changes: 5 additions & 0 deletions check/src/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,8 @@ pub const PROJECT_HASH_SECTION_NAME: &str = "project_hash";

/// Name of the toolchain file used to specify the Rust toolchain version for a project.
pub const TOOLCHAIN_FILE_NAME: &str = "rust-toolchain.toml";

/// Base Rust image version to be used for reproducible builds. This simply installs cargo and the Rust
/// compiler, but the user will specify the exact version of the Rust toolchain to use for building within
/// the docker container.
pub const RUST_BASE_IMAGE_VERSION: &str = "1.79.0";
3 changes: 1 addition & 2 deletions check/src/deploy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// For licensing, see https://github.com/OffchainLabs/cargo-stylus/blob/main/licenses/COPYRIGHT.md

#![allow(clippy::println_empty_string)]

use crate::{
check::{self, ProgramCheck},
constants::ARB_WASM_H160,
Expand Down Expand Up @@ -140,7 +139,7 @@ impl DeployConfig {
greyln!("deployed code at address: {address}");
}
let tx_hash = receipt.transaction_hash.debug_lavender();
greyln!("Deployment tx hash: {tx_hash}");
greyln!("deployment tx hash: {tx_hash}");
println!(
r#"we recommend running cargo stylus cache --address={} to cache your activated program in ArbOS.
Cached programs benefit from cheaper calls. To read more about the Stylus program cache, see
Expand Down
92 changes: 71 additions & 21 deletions check/src/docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
use std::path::PathBuf;
use std::process::{Command, Stdio};

use cargo_stylus_util::color::Color;
use eyre::{bail, eyre, Result};

use crate::constants::TOOLCHAIN_FILE_NAME;
use crate::constants::{RUST_BASE_IMAGE_VERSION, TOOLCHAIN_FILE_NAME};
use crate::macros::greyln;
use crate::project::extract_toolchain_channel;

fn version_to_image_name(version: &str) -> String {
Expand All @@ -20,16 +22,38 @@
.arg(name)
.output()
.map_err(|e| eyre!("failed to execute Docker command: {e}"))?;

if !output.status.success() {
let stderr = std::str::from_utf8(&output.stderr)
.map_err(|e| eyre!("failed to read Docker command stderr: {e}"))?;
if stderr.contains("Cannot connect to the Docker daemon") {
println!(
r#"Cargo stylus deploy|check|verify run in a Docker container by default to ensure deployments
are reproducible, but Docker is not found in your system. Please install Docker if you wish to create
a reproducible deployment, or opt out by using the --no-verify flag for local builds"#
);
bail!("Docker not running");
}
bail!(stderr.to_string())
}

Ok(output.stdout.iter().filter(|c| **c == b'\n').count() > 1)
}

fn create_image(version: &str) -> Result<()> {
let name = version_to_image_name(version);
let name = version_to_image_name(&version);

Check warning on line 44 in check/src/docker.rs

View workflow job for this annotation

GitHub Actions / clippy

this expression creates a reference which is immediately dereferenced by the compiler

warning: this expression creates a reference which is immediately dereferenced by the compiler --> check/src/docker.rs:44:38 | 44 | let name = version_to_image_name(&version); | ^^^^^^^^ help: change this to: `version` | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#needless_borrow = note: `#[warn(clippy::needless_borrow)]` on by default
if image_exists(&name)? {
return Ok(());
}
let toolchain_file_path = PathBuf::from(".").as_path().join(TOOLCHAIN_FILE_NAME);
let toolchain_channel = extract_toolchain_channel(&toolchain_file_path)?;
let cargo_stylus_version = env!("CARGO_PKG_VERSION");
let cargo_stylus_version: String = cargo_stylus_version
.chars()
.filter(|c| c.is_alphanumeric() || *c == '-' || *c == '.')
.collect();
println!(
"Building Docker image for cargo-stylus version {} and Rust toolchain {}",
cargo_stylus_version, version,
);
let mut child = Command::new("docker")
.arg("build")
.arg("-t")
Expand All @@ -42,20 +66,20 @@
write!(
child.stdin.as_mut().unwrap(),
"\
FROM rust:{} as builder\n\
RUN rustup toolchain install {} && rustup default {}
FROM --platform=linux/amd64 rust:{} as builder\n\
RUN rustup toolchain install {}-x86_64-unknown-linux-gnu
RUN rustup default {}-x86_64-unknown-linux-gnu
RUN rustup target add wasm32-unknown-unknown
RUN rustup target add wasm32-wasi
RUN rustup target add aarch64-unknown-linux-gnu
RUN rustup target add x86_64-unknown-linux-gnu
RUN cargo install cargo-stylus
RUN cargo install --force cargo-stylus-check
RUN cargo install --force cargo-stylus-replay
RUN cargo install --force cargo-stylus-cgen
RUN cargo install cargo-stylus-check --version {} --force
RUN cargo install cargo-stylus --version {} --force
",
RUST_BASE_IMAGE_VERSION,
version,
version,
toolchain_channel,
toolchain_channel,
cargo_stylus_version,
cargo_stylus_version,
)?;
child.wait().map_err(|e| eyre!("wait failed: {e}"))?;
Ok(())
Expand All @@ -79,21 +103,47 @@
.arg(name)
.args(command_line)
.spawn()
.map_err(|e| eyre!("failed to execure Docker command: {e}"))?
.map_err(|e| eyre!("failed to execute Docker command: {e}"))?
.wait()
.map_err(|e| eyre!("wait failed: {e}"))?;
Ok(())
}

pub fn run_reproducible(version: &str, command_line: &[String]) -> Result<()> {
let version: String = version
.chars()
.filter(|c| c.is_alphanumeric() || *c == '.')
.collect();
pub fn run_reproducible(command_line: &[String]) -> Result<()> {
verify_valid_host()?;
let toolchain_file_path = PathBuf::from(".").as_path().join(TOOLCHAIN_FILE_NAME);
let toolchain_channel = extract_toolchain_channel(&toolchain_file_path)?;
greyln!(
"Running reproducible Stylus command with toolchain {}",
toolchain_channel.mint()
);
let mut command = vec!["cargo", "stylus"];
for s in command_line.iter() {
command.push(s);
}
create_image(&version)?;
run_in_docker_container(&version, &command)
create_image(&toolchain_channel)?;
run_in_docker_container(&toolchain_channel, &command)
}

fn verify_valid_host() -> Result<()> {
let Ok(os_type) = sys_info::os_type() else {
bail!("unable to determine host OS type");
};
if os_type == "Windows" {
// Check for WSL environment
let Ok(kernel_version) = sys_info::os_release() else {
bail!("unable to determine kernel version");
};
if kernel_version.contains("microsoft") || kernel_version.contains("WSL") {
greyln!("Detected Windows Linux Subsystem host");
} else {
bail!(
"Reproducible cargo stylus commands on Windows are only supported \
in Windows Linux Subsystem (WSL). Please install within WSL. \
To instead opt out of reproducible builds, add the --no-verify \
flag to your commands."
);
}
}
Ok(())
}
Loading
Loading