From c056bc0bd000b962ca846cd3a06cbd9f6835d1c3 Mon Sep 17 00:00:00 2001 From: Justin W Smith <103147162+justsmth@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:32:17 -0500 Subject: [PATCH] Handle spaces in source/build paths (#608) * Handle spaces in source/build paths * Test build w/ spaces in paths * Skip test for macos-14-xlarge w/ FIPS --- .github/workflows/compilers.yml | 154 ++++++++++++++++++ .github/workflows/tests.yml | 189 +++++++---------------- aws-lc-fips-sys/builder/cmake_builder.rs | 4 +- aws-lc-sys/builder/cc_builder.rs | 2 +- aws-lc-sys/builder/cmake_builder.rs | 20 ++- 5 files changed, 232 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/compilers.yml diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml new file mode 100644 index 00000000000..0c244d52f74 --- /dev/null +++ b/.github/workflows/compilers.yml @@ -0,0 +1,154 @@ +name: compilers + +on: + push: + branches: + - '*' + - '!generate/aws-lc-*' + pull_request: + branches: + - '*' +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: 1 + # We can pin the version if nightly is too unstable. + # Otherwise, we test against the latest version. + RUST_NIGHTLY_TOOLCHAIN: nightly + +jobs: + + aws-lc-rs-2004-gcc: + if: github.repository_owner == 'aws' + name: GCC ${{ matrix.gcc_version }} - CMake ${{ matrix.cmake }} - FIPS ${{ matrix.fips }} + runs-on: ubuntu-20.04 + env: + AWS_LC_SYS_CMAKE_BUILDER: ${{ matrix.cmake }} + strategy: + fail-fast: false + matrix: + cmake: [ '0', '1' ] + gcc_version: [ '7', '8' ] + fips: [ '0', '1' ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + - name: Set up GCC + uses: egor-tensin/setup-gcc@v1.3 + with: + version: ${{ matrix.gcc_version }} + platform: x64 + - if: matrix.fips == '1' + uses: actions/setup-go@v4 + with: + go-version: '>=1.18' + - name: Run cargo test (debug) + run: cargo test -p aws-lc-rs --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} + - name: Run cargo test (release) + run: cargo test -p aws-lc-rs --release --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} + # The steps below verify that we're successfully using `-ffile-prefix-map` + # to remove build environment paths from the resulting library. + - if: ${{ matrix.gcc_version == '8' }} + name: Verify paths found in debug build + run: | + DEBUG_LIBCRYPTO=$(find ./target/debug -name "libaws_lc_*_crypto.a") + if strings ${DEBUG_LIBCRYPTO} | grep runner; then + exit 0; # SUCCESS + else + exit 1; # FAIL - we expected to find "runner" (i.e., a path) + fi + # TODO: Due to the nature of the FIPS build (e.g., its dynamic generation of + # assembly files and its custom compilation commands within CMake), not all + # source paths are stripped from the resulting binary. + - if: ${{ matrix.gcc_version == '8' && matrix.fips == '0' }} + name: Verify paths not found in release build + run: | + RELEASE_LIBCRYPTO=$(find ./target/release -name "libaws_lc_*_crypto.a") + if strings ${RELEASE_LIBCRYPTO} | grep runner; then + exit 1; # FAIL - we did not expect to find "runner" (i.e., a path) + else + exit 0; # SUCCESS + fi + + aws-lc-rs-1804-gcc: + if: github.repository_owner == 'aws' + name: GCC ${{ matrix.gcc_version }} - CMake ${{ matrix.cmake }} - FIPS ${{ matrix.fips }} + runs-on: ubuntu-20.04 + container: + image: ubuntu:18.04 + env: + AWS_LC_SYS_CMAKE_BUILDER: ${{ matrix.cmake }} + strategy: + fail-fast: false + matrix: + cmake: [ '0', '1' ] + gcc_version: [ '4.8', '5', '6' ] + fips: [ '0', '1' ] + steps: + - run: | + apt-get update + apt-get install -y ca-certificates + apt-get install -y cmake curl sudo + apt-get install -y --no-install-recommends gpg-agent software-properties-common + apt-add-repository --yes ppa:git-core/ppa + add-apt-repository --yes ppa:longsleep/golang-backports + apt-get update + apt-get install -y build-essential git golang-go + curl -L -O -J https://github.com/PowerShell/PowerShell/releases/download/v7.2.23/powershell_7.2.23-1.deb_amd64.deb + dpkg -i powershell_7.2.23-1.deb_amd64.deb + apt-get install -f + rm powershell_7.2.23-1.deb_amd64.deb + - name: Checkout + run: | + git config --global --add safe.directory '*' + git clone --recursive ${{ github.server_url }}/${{ github.repository }}.git . + git fetch origin ${{ github.sha }} + git checkout --recurse-submodules -b ci-job ${{ github.sha }} + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + - name: Set up GCC + uses: egor-tensin/setup-gcc@v1.3 + with: + version: ${{ matrix.gcc_version }} + platform: x64 + - name: Run cargo test (debug) + run: cargo test -p aws-lc-rs --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} + - name: Run cargo test (release) + run: cargo test -p aws-lc-rs --release --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} + + aws-lc-rs-c-std-test: + if: github.repository_owner == 'aws' + name: C-std ${{ matrix.os }} - ${{ matrix.c_std }} - Force CMake ${{ matrix.cmake }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + rust: [ stable ] + os: [ ubuntu-latest, macos-13, macos-14-xlarge, windows-latest ] + c_std: [ "11", "99" ] + cmake: [ '0', '1' ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: stable + - run: | + echo 'export AWS_LC_SYS_CMAKE_BUILDER=${{ matrix.cmake }}' >> "$GITHUB_ENV" + - name: Run cargo test + working-directory: ./aws-lc-rs + env: + AWS_LC_SYS_PREBUILT_NASM: 1 + AWS_LC_SYS_C_STD: ${{ matrix.c_std }} + run: cargo test --all-targets --features unstable diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a2e25b0bed..4d6ac440c00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,139 +53,6 @@ jobs: working-directory: ./aws-lc-rs-testing run: cargo test --all-targets - aws-lc-rs-1804-gcc: - if: github.repository_owner == 'aws' - name: GCC ${{ matrix.gcc_version }} - CMake ${{ matrix.cmake }} - FIPS ${{ matrix.fips }} - runs-on: ubuntu-20.04 - container: - image: ubuntu:18.04 - env: - AWS_LC_SYS_CMAKE_BUILDER: ${{ matrix.cmake }} - strategy: - fail-fast: false - matrix: - cmake: [ '0', '1' ] - gcc_version: [ '4.8', '5', '6' ] - fips: [ '0', '1' ] - steps: - - run: | - apt-get update - apt-get install -y ca-certificates - apt-get install -y cmake curl sudo - apt-get install -y --no-install-recommends gpg-agent software-properties-common - apt-add-repository --yes ppa:git-core/ppa - add-apt-repository --yes ppa:longsleep/golang-backports - apt-get update - apt-get install -y build-essential git golang-go - curl -L -O -J https://github.com/PowerShell/PowerShell/releases/download/v7.2.23/powershell_7.2.23-1.deb_amd64.deb - dpkg -i powershell_7.2.23-1.deb_amd64.deb - apt-get install -f - rm powershell_7.2.23-1.deb_amd64.deb - - name: Checkout - run: | - git config --global --add safe.directory '*' - git clone --recursive ${{ github.server_url }}/${{ github.repository }}.git . - git fetch origin ${{ github.sha }} - git checkout --recurse-submodules -b ci-job ${{ github.sha }} - - uses: dtolnay/rust-toolchain@master - id: toolchain - with: - toolchain: stable - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1.3 - with: - version: ${{ matrix.gcc_version }} - platform: x64 - - name: Run cargo test (debug) - run: cargo test -p aws-lc-rs --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} - - name: Run cargo test (release) - run: cargo test -p aws-lc-rs --release --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} - - aws-lc-rs-2004-gcc: - if: github.repository_owner == 'aws' - name: GCC ${{ matrix.gcc_version }} - CMake ${{ matrix.cmake }} - FIPS ${{ matrix.fips }} - runs-on: ubuntu-20.04 - env: - AWS_LC_SYS_CMAKE_BUILDER: ${{ matrix.cmake }} - strategy: - fail-fast: false - matrix: - cmake: [ '0', '1' ] - gcc_version: [ '7', '8' ] - fips: [ '0', '1' ] - steps: - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - uses: dtolnay/rust-toolchain@master - id: toolchain - with: - toolchain: stable - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1.3 - with: - version: ${{ matrix.gcc_version }} - platform: x64 - - if: matrix.fips == '1' - uses: actions/setup-go@v4 - with: - go-version: '>=1.18' - - name: Run cargo test (debug) - run: cargo test -p aws-lc-rs --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} - - name: Run cargo test (release) - run: cargo test -p aws-lc-rs --release --all-targets --no-default-features --features ${{ (matrix.fips == '0' && 'unstable,aws-lc-sys') || 'unstable,fips' }} - # The steps below verify that we're successfully using `-ffile-prefix-map` - # to remove build environment paths from the resulting library. - - if: ${{ matrix.gcc_version == '8' }} - name: Verify paths found in debug build - run: | - DEBUG_LIBCRYPTO=$(find ./target/debug -name "libaws_lc_*_crypto.a") - if strings ${DEBUG_LIBCRYPTO} | grep runner; then - exit 0; # SUCCESS - else - exit 1; # FAIL - we expected to find "runner" (i.e., a path) - fi - # TODO: Due to the nature of the FIPS build (e.g., its dynamic generation of - # assembly files and its custom compilation commands within CMake), not all - # source paths are stripped from the resulting binary. - - if: ${{ matrix.gcc_version == '8' && matrix.fips == '0' }} - name: Verify paths not found in release build - run: | - RELEASE_LIBCRYPTO=$(find ./target/release -name "libaws_lc_*_crypto.a") - if strings ${RELEASE_LIBCRYPTO} | grep runner; then - exit 1; # FAIL - we did not expect to find "runner" (i.e., a path) - else - exit 0; # SUCCESS - fi - - aws-lc-rs-c-std-test: - if: github.repository_owner == 'aws' - name: C-std ${{ matrix.os }} - ${{ matrix.c_std }} - Force CMake ${{ matrix.cmake }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - rust: [ stable ] - os: [ ubuntu-latest, macos-13, macos-14-xlarge, windows-latest ] - c_std: [ "11", "99" ] - cmake: [ '0', '1' ] - steps: - - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - uses: dtolnay/rust-toolchain@master - id: toolchain - with: - toolchain: stable - - run: | - echo 'export AWS_LC_SYS_CMAKE_BUILDER=${{ matrix.cmake }}' >> "$GITHUB_ENV" - - name: Run cargo test - working-directory: ./aws-lc-rs - env: - AWS_LC_SYS_PREBUILT_NASM: 1 - AWS_LC_SYS_C_STD: ${{ matrix.c_std }} - run: cargo test --all-targets --features unstable - bindgen-test: if: github.repository_owner == 'aws' name: aws-lc-rs bindgen-tests @@ -624,3 +491,59 @@ jobs: - name: aws-lc-fips-sys bindgen build working-directory: ./aws-lc-fips-sys run: cargo test --features bindgen + + source-path-has-spaces: + if: github.repository_owner == 'aws' + name: + "Spaces in path - ${{ matrix.os }} - FIPS: ${{ matrix.fips }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-13, macos-14-xlarge, windows-latest ] + fips: [ 0, 1 ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + path: "path has spaces/aws-lc-rs" + - if: ${{ matrix.os == 'windows-latest' }} + uses: ilammy/setup-nasm@v1 + - uses: dtolnay/rust-toolchain@stable + - uses: seanmiddleditch/gha-setup-ninja@v5 + - uses: actions/setup-go@v4 + with: + go-version: '>=1.18' + # FIPS builds for ARM platforms (e.g, macos-14-xlarge) will fail until upstream change is made: + # https://github.com/aws/aws-lc/pull/2005 + - if: matrix.os != 'macos-14-xlarge' || matrix.fips != 1 + name: Run cargo test + working-directory: "path has spaces/aws-lc-rs" + run: cargo test --tests -p aws-lc-rs --no-default-features --features ${{ (matrix.fips == 1 && 'fips') || 'aws-lc-sys' }} + + cmake-source-path-has-spaces: + if: github.repository_owner == 'aws' + name: + "Spaces in path (CMake) - ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + env: + AWS_LC_SYS_CMAKE_BUILDER: 1 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-13, macos-14-xlarge ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + path: "path has spaces/aws-lc-rs" + - if: ${{ matrix.os == 'windows-latest' }} + uses: ilammy/setup-nasm@v1 + - uses: dtolnay/rust-toolchain@stable + - uses: seanmiddleditch/gha-setup-ninja@v5 + - uses: actions/setup-go@v4 + with: + go-version: '>=1.18' + - name: Run cargo test + working-directory: "path has spaces/aws-lc-rs" + run: cargo test --tests -p aws-lc-rs --features bindgen diff --git a/aws-lc-fips-sys/builder/cmake_builder.rs b/aws-lc-fips-sys/builder/cmake_builder.rs index e168298c8f4..89662a97003 100644 --- a/aws-lc-fips-sys/builder/cmake_builder.rs +++ b/aws-lc-fips-sys/builder/cmake_builder.rs @@ -121,13 +121,13 @@ impl CmakeBuilder { if parent_dir.is_some() && (target_family() == "unix" || target_env() == "gnu") { let parent_dir = parent_dir.unwrap(); let cc_build = cc::Build::new(); - let flag = format!("-ffile-prefix-map={}=", parent_dir.display()); + let flag = format!("\"-ffile-prefix-map={}=\"", parent_dir.display()); if let Ok(true) = cc_build.is_flag_supported(&flag) { emit_warning(&format!("Using flag: {}", &flag)); cmake_cfg.asmflag(&flag); cmake_cfg.cflag(&flag); } else { - let flag = format!("-fdebug-prefix-map={}=", parent_dir.display()); + let flag = format!("\"-fdebug-prefix-map={}=\"", parent_dir.display()); if let Ok(true) = cc_build.is_flag_supported(&flag) { emit_warning(&format!("Using flag: {}", &flag)); cmake_cfg.asmflag(&flag); diff --git a/aws-lc-sys/builder/cc_builder.rs b/aws-lc-sys/builder/cc_builder.rs index dd016003e36..4364dfac0e1 100644 --- a/aws-lc-sys/builder/cc_builder.rs +++ b/aws-lc-sys/builder/cc_builder.rs @@ -188,7 +188,7 @@ impl CcBuilder { env::set_var("CFLAGS", cflags); } - if target_os() == "macos" && target_arch() == "x86_64" { + if target_os() == "macos" { // This compiler error has only been seen on MacOS x86_64: // ``` // clang: error: overriding '-mmacosx-version-min=13.7' option with '--target=x86_64-apple-macosx14.2' [-Werror,-Woverriding-t-option] diff --git a/aws-lc-sys/builder/cmake_builder.rs b/aws-lc-sys/builder/cmake_builder.rs index a5a789acdc2..16a7ed35bf7 100644 --- a/aws-lc-sys/builder/cmake_builder.rs +++ b/aws-lc-sys/builder/cmake_builder.rs @@ -103,7 +103,25 @@ impl CmakeBuilder { self.output_lib_type, ); let mut cflags = OsString::new(); - cflags.push(cc_builder.prepare_builder().get_compiler().cflags_env()); + let compiler = cc_builder.prepare_builder().get_compiler(); + let args = compiler.args(); + for (i, arg) in args.iter().enumerate() { + if i > 0 { + cflags.push(" "); + } + if let Some(arg) = arg.to_str() { + if arg.contains(' ') { + cflags.push("\""); + cflags.push(arg); + cflags.push("\""); + } else { + cflags.push(arg); + } + } else { + cflags.push(arg); + } + } + if !get_cflags().is_empty() { cflags.push(" "); cflags.push(get_cflags());