Skip to content

fix: only first sync task uses multi-threading #582

fix: only first sync task uses multi-threading

fix: only first sync task uses multi-threading #582

Workflow file for this run

# SPDX-FileCopyrightText: © Vegard IT GmbH (https://vegardit.com)
# SPDX-FileContributor: Sebastian Thomschke (Vegard IT GmbH)
# SPDX-License-Identifier: Apache-2.0
# SPDX-ArtifactOfProjectHomePage: https://github.com/vegardit/copycat
#
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
name: Build
on:
push:
branches-ignore: # build all branches except:
- 'dependabot/**' # prevent GHA triggered twice (once for commit to the branch and once for opening/syncing the PR)
tags-ignore: # don't build tags
- '**'
paths-ignore:
- '**/*.adoc'
- '**/*.md'
- '.editorconfig'
- '.git*'
- '.github/*.yml'
- '.github/workflows/stale.yml'
pull_request:
paths-ignore:
- '**/*.adoc'
- '**/*.md'
- '.editorconfig'
- '.git*'
- '.github/*.yml'
workflow_dispatch:
# https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/
inputs:
additional_maven_args:
description: 'Additional Maven Args'
required: false
default: ''
defaults:
run:
shell: bash
env:
# TODO don't upgrade as newer versions break Windows builds https://github.com/oracle/graal/issues/4340
GRAALVM_VERSION: 21.3.3.1 # https://github.com/graalvm/graalvm-ce-builds/releases
JAVA_VERSION: 17
RELEASE_NAME: "snapshot"
jobs:
###########################################################
maven-build:
###########################################################
runs-on: ubuntu-latest
steps:
- name: Show environment variables
run: env | sort
- name: Git Checkout
uses: actions/checkout@v4 # https://github.com/actions/checkout
- name: "Install: JDK ${{ env.JAVA_VERSION }}"
uses: actions/setup-java@v4 # https://github.com/actions/setup-java
with:
distribution: temurin
java-version: ${{ env.JAVA_VERSION }}
- name: "Cache: Restore"
id: cache-restore
if: ${{ !env.ACT }} # https://github.com/nektos/act#skipping-steps
uses: actions/cache/restore@v4
with:
path: |
~/.m2/bin
~/.m2/repository
!~/.m2/repository/com/vegardit/copycat
key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
- name: Test with Maven
if: ${{ github.ref_name != 'main' || env.ACT }}
env:
GITHUB_USER: ${{ github.actor }}
GITHUB_API_KEY: ${{ github.token }}
MAY_CREATE_RELEASE: false
run: |
bash .ci/build.sh ${{ github.event.inputs.additional_maven_args }}
- name: Build with Maven
if: ${{ github.ref_name == 'main' && !env.ACT }}
env:
GITHUB_USER: ${{ github.actor }}
GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
MAY_CREATE_RELEASE: true
run: |
set -eu
# https://github.community/t/github-actions-bot-email-address/17204
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
bash .ci/build.sh ${{ github.event.inputs.additional_maven_args }}
- name: Test minimized fat jar
run: |
set -ex
mv target/copycat-*-fat-minimized.jar target/copycat-fat.jar
java -jar target/copycat-fat.jar --help
java -jar target/copycat-fat.jar -V
java -jar target/copycat-fat.jar sync target/classes target/classes.copy -v
- name: "Share: maven-build-artifacts"
uses: actions/upload-artifact@v4
with:
name: maven-build-artifacts
path: |
target/copycat-fat.jar
target/picocli-reflections.json
target/bash/bashcompletion.sh
##################################################
# Cache Update
# See https://github.com/actions/cache/issues/342
##################################################
- name: "Cache: Delete Previous"
if: ${{ steps.cache-restore.outputs.cache-hit && !env.ACT }}
env:
GH_TOKEN: ${{ github.token }}
run: |
gh extension install actions/gh-actions-cache
# "|| true" is to avoid "Error: Resource not accessible by integration" from failing the job
gh actions-cache delete ${{ steps.cache-restore.outputs.cache-primary-key }} --confirm || true
- name: "Cache: Update"
uses: actions/cache/save@v4
if: ${{ always() && !cancelled() && !env.ACT }} # save cache even fails
with:
path: |
~/.m2/bin
~/.m2/repository
!~/.m2/repository/com/vegardit/maven
!~/.m2/repository/*SNAPSHOT*
key: ${{ steps.cache-restore.outputs.cache-primary-key }}
###########################################################
native-image:
###########################################################
runs-on: ${{ matrix.os }}
needs: [maven-build]
strategy:
fail-fast: false
matrix:
include:
- arch: "linux-amd64"
os: "ubuntu-latest"
graal_install_root: "/opt/graalvm"
- arch: "linux-arm64"
os: "ubuntu-latest"
graal_install_root: "/opt/graalvm"
- arch: "darwin-amd64"
os: "macos-latest"
graal_install_root: "graalvm"
- arch: "windows-amd64"
os: "windows-latest"
graal_install_root: "graalvm"
env:
# https://www.graalvm.org/21.3/reference-manual/native-image/Options/
# -H:NativeLinkerOption=-no-pie -> do not to generate Position Independent Executables (PIE)
NATIVE_IMAGE_ARGS:
-H:NativeLinkerOption=-no-pie
-H:ReflectionConfigurationFiles=picocli-reflections.json
-H:Log=registerResource:3
-H:+ReportExceptionStackTraces
-H:+RemoveUnusedSymbols
-H:ExcludeResources=com/sun/.*.properties
--exclude-config copycat-fat.jar META-INF/native-image/net.sf.jstuff/.*.json
--exclude-config copycat-fat.jar META-INF/native-image/jansi/.*.json
--allow-incomplete-classpath
--no-fallback
--no-server
--verbose
-Dfile.encoding=UTF-8
--class-path native-image-metadata
--class-path copycat-fat.jar
com.vegardit.copycat.CopyCatMain
copycat-${{ matrix.arch }}
steps:
- name: "Cache: GraalVM binaries"
uses: actions/cache@v4
with:
path: |
${{ matrix.graal_install_root }}
key: ${{ matrix.arch }}-graalvm-${{ env.GRAALVM_VERSION }}-java-${{ env.JAVA_VERSION }}
- name: "Create arm64 container"
if: matrix.arch == 'linux-arm64'
run: |
sudo apt-get update -y
sudo apt-get install --no-install-recommends -y qemu-user-static
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
docker container create --name arm64 --tty \
-v /opt:/opt:rw \
-v $PWD:/workspace:rw \
arm64v8/ubuntu:22.04
docker container start arm64
docker exec --tty arm64 apt-get update -y
docker exec --tty arm64 apt-get install -y curl sudo
- name: "Install: GraalVM"
run: |
set -eu
case "${{ matrix.arch }}" in
linux-amd64)
sudo apt-get install --no-install-recommends -y gcc g++ libz-dev
GRAAL_VM_HOME=${{ matrix.graal_install_root }}
if [ ! -e $GRAAL_VM_HOME/bin/native-image ]; then
rm -rf $GRAAL_VM_HOME
mkdir -p $GRAAL_VM_HOME
curl -fL "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.GRAALVM_VERSION }}/graalvm-ce-java${{ env.JAVA_VERSION}}-linux-amd64-${{ env.GRAALVM_VERSION }}.tar.gz" \
| tar zxv -C $GRAAL_VM_HOME --strip-components=1 \
--exclude=*/bin/jvisualvm \
--exclude=*/lib/src.zip \
--exclude=*/lib/visualvm
$GRAAL_VM_HOME/bin/gu install native-image
fi
$GRAAL_VM_HOME/bin/native-image --version
echo "JAVA_HOME=$GRAAL_VM_HOME" >> $GITHUB_ENV
echo "$GRAAL_VM_HOME/bin" >> $GITHUB_PATH
;;
linux-arm64)
cat <<EOF > install_graalvm.sh
set -eu
sudo apt-get install --no-install-recommends -y gcc g++ libz-dev
GRAAL_VM_HOME=${{ matrix.graal_install_root }}
if [ ! -e \$GRAAL_VM_HOME/bin/native-image ]; then
rm -rf \$GRAAL_VM_HOME
mkdir -p \$GRAAL_VM_HOME
curl -fL "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.GRAALVM_VERSION }}/graalvm-ce-java${{ env.JAVA_VERSION }}-linux-aarch64-${{ env.GRAALVM_VERSION }}.tar.gz" \
| tar zxv -C \$GRAAL_VM_HOME --strip-components=1 \
--exclude=*/bin/jvisualvm \
--exclude=*/lib/src.zip \
--exclude=*/lib/visualvm
\$GRAAL_VM_HOME/bin/gu install native-image
fi
\$GRAAL_VM_HOME/bin/native-image --version
EOF
docker exec --tty -w /workspace arm64 /bin/bash ./install_graalvm.sh
;;
darwin-amd64)
GRAAL_VM_HOME=${{ matrix.graal_install_root }}/Contents/Home
if [ ! -e $GRAAL_VM_HOME/bin/native-image ]; then
rm -rf ${{ matrix.graal_install_root }}
mkdir ${{ matrix.graal_install_root }}
curl -fL "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.GRAALVM_VERSION }}/graalvm-ce-java${{ env.JAVA_VERSION }}-darwin-amd64-${{ env.GRAALVM_VERSION }}.tar.gz" \
| tar zxv -C graalvm --strip-components=1 \
--exclude=*/bin/jvisualvm \
--exclude=*/lib/src.zip \
--exclude=*/lib/visualvm
$GRAAL_VM_HOME/bin/gu install native-image
fi
$GRAAL_VM_HOME/bin/native-image --version
echo "JAVA_HOME=$GRAAL_VM_HOME" >> $GITHUB_ENV
echo "$GRAAL_VM_HOME/bin" >> $GITHUB_PATH
;;
windows-amd64)
GRAAL_VM_HOME=${{ matrix.graal_install_root }}
if [ ! -e $GRAAL_VM_HOME/bin/native-image ]; then
rm -rf $GRAAL_VM_HOME
echo "Downloading GraalVM..."
curl -fL -o graalvm.zip https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${{ env.GRAALVM_VERSION }}/graalvm-ce-java${{ env.JAVA_VERSION }}-windows-amd64-${{ env.GRAALVM_VERSION }}.zip
echo "Extracting GraalVM..."
7z x graalvm.zip
mv graalvm-ce-* $GRAAL_VM_HOME
$GRAAL_VM_HOME/bin/gu.cmd install native-image
fi
$GRAAL_VM_HOME/bin/native-image.cmd --version
echo "JAVA_HOME=$(cygpath -wa $GRAAL_VM_HOME)" >> $GITHUB_ENV
echo "$(cygpath -wa $GRAAL_VM_HOME/bin)" >> $GITHUB_PATH
;;
esac
- name: "Install: binutils (strip)"
run: |
case "${{ matrix.arch }}" in
linux-amd64)
sudo apt-get --no-install-recommends install -y binutils
;;
linux-arm64)
docker exec --tty arm64 \
sudo apt-get --no-install-recommends install -y binutils
;;
esac
- name: "Install: UPX"
run: |
case "${{ matrix.arch }}" in
linux-amd64)
mkdir /opt/upx
upx_download_url=$(curl -fsSL -H "Authorization: token ${{ github.token }}" https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep amd64_linux.tar.xz | cut "-d\"" -f4)
echo "Downloading [$upx_download_url]..."
curl -fL $upx_download_url | tar Jxv -C /opt/upx --strip-components=1
;;
linux-arm64)
mkdir /opt/upx
upx_download_url=$(curl -fsSL -H "Authorization: token ${{ github.token }}" https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep arm64_linux.tar.xz | cut "-d\"" -f4)
echo "Downloading [$upx_download_url]..."
curl -fL $upx_download_url | tar Jxv -C /opt/upx --strip-components=1
;;
darwin-amd64)
brew install upx
;;
windows-amd64)
# first choco install in a build takes 3+ minutes
# choco install -y upx
upx_download_url=$(curl -fsSL -H "Authorization: token ${{ github.token }}" https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep win64.zip | cut "-d\"" -f4)
echo "Downloading [$upx_download_url]..."
curl -fL -o /tmp/upx.zip $upx_download_url
echo "Extracting upx.zip..."
mkdir /tmp/upx
7z e /tmp/upx.zip -o/tmp/upx *.exe -r
echo "$(cygpath -wa /tmp/upx)" >> $GITHUB_PATH
;;
esac
- name: "Get: maven-build-artifacts"
uses: actions/download-artifact@v4
with:
name: maven-build-artifacts
- name: Enable Developer Command Prompt
if: matrix.arch == 'windows-amd64'
# makes cl.exe available on PATH
# https://github.com/marketplace/actions/enable-developer-command-prompt
uses: ilammy/msvc-dev-cmd@v1
- name: "Collect metadata"
run: |
set -eu
# https://www.graalvm.org/22.3/reference-manual/native-image/metadata/AutomaticMetadataCollection/#tracing-agent
cat <<'EOF' > collect_metadata.sh
set -eu
rm -rf native-image-metadata
function traceCall() {
echo "Tracing copycat ${@}..."
java -agentlib:native-image-agent=config-merge-dir=native-image-metadata/META-INF/native-image/ -jar copycat-fat.jar ${@} >/dev/null
}
rm -rf workdir
mkdir -p workdir/source/dir1
touch workdir/source/dir1/file1
traceCall sync workdir/source workdir/target
traceCall --help
traceCall --help
traceCall --version
traceCall sync --help
traceCall watch --help
EOF
case "${{ matrix.arch }}" in
linux-arm64)
docker exec --tty -w /workspace arm64 /bin/bash -c "
export JAVA_HOME=${{ matrix.graal_install_root }}
export PATH=${{ matrix.graal_install_root }}/bin:\$PATH
bash collect_metadata.sh
"
;;
*)
bash collect_metadata.sh
;;
esac
- name: "Build executable"
run: |
case "${{ matrix.arch }}" in
linux-amd64)
/usr/bin/gcc -v
native-image \
-H:+StaticExecutableWithDynamicLibC `# https://www.graalvm.org/21.3/reference-manual/native-image/StaticImages/#build-mostly-static-native-image` \
--report-unsupported-elements-at-runtime `# avoid: Unsupported type sun.awt.X11.XBaseWindow is reachable` \
${{ env.NATIVE_IMAGE_ARGS }}
;;
linux-arm64)
docker exec --tty -w /workspace arm64 /bin/bash -c "
export JAVA_HOME=${{ matrix.graal_install_root }}
export PATH=${{ matrix.graal_install_root }}/bin:\$PATH
/usr/bin/gcc -v
native-image \
-H:+StaticExecutableWithDynamicLibC `# https://www.graalvm.org/21.3/reference-manual/native-image/StaticImages/#build-mostly-static-native-image` \
--report-unsupported-elements-at-runtime `# avoid: Unsupported type sun.awt.X11.XBaseWindow is reachable` \
-Djdk.lang.Process.launchMechanism=vfork `# https://github.com/oracle/graal/issues/4143`\
-H:-CheckToolchain `#https://github.com/oracle/graal/issues/4143`\
${{ env.NATIVE_IMAGE_ARGS }}
"
;;
darwin-amd64)
# not using "--static" because of error: DARWIN does not support building static executable images.
native-image ${{ env.NATIVE_IMAGE_ARGS }}
;;
windows-amd64)
# https://www.graalvm.org/21.3/reference-manual/native-image/StaticImages/
native-image.cmd --static ${{ env.NATIVE_IMAGE_ARGS }}
;;
esac
- name: "Test executable"
run: |
case "${{ matrix.arch }}" in
linux-arm64)
docker exec --tty -w /workspace arm64 /bin/bash -c "
chmod u+x copycat-${{ matrix.arch }} &&
./copycat-${{ matrix.arch }} --version &&
./copycat-${{ matrix.arch }} --help &&
./copycat-${{ matrix.arch }} sync --help &&
./copycat-${{ matrix.arch }} watch --help
"
;;
*)
chmod u+x copycat-${{ matrix.arch }}
./copycat-${{ matrix.arch }} --version
./copycat-${{ matrix.arch }} --help
./copycat-${{ matrix.arch }} sync --help
./copycat-${{ matrix.arch }} --help
;;
esac
- name: "Compress executable"
run: |
set -eu
case "${{ matrix.arch }}" in
linux-amd64)
strip --strip-unneeded copycat-${{ matrix.arch }}
/opt/upx/upx -v -9 --no-progress copycat-${{ matrix.arch }}
;;
linux-arm64)
docker exec --tty -w /workspace arm64 /bin/bash -c "
strip --strip-unneeded copycat-${{ matrix.arch }} &&
/opt/upx/upx -v -9 --no-progress copycat-${{ matrix.arch }}
"
;;
darwin-amd64)
# https://www.unix.com/man-page/osx/1/strip/
strip copycat-${{ matrix.arch }}
upx -v -9 --no-progress copycat-${{ matrix.arch }}
;;
windows-amd64)
cp copycat-${{ matrix.arch }}.exe copycat-${{ matrix.arch }}-uncompressed.exe
upx -v -9 --no-progress copycat-${{ matrix.arch }}.exe
;;
esac
- name: "Test compressed executable"
run: |
case "${{ matrix.arch }}" in
linux-arm64)
docker exec --tty -w /workspace arm64 /bin/bash -c "
chmod u+x copycat-${{ matrix.arch }} &&
./copycat-${{ matrix.arch }} --version &&
./copycat-${{ matrix.arch }} --help &&
./copycat-${{ matrix.arch }} sync --help &&
./copycat-${{ matrix.arch }} watch --help
"
;;
*)
chmod u+x copycat-${{ matrix.arch }}
./copycat-${{ matrix.arch }} --version
./copycat-${{ matrix.arch }} --help
./copycat-${{ matrix.arch }} sync --help
./copycat-${{ matrix.arch }} --help
;;
esac
- name: "Share: native binary"
uses: actions/upload-artifact@v4
if: matrix.arch != 'windows-amd64'
with:
name: binaries-${{ matrix.arch }}
path: copycat-${{ matrix.arch }}
- name: "Share: native binary"
uses: actions/upload-artifact@v4
if: matrix.arch == 'windows-amd64'
with:
name: binaries-${{ matrix.arch }}
path: |
copycat-${{ matrix.arch }}.exe
copycat-${{ matrix.arch }}-uncompressed.exe
###########################################################
publish-release:
###########################################################
runs-on: ubuntu-latest
needs:
- native-image
if: ${{ github.ref_name == 'main' && !github.event.act }}
steps:
- name: "SCM Checkout"
# only required by "hub release create" to prevent "fatal: Not a git repository"
uses: actions/checkout@v4 #https://github.com/actions/checkout
- name: "Get: all build artifacts"
uses: actions/download-artifact@v4
with:
path: artifacts
merge-multiple: true
- name: "Delete previous '${{ env.RELEASE_NAME }}' release"
if: env.RELEASE_NAME == 'snapshot'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://cli.github.com/manual/gh_release_delete
run: |
GH_DEBUG=1 gh release delete "$RELEASE_NAME" --yes --cleanup-tag || true
- name: "Create '${{ env.RELEASE_NAME }}' Release"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# https://cli.github.com/manual/gh_release_create
run: |
GH_DEBUG=1 gh release create "$RELEASE_NAME" \
--latest \
--notes "${{ github.event.head_commit.message }}" \
--target "${{ github.sha }}" \
artifacts/copycat-fat.jar#copycat-${{ env.RELEASE_NAME }}-fat.jar \
artifacts/bash/bashcompletion.sh#copycat-${{ env.RELEASE_NAME }}-bashcompletion.sh \
artifacts/copycat-linux-amd64#copycat-${{ env.RELEASE_NAME }}-linux-amd64 \
artifacts/copycat-linux-arm64#copycat-${{ env.RELEASE_NAME }}-linux-arm64 \
artifacts/copycat-darwin-amd64#copycat-${{ env.RELEASE_NAME }}-darwin-amd64 \
artifacts/copycat-windows-amd64.exe#copycat-${{ env.RELEASE_NAME }}-windows-amd64.exe \
artifacts/copycat-windows-amd64-uncompressed.exe#copycat-${{ env.RELEASE_NAME }}-windows-amd64-uncompressed.exe
- name: "Delete intermediate build artifacts"
uses: geekyeggo/delete-artifact@v5 # https://github.com/GeekyEggo/delete-artifact/
with:
name: "*"
failOnError: false