Merge pull request #8715 from Agoric/mfig-build-uses-hashes

build(agd): rely on checksums, not timestamps

.github/actions/restore-node/action.yml

@@ -72,13 +72,17 @@ runs:
     - name: Move Endo checkout outside the working directory
       id: endo-sha
       run: |-
-        set -e
+        set -ex
         if test -e ./replacement-endo; then
           mv ./replacement-endo ~/endo
-          echo "sha=$(cd ~/endo && git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          sha=$(cd ~/endo && git rev-parse HEAD)
         else
-          echo "sha=NOPE" >> $GITHUB_OUTPUT
+          sha=NOPE
         fi
+        echo "sha=$sha" >> $GITHUB_OUTPUT
+        cd "${{ inputs.path }}"
+        echo "$sha" > endo-sha.txt
+        git add endo-sha.txt
       shell: bash
     - name: merge endo integration branch
       id: endo-integration-merge
@@ -99,7 +103,9 @@ runs:
       with:
         node-version: ${{ inputs.node-version }}
         cache: yarn
-        cache-dependency-path: ${{ inputs.path }}/yarn.lock
+        cache-dependency-path: |
+          ${{ inputs.path }}/yarn.lock
+          ${{ inputs.path }}/endo-sha.txt
     - uses: kenchan0130/actions-system-info@master
       id: system-info
     - name: restore built files
@@ -113,7 +119,7 @@ runs:
     - name: yarn install
       working-directory: ${{ inputs.path }}
       run: |-
-        set -e
+        set -ex
         if test "${{ inputs.xsnap-random-init }}" != 0 && test -d /etc/apt; then
           # Need libbsd-dev, as it's referenced in xsnapPlatform.h
           sudo apt-get update
@@ -124,8 +130,45 @@ runs:
           scripts/get-packed-versions.sh ~/endo | scripts/resolve-versions.sh
         fi
         yarn install
-        mkdir -p node_modules/.cache/agoric
-        date > node_modules/.cache/agoric/yarn-installed
+
+        if ! cmp -s <(git cat-file blob HEAD:package.json) package.json; then
+          # In the event that the package.json has been modified by Endo
+          # replacements, we need to have a yarn-installed.sum that matches
+          # the unmodified package.json.  As long as we don't explicitly `yarn
+          # install` anywhere other than in this action and in the bin/agd
+          # script, we should be able to reuse even Endo-overridden built caches
+          # successfully.
+          git checkout HEAD -- package.json
+        fi
+
+        function diffsha1() {
+          stamp=$1
+          shift
+          find ${1+"$@"} -exec sha1sum {} \; | sort +1 > "$stamp.new" || true
+          if test ! -s "$stamp.new"; then
+            echo "No new dependencies found for $stamp" 1>&2
+            return 0
+          fi
+          diff -u "$stamp" "$stamp.new" || return 1
+          return 0
+        }
+
+        # Find the current list of package.jsons.
+        files=( package.json )
+        while IFS= read -r line; do
+          files+=( "$line" )
+        done < <(yarn -s workspaces info |
+          sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }')
+
+        STAMPS=node_modules/.cache/agoric
+
+        mkdir -p "$STAMPS"
+        sum="$STAMPS/yarn-installed.sum"
+        diffsha1 "$sum" "${files[@]}" || {
+          mv "$sum.new" "$sum"
+        }
+        rm -f "$sum.new"
+
         if test -e ~/endo; then
           # Stage the redirected `yarn install` consequences.
           git add package.json yarn.lock
@@ -136,7 +179,7 @@ runs:
     - name: yarn build
       working-directory: ${{ inputs.path }}
       run: |-
-        set -e
+        set -ex
         yarn build
         mkdir -p node_modules/.cache/agoric
         date > node_modules/.cache/agoric/yarn-built

bin/agd

@@ -19,6 +19,18 @@ function fatal() {
   exit 1
 }
 
+function diffsha1() {
+  stamp=$1
+  shift
+  find ${1+"$@"} -exec sha1sum {} \; | sort +1 > "$stamp.new" || true
+  if test ! -s "$stamp.new"; then
+    echo "No new dependencies found for $stamp" 1>&2
+    return 0
+  fi
+  diff -u "$stamp" "$stamp.new" || return 1
+  return 0
+}
+
 GVM_URL=${GVM_URL-https://github.com/devnw/gvm/releases/download/latest/gvm}
 NVM_GIT_REPO=${NVM_GIT_REPO-https://github.com/nvm-sh/nvm.git}
 
@@ -31,7 +43,8 @@ thisdir=$(cd "$(dirname -- "$real0")" >/dev/null && pwd -P)
 source "$thisdir/../repoconfig.sh"
 
 if test "${1-''}" = build; then
-  BUILD_ONLY=true
+  do_not_build=false
+  only_build=true
   # Enable shell debugging.
   set -x
   case "${2-''}" in
@@ -40,14 +53,22 @@ if test "${1-''}" = build; then
     ;;
   esac
 else
-  BUILD_ONLY=false
+  case ${NO_BUILD-false} in
+  true | yes | y | 1)
+    do_not_build=true
+    ;;
+  *)
+    do_not_build=false
+    ;;
+  esac
+  only_build=false
 fi
 
-NEED_NODEJS=$BUILD_ONLY
+need_nodejs=$only_build
 case $@ in
-*" start" | *" start "*)
+start | *" start" | *" start "*)
   # We're starting the daemon, so we need Node.js.
-  NEED_NODEJS=true
+  need_nodejs=true
   ;;
 esac
 
@@ -61,7 +82,7 @@ else
   SKIP_DOWNLOAD=true
 fi
 
-if $NEED_NODEJS; then
+if $need_nodejs; then
   # We need to get node at the script top level because it's used by the daemon
   # as well.
   case $(node --version 2>/dev/null) in
@@ -93,12 +114,13 @@ fi
 (
   # Send the build output to stderr unless we're only building.  This prevents
   # the daemon's stdout from being polluted with build output.
-  $BUILD_ONLY || exec 1>&2
+  $only_build || exec 1>&2
 
   cd "$thisdir/.."
+  test -d "$STAMPS" || echo "Creating $STAMPS" 1>&2
   mkdir -p "$STAMPS"
 
-  if $NEED_NODEJS; then
+  if $need_nodejs; then
     lazy_yarn() {
       yarn --version 2>/dev/null 1>&2 || {
         # Auto-download the Yarn version we need, if allowed.
@@ -109,14 +131,8 @@ fi
       yarn "$@"
     }
 
-    # Check if any package.json is newer than the installation stamp.
-    stamp=$STAMPS/yarn-installed
-    if test -e "$stamp"; then
-      print=( -newer "$stamp" )
-    else
-      print=()
-    fi
-    print+=( -print )
+    # Check if any package.json sums are different.
+    sum=$STAMPS/yarn-installed.sum
 
     # Find the current list of package.jsons.
     files=( package.json )
@@ -125,13 +141,12 @@ fi
     done < <(lazy_yarn -s workspaces info |
       sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }')
 
-    src=$(find "${files[@]}" "${print[@]}" | head -1 || true)
-    test -z "$src" || {
-      echo "At least $src is newer than node_modules"
-      rm -f "$STAMPS/yarn-built"
+    diffsha1 "$sum" "${files[@]}" || { echo "$sum has changed"; $do_not_build; } || {
+      rm -f "$sum" "$STAMPS/yarn-built"
       lazy_yarn install
-      date > "$stamp"
+      mv "$sum.new" "$sum"
     }
+    rm -f "$sum.new"
 
     stamp=$STAMPS/yarn-built
     test -e "$stamp" || {
@@ -141,17 +156,12 @@ fi
     }
   fi
 
-  stamp=$GOLANG_DAEMON
-  if test -e "$stamp"; then
-    print=( -newer "$stamp" )
-  else
-    print=()
-  fi
-  print+=( -print )
-  src=$(find "$GOLANG_DIR" \( ! -name '*_test.go' -name '*.go' -o -name '*.cc' -o -name 'go.*' \) "${print[@]}" | head -1 || true)
-  test -z "$src" || {
-    echo "At least $src is newer than $stamp"
-
+  sum=$STAMPS/golang-built.sum
+  diffsha1 "$sum" "$GOLANG_DIR" \( \
+    ! -name '*_test.go' -name '*.go' \
+    -o -name '*.cc' \
+    -o -name 'go.*' \) || { echo "$GOLANG_DIR $sum has changed"; $do_not_build; } || {
+    rm -f "$sum"
     (
       # Run this build in another subshell in case we had to modify the path.
       case $(go version 2>/dev/null) in
@@ -182,6 +192,8 @@ fi
       make
     )
   }
+  test -f "$sum" || mv -f "$sum.new" "$sum"
+  rm -f "$sum.new"
 )
 
 # the xsnap binary lives in a platform-specific directory
@@ -196,7 +208,7 @@ esac
 xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n)
 [[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}"
 
-if $BUILD_ONLY; then
+if $only_build; then
   echo "Build complete." 1>&2
   exit 0
 fi

golang/cosmos/package.json

@@ -5,7 +5,7 @@
   "parsers": {
     "js": "mjs"
   },
-  "main": "src/index.cjs",
+  "main": "index.cjs",
   "engines": {
     "node": ">=14.15.0"
   },

packages/deployment/Dockerfile.sdk

@@ -63,6 +63,9 @@ RUN \
 
 # Remove dev dependencies.
 RUN rm -rf packages/xsnap/moddable packages/xsnap/xsnap-native/build/tmp
+RUN mkdir golang/tmp && \
+  mv golang/cosmos/package.json golang/cosmos/build golang/cosmos/index.cjs golang/tmp/ && \
+  rm -rf golang/cosmos && mv golang/tmp golang/cosmos
 # FIXME: This causes bundling differences.  https://github.com/endojs/endo/issues/919
 # RUN yarn install --frozen-lockfile --production --network-timeout 100000
 

packages/deployment/ansible/config.yml

@@ -7,7 +7,6 @@
   vars:
     - service: ag-chain-cosmos
     - APPDIR: "{{lookup('pipe', 'pwd')}}/../.."
-    - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd"
   roles:
     - stop
     - copy

packages/deployment/ansible/copy.yml

@@ -7,6 +7,5 @@
   vars:
     - service: ag-chain-cosmos
     - APPDIR: "{{lookup('pipe', 'pwd')}}/../.."
-    - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd"
   roles:
     - copy

packages/deployment/ansible/prepare-cosmos.yml

@@ -7,7 +7,6 @@
   vars:
     - service: ag-chain-cosmos
     - CHAIN_NAME: "{{ lookup('file', SETUP_HOME + '/' + service + '/chain-name.txt') }}"
-    - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd"
     - APPDIR: "{{lookup('pipe', 'pwd')}}/../.."
   roles:
     - copy

packages/deployment/ansible/roles/copy/tasks/main.yml

@@ -1,12 +1,4 @@
 ---
-- name: Synchronize helper
-  synchronize:
-    src: '{{HELPER_BINARY}}'
-    dest: /usr/local/bin/
-    mode: push
-    use_ssh_args: true
-  when: HELPER_BINARY is defined
-
 - name: 'Synchronize Agoric SDK {{ AGORIC_SDK }}'
   synchronize:
     src: '{{ AGORIC_SDK }}/'
@@ -19,6 +11,12 @@
     mode: push
     use_ssh_args: true
 
+- name: 'Symlink agd'
+  file:
+    src: '/usr/src/agoric-sdk/bin/agd'
+    dest: '/usr/local/bin/agd'
+    state: link
+
 - name: 'Symlink ag-chain-cosmos'
   file:
     src: '/usr/src/agoric-sdk/packages/cosmic-swingset/bin/ag-chain-cosmos'

packages/deployment/ansible/roles/cosmos-genesis/files/faucet-helper.sh

@@ -30,7 +30,7 @@ show-faucet-address)
 esac
 
 networkName=$(basename "$thisdir")
-if netconfig=$(curl "https://$networkName.agoric.net/network-config"); then
+if netconfig=$(curl -s "https://$networkName.agoric.net/network-config"); then
   chainName=$(echo "$netconfig" | jq -r .chainName)
   read -r -a origRpcAddrs <<<"$(echo "$netconfig" | jq -r .rpcAddrs[])"
 else

packages/deployment/scripts/integration-test.sh

@@ -29,6 +29,8 @@ else
   echo "Running chain without loadgen" >&2
 fi
 
+set -x
+
 SOLO_ADDR=
 VAT_CONFIG=
 RESULTSDIR=${RESULTSDIR-"$NETWORK_NAME/results"}
@@ -69,7 +71,7 @@ then
   cp ag-chain-cosmos/data/genesis.json "$RESULTSDIR/genesis.json"
   cp "$AG_SETUP_COSMOS_HOME/ag-chain-cosmos/data/genesis.json" "$RESULTSDIR/genesis.json"
   cd "$LOADGEN"
-  SOLO_COINS=40000000000uist PATH="$thisdir/../bin:$PATH" \
+  SOLO_COINS=40000000000uist PATH="$thisdir/../bin:$SDK_SRC/bin:$PATH" \
     "$AG_SETUP_COSMOS_HOME/faucet-helper.sh" add-egress loadgen "$SOLO_ADDR"
   SLOGSENDER=@agoric/telemetry/src/otel-trace.js SOLO_SLOGSENDER="" \
   SLOGSENDER_FAIL_ON_ERROR=1 SLOGSENDER_AGENT=process \

scripts/registry.sh

@@ -64,12 +64,16 @@ publish() {
     yarn build
     git commit --allow-empty -am "chore: prepare for publishing"
 
+    yarn lerna run build:types
+
     # Publish the packages to our local service.
     # without concurrency until https://github.com/Agoric/agoric-sdk/issues/8091
     yarn lerna publish --concurrency 1 prerelease --exact \
       --dist-tag="$DISTTAG" --preid=dev \
       --no-push --no-git-reset --no-git-tag-version --no-verify-access --yes
 
+    yarn lerna run clean:types
+
     # Change any version prefices to an exact match, and merge our versions.
     VERSIONSHASH=$(jq --argfile versions <(popd >/dev/null && git cat-file blob "$VERSIONSHASH") \
       '[to_entries[] | { key: .key, value: (.value | sub("^[~^]"; "")) }]

scripts/smoketest-binaries.sh

@@ -4,7 +4,7 @@
 
 binaries=( "agd version" "agoric -V" "ag-chain-cosmos version" )
 for binary in "${binaries[@]}"; do
-   echo "Checking $binary a"
+   echo "Checking $binary"
    $binary
    ec=$?
    if [[ $ec -ne 0 ]]; then