Cleanup artifacts before upload

We remove the files which were already present when the job started.

This means we need all the transitive deps in `need`

.gitlab-ci.yml

@@ -60,6 +60,7 @@ before_script:
   - opam config list
   - dune printenv --root .
   - dev/tools/check-cachekey.sh
+  - dev/tools/list-potential-artifacts.sh > downloaded_artifacts.txt
   - dev/ci/gitlab-section.sh end before_script
 
 # Regular "release" build of Coq, with final installed layout
@@ -190,6 +191,9 @@ before_script:
     # set CI_TARGETS from job name if not already provided, then print
     - echo CI_TARGETS = ${CI_TARGETS:=${CI_JOB_NAME#*:ci-}}
     - for target in $CI_TARGETS; do dev/ci/ci-wrapper.sh "$target"; done
+  after_script:
+    - dev/tools/list-potential-artifacts.sh > available_artifacts.txt
+    - dev/tools/cleanup-artifacts downloaded_artifacts.txt available_artifacts.txt
   artifacts:
     name: "$CI_JOB_NAME"
     paths:
@@ -620,6 +624,11 @@ library:ci-bedrock2_examples:
   variables:
     NJOBS: "1"
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-coqutil
+  - library:ci-kami
+  - library:ci-riscv_coq
   - library:ci-bedrock2
   stage: build-3+
   timeout: 2h
@@ -665,6 +674,9 @@ library:ci-coqtail:
 library:ci-coquelicot:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
   - library:ci-mathcomp
   stage: build-3+
 
@@ -691,6 +703,9 @@ library:ci-ext_lib:
 library:ci-fcsl_pcm:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
   - library:ci-mathcomp
   stage: build-2
 
@@ -699,6 +714,12 @@ library:ci-fiat_crypto:
   variables:
     COQEXTRAFLAGS: "-async-proofs-tac-j 0"
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-coqutil
+  - library:ci-kami
+  - library:ci-riscv_coq
+  - library:ci-bedrock2
   - library:ci-coqprime
   - library:ci-rupicola
   - plugin:ci-rewriter
@@ -717,6 +738,15 @@ library:ci-fiat_crypto_legacy:
 library:ci-fiat_crypto_ocaml:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-coqutil
+  - library:ci-kami
+  - library:ci-riscv_coq
+  - library:ci-bedrock2
+  - library:ci-coqprime
+  - library:ci-rupicola
+  - plugin:ci-rewriter
   - library:ci-fiat_crypto
   stage: build-3+
   artifacts:
@@ -793,12 +823,18 @@ library:ci-mczify:
 library:ci-finmap:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
   - library:ci-mathcomp
   stage: build-3+
 
 library:ci-bigenough:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
   - library:ci-mathcomp
   stage: build-3+
 
@@ -829,6 +865,10 @@ library:ci-itree:
 library:ci-itree_io:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-ext_lib
+  - library:ci-paco
   - library:ci-simple_io
   - library:ci-itree
   stage: build-3+
@@ -901,13 +941,25 @@ library:ci-mathcomp_word:
 library:ci-jasmin:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
+  - library:ci-mathcomp
   - library:ci-mathcomp_word
   stage: build-3+
 
 library:ci-http:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
+  - library:ci-mathcomp
   - library:ci-menhir
+  - library:ci-ext_lib
+  - library:ci-simple_io
+  - library:ci-paco
+  - library:ci-itree
   - library:ci-itree_io
   - plugin:ci-quickchick
   stage: build-3+
@@ -1036,12 +1088,19 @@ plugin:ci-quickchick:
   extends: .ci-template-flambda
   needs:
   - library:ci-simple_io
+  - plugin:ci-elpi_hb
   - library:ci-mathcomp
   stage: build-3+
 
 plugin:ci-quickchick_test:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-ext_lib
+  - library:ci-simple_io
+  - plugin:ci-elpi_hb
+  - library:ci-mathcomp
   - plugin:ci-quickchick
   stage: build-3+
 
@@ -1069,6 +1128,11 @@ library:ci-riscv_coq:
 library:ci-rupicola:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - library:ci-coqutil
+  - library:ci-kami
+  - library:ci-riscv_coq
   - library:ci-bedrock2
   stage: build-3+
 
@@ -1088,6 +1152,9 @@ plugin:ci-smtcoq:
 plugin:ci-smtcoq_trakt:
   extends: .ci-template-flambda
   needs:
+  - build:edge+flambda
+  - library:ci-stdlib+flambda
+  - plugin:ci-elpi_hb
   - library:ci-trakt
   stage: build-3+
 

dev/tools/cleanup-artifacts.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+before=$1
+after=$2
+
+# https://unix.stackexchange.com/questions/418429/find-intersection-of-lines-in-two-files
+awk 'BEGIN{while( (getline k < "'"$before"'")>0 ){a[k]}} $0 in a' "$after" |
+  xargs rm

dev/tools/list-potential-artifacts.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ -d _install_ci ]; then
+  find _install_ci -type f | sort
+fi