Fully rewrite the golden tests runner using Alcotest (#169)

colis-anr · May 12, 2023 · d678f96 · d678f96
2 parents 015a3a0 + f8cf93c
commit d678f96
Show file tree

Hide file tree

Showing 11 changed files with 206 additions and 237 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,15 +25,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {os: ubuntu, ocaml: 4.04.x}
-          - {os: ubuntu, ocaml: 4.05.x}
-          - {os: ubuntu, ocaml: 4.06.x}
-          - {os: ubuntu, ocaml: 4.07.x}
-          - {os: ubuntu, ocaml: 4.08.x}  ## oldest version with 0install
-          - {os: ubuntu, ocaml: 4.08.x, lower-bounds: lower-bounds, doc: no-doc}
-          - {os: ubuntu, ocaml: 4.09.x}
-          - {os: ubuntu, ocaml: 4.10.x}
-          - {os: ubuntu, ocaml: 4.11.x}
+          - {os: ubuntu, ocaml: 4.11.x}  ## in Debian stable (as of 2023-05-11)
+          - {os: ubuntu, ocaml: 4.11.x, lower-bounds: lower-bounds, doc: no-doc}
           - {os: ubuntu, ocaml: 4.12.x}
           - {os: ubuntu, ocaml: 4.13.x}
           # {os: ubuntu, ocaml: 4.14.x}  ## main supported version
@@ -103,11 +96,14 @@ jobs:
         run: |
           ## Based on: https://sim642.eu/blog/2022/03/13/ocaml-dependencies-lower-bounds-ci/
           ## Note the current version of `ocaml-base-compiler`
-          obc=ocaml-base-compiler.$(opam info --color=never --field=version ocaml-base-compiler)
+          obc=ocaml-base-compiler.$(opam exec -- ocamlc --version)
           ## Ask `opam-0install` to get us the oldest possible dependencies for
           ## Morbig, while keeping the exact same version of
           ## `ocaml-base-compiler`.
-          opam exec -- opam-0install --prefer-oldest $OPAM_FLAGS morbig $obc > lower-bounds
+          opam exec -- \
+              opam-0install --prefer-oldest $OPAM_FLAGS morbig.dev $obc \
+                  | sed 's|morbig.dev||' \
+                  > lower-bounds
           ## Log what we are aiming for
           printf '\e[36m<><>\e[0m Packages to install \e[36m><><><><><><><><><><><><><><><><><><><><><><><><><><><>\e[0m\n'
           printf '`opam-0install` suggests installing the following packages:\n'
@@ -138,8 +134,17 @@ jobs:
         run: opam exec -- make examples
 
       - name: Try uninstalling
+        if: matrix.lower-bounds != 'lower-bounds'
         run: opam exec -- make uninstall
 
+      - name: (Lower bounds) Try uninstalling
+        ## NOTE: Old versions of Dune (< 3.1.0) fail when they cannot remove
+        ## `bin` or `lib/morbig` in the prefix. But `bin` is very rarely empty.
+        ## Until it is reasonable to require Dune >= 3.1.0, we keep this special
+        ## case for the part of the CI interested in lower bounds.
+        if: matrix.lower-bounds == 'lower-bounds'
+        run: opam exec -- make uninstall || true
+
       - name: Try cleaning
         run: opam exec -- make clean
 
@@ -180,12 +185,16 @@ jobs:
         run: make
 
       ## No odoc in Debian stable, sadly, so no documentation!
+      ## REVIEW: Ubuntu 22.10 does have a package `ocaml-odoc` though.
 
       # - name: Try building documentation
       #   run: make doc
 
-      - name: Run tests
-        run: make check
+      ## Ubuntu 22.10 only has Alcotest 1.6 which is not recent enough for our
+      ## tests. Therefore, tests are disabled on this platform.
+
+      # - name: Run tests
+      #   run: make check
 
       - name: Try installing
         run: sudo make install
@@ -196,7 +205,7 @@ jobs:
       - name: Try uninstalling
         run: sudo make uninstall
 
-      ## Not working at the moment (2023-03-31), `dune clean` complains about:
+      ## Not working (as of 2023-03-31), `dune clean` complains about:
       ##
       ##     Error: unlink: _build/default/.dune/configurator: Permission denied
       ##
@@ -215,7 +224,7 @@ jobs:
         include:
           - {tag: alpine, deploy-image: deploy-image}
           - {tag: archlinux}
-          - {tag: centos}
+          - {tag: centos, tests: no-tests}  ## only has Alcotest 1.6 (as of 2023-05-11)
           - {tag: debian}
           - {tag: debian-testing}
           - {tag: debian-unstable}
@@ -304,6 +313,10 @@ jobs:
         run: |
           nix develop .#${{ matrix.package }} --command make
 
+      - name: Build documentation in development Shell
+        run: |
+          nix develop .#${{ matrix.package }} --command make doc
+
       - name: Run tests in development Shell
         run: |
           nix develop .#${{ matrix.package }} --command make check

diff --git a/.nix/with-nixpkgs.nix b/.nix/with-nixpkgs.nix
@@ -22,6 +22,12 @@
           ppx_deriving_yojson
           visitors
           yojson
+
+          ## doc inputs
+          odoc
+
+          ## check inputs
+          alcotest
         ];
 
         buildPhase = ''

diff --git a/.nix/with-opam-nix.nix b/.nix/with-opam-nix.nix
@@ -3,9 +3,10 @@
     ## NOTE: The use of `./..` matters because the path is taken as relative to
     ## the current file, and therefore to `.nix/`.
     let
-      scope =
-        inputs'.opam-nix.lib.buildOpamProject { inherit pkgs; } "morbig" ./.. {
-          ocaml-base-compiler = "*";
-        };
+      scope = inputs'.opam-nix.lib.buildOpamProject {
+        inherit pkgs;
+        resolveArgs.with-doc = true;
+        resolveArgs.with-test = true;
+      } "morbig" ./.. { ocaml-base-compiler = "*"; };
     in { packages.with-opam-nix = scope.morbig // { inherit scope; }; };
 }
diff --git a/Dockerfile b/Dockerfile
@@ -30,7 +30,7 @@ RUN if $doc; then opam depext -i odoc; fi
 ## `--build-arg tests=true`.
 
 ARG tests=false
-RUN if $tests; then opam depext -i conf-jq; fi
+RUN if $tests; then opam depext -i conf-jq alcotest; fi
 
 ## Work in /home/opam/morbig, copy all the file there with the right
 ## owner and group.

diff --git a/Makefile b/Makefile
@@ -32,11 +32,8 @@ install: build
 uninstall: build
 	dune uninstall $(INSTALL_ARGS)
 
-check: build
-	@ output=$$(./tests/golden/run 2>&1) ;     \
-	  status=$$? ;                      \
-	  echo "$$output" | tee tests.org ; \
-	  exit $$status
+check:
+	dune test
 
 examples:
 	find examples -name 'Makefile' | \
@@ -47,8 +44,6 @@ clean:
 	rm -f src/c/dune
 	dune clean
 	rm -f bin lib doc
-	rm -f tests.org
-	tests/golden/run clean || true
 
 PACKAGE=$(shell echo morbig-`cat VERSION`)
 

diff --git a/dune-project b/dune-project
@@ -25,6 +25,7 @@
 
  (depends
   (conf-jq              :with-test)
+  (alcotest             (and :with-test (>= "1.7.0")))
   (menhir               (>= "20200211"))
   (ocaml                (>= "4.04"))
   (odoc                 :with-doc)

diff --git a/flake.lock b/flake.lock
diff --git a/morbig.opam b/morbig.opam
@@ -21,6 +21,7 @@ bug-reports: "https://github.com/colis-anr/morbig/issues"
 depends: [
   "dune" {>= "2.5"}
   "conf-jq" {with-test}
+  "alcotest" {with-test & >= "1.7.0"}
   "menhir" {>= "20200211"}
   "ocaml" {>= "4.04"}
   "odoc" {with-doc}

diff --git a/tests/golden/dune b/tests/golden/dune
@@ -0,0 +1,7 @@
+(test
+ (name run)
+ (libraries alcotest)
+ (deps
+  (source_tree .))
+ (action
+  (run %{test} --morbig %{bin:morbig})))