Comprehensive test suite

.cirrus.yml

@@ -3,21 +3,8 @@ task:
   timeout_in: 5m
   container:
     image: gcc:latest
-  env:
-    ASAN_OPTIONS: detect_leaks=0
-  bootstrap_script:
-    - sh bootstrap.sh
-  test_build_bob_with_bob_script:
-    - $(realpath .bootstrap/bob) build
-  test_run_bob_with_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) run build
-  test_install_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) install
-  test_run_installed_bob_script:
-    - rm -rf .bob
-    - bob build
+  tests_script:
+    - sh tests.sh
   amd64_artifacts:
     path: ".bob/prefix/bin/bob"
 
@@ -26,21 +13,8 @@ task:
   timeout_in: 5m
   arm_container:
     image: gcc:latest
-  env:
-    ASAN_OPTIONS: detect_leaks=0
-  bootstrap_script:
-    - sh bootstrap.sh
-  test_build_bob_with_bob_script:
-    - $(realpath .bootstrap/bob) build
-  test_run_bob_with_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) run build
-  test_install_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) install
-  test_run_installed_bob_script:
-    - rm -rf .bob
-    - bob build
+  tests_script:
+    - sh tests.sh
   arm64_artifacts:
     path: ".bob/prefix/bin/bob"
 
@@ -54,23 +28,7 @@ compute_engine_instance:
 task:
   name: Test Bob with Clang on FreeBSD (x86_64)
   timeout_in: 30m
-  env:
-    ASAN_OPTIONS: detect_leaks=0
-  setup_script:
-    - uname -a
-    - df -h
-  bootstrap_script:
-    - sh bootstrap.sh
-  test_build_bob_with_bob_script:
-    - $(realpath .bootstrap/bob) build
-  test_run_bob_with_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) run build
-  test_install_bob_script:
-    - rm -rf .bob
-    - $(realpath .bootstrap/bob) install
-  test_run_installed_bob_script:
-    - rm -rf .bob
-    - bob build
+  tests_script:
+    - sh tests.sh
   amd64_artifacts:
     path: ".bob/prefix/bin/bob"

.github/workflows/tests.yml

@@ -5,31 +5,15 @@ on: [push, workflow_dispatch]
 jobs:
   run_tests:
     runs-on: ubuntu-latest
-    env:
-      ASAN_OPTIONS: detect_leaks=0
     steps:
       - uses: actions/checkout@v4
       - name: Set up compiler
         uses: egor-tensin/setup-clang@v1
         with:
           version: latest
           platform: x64
-      - name: Bootstrap
-        run: sh bootstrap.sh
-      - name: Test building Bob using Bob
-        run: .bootstrap/bob build
-      - name: Test running the version of Bob built with Bob
-        run: |
-          rm -rf .bob
-          .bootstrap/bob run build
-      - name: Test installing Bob
-        run: |
-          rm -rf .bob
-          sudo -E .bootstrap/bob install
-      - name: Test running the installed version of Bob
-        run: |
-          rm -rf .bob
-          bob build
+      - name: Run tests
+        run: sh tests.sh
       - name: Set up tmate session
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3
@@ -43,26 +27,10 @@ jobs:
       matrix:
         os: [macos-13, macos-latest] # 'macos-13' is Intel, 'macos-latest' is M1.
     runs-on: ${{ matrix.os }}
-    env:
-      ASAN_OPTIONS: detect_leaks=0
     steps:
       - uses: actions/checkout@v4
-      - name: Bootstrap
-        run: sh bootstrap.sh
-      - name: Test building Bob using Bob
-        run: .bootstrap/bob build
-      - name: Test running the version of Bob built with Bob
-        run: |
-          rm -rf .bob
-          .bootstrap/bob run build
-      - name: Test installing Bob
-        run: |
-          rm -rf .bob
-          sudo -E .bootstrap/bob install
-      - name: Test running the installed version of Bob
-        run: |
-          rm -rf .bob
-          bob build
+      - name: Run tests
+        run: sh tests.sh
       - name: Set up tmate session
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3

.gitignore

@@ -10,3 +10,4 @@ compile_commands.json
 .cache
 .bob
 .bootstrap
+.test-out

README.md

@@ -51,3 +51,11 @@ To install a project with Bob, run:
 ```console
 bob install
 ```
+
+## Testing
+
+To run the Bob tests, simply run:
+
+```console
+sh tests.sh
+```

src/class/cc.c

@@ -60,7 +60,7 @@ static void get_include_deps(compile_task_t* task, char* cc) {
 	// -MT "": Exclude the source file from the output.
 	// TODO Can we do this in parallel easily with 'cmd_exec_async'?
 
-	cmd_t __attribute__((cleanup(cmd_free))) cmd;
+	cmd_t __attribute__((cleanup(cmd_free))) cmd = {0};
 	cmd_create(&cmd, cc, "-fdiagnostics-color=always", "-MM", "-MT", "", task->src, NULL);
 	add_flags(&cmd, task);
 

src/class/fs.c

@@ -10,6 +10,7 @@
 #include <errno.h>
 #include <fts.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
 static flamingo_val_t* list_val = NULL;

src/frugal.c

@@ -59,6 +59,9 @@ bool frugal_flags(flamingo_val_t* flags, char* out) {
 
 	fclose(f);
 
+	// Note that we want to compare the order of these flags too, so just compare the strings.
+	// There could be some cases where flipping the order of flags could change the build, which is perhaps rare but let's not try to be too smart here.
+
 	if (strcmp(prev_flag_str, flag_str) != 0) {
 		goto write_out;
 	}

src/install.c

@@ -14,7 +14,7 @@
 #include <libgen.h>
 
 static flamingo_val_t* install_map = NULL;
-static char const* prefix;
+static char const* prefix = NULL;
 
 int setup_install_map(flamingo_t* flamingo, char const* _prefix) {
 	// Find the install map.
@@ -183,12 +183,12 @@ static int install_single(flamingo_val_t* key_val, char* val, bool installing_co
 }
 
 int install_all(char const* _prefix) {
-	assert(_prefix == prefix);
-
 	if (install_map == NULL || prefix == NULL) {
 		return 0;
 	}
 
+	assert(_prefix == prefix);
+
 	for (size_t i = 0; i < install_map->map.count; i++) {
 		flamingo_val_t* const val_val = install_map->map.vals[i];
 		char* const val = strndup(val_val->str.str, val_val->str.size);

src/logging.c

@@ -10,7 +10,6 @@
 
 #include <assert.h>
 #include <err.h>
-#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

src/logging.h

@@ -6,7 +6,6 @@
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdio.h>
-#include <stdlib.h>
 
 // external variables
 
@@ -56,5 +55,5 @@ void log_already_done(char const* cookie, char const* prefix, char const* past);
 #define LOG_FATAL(...) vlog(stderr, "💀 " BOLD PURPLE, __VA_ARGS__)
 #define LOG_ERROR(...) vlog(stderr, "🛑 " BOLD RED, __VA_ARGS__)
 #define LOG_WARN(...) vlog(stderr, "🚸 " REGULAR YELLOW, __VA_ARGS__)
-#define LOG_INFO(...) vlog(stderr, "👷 " REGULAR BLUE, __VA_ARGS__)
-#define LOG_SUCCESS(...) vlog(stderr, "✅ " REGULAR GREEN, __VA_ARGS__)
+#define LOG_INFO(...) vlog(stdout, "👷 " REGULAR BLUE, __VA_ARGS__)
+#define LOG_SUCCESS(...) vlog(stdout, "✅ " REGULAR GREEN, __VA_ARGS__)

tests.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+. tests/common.sh
+
+echo "Bootstrapping Bob..."
+sh bootstrap.sh
+
+echo "Installing Bob..."
+SUDO .bootstrap/bob install > /dev/null
+
+# Actually run the tests.
+
+all_passed=1
+
+for test in $(ls -p tests | grep -v /); do
+	if [ $test = "common.sh" ]; then
+		continue
+	fi
+
+	if [ -d $test ]; then
+		continue
+	fi
+
+	rm -rf .bob $TEST_OUT
+	mkdir -p $TEST_OUT
+
+	echo -n "Running test $test... "
+	sh tests/$test > /dev/null
+
+	if [ $? = 0 ]; then
+		echo "PASSED"
+	else
+		echo "FAILED"
+		all_passed=0
+	fi
+done
+
+if [ $all_passed = 0 ]; then
+	echo "TESTS FAILED!" >&2
+	exit 1
+else
+	echo "ALL TESTS PASSED!"
+fi

tests/bob.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+
+. tests/common.sh
+
+# Test building Bob itself.
+
+bob run -o $TEST_OUT/test-bob build
+bob run -o $TEST_OUT/test-bob run -o $TEST_OUT/test-bob2 build

tests/chown.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+. tests/common.sh
+
+# Regression test for the following PR:
+# https://github.com/inobulles/bob/pull/69
+# Basically, there's this feature which chown's generated files to the owner of the project directory when running Bob as root.
+# The issue though is that, before #69, even installed files would be chown'd to the owner of the project directory.
+# The result was that on FreeBSD, the ldconfig service would refuse to cache the libraries in /usr/local/lib because they weren't owned by root, which is obviously very bad as nothing that linked against those would work.
+
+# Make sure everything is clean.
+
+INSTALL_PATH=/usr/local/share/bob_chown_test.fl
+SUDO rm -rf tests/chown/.bob $INSTALL_PATH
+
+# Create a user.
+
+USER=bobtestuser
+
+if [ $(uname) = "FreeBSD" ]; then
+	SUDO pw adduser -n $USER
+elif [ $(uname) = "Linux" ]; then
+	SUDO useradd -m $USER
+elif [ $(uname) = "Darwin" ]; then
+	out=$(SUDO sysadminctl -addUser $USER 2>&1)
+
+	if [ $? != 0 ]; then
+		echo "Failed to create user: $out" >&2
+		exit 1
+	fi
+else
+	echo "Unsupported OS (don't know how to create a user)." >&2
+	exit 1
+fi
+
+# Chown the project directory to that user.
+
+SUDO chown -R bobtestuser tests/chown
+SUDO bob -C tests/chown build
+
+if [ ! -z "$(find tests/chown ! -user bobtestuser)" ]; then
+	echo "Some files are not owned by bobtestuser in tests/chown." >&2
+	exit 1
+fi
+
+SUDO bob -C tests/chown install
+
+# XXX Annoyingly, 'stat -f %Su' doesn't work on Linux (you have to use 'stat -c %U').
+
+owner=$(ls -ld /usr/local/share/bob_chown_test.fl | awk '{print $3}')
+
+if [ $owner != "root" ]; then
+	echo "Installed files are not owned by root." >&2
+	exit 1
+fi

tests/chown/build.fl

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2024 Aymeric Wibo
+
+import bob
+
+install = {
+	"build.fl": "share/bob_chown_test.fl",
+}

tests/common.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+export ASAN_OPTIONS=detect_leaks=0 # XXX For now, let's not worry about leaks.
+export TEST_OUT=.test-out
+
+# Find doas or sudo.
+
+if [ $(id -u) = 0 ]; then
+	alias SUDO=""
+elif [ $(uname) = "Linux" ]; then
+	# XXX Linux is annoying, 'which -s' doesn't exist.
+	alias SUDO="sudo -E"
+elif which -s doas; then
+	alias SUDO="doas"
+elif which -s sudo; then
+	alias SUDO="sudo -E"
+else
+	echo "No sudo or doas found." >&2
+	exit 1
+fi