Add zx0 compression support

zx0 use demonstrated by compression of Background asset which is now a smaller INCBIN than the native uncompressed RAW asset. It is decompressed into a BSS bitplanes area, which is ALLOC'ed at runtime making the executable file size considerably smaller. Also improved assets workflow so that incremental updates can take place without reprocessing all assets
neildavis · Nov 21, 2024 · 26ae3ff · 26ae3ff
1 parent 33eec32
commit 26ae3ff
Show file tree

Hide file tree

Showing 9 changed files with 223 additions and 90 deletions.
diff --git a/.github/workflows/amiga_demo.yml b/.github/workflows/amiga_demo.yml
@@ -32,9 +32,15 @@ jobs:
           cd ipng2iff
           cargo build -r
           sudo cp target/release/ipng2iff /usr/local/bin/
+      - name: Install salvador (zx0 packer)
+        run: |
+            git clone https://github.com/emmanuel-marty/salvador.git
+            cd salvador
+            make -j$(nproc)
+            sudo cp salvador /usr/local/bin/
       - name: Build ADF using make
         run: |
-          make adf XDF_TOOL=venv/bin/xdftool -j$(nproc)
+          make adf XDF_TOOL=venv/bin/xdftool
       - name: Upload ADF artifact(s)
         uses: actions/upload-artifact@v4
         with:

diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@ uae/dh0/main
 assets/*.png
 assets/*.iff
 assets/*.raw
+assets/*.zx0
 # src that we don't care about
 include/*_palette.i
 # compiled tools

diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -21,6 +21,7 @@
 				"exefilename": "../uae/dh0/main",
 				"entrypoint": "main.s",
 				"args": [
+					"-mrel",
 					"-bamigahunk",
 					"-Bstatic"
 				]

diff --git a/Makefile b/Makefile
@@ -43,6 +43,9 @@ ifneq ($(DEBUG),1)
 LINK_ARGS += -s 
 endif
 
+# ZX0 compressor
+ZX0 := salvador -v
+
 # Source files
 SRC_DIR := ./src
 MAIN_SRC := $(SRC_DIR)/main.s
@@ -57,20 +60,30 @@ OBJS := $(patsubst $(SRC_DIR)/%,$(BUILD_DIR)/%,$(ASM_SRCS:.s=.o))
 ASSETS_DIR := ./assets
 GIMP_ASSETS := $(wildcard $(ASSETS_DIR)/*.xcf)
 RAW_ASSETS := $(GIMP_ASSETS:.xcf=.raw)
+# ZX0 Assets are RAW_ASSETS compressed with ZX0 (salvador)
+ZX0_ASSETS := $(RAW_ASSETS:.raw=_raw.zx0)
 PALETTE_DIR := ./include
 
 # The target ADF dir
 ADF_DIR := ./uae/dh0
 # The Target Binary Program
 TARGET := $(ADF_DIR)/main
 
+# Generic rule to create a RAW asset from XCF
+$(ASSETS_DIR)/%.raw: $(ASSETS_DIR)/%.xcf
+	@./scripts/convert_assets_to_raw.sh -x -p -r -s -i $(PALETTE_DIR) "$<"
+
+# Generic rule to compress a RAW asset using zx0
+$(ASSETS_DIR)/%_raw.zx0: $(ASSETS_DIR)/%.raw
+	$(ZX0) "$<" "$@" 
+
 # Generic rule to assemble a 68k asm source file (../src/*.cpp) into an object file (*.o)
 $(BUILD_DIR)/%.o: $(SRC_DIR)/%.s $(ASM)
 #	@echo '(${ASM}) Assembling source file: $<'
 	$(ASM) $(ASM_ARGS) -o "$@" "$<"
 
 # Link executable
-$(TARGET): $(LD) $(RAW_ASSETS) $(BUILD_DIR) $(MAIN_OBJ) $(OBJS)
+$(TARGET): $(LD) $(RAW_ASSETS) $(ZX0_ASSETS) $(BUILD_DIR) $(MAIN_OBJ) $(OBJS)
 #	@echo '(${LD}) Linking target: $@'
 	$(LD) $(LINK_ARGS) -o "$@" $(MAIN_OBJ) $(OBJS)
 	@echo 'Finished linking target: $@'
@@ -79,10 +92,6 @@ $(TARGET): $(LD) $(RAW_ASSETS) $(BUILD_DIR) $(MAIN_OBJ) $(OBJS)
 $(BUILD_DIR):
 	@mkdir -p $(BUILD_DIR)
 
-# RAW assets
-$(RAW_ASSETS):
-	@./scripts/convert_assets_to_raw.sh -x -p -r -s -i $(PALETTE_DIR)
-
 # TOOLS - vasm
 $(ASM): $(VASM_DIR)
 	@echo 'Building vasmm68k_mot in $(VASM_DIR)...'
@@ -105,7 +114,7 @@ all: $(TARGET)
 
 tools: $(ASM) $(LD)
 
-assets: $(RAW_ASSETS)
+assets: $(RAW_ASSETS) $(ZX0_ASSETS)
 
 ADF_FILE := amiga_demo.adf
 ADF_VOLUME_NAME := 'Amiga Demo'
@@ -138,6 +147,7 @@ clean_assets:
 	@rm -f $(ASSETS_DIR)/*.png
 	@rm -f $(ASSETS_DIR)/*.iff
 	@rm -f $(ASSETS_DIR)/*.raw
+	@rm -f $(ASSETS_DIR)/*.zx0
 
 # clean EVERYTHING
 clean_all: clean clean_tools clean_assets clean_adf

diff --git a/README.md b/README.md
@@ -12,6 +12,7 @@ under [their own licenses](./tools/LICENSE.md).
 The following features are provided and demonstrated:
 
 * Conversion of [GIMP](https://www.gimp.org/) authored image assets (*`.xcf`) into `.png`, `.iff` and `.raw` (interleaved) formats
+* Compression ('packing') or raw assets using the '`zx0`' format
 * Generation of palette (`COLORxx` register) data for image assets in copper list format
 * Host compilation of assembler (`vasm`) and linker (`vlink`) tools included.
 * Building to a [UAE](https://en.wikipedia.org/wiki/UAE_(emulator)) emulated hard drive (`dh0`) folder
@@ -24,18 +25,21 @@ for CI/CD automated building in the the cloud.
 
 * Only RAW files with 'interleaved' bitplanes data are generated (no 'back-to-back' support)
 * Only images for low resolution (non-EHB) mode apps are supported.
-* No compression/packing support.
+* Only `zx0` compression/packing support.
 * No support for attached sprites palette generation
 * No special treatment for AGA
 * Bare bones 'no frills' bootable AmigaDOS ADFs. i.e. no loading messages etc.
 
 ## Demo App ##
 
 This repo contains source code for a simple Amiga demo using Bitplanes (playfield), Blitter objects (BOBs) and Sprites.
+
 This demo was made using samples of [example code](https://www.edsa.uk/blog/downloads) from the excellent book
 ['Bare-Metal Amiga Programming'](https://www.edsa.uk/blog/bare-metal-amiga-programming)
 by E. Th. van den Oosterkamp, and used under his permissive license terms.
 
+This demo also includes m68k asm `zx0` decompression code from [`salvador`](https://github.com/emmanuel-marty/salvador) by Emmanuel Marty, also used under permissive license terms.
+
 The app was also developed in the equally excellent
 [Amiga Assembly](https://marketplace.visualstudio.com/items?itemName=prb28.amiga-assembly)
 extension for [Visual Studio Code](https://code.visualstudio.com/) (VSCode).
@@ -98,6 +102,22 @@ It is part of the [`netpbm`](https://netpbm.sourceforge.net/) toolkit.
 sudo apt install netpbm
 ```
 
+### salvador ###
+[salvador](https://github.com/emmanuel-marty/salvador) is used to compress `*.raw` images into `*_raw.zx0` packed files in the `zx0` format to help save space.
+
+`salvador` has to be built from source.
+```sh
+git clone https://github.com/emmanuel-marty/salvador
+cd salvador
+make
+```
+
+Once `salvador` has built, you need to move it to somewhere where it can be found in your `$PATH`. e.g.
+
+```sh
+sudo cp salvador /usr/local/bin
+```
+
 ### amitools ###
 
 [amitools](https://pypi.org/project/amitools/) is used to create floppy disk (`ADF`) images using
@@ -212,7 +232,7 @@ Passing `-h` (or `--help`) to the script shows usage information:
 
 ```none
 $ ./scripts/convert_assets_to_raw.sh -h
-usage: convert_assets_to_raw.sh [options]
+usage: convert_assets_to_raw.sh [options] [input file]
 
 Options:
  
@@ -236,6 +256,8 @@ Options:
      Include generation of PNG files from XCFs. 
 ```
 
+If no `input_file` is specified, the script will works as a wildcard selecting all applicable files in the specified (or default) 'assets' directory.
+
 ## GitHub Actions Workflow ##
 
 This repository includes a [GitHub Actions](https://docs.github.com/en/actions) (GHA)

diff --git a/scripts/convert_assets_to_raw.sh b/scripts/convert_assets_to_raw.sh
@@ -26,13 +26,13 @@ script_path="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 # Get script name
 script_name="$(basename "$0")"
 
-# Set derault input/output dirs
+# Set default input/output dirs
 asset_dir=$(realpath "$script_path/../assets")
 inc_dir="$asset_dir"
 
-# Function to show ussage information and exit
+# Function to show usage information and exit
 show_usage() {
-  echo "usage: ${script_name} [options]"
+  echo "usage: ${script_name} [options] [input file]"
   echo -e  "\nOptions:\n" \
             "\n -a,--asset-dir <dir name>\n" \
             "    Use dir <dir name> to find source assets and place output assets.\n" \
@@ -102,7 +102,7 @@ while [[ $# > 0 ]]; do
         -r | --iff-raw)     skip_raw=false                  ;;
         -s | --inc-pal)     skip_palette=false              ;;
         -x | --xcf-png)     skip_png=false                  ;;
-        *) echo "Unrecognized option ${1} ignored"          ;;
+        *) input_file="${1}"                                ;;
     esac
     shift
 done
@@ -112,6 +112,15 @@ if $show_usage; then
   exit 0
 fi
 
+
+if [[ -z "$input_file" ]]; then
+    input_file="${asset_dir}/*"
+else
+    input_file="$(echo ${input_file%.*})"
+fi
+
+echo "INPUT FILE(S): ${input_file}"
+
 # Check to make sure we have the tools we need available in $PATH
 rgb2iff_cmd="ipng2iff"
 ilbm_cmd="ilbmtoppm"
@@ -154,12 +163,12 @@ if [[ ! -d "$inc_dir" ]]; then
 fi
 inc_dir=$(realpath "$inc_dir")
 
-# Convert all *.xcf files in assets/GIMP into PNG files in assets/PNG
+# Convert *.xcf files in assets/GIMP into PNG files in assets/PNG
 if $skip_png; then
     echo "XCF->PNG generation skipped. Specify --xcf-png option to include"
 else
-    for xcf_file in "$asset_dir"/*.xcf; do
-        png_file="$asset_dir/$(basename $xcf_file .xcf).png"
+    for xcf_file in ${input_file}.xcf; do
+        png_file="$(dirname $xcf_file)/$(basename $xcf_file .xcf).png"
         echo -e "\nConverting: XCF --> PNG\n<-- $xcf_file\n--> $png_file"
         if ! $dry_run; then
             # Write PNG file
@@ -168,13 +177,13 @@ else
     done
 fi
 
-# Convert all *.png files in assets/PNG into IFF/ILBM *.iff files in assets/IFF
+# Convert *.png files in assets/PNG into IFF/ILBM *.iff files in assets/IFF
 if $skip_iff; then
     echo "PNG->IFF generation skipped. Specify --png-iff option to include"
 else
-    for png_file in "$asset_dir"/*.png; do
+    for png_file in ${input_file}.png; do
         png_res=$(file "$png_file" | grep -oP '([[:digit:]]+[[:blank:]]*x[[:blank:]]*[[:digit:]]+)' | sed 's/ //g')    
-        iff_file="$asset_dir/$(basename $png_file .png).iff"
+        iff_file="$(dirname $png_file)/$(basename $png_file .png).iff"
         echo -e "\nConverting: PNG --> IFF\n<-- $png_file ($png_res)\n--> $iff_file"
         if  ! $dry_run; then
             # Write IFF file
@@ -223,7 +232,7 @@ get_ilbm_info() {
     done <<<$(echo "$ilbm_cmap")
 }
 
-# Convert all *.iff files in assets/IFF to raw (interleaved) plane data in RAW
+# Convert *.iff files in assets/IFF to raw (interleaved) plane data in RAW
 process_ilbm=false
 if $skip_raw; then
     echo "IFF->RAW conversion skipped. Specify --iff-raw option to include"
@@ -236,10 +245,10 @@ else
     process_ilbm=true
 fi
 if $process_ilbm; then
-    for iff_file in "$asset_dir"/*.iff; do
+    for iff_file in ${input_file}.iff; do
         # grab some metadata on the IFF file
         get_ilbm_info
-        raw_file="$asset_dir/$(basename $iff_file .iff).raw"
+        raw_file="$(dirname $iff_file)/$(basename $iff_file .iff).raw"
         # Only write one palette file for each sprite pair
         case "{$iff_file,,}" in
         *spr0* | *spr1*)   palette_file="$inc_dir/sprites_01_palette.i"     ;; # Sprites 0 & 1 palette file
@@ -278,13 +287,13 @@ fi
 
 # Delete intermediary files if requested:
 if $delete_int_files; then
-    for png_file in "$asset_dir"/*.png; do
+    for png_file in ${input_file}.png; do
         echo "Removing $png_file"
         if ! $dry_run; then
             rm "$png_file"
         fi
     done
-    for iff_file in "$asset_dir"/*.iff; do
+    for iff_file in ${input_file}.iff; do
         echo "Removing $iff_file"
         if ! $dry_run; then
             rm "$iff_file"

diff --git a/src/bobs.s b/src/bobs.s
@@ -72,7 +72,7 @@ MoveBOB::
 ; 		D1.W - Y pos (vert) 
 
 PrepBOB::
-             LEA.L      Background(PC),a1               ; APTR interleaved playfield
+             LEA.L      Bitplanes(PC),a1                ; APTR interleaved playfield
              MULU       #80,d1                          ; Convert Y pos into offset
              ADD.L      d1,a1                           ; Add offset to destination
              AND.W      #$FFF0,d0                       ; Position without shift
@@ -104,7 +104,7 @@ PrepBOB::
 ; 		D1.W - Y pos (vert) 
 
 PlaceBOB:
-             LEA.L      Background(PC),a2               ; APTR interleaved playfield
+             LEA.L      Bitplanes(PC),a2                ; APTR interleaved playfield
              MULU       #80,d1                          ; Convert Y pos into offset
              ADD.L      d1,a2                           ; Add offset to destination
              EXT.L      d0                              ; Clear top bits of D0
@@ -120,8 +120,8 @@ PlaceBOB:
 
              MOVE.L     a1,BLTAPT(a5)                   ; Source A = Mask
              MOVE.L     a0,BLTBPT(a5)                   ; Source B = Object
-             MOVE.L     a2,BLTCPT(a5)                   ; Source C = Background
-             MOVE.L     a2,BLTDPT(a5)                   ; Destination = Background
+             MOVE.L     a2,BLTCPT(a5)                   ; Source C = Bitplanes
+             MOVE.L     a2,BLTDPT(a5)                   ; Destination = Bitplanes
              MOVE.W     #$FFFF,BLTAFWM(a5)              ; No first word masking
              MOVE.W     #$FFFF,BLTALWM(a5)              ; No last word masking
              MOVE.W     d0,BLTCON1(a5)                  ; Use shift for source B
@@ -142,7 +142,7 @@ PlaceBOB:
 ; 		D1.W - Y pos (vert) 
 
 ClearBOB:
-             LEA.L      Background(PC),a1               ; APTR interleaved playfield
+             LEA.L      Bitplanes(PC),a1                ; APTR interleaved playfield
              MULU       #80,d1                          ; Convert Y pos into offset
              ADD.L      d1,a1                           ; Add offset to destination
              AND.W      #$FFF0,d0                       ; Position without shift