feat: build and release libcmt as a shared object

.github/workflows/main.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Create debian package control file
         run: make control
 
+      - name: Create version file
+        run: make package.json
+
       - name: Buildx setup
         uses: docker/setup-buildx-action@v2
 

Dockerfile

@@ -153,11 +153,13 @@ ARG STAGING_BASE=${BUILD_BASE}/_install
 ARG STAGING_DEBIAN=${STAGING_BASE}/DEBIAN
 ARG STAGING_SBIN=${STAGING_BASE}/usr/sbin
 ARG STAGING_BIN=${STAGING_BASE}/usr/bin
+ARG STAGING_SHARE=${STAGING_BASE}/usr/share/machine-emulator-tools
 
 RUN mkdir -p ${STAGING_DEBIAN} ${STAGING_SBIN} ${STAGING_BIN} ${STAGING_BASE}/etc && \
     echo "cartesi-machine" > ${staging_base}/etc/hostname
 
 COPY control ${STAGING_DEBIAN}/control
+COPY package.json ${STAGING_SHARE}/package.json
 COPY postinst ${STAGING_DEBIAN}/postinst
 
 COPY --from=builder ${BUILD_BASE}/tools/sys-utils/cartesi-init/cartesi-init ${STAGING_SBIN}

Makefile

@@ -31,7 +31,7 @@ LINUX_HEADERS_URLPATH := https://github.com/cartesi/image-kernel/releases/downlo
 
 all: fs
 
-build: control
+build: control package.json
 	@docker buildx build --load \
 		--build-arg TOOLS_DEB=$(TOOLS_DEB) \
 		--build-arg IMAGE_KERNEL_VERSION=$(IMAGE_KERNEL_VERSION) \
@@ -49,8 +49,10 @@ copy:
 
 $(TOOLS_DEB) deb: build
 
-control: Makefile control.template
-	@sed 's|ARG_VERSION|$(VERSION)|g' control.template > control
+control: Makefile control.in
+	@sed 's|ARG_VERSION|$(VERSION)|g' control.in > control
+package.json: Makefile package.json.in
+	@sed 's|ARG_VERSION|$(VERSION)|g' package.json.in > package.json
 
 $(TOOLS_ROOTFS) fs: $(TOOLS_DEB)
 	@docker buildx build --platform=linux/riscv64 \

package.json.in

@@ -0,0 +1,4 @@
+{
+    "name": "machine-emulator-tools",
+    "version": "ARG_VERSION"
+}

sys-utils/libcmt/Makefile

@@ -27,7 +27,7 @@ TOOLCHAIN_PREFIX ?= riscv64-linux-gnu-
 TARGET_CC := $(TOOLCHAIN_PREFIX)gcc
 TARGET_AR := $(TOOLCHAIN_PREFIX)ar
 COMMON_CFLAGS := -Wvla -O2 -g -Wall -pedantic -Wextra -Isrc \
-                 -fno-strict-aliasing -fno-strict-overflow
+                 -fno-strict-aliasing -fno-strict-overflow -fPIC
 TARGET_CFLAGS := $(COMMON_CFLAGS) -ftrivial-auto-var-init=zero -Wstrict-aliasing=3
 CFLAGS := $(COMMON_CFLAGS)
 CC := gcc
@@ -68,6 +68,7 @@ libcmt_SRC := \
 libcmt_OBJDIR    := build/lib
 libcmt_OBJ       := $(patsubst %.c,$(libcmt_OBJDIR)/%.o,$(libcmt_SRC))
 libcmt_LIB       := $(libcmt_OBJDIR)/libcmt.a
+libcmt_SO        := $(libcmt_OBJDIR)/libcmt.so
 
 $(libcmt_OBJ): $(libcmt_OBJDIR)/%.o: %.c
 	@mkdir -p $(@D)
@@ -76,12 +77,16 @@ $(libcmt_OBJ): $(libcmt_OBJDIR)/%.o: %.c
 $(libcmt_LIB): $(libcmt_OBJ)
 	$(TARGET_AR) rcs $@ $^
 
-libcmt: $(libcmt_LIB)
-install: $(libcmt_LIB)
+$(libcmt_SO): $(libcmt_OBJ)
+	$(TARGET_CC) -shared -o $@ $^
+
+libcmt: $(libcmt_LIB) $(libcmt_SO)
+install: $(libcmt_LIB) $(libcmt_SO) build/ffi.h
 	mkdir -p $(TARGET_DESTDIR)$(TARGET_PREFIX)/lib
-	cp -f $< $(TARGET_DESTDIR)$(TARGET_PREFIX)/lib
+	cp -f $(libcmt_LIB) $(libcmt_SO) $(TARGET_DESTDIR)$(TARGET_PREFIX)/lib
 	mkdir -p $(TARGET_DESTDIR)$(TARGET_PREFIX)/include/libcmt/
 	cp -f src/*.h $(TARGET_DESTDIR)$(TARGET_PREFIX)/include/libcmt/
+	cp -f build/ffi.h $(TARGET_DESTDIR)$(TARGET_PREFIX)/include/libcmt/
 	mkdir -p $(TARGET_DESTDIR)$(TARGET_PREFIX)/lib/pkgconfig
 	sed -e 's|@ARG_PREFIX@|$(TARGET_PREFIX)|g' src/libcmt.pc > $(TARGET_DESTDIR)$(TARGET_PREFIX)/lib/pkgconfig/libcmt.pc
 
@@ -103,6 +108,7 @@ mock_SRC := \
 mock_OBJDIR := build/mock
 mock_OBJ    := $(patsubst %.c,$(mock_OBJDIR)/%.o,$(mock_SRC))
 mock_LIB    := $(mock_OBJDIR)/libcmt.a
+mock_SO     := $(mock_OBJDIR)/libcmt.so
 
 $(mock_OBJ): $(mock_OBJDIR)/%.o: %.c
 	@mkdir -p $(@D)
@@ -111,13 +117,17 @@ $(mock_OBJ): $(mock_OBJDIR)/%.o: %.c
 $(mock_LIB): $(mock_OBJ)
 	$(AR) rcs $@ $^
 
-mock: $(mock_LIB)
+$(mock_SO): $(mock_OBJ)
+	$(CC) -shared -o $@ $^
+
+mock: $(mock_LIB) $(mock_SO)
 
-install-mock: $(mock_LIB)
+install-mock: $(mock_LIB) $(mock_SO) build/ffi.h
 	mkdir -p $(DESTDIR)$(PREFIX)/lib
-	cp -f $< $(DESTDIR)$(PREFIX)/lib
+	cp -f $(mock_LIB) $(mock_SO) $(DESTDIR)$(PREFIX)/lib
 	mkdir -p $(DESTDIR)$(PREFIX)/include/libcmt/
 	cp -f src/*.h $(DESTDIR)$(PREFIX)/include/libcmt/
+	cp -f build/ffi.h $(DESTDIR)$(PREFIX)/include/libcmt/
 	mkdir -p $(DESTDIR)$(PREFIX)/lib/pkgconfig
 	sed -e 's|@ARG_PREFIX@|$(PREFIX)|g' src/libcmt_mock.pc > $(DESTDIR)$(PREFIX)/lib/pkgconfig/libcmt.pc
 
@@ -173,6 +183,8 @@ $(tools_OBJDIR)/funsel: tools/funsel.c $(mock_LIB)
 
 tools: $(tools_BINS)
 
+build/ffi.h: src/buf.h src/abi.h src/keccak.h src/merkle.h src/io.h src/rollup.h
+	cat $^ | sh tools/prepare-ffi.sh > $@
 #-------------------------------------------------------------------------------
 LINTER_IGNORE_SOURCES=src/io.c
 LINTER_IGNORE_HEADERS=

sys-utils/libcmt/README.md

@@ -59,7 +59,7 @@ The (verifiable) outputs root hash:
 advance.outputs_root_hash.bin
 ```
 
-inputs must follow this syntax, a comma separated list of reason number followed by a filepath:
+Inputs must follow this syntax, a comma separated list of reason number followed by a file path:
 ```
 CMT_INPUTS="<reason-number> ':' <filepath> ( ',' <reason-number> ':' <filepath> ) *"
 ```
@@ -76,7 +76,7 @@ CMT_DEBUG=yes ./application
 ## generating inputs
 
 Inputs and Outputs are expected to be EVM-ABI encoded. Encoding and decoding
-can be acheived multiple ways, including writing tools with this library. A
+can be achieved multiple ways, including writing tools with this library. A
 simple way to generate testing data is to use the @p cast tool from
 [foundry](http://book.getfoundry.sh/reference/cast/cast.html) and `xxd`.
 
@@ -132,7 +132,80 @@ cast calldata-decode "Notice(bytes)" 0x`xxd -p -c0 "$1"` | (
 # binds
 
 This library is intented to be used with programming languages other than C.
-They acheive this by different means.
+They achieve this by different means.
+
+## Python
+
+Python has multiple Foreign Function Interface (FFI) options for interoperability with C.
+This example uses CFFI and the `libcmt` mock. Which is assumed to be installed at `./libcmt-0.1.0`.
+
+This document uses the [main mode](https://cffi.readthedocs.io/en/latest/overview.html#main-mode-of-usage) of CFFI and works in two steps: `build`, then `use`.
+
+### Build
+
+The `build` step has the objective of creating a python module for libcmt.
+To achieve this we'll use `libcmt/ffi.h` and the script below.
+Paths may need adjustments.
+
+```
+import os
+from cffi import FFI
+
+ffi = FFI()
+with open(os.path.join(os.path.dirname(__file__), "../libcmt-0.1.0/usr/include/libcmt/ffi.h")) as f:
+    ffi.cdef(f.read())
+ffi.set_source("pycmt",
+"""
+#include "libcmt/rollup.h"
+""",
+     include_dirs=["libcmt-0.1.0/usr/include"],
+     library_dirs=["libcmt-0.1.0/usr/lib"],
+     libraries=['cmt'])
+
+if __name__ == "__main__":
+    ffi.compile(verbose=True)
+```
+
+### Use
+
+With the module built, we can import it with python and use its functions.
+This example uses the raw bindings and just serves to illustrate the process.
+Better yet would be to wrap these functions into a more "pythonic" API.
+
+`LD_LIBRARY_PATH=libcmt-0.1.0/lib/ python`
+
+```
+#!/usr/bin/env python
+
+# run this file only after building pycmt!
+import os
+from pycmt.lib import ffi, lib
+
+r = ffi.new("cmt_rollup_t[1]")
+assert(lib.cmt_rollup_init(r) == 0)
+
+address = ffi.new("uint8_t[20]", b"1000000000000")
+value   = ffi.new("uint8_t[32]", b"0000000000001")
+data    = b"hello world"
+index   = ffi.new("uint64_t *")
+print(os.strerror(-lib.cmt_rollup_emit_voucher(r,
+                            20, address,
+                            32, value,
+                            len(data), data, index)))
+
+ffi.gc(r, lib.cmt_rollup_fini)
+```
+
+Common errors such as the one below indicate that python couldn't find libcmt,
+make sure `LD_LIBRARY_PATH` points to the correct directory, or better yet, link
+against the static library.
+
+```
+>>> import pycmt
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+ImportError: libcmt.so: cannot open shared object file: No such file or directory
+```
 
 ## Go
 

sys-utils/libcmt/tools/prepare-ffi.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+sed \
+	-e '/\/\*/,/\*\//d' \
+	-e '/#if\s/,/#endif/d' \
+	-e '/#define/d' \
+	-e '/#endif/d' \
+	-e '/#ifndef/d' \
+	-e '/#include/d' \