Merge branch 'main' into tudor/feat-docker-action

.buildconfig

@@ -2,5 +2,3 @@ PG_VERSION=16.4
 SDK_VERSION=3.1.72.3bi
 WASI_SDK_VERSION=24.0.4
 SDKROOT=/opt/python-wasm-sdk
-
-

.github/workflows/build_wasm_postgres.yml

@@ -47,7 +47,7 @@ jobs:
             /tmp/sdk/postgres-*.tar.gz
             postgres
             postgresql-*
-          key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }}
+          key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'extra/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }}
 
       - name: Install python-wasm-sdk for emsdk/wasi+prebuilts
         if: steps.cache-restore.outputs.cache-hit != 'true'

cibuild.sh

@@ -472,8 +472,8 @@ ________________________________________________________________________________
 
                 # debug CI does not use pnpm/npm for building pg, so call the typescript build
                 # part from here
-                pnpm --filter "pglite^..." build || exit 450
-
+                #pnpm --filter "pglite^..." build || exit 450
+                pnpm run build:js || exit 476
                 pnpm pack || exit 31
                 packed=$(echo -n electric-sql-pglite-*.tgz)
 

cibuild/getsyms.py

@@ -3,13 +3,13 @@
 import sys
 import os
 
+
 def dbg(*argv, **kw):
-    kw.setdefault('file',sys.stderr)
-    return print(*argv,**kw)
+    kw.setdefault("file", sys.stderr)
+    return print(*argv, **kw)
 
 
-SNIFF="clock_gettime"
-SNIFF=""
+SNIFF = ""
 
 
 # we use output from wasi `wasm-objdump -x` run
@@ -20,8 +20,8 @@ def dbg(*argv, **kw):
 SECTION = sys.argv[-1].lower()
 
 if SECTION.startswith("export"):
-    BEGIN= "Export["
-    END="Start:"
+    BEGIN = "Export["
+    END = "Start:"
     SECTION = "exports"
     WAY = "-> "
 
@@ -32,126 +32,139 @@ def dbg(*argv, **kw):
                 print(f"_{line}")
 
 else:
-    BEGIN= "Import["
-    END="Function["
+    BEGIN = "Import["
+    END = "Function["
     SECTION = "imports"
     WAY = "<- "
 
-with open(os.environ.get("OBJDUMP", "dump.wasm-objdump"),'r') as wasmdump:
+with open(os.environ.get("OBJDUMP", "dump.wasm-objdump"), "r") as wasmdump:
     for line in wasmdump.readlines():
         line = line.strip()
 
         if not line:
             continue
         if SNIFF:
-            if line.find(SNIFF)>=0:
-               dbg(line, recording)
+            if line.find(SNIFF) >= 0:
+                dbg(line, recording)
 
         if recording:
             if line.startswith(END):
-                dbg("-"*40)
+                dbg("-" * 40)
                 recording = False
                 break
 
             exports.append(line)
-            if line[0]!='-':
+            if line[0] != "-":
                 dbg(line)
 
         else:
-            if line[0]=='-':
+            if line[0] == "-":
                 continue
 
             if line.startswith(BEGIN):
-                dbg("\n\n#",line)
+                dbg("\n\n#", line)
                 recording = True
                 continue
-            dbg("skip",line)
+            dbg("skip", line)
 
 dbg(f"found {len(exports)} {SECTION}")
 
 if 1:
     badchars = '"<> '
-    VERBOSE = '-v' in sys.argv
+    VERBOSE = "-v" in sys.argv
     REPORT = []
     for line in exports:
-        typ, header = line.split('] ',1)
-        if typ.startswith('- func['):
+        typ, header = line.split("] ", 1)
+        if typ.startswith("- func["):
             typ = "def"
-        elif typ.startswith('- global['):
+        elif typ.startswith("- global["):
             pass
             typ = "var"
-        elif typ.startswith('- memory['):
+        elif typ.startswith("- memory["):
             pass
             typ = "mem"
-        elif typ.startswith('- table['):
+        elif typ.startswith("- table["):
             pass
             typ = "tbl"
 
         if SNIFF:
-            if line.find(SNIFF)>=0:
-                dbg(line)
+            if line.find(SNIFF) >= 0:
+                dbg(
+                    f"""
+
+-------------------------------------------------------------------------------------
+
+{line=}
+{typ=}
+
+
+-------------------------------------------------------------------------------------
+
+"""
+                )
 
         try:
-            if typ in ('def','var'):
+            if typ in ("def", "var"):
                 left, right = header.rsplit(WAY, 1)
                 left = left.strip(badchars)
                 right = right.strip(badchars)
                 # GOT.mem. GOT.func. env.
                 clean = False
-                for clean in ('.', ' <'):
-                    if left.find(clean)>=0:
+                for clean in (".", " <"):
+                    if left.find(clean) >= 0:
                         _, left = left.rsplit(clean, 1)
                         _, right = right.rsplit(".", 1)
                         clean = True
                         break
                 if clean:
                     left = left.strip(badchars)
-                if right.find('.')>0:
+                if right.find(".") > 0:
                     _, right = right.rsplit(".", 1)
                 right = right.strip(badchars)
 
-                if left.find('=')> 0:
+                if left.find("=") > 0:
                     left = right
 
-                if left.find('=')> 0:
+                if left.find("=") > 0:
                     left = ""
 
                 if not left:
                     left = right
 
                 if SNIFF:
-                    if line.find(SNIFF)>=0:
+                    if line.find(SNIFF) >= 0:
                         dbg(f"{left=} {right=} '{line}'")
 
-
-                if left.find('=')> 0:
+                if left.find("=") > 0:
                     left = ""
-                elif left.find('::')> 0:
+                elif left.find("::") > 0:
                     if VERBOSE:
                         raise Exception("bad export (c++)")
-                    continue
-                elif left.find(' ')> 0:
+                    # continue
+                elif left.find(" ") > 0:
                     if VERBOSE:
                         raise Exception("bad export (space)")
                     continue
 
-
                 if VERBOSE:
-                    demangle =  os.popen(f'c++filt {right}').read().strip()
-                    if not left or demangle==left:
+                    demangle = os.popen(f"c++filt {right}").read().strip()
+                    if not left or demangle == left:
                         dbg(typ, right, "# right")
-                    elif demangle!=right:
-                        dbg(typ,left,"#", demangle)
+                    elif demangle != right:
+                        dbg(typ, left, "#", demangle)
                     else:
-                        dbg(typ,"LEFT=", left, "RIGHT=", demangle)
+                        dbg(typ, "LEFT=", left, "RIGHT=", demangle)
 
                 if (right or left) not in REPORT:
                     REPORT.append(right or left)
         except Exception as e:
             dbg("ERROR", typ, header, e)
 
-    dbg(len(REPORT),"unique")
+    dbg(len(REPORT), "unique")
+    NODUNDER = []
 
     for rep in REPORT:
-        print(f"_{rep}")
-
+        if rep in NODUNDER:
+            print(f"{rep}")
+        else:
+            print(f"_{rep}")

cibuild/linkexport.sh

@@ -1,11 +1,12 @@
 # this only runs when wasm-objdump is working and OBJDUMP not set to false
 
 echo "============= link export : begin ==============="
-emcc $EMCC_WEB -fPIC -sMAIN_MODULE=1 -O0 \
- -D__PYDK__=1 -DPREFIX=${PGROOT} \
- -sTOTAL_MEMORY=${TOTAL_MEMORY} -sEXPORT_ALL -sSTACK_SIZE=4MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sGLOBAL_BASE=${CMA_MB}MB \
- -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
- -lnodefs.js -lidbfs.js \
+
+# -O0 -sEXPORT_ALL should make all symbols visible without any mangling.
+COPT="-O0 -g3" \
+ emcc \
+ $EMCC_WEB -fPIC -sMAIN_MODULE=1 -sEXPORT_ALL -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
+ -DPREFIX=${PGROOT} -lnodefs.js -lidbfs.js \
  -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \
  $PGPRELOAD \
  -o postgres.html $PG_O $PG_L || exit 14

cibuild/linkwasi.sh

@@ -7,6 +7,7 @@ WASI_CFLAGS="-DPATCH_PG_DEBUG=/tmp/pglite/include/pg_debug.h -DPREFIX=/tmp/pglit
  -Ipostgresql/src/backend \
  -c -o build/postgres/wasi_dlfcn.o \
  -Ibuild/postgres/src/include patches/wasi_dlfcn.c || exit 8
+
 #  -L./build/postgres/src/backend/snowball -ldict_snowball
 # ./build/postgres/src/backend/snowball/dict_snowball.o
 # ./build/postgres/src/backend/snowball/libdict_snowball.a
@@ -770,7 +771,7 @@ llvm-ar cr ../../libpglite.a $PGOBJ
 
 # just linking
 
-
+# -Wl,--no-entry -mexec-model=reactor
 
 $CC -o postgres \
  -fno-strict-aliasing \
@@ -784,11 +785,13 @@ $CC -o postgres \
     ../../src/pl/plpgsql/src/libplpgsql.a \
  -lz -lm -lwasi-emulated-mman -lwasi-emulated-signal -lc \
  -Wl,--export=pg_initdb \
+ -Wl,--export=setup \
+ -Wl,--export=loop \
  -Wl,--export=interactive_one \
  -Wl,--export=use_socketfile \
  -Wl,--export=interactive_write \
- -Wl,--export=interactive_read
- -Wl,--global-base=33554432
+ -Wl,--export=interactive_read \
+ -Wl,--global-base=33333333
 
 cp -vf postgres postgres.wasi || exit 192
 #cp -vf postgres.wasi /tmp/pglite/bin/postgres.wasi

cibuild/linkweb.sh

@@ -99,20 +99,15 @@ pushd src/backend
     if ${PGES6:-true}
     then
         # es6
-        MODULE="-g3 -O0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" #OK
-        MODULE="-g0 -Os -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # no plpgsql 7.2M
-        MODULE="-g0 -O2 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" #OK 7.4M
-        #MODULE="-g0 -O3 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # NO
-        # MODULE="-g0 -Os -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # NO  08-23 3.1.65
         MODULE="$LDEBUG --closure 0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module"
-
+        export COPTS="-O2 -g0"
     else
         export COPTS="-O0 -g3"
         # local debug always fast build
         MODULE="-sMODULARIZE=0 -sEXPORT_ES6=0"
     fi
 
-    MODULE="$MODULE  --shell-file ${WORKSPACE}/tests/repl.html"
+    MODULE="$MODULE --shell-file ${WORKSPACE}/tests/repl.html"
     # closure -sSIMPLE_OPTIMIZATION ?
 
     # =======================================================

cibuild/pg-git.sh

@@ -42,7 +42,6 @@ else
     if $WASI
     then
 
-        echo "TODO $(which clang)"
     CNF="${PGSRC}/configure --prefix=${PGROOT} \
  --disable-spinlocks --disable-atomics \
  --without-zlib --disable-largefile --without-llvm \
@@ -55,7 +54,7 @@ else
      ZIC=/usr/sbin/zic \
      CC=wasi-c \
      CXX=wasi-c++ \
-     CONFIG_SITE==${PGDATA}/config.site CONFIGURE=true \
+     CONFIG_SITE=${PGDATA}/config.site \
      $CNF \
      --host=$(arch) --target=wasm32-unknown-wasi --with-template=wasi
     then