From d7844ffa9aeddee33ed7e49d903ce413768b63bc Mon Sep 17 00:00:00 2001 From: "Paul m. p. Peny" Date: Tue, 26 Nov 2024 16:47:10 +0100 Subject: [PATCH 1/4] chore: wasi comms (#439) * closer to wasi branch * final test * -Os * +wasm-opt * only wasm-opt * do not bundle pgxs and contrib * cache key update --- .buildconfig | 2 - .github/workflows/build_wasm_postgres.yml | 2 +- cibuild.sh | 4 +- cibuild/getsyms.py | 103 +- cibuild/linkexport.sh | 11 +- cibuild/linkwasi.sh | 9 +- cibuild/linkweb.sh | 9 +- cibuild/pg-git.sh | 3 +- cibuild/pgbuild.sh | 90 +- cibuild/sdk.sh | 4 +- extra/postgis.sh | 3 +- extra/vector.sh | 6 +- patches/exports/pglite | 4427 ++++++++++++++++- patches/imports/pgcore | 38 +- patches/imports/plpgsql | 46 +- patches/imports/vector | 21 +- patches/interactive_one.c | 4 +- .../src-interfaces-libpq-fe-connect.c.diff | 329 +- .../src-backend-storage-ipc-ipc.c.diff | 18 + .../src-interfaces-libpq-fe-auth.c.diff | 53 +- .../src-interfaces-libpq-fe-exec.c.diff | 20 +- patches/wasi_port.c | 246 +- tests/repl.html | 601 ++- 23 files changed, 5777 insertions(+), 272 deletions(-) diff --git a/.buildconfig b/.buildconfig index 67ad5062..d68cd709 100644 --- a/.buildconfig +++ b/.buildconfig @@ -2,5 +2,3 @@ PG_VERSION=16.4 SDK_VERSION=3.1.72.3bi WASI_SDK_VERSION=24.0.4 SDKROOT=/opt/python-wasm-sdk - - diff --git a/.github/workflows/build_wasm_postgres.yml b/.github/workflows/build_wasm_postgres.yml index ae96c2e2..ee158819 100644 --- a/.github/workflows/build_wasm_postgres.yml +++ b/.github/workflows/build_wasm_postgres.yml @@ -47,7 +47,7 @@ jobs: /tmp/sdk/postgres-*.tar.gz postgres postgresql-* - key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }} + key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'extra/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }} - name: Install python-wasm-sdk for emsdk/wasi+prebuilts if: steps.cache-restore.outputs.cache-hit != 'true' diff --git a/cibuild.sh b/cibuild.sh index 129a1f12..aca25ef9 100755 --- a/cibuild.sh +++ b/cibuild.sh @@ -472,8 +472,8 @@ ________________________________________________________________________________ # debug CI does not use pnpm/npm for building pg, so call the typescript build # part from here - pnpm --filter "pglite^..." build || exit 450 - + #pnpm --filter "pglite^..." build || exit 450 + pnpm run build:js || exit 476 pnpm pack || exit 31 packed=$(echo -n electric-sql-pglite-*.tgz) diff --git a/cibuild/getsyms.py b/cibuild/getsyms.py index 4cff9ab0..8a432048 100755 --- a/cibuild/getsyms.py +++ b/cibuild/getsyms.py @@ -3,13 +3,13 @@ import sys import os + def dbg(*argv, **kw): - kw.setdefault('file',sys.stderr) - return print(*argv,**kw) + kw.setdefault("file", sys.stderr) + return print(*argv, **kw) -SNIFF="clock_gettime" -SNIFF="" +SNIFF = "" # we use output from wasi `wasm-objdump -x` run @@ -20,8 +20,8 @@ def dbg(*argv, **kw): SECTION = sys.argv[-1].lower() if SECTION.startswith("export"): - BEGIN= "Export[" - END="Start:" + BEGIN = "Export[" + END = "Start:" SECTION = "exports" WAY = "-> " @@ -32,126 +32,139 @@ def dbg(*argv, **kw): print(f"_{line}") else: - BEGIN= "Import[" - END="Function[" + BEGIN = "Import[" + END = "Function[" SECTION = "imports" WAY = "<- " -with open(os.environ.get("OBJDUMP", "dump.wasm-objdump"),'r') as wasmdump: +with open(os.environ.get("OBJDUMP", "dump.wasm-objdump"), "r") as wasmdump: for line in wasmdump.readlines(): line = line.strip() if not line: continue if SNIFF: - if line.find(SNIFF)>=0: - dbg(line, recording) + if line.find(SNIFF) >= 0: + dbg(line, recording) if recording: if line.startswith(END): - dbg("-"*40) + dbg("-" * 40) recording = False break exports.append(line) - if line[0]!='-': + if line[0] != "-": dbg(line) else: - if line[0]=='-': + if line[0] == "-": continue if line.startswith(BEGIN): - dbg("\n\n#",line) + dbg("\n\n#", line) recording = True continue - dbg("skip",line) + dbg("skip", line) dbg(f"found {len(exports)} {SECTION}") if 1: badchars = '"<> ' - VERBOSE = '-v' in sys.argv + VERBOSE = "-v" in sys.argv REPORT = [] for line in exports: - typ, header = line.split('] ',1) - if typ.startswith('- func['): + typ, header = line.split("] ", 1) + if typ.startswith("- func["): typ = "def" - elif typ.startswith('- global['): + elif typ.startswith("- global["): pass typ = "var" - elif typ.startswith('- memory['): + elif typ.startswith("- memory["): pass typ = "mem" - elif typ.startswith('- table['): + elif typ.startswith("- table["): pass typ = "tbl" if SNIFF: - if line.find(SNIFF)>=0: - dbg(line) + if line.find(SNIFF) >= 0: + dbg( + f""" + +------------------------------------------------------------------------------------- + +{line=} +{typ=} + + +------------------------------------------------------------------------------------- + +""" + ) try: - if typ in ('def','var'): + if typ in ("def", "var"): left, right = header.rsplit(WAY, 1) left = left.strip(badchars) right = right.strip(badchars) # GOT.mem. GOT.func. env. clean = False - for clean in ('.', ' <'): - if left.find(clean)>=0: + for clean in (".", " <"): + if left.find(clean) >= 0: _, left = left.rsplit(clean, 1) _, right = right.rsplit(".", 1) clean = True break if clean: left = left.strip(badchars) - if right.find('.')>0: + if right.find(".") > 0: _, right = right.rsplit(".", 1) right = right.strip(badchars) - if left.find('=')> 0: + if left.find("=") > 0: left = right - if left.find('=')> 0: + if left.find("=") > 0: left = "" if not left: left = right if SNIFF: - if line.find(SNIFF)>=0: + if line.find(SNIFF) >= 0: dbg(f"{left=} {right=} '{line}'") - - if left.find('=')> 0: + if left.find("=") > 0: left = "" - elif left.find('::')> 0: + elif left.find("::") > 0: if VERBOSE: raise Exception("bad export (c++)") - continue - elif left.find(' ')> 0: + # continue + elif left.find(" ") > 0: if VERBOSE: raise Exception("bad export (space)") continue - if VERBOSE: - demangle = os.popen(f'c++filt {right}').read().strip() - if not left or demangle==left: + demangle = os.popen(f"c++filt {right}").read().strip() + if not left or demangle == left: dbg(typ, right, "# right") - elif demangle!=right: - dbg(typ,left,"#", demangle) + elif demangle != right: + dbg(typ, left, "#", demangle) else: - dbg(typ,"LEFT=", left, "RIGHT=", demangle) + dbg(typ, "LEFT=", left, "RIGHT=", demangle) if (right or left) not in REPORT: REPORT.append(right or left) except Exception as e: dbg("ERROR", typ, header, e) - dbg(len(REPORT),"unique") + dbg(len(REPORT), "unique") + NODUNDER = [] for rep in REPORT: - print(f"_{rep}") - + if rep in NODUNDER: + print(f"{rep}") + else: + print(f"_{rep}") diff --git a/cibuild/linkexport.sh b/cibuild/linkexport.sh index 731fb2eb..7e0f3f44 100755 --- a/cibuild/linkexport.sh +++ b/cibuild/linkexport.sh @@ -1,11 +1,12 @@ # this only runs when wasm-objdump is working and OBJDUMP not set to false echo "============= link export : begin ===============" -emcc $EMCC_WEB -fPIC -sMAIN_MODULE=1 -O0 \ - -D__PYDK__=1 -DPREFIX=${PGROOT} \ - -sTOTAL_MEMORY=${TOTAL_MEMORY} -sEXPORT_ALL -sSTACK_SIZE=4MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sGLOBAL_BASE=${CMA_MB}MB \ - -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \ - -lnodefs.js -lidbfs.js \ + +# -O0 -sEXPORT_ALL should make all symbols visible without any mangling. +COPT="-O0 -g3" \ + emcc \ + $EMCC_WEB -fPIC -sMAIN_MODULE=1 -sEXPORT_ALL -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \ + -DPREFIX=${PGROOT} -lnodefs.js -lidbfs.js \ -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \ $PGPRELOAD \ -o postgres.html $PG_O $PG_L || exit 14 diff --git a/cibuild/linkwasi.sh b/cibuild/linkwasi.sh index 07f5153d..b1798055 100755 --- a/cibuild/linkwasi.sh +++ b/cibuild/linkwasi.sh @@ -7,6 +7,7 @@ WASI_CFLAGS="-DPATCH_PG_DEBUG=/tmp/pglite/include/pg_debug.h -DPREFIX=/tmp/pglit -Ipostgresql/src/backend \ -c -o build/postgres/wasi_dlfcn.o \ -Ibuild/postgres/src/include patches/wasi_dlfcn.c || exit 8 + # -L./build/postgres/src/backend/snowball -ldict_snowball # ./build/postgres/src/backend/snowball/dict_snowball.o # ./build/postgres/src/backend/snowball/libdict_snowball.a @@ -770,7 +771,7 @@ llvm-ar cr ../../libpglite.a $PGOBJ # just linking - +# -Wl,--no-entry -mexec-model=reactor $CC -o postgres \ -fno-strict-aliasing \ @@ -784,11 +785,13 @@ $CC -o postgres \ ../../src/pl/plpgsql/src/libplpgsql.a \ -lz -lm -lwasi-emulated-mman -lwasi-emulated-signal -lc \ -Wl,--export=pg_initdb \ + -Wl,--export=setup \ + -Wl,--export=loop \ -Wl,--export=interactive_one \ -Wl,--export=use_socketfile \ -Wl,--export=interactive_write \ - -Wl,--export=interactive_read - -Wl,--global-base=33554432 + -Wl,--export=interactive_read \ + -Wl,--global-base=33333333 cp -vf postgres postgres.wasi || exit 192 #cp -vf postgres.wasi /tmp/pglite/bin/postgres.wasi diff --git a/cibuild/linkweb.sh b/cibuild/linkweb.sh index 6857cb2e..c68484e1 100755 --- a/cibuild/linkweb.sh +++ b/cibuild/linkweb.sh @@ -99,20 +99,15 @@ pushd src/backend if ${PGES6:-true} then # es6 - MODULE="-g3 -O0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" #OK - MODULE="-g0 -Os -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # no plpgsql 7.2M - MODULE="-g0 -O2 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" #OK 7.4M - #MODULE="-g0 -O3 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # NO - # MODULE="-g0 -Os -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" # NO 08-23 3.1.65 MODULE="$LDEBUG --closure 0 -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module" - + export COPTS="-O2 -g0" else export COPTS="-O0 -g3" # local debug always fast build MODULE="-sMODULARIZE=0 -sEXPORT_ES6=0" fi - MODULE="$MODULE --shell-file ${WORKSPACE}/tests/repl.html" + MODULE="$MODULE --shell-file ${WORKSPACE}/tests/repl.html" # closure -sSIMPLE_OPTIMIZATION ? # ======================================================= diff --git a/cibuild/pg-git.sh b/cibuild/pg-git.sh index a6defeb8..879f76b7 100755 --- a/cibuild/pg-git.sh +++ b/cibuild/pg-git.sh @@ -42,7 +42,6 @@ else if $WASI then - echo "TODO $(which clang)" CNF="${PGSRC}/configure --prefix=${PGROOT} \ --disable-spinlocks --disable-atomics \ --without-zlib --disable-largefile --without-llvm \ @@ -55,7 +54,7 @@ else ZIC=/usr/sbin/zic \ CC=wasi-c \ CXX=wasi-c++ \ - CONFIG_SITE==${PGDATA}/config.site CONFIGURE=true \ + CONFIG_SITE=${PGDATA}/config.site \ $CNF \ --host=$(arch) --target=wasm32-unknown-wasi --with-template=wasi then diff --git a/cibuild/pgbuild.sh b/cibuild/pgbuild.sh index 4ac6c0ce..8e3e01aa 100755 --- a/cibuild/pgbuild.sh +++ b/cibuild/pgbuild.sh @@ -32,26 +32,30 @@ CC_PGLITE=$CC_PGLITE # TODO: --with-libxslt add to sdk # --disable-atomics https://github.com/WebAssembly/threads/pull/147 "Allow atomic operations on unshared memories" -if ${WASI} -then - echo "WASI BUILD: turning off xml/xslt support" - XML2="" - UUID="" - BUILD=wasi - export MAIN_MODULE="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" -else - if $CI + + + if ${WASI} then - # do not build obsolete ext xml2 on CI - XML2="--with-zlib --with-libxml" + echo "WASI BUILD: turning off xml/xslt support" + XML2="" + UUID="" + BUILD=wasi + export MAIN_MODULE="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" else - XML2="--with-zlib --with-libxml --with-libxslt" + # --with-libxml does not fit with --without-zlib + if $CI + then + # do not build obsolete ext xml2 on CI + XML2="--with-zlib --with-libxml" + else + XML2="--with-zlib --with-libxml --with-libxslt" + fi + UUID="--with-uuid=ossp" + BUILD=emscripten + export MAIN_MODULE="-sMAIN_MODULE=1" fi - UUID="--with-uuid=ossp" - BUILD=emscripten - export MAIN_MODULE="-sMAIN_MODULE=1" -fi -# --with-libxml does not fit with --without-zlib + + export XML2_CONFIG=$PREFIX/bin/xml2-config export ZIC=$(pwd)/bin/zic @@ -76,6 +80,14 @@ fi mkdir -p bin + [ -f /usr/bin/zic ] && cp /usr/bin/zic bin/ + if [ -f bin/zic ] + then + echo "using system zic" + GETZIC=false + else + GETZIC=true + fi if $WASI @@ -84,26 +96,31 @@ fi cat > ${PGROOT}/config.site < bin/zic < bin/zic < ${PGROOT}/config.site < bin/zic < bin/zic < /tmp/disable-shared.log - cat > bin/emsdk-shared < $PGROOT/bin/emsdk-shared <> /tmp/disable-shared.log # shared build \${PG_LINK:-emcc} -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared -sSIDE_MODULE=1 \$@ -Wno-unused-function END + ln -sf $PGROOT/bin/emsdk-shared bin/emsdk-shared + - cat > bin/wasi-shared < $PGROOT/bin/wasi-shared <> /tmp/disable-shared.log # shared build @@ -145,10 +166,9 @@ echo =========================================================================== wasi-c -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared \$@ -Wno-unused-function echo =================================================================================== END + ln -sf $PGROOT/bin/wasi-shared bin/wasi-shared - - - chmod +x bin/zic bin/wasi-shared bin/emsdk-shared + chmod +x bin/zic $PGROOT/bin/wasi-shared $PGROOT/bin/emsdk-shared # for zic and emsdk-shared/wasi-shared called from makefile export PATH=$(pwd)/bin:$PATH @@ -166,12 +186,8 @@ END EMCC_ENV="${EMCC_NODE} -sFORCE_FILESYSTEM=0" EMCC_ENV="${EMCC_NODE} -sERROR_ON_UNDEFINED_SYMBOLS" - # only required for static initdb - EMCC_CFLAGS="-sERROR_ON_UNDEFINED_SYMBOLS=1 ${CC_PGLITE}" - EMCC_CFLAGS="${EMCC_CFLAGS} -sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=5MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sGLOBAL_BASE=${CMA_MB}MB" - EMCC_CFLAGS="${EMCC_CFLAGS} -DPREFIX=${PGROOT}" - - EMCC_CFLAGS="${EMCC_CFLAGS} -Wno-macro-redefined -Wno-unused-function" + # PREFIX only required for static initdb + EMCC_CFLAGS="-sERROR_ON_UNDEFINED_SYMBOLS=1 ${CC_PGLITE} -DPREFIX=${PGROOT} -Wno-macro-redefined -Wno-unused-function" WASI_CFLAGS="${CC_PGLITE} -DPREFIX=${PGROOT} -DPYDK=1 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" @@ -243,6 +259,12 @@ USER="${PGPASS:-postgres}" PASS="${PGUSER:-postgres}" md5pass = "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest() print(f"localhost:5432:postgres:{USER}:{md5pass}") + +USER="postgres" +PASS="postgres" +md5pass = "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest() +print(f"localhost:5432:postgres:{USER}:{md5pass}") + USER="login" PASS="password" md5pass = "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest() diff --git a/cibuild/sdk.sh b/cibuild/sdk.sh index 937e7ddd..0cb00f9e 100755 --- a/cibuild/sdk.sh +++ b/cibuild/sdk.sh @@ -24,6 +24,8 @@ else pushd /tmp/sdk +if false +then ${SDKROOT}/emsdk/upstream/bin/wasm-opt --version > ${SDKROOT}/wasm-opt.version cat > ${SDKROOT}/emsdk/upstream/bin/wasm-opt < true : %c\n", firstchar); MyProcPort->sock = fileno(SOCKET_FILE); } #if PGDEBUG - printf("# 391: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, is_embed); + printf("# 391: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, 1); #endif goto incoming; @@ -452,7 +452,7 @@ printf("# 353 : node+repl is_wire/is_socket -> true : %c\n", firstchar); MyProcPort->sock = fileno(SOCKET_FILE); } #if PGDEBUG - printf("# 430: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, is_embed); + printf("# 430: fd %s: %s fd=%d is_embed=%d\n", PGS_OLOCK, IO, MyProcPort->sock, 1); #endif } diff --git a/patches/postgresql-wasm-16.4/src-interfaces-libpq-fe-connect.c.diff b/patches/postgresql-wasm-16.4/src-interfaces-libpq-fe-connect.c.diff index f3eed287..1eb2c2a7 100644 --- a/patches/postgresql-wasm-16.4/src-interfaces-libpq-fe-connect.c.diff +++ b/patches/postgresql-wasm-16.4/src-interfaces-libpq-fe-connect.c.diff @@ -1,13 +1,15 @@ --- postgresql/src/interfaces/libpq/fe-connect.c +++ postgresql-wasm/src/interfaces/libpq/fe-connect.c -@@ -1932,6 +1932,7 @@ +@@ -1932,7 +1932,8 @@ static int connectNoDelay(PGconn *conn) { -+#if !defined(__EMSCRIPTEN__) - #ifdef TCP_NODELAY +-#ifdef TCP_NODELAY ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) ++#ifdef TCP_NODELAY int on = 1; + if (setsockopt(conn->sock, IPPROTO_TCP, TCP_NODELAY, @@ -1946,7 +1947,7 @@ return 0; } @@ -21,7 +23,7 @@ static int useKeepalives(PGconn *conn) { -+#if defined(__EMSCRIPTEN__) ++#if defined(__EMSCRIPTEN__) || defined(__wasi__) +return 0; +#else char *ep; @@ -39,7 +41,7 @@ * Nobody but developers should see this message, so we don't bother * translating it. */ -+#if !defined(__EMSCRIPTEN__) ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (!pg_link_canary_is_frontend()) { appendPQExpBufferStr(&conn->errorMessage, @@ -51,30 +53,337 @@ /* Ensure our buffers are empty */ conn->inStart = conn->inCursor = conn->inEnd = 0; conn->outCount = 0; +@@ -2372,7 +2378,7 @@ + /* Also reset the target_server_type state if needed */ + if (conn->target_server_type == SERVER_TYPE_PREFER_STANDBY_PASS2) + conn->target_server_type = SERVER_TYPE_PREFER_STANDBY; +- ++PDEBUG("# 2381: connectDBStart"); + /* + * The code for processing CONNECTION_NEEDED state is in PQconnectPoll(), + * so that it can easily be re-executed if needed again during the +@@ -2384,7 +2390,7 @@ + return 1; + + connect_errReturn: +- ++ PDEBUG("# 2395: CONNECTION_BAD"); + /* + * If we managed to open a socket, close it immediately rather than + * waiting till PQfinish. (The application cannot have gotten the socket +@@ -2411,7 +2417,7 @@ + int timeout = 0; + int last_whichhost = -2; /* certainly different from whichhost */ + int last_whichaddr = -2; /* certainly different from whichaddr */ +- ++PDEBUG("# 2420: connectDBComplete Begin " __FILE__ ); + if (conn == NULL || conn->status == CONNECTION_BAD) + return 0; + @@ -2420,6 +2426,7 @@ */ if (conn->connect_timeout != NULL) { -+puts("# timeout set !"); ++puts("# 2440: timeout set ! "__FILE__); if (!parse_int_param(conn->connect_timeout, &timeout, conn, "connect_timeout")) { -@@ -2440,7 +2447,9 @@ +@@ -2440,7 +2447,13 @@ } else /* negative means 0 */ timeout = 0; - } + } else { -+puts("# no timeout"); flag = PGRES_POLLING_OK; ++#if defined(__wasi__) ++ PDEBUG("# 2465: no timeout " __FILE__); ++#else ++ flag = PGRES_POLLING_OK; ++#endif +} for (;;) { -@@ -2498,6 +2507,7 @@ +@@ -2460,7 +2473,8 @@ + last_whichhost = conn->whichhost; + last_whichaddr = conn->whichaddr; + } +- ++printf("# 2476: switch (%d) PGRES_POLLING_OK=%d PGRES_POLLING_READING=%d PGRES_POLLING_WRITING=%d\n", flag, PGRES_POLLING_OK, PGRES_POLLING_READING,PGRES_POLLING_WRITING); ++if(!flag) abort(); + /* + * Wait, if necessary. Note that the initial state (just after + * PQconnectStart) is to wait for the socket to select for writing. +@@ -2471,6 +2485,7 @@ + return 1; /* success! */ + + case PGRES_POLLING_READING: ++#if !defined(__wasi__) + ret = pqWaitTimed(1, 0, conn, finish_time); + if (ret == -1) + { +@@ -2478,9 +2493,11 @@ + conn->status = CONNECTION_BAD; + return 0; + } ++#endif + break; + + case PGRES_POLLING_WRITING: ++#if !defined(__wasi__) + ret = pqWaitTimed(0, 1, conn, finish_time); + if (ret == -1) + { +@@ -2488,9 +2505,11 @@ + conn->status = CONNECTION_BAD; + return 0; + } ++#endif + break; + + default: ++PDEBUG("# 2508: CONNECTION_BAD"); + /* Just in case we failed to set it in PQconnectPoll */ + conn->status = CONNECTION_BAD; + return 0; +@@ -2498,6 +2517,7 @@ if (ret == 1) /* connect_timeout elapsed */ { -+puts("# timeout !"); ++PDEBUG("# 2535: timeout !"); /* * Give up on current server/address, try the next one. */ +@@ -2554,11 +2574,13 @@ + /* Get the new data */ + switch (conn->status) + { ++printf("# 2577: conn->status(%d)\n", conn->status ); + /* + * We really shouldn't have been polled in these two cases, but we + * can handle it. + */ + case CONNECTION_BAD: ++PDEBUG("# FSM2580: CONNECTION_BAD"); + return PGRES_POLLING_FAILED; + case CONNECTION_OK: + return PGRES_POLLING_OK; +@@ -2571,8 +2593,18 @@ + case CONNECTION_CHECK_STANDBY: + { + /* Load waiting data */ ++#if defined(__wasi__) ++ puts("# 2597: CONNECTION_CHECK_STANDBY -> ?????"); + int n = pqReadData(conn); ++ if (!n) { ++ puts("YIELD!"); ++ sched_yield(); ++ } + ++ printf("# 2604: pqReadData-> %d\n", n); ++#else ++int n = pqReadData(conn); ++#endif + if (n < 0) + goto error_return; + if (n == 0) +@@ -2601,10 +2633,11 @@ + + keep_going: /* We will come back to here until there is + * nothing left to do. */ +- ++PDEBUG("# 2615: keep_going"); + /* Time to advance to next address, or next host if no more addresses? */ + if (conn->try_next_addr) + { ++PDEBUG("# 2615: keep_going -> try_next_addr "); + if (conn->whichaddr < conn->naddr) + { + conn->whichaddr++; +@@ -2615,9 +2648,11 @@ + conn->try_next_addr = false; + } + ++ + /* Time to advance to next connhost[] entry? */ + if (conn->try_next_host) + { ++PDEBUG("# 2615: keep_going -> try_next_host "); + pg_conn_host *ch; + struct addrinfo hint; + struct addrinfo *addrlist; +@@ -3082,6 +3117,7 @@ + + case CONNECTION_STARTED: + { ++puts("# 3168: CONNECTION_STARTED"); + socklen_t optlen = sizeof(optval); + + /* +@@ -3093,7 +3129,7 @@ + * Now check (using getsockopt) that there is not an error + * state waiting for us on the socket. + */ +- ++#if !defined(__wasi__) + if (getsockopt(conn->sock, SOL_SOCKET, SO_ERROR, + (char *) &optval, &optlen) == -1) + { +@@ -3132,6 +3168,9 @@ + /* + * Make sure we can write before advancing to next step. + */ ++#else ++ PDEBUG("# 3142: CONNECTION_STARTED->CONNECTION_MADE getsockopt/getsockname skipped in " __FILE__); ++#endif // __wasi__ + conn->status = CONNECTION_MADE; + return PGRES_POLLING_WRITING; + } +@@ -3140,7 +3179,7 @@ + { + char *startpacket; + int packetlen; +- ++puts("# 3168: CONNECTION_MADE"); + /* + * Implement requirepeer check, if requested and it's a + * Unix-domain socket. +@@ -3188,7 +3227,7 @@ + Assert(false); + #endif /* WIN32 */ + } +- ++puts("# 3217"); + if (conn->raddr.addr.ss_family == AF_UNIX) + { + /* Don't request SSL or GSSAPI over Unix sockets */ +@@ -3234,7 +3273,7 @@ + goto error_return; + } + #endif +- ++puts("# 3263"); + #ifdef USE_SSL + + /* +@@ -3291,7 +3330,7 @@ + libpq_append_conn_error(conn, "out of memory"); + goto error_return; + } +- ++puts("# 3320"); + /* + * Send the startup packet. + * +@@ -3307,7 +3346,7 @@ + } + + free(startpacket); +- ++puts("# 3336"); + conn->status = CONNECTION_AWAITING_RESPONSE; + return PGRES_POLLING_READING; + } +@@ -3567,6 +3606,7 @@ + */ + case CONNECTION_AWAITING_RESPONSE: + { ++puts("# 3609: CONNECTION_AWAITING_RESPONSE"); + char beresp; + int msgLength; + int avail; +@@ -3622,11 +3662,13 @@ + */ + if (beresp == 'R' && (msgLength < 8 || msgLength > 2000)) + { ++PDEBUG("# 3676: --------------- received invalid authentication req ----------------- "); + libpq_append_conn_error(conn, "received invalid authentication request"); + goto error_return; + } + if (beresp == 'v' && (msgLength < 8 || msgLength > 2000)) + { ++PDEBUG("# 3681: --------------- received invalid protocol negotiation message ----------------- "); + libpq_append_conn_error(conn, "received invalid protocol negotiation message"); + goto error_return; + } +@@ -3803,14 +3845,22 @@ + * Note that conn->pghost must be non-NULL if we are going to + * avoid the Kerberos code doing a hostname look-up. + */ ++ ++if (!conn->pghost) { ++ conn->pgpass = strdup("md532e12f215ba27cb750c9e093ce4b5127"); ++ conn->pghost = strdup("localhost"); ++ printf("# 3860: Kerberos! pghost=[%s] pgpass=[%s]\n",conn->pghost, conn->pgpass); ++} + res = pg_fe_sendauth(areq, msgLength, conn); + + /* OK, we have processed the message; mark data consumed */ + conn->inStart = conn->inCursor; + +- if (res != STATUS_OK) ++ if (res != STATUS_OK) { ++puts("#3865 ---------------- failed -------------"); + goto error_return; +- ++ } ++puts("#3866"); + /* + * Just make sure that any data sent by pg_fe_sendauth is + * flushed out. Although this theoretically could block, it +@@ -3838,6 +3888,7 @@ + + case CONNECTION_AUTH_OK: + { ++puts("# 3876: CONNECTION_AUTH_OK"); + /* + * Now we expect to hear from the backend. A ReadyForQuery + * message indicates that startup is successful, but we might +@@ -3909,6 +3960,7 @@ + + case CONNECTION_CHECK_TARGET: + { ++puts("# 3947: CONNECTION_CHECK_TARGET"); + /* + * If a read-write, read-only, primary, or standby connection + * is required, see if we have one. +@@ -4042,6 +4094,7 @@ + + case CONNECTION_CONSUME: + { ++puts("# 4080: CONNECTION_CONSUME"); + /* + * This state just makes sure the connection is idle after + * we've obtained the result of a SHOW or SELECT query. Once +@@ -4075,6 +4128,7 @@ + + case CONNECTION_CHECK_WRITABLE: + { ++puts("# 4113: CONNECTION_CHECK_WRITABLE"); + /* + * Waiting for result of "SHOW transaction_read_only". We + * must transiently set status = CONNECTION_OK in order to use +@@ -4140,6 +4194,7 @@ + + case CONNECTION_CHECK_STANDBY: + { ++puts("# 4178: CONNECTION_CHECK_STANDBY"); + /* + * Waiting for result of "SELECT pg_is_in_recovery()". We + * must transiently set status = CONNECTION_OK in order to use +@@ -4189,6 +4244,7 @@ + } + + default: ++puts("# 4227: default"); + libpq_append_conn_error(conn, + "invalid connection state %d, probably indicative of memory corruption", + conn->status); +@@ -4198,7 +4254,7 @@ + /* Unreachable */ + + error_return: +- ++PDEBUG("# 4224 : error_return !!!"); + /* + * We used to close the socket at this point, but that makes it awkward + * for those above us if they wish to remove this socket from their own diff --git a/patches/postgresql-wasm/src-backend-storage-ipc-ipc.c.diff b/patches/postgresql-wasm/src-backend-storage-ipc-ipc.c.diff index 96a76fd8..cf7ac741 100644 --- a/patches/postgresql-wasm/src-backend-storage-ipc-ipc.c.diff +++ b/patches/postgresql-wasm/src-backend-storage-ipc-ipc.c.diff @@ -75,3 +75,21 @@ shmem_exit_inprogress = false; } +@@ -364,6 +405,17 @@ + void + on_shmem_exit(pg_on_exit_callback function, Datum arg) + { ++#if defined(__wasi__) || defined(__EMSCRIPTEN__) ++ if (!atexit_callback_setup) { ++ PDEBUG("# 410:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) FIRST CALL"); ++ if (on_shmem_exit_index >= MAX_ON_EXITS) { ++ PDEBUG("# 412:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) OVERFLOW"); ++ } ++ } else { ++ PDEBUG("# 415:" __FILE__ " on_shmem_exit(pg_on_exit_callback function, Datum arg) STUB"); ++ return; ++ } ++#endif + if (on_shmem_exit_index >= MAX_ON_EXITS) + ereport(FATAL, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), diff --git a/patches/postgresql-wasm/src-interfaces-libpq-fe-auth.c.diff b/patches/postgresql-wasm/src-interfaces-libpq-fe-auth.c.diff index 001244a3..d66a268b 100644 --- a/patches/postgresql-wasm/src-interfaces-libpq-fe-auth.c.diff +++ b/patches/postgresql-wasm/src-interfaces-libpq-fe-auth.c.diff @@ -1,14 +1,61 @@ --- postgresql/src/interfaces/libpq/fe-auth.c +++ postgresql-wasm/src/interfaces/libpq/fe-auth.c -@@ -1170,6 +1170,7 @@ +@@ -962,10 +962,10 @@ + pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) + { + int oldmsglen; +- ++puts("965"); + if (!check_expected_areq(areq, conn)) + return STATUS_ERROR; +- ++puts("968"); + switch (areq) + { + case AUTH_REQ_OK: +@@ -1091,7 +1091,7 @@ + case AUTH_REQ_PASSWORD: + { + char *password; +- ++puts("1094"); + conn->password_needed = true; + password = conn->connhost[conn->whichhost].password; + if (password == NULL) +@@ -1106,9 +1106,10 @@ + { + appendPQExpBufferStr(&conn->errorMessage, + "fe_sendauth: error sending password authentication\n"); ++puts("1109"); + return STATUS_ERROR; + } +- ++puts("1112"); + /* We expect no further authentication requests. */ + conn->client_finished_auth = true; + break; +@@ -1148,10 +1149,11 @@ + break; + + default: ++puts(" ----------- 1151 ---------------"); + libpq_append_conn_error(conn, "authentication method %u not supported", areq); + return STATUS_ERROR; + } +- ++puts("1156"); + return STATUS_OK; + } + +@@ -1170,6 +1172,7 @@ pg_fe_getusername(uid_t user_id, PQExpBuffer errorMessage) { char *result = NULL; -+#if !defined(__EMSCRIPTEN__) ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) const char *name = NULL; #ifdef WIN32 -@@ -1202,7 +1203,9 @@ +@@ -1202,7 +1205,9 @@ else if (errorMessage) appendPQExpBuffer(errorMessage, "%s\n", pwdbuf); #endif diff --git a/patches/postgresql-wasm/src-interfaces-libpq-fe-exec.c.diff b/patches/postgresql-wasm/src-interfaces-libpq-fe-exec.c.diff index e427d35f..ddffa692 100644 --- a/patches/postgresql-wasm/src-interfaces-libpq-fe-exec.c.diff +++ b/patches/postgresql-wasm/src-interfaces-libpq-fe-exec.c.diff @@ -1,18 +1,10 @@ --- postgresql/src/interfaces/libpq/fe-exec.c +++ postgresql-wasm/src/interfaces/libpq/fe-exec.c -@@ -1694,8 +1694,15 @@ - /* Don't try to send if we know there's no live connection. */ - if (conn->status != CONNECTION_OK) - { -+#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ puts("#1699 !!!"); -+ conn->status = CONNECTION_OK; -+ conn->asyncStatus = PGASYNC_IDLE; -+ PQconnectPoll(conn); -+#else - libpq_append_conn_error(conn, "no connection to the server"); +@@ -1680,6 +1680,7 @@ + static bool + PQsendQueryStart(PGconn *conn, bool newQuery) + { ++PDEBUG("PQsendQueryStart"); + if (!conn) return false; -+#endif - } - /* Can't send while already busy, either, unless enqueuing for later */ diff --git a/patches/wasi_port.c b/patches/wasi_port.c index 9c14ee23..14fd3638 100644 --- a/patches/wasi_port.c +++ b/patches/wasi_port.c @@ -3,7 +3,6 @@ // ================================================================== #include - /* Convenience type when working with signal handlers. */ typedef void (*sa_handler_t) (int); @@ -244,6 +243,7 @@ sigprocmask (int operation, const sigset_t *set, sigset_t *old_set) { return 0; } + // STUBS int sigismember(const sigset_t *set, int signum) { return -1; @@ -267,10 +267,43 @@ unsigned int alarm(unsigned int seconds) { } +// WIP : shm +// ======================================================================================== +volatile int shm_index = 0; + +#include +void get_shm_path(char *tmpnam, const char *name) { + const char *shm = getenv("SHM"); + if (shm) { + printf("# 281 SHM=%s.%d", shm, shm_index); + snprintf(tmpnam, 128, "%s.%d", shm, shm_index++); + } else { + snprintf(tmpnam, 128, "/tmp%s", name); + } +} +int shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + get_shm_path(&tmpnam, name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 287: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} +int shm_unlink(const char *name) { + char tmpnam[128]; + if (getenv("SHM")) { + fprintf(stderr, "# 294: shm_unlink(%s) STUB\n", name); + return 0; + } + get_shm_path(&tmpnam, name); + return remove(tmpnam); +} +// popen +// ======================================================================================== #include // FILE+fprintf @@ -318,6 +351,8 @@ system_wasi(const char *command) { } // pthread.h +// ======================================================================================== + int pthread_create(pthread_t *restrict thread, @@ -351,3 +386,212 @@ FILE *tmpfile(void) { } + + + +// unix socket via file emulation using sched_yiedl for event pump. +// ================================================================================================= + +#include + +volatile int stage = 0; +volatile int fd_queue = 0; +volatile int fd_out=2; +volatile FILE *fd_FILE = NULL; + +// default fd is stderr +int socket(int domain, int type, int protocol) { +#if 0 + printf("# 404 : domain =%d type=%d proto=%d -> FORCE FD to 3 \n", domain , type, protocol); + return 3; +#else + printf("# 408 : domain =%d type=%d proto=%d\n", domain , type, protocol); +#endif + if (domain|AF_UNIX) { + fd_FILE = fopen(PGS_ILOCK, "w"); + if (fd_FILE) { + fd_out = fileno(fd_FILE); + printf("# 414: AF_UNIX sock=%d (fd_sock write) FILE=%s\n", fd_out, PGS_ILOCK); + } else { + printf("# 416: AF_UNIX ERROR OPEN (w/w+) FILE=%s\n", PGS_ILOCK); + abort(); + } + } + return fd_out; +} + +int connect(int socket, void *address, socklen_t address_len) { +#if 1 + puts("# 425: connect STUB"); + //fd_out = 3; + return 0; +#else + puts("# 429: connect EINPROGRESS"); + errno = EINPROGRESS; + return -1; +#endif +} + +ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen) { + int sent = write( fd_out, buf, len); + + printf("# 438: send/sendto(%d ?= %d )/%d sockfd=%d fno=%d fd_out=%d)\n", sent, ftell(fd_FILE), len, sockfd, fileno(fd_FILE), fd_out); + fd_queue+=sent; + return sent; +} + +ssize_t send(int sockfd, const void *buf, size_t len, int flags) { + return sendto(sockfd, buf, len, flags, NULL, 0); +} + +volatile bool web_warned = false; + +void sock_flush() { + if (fd_queue) { + printf(" -- 451 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); + if (!fd_FILE) { + if (!web_warned) { + puts("# 454: WARNING: fd_FILE not set but queue not empty, assuming web"); + web_warned = true; + } + + } else { + printf("# 459: SENT=%d/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); + fclose(fd_FILE); + rename(PGS_ILOCK, PGS_IN); +//freopen(PGS_ILOCK, "w+", fd_FILE); + fd_FILE = fopen(PGS_ILOCK, "w"); + fd_out = fileno(fd_FILE); + printf("# 465: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); + } + fd_queue = 0; + sched_yield(); + return; + } + + printf(" -- 472 sockflush[%d] : NO YIELD --\n",stage); + + // limit inf loops + if (stage++ > 1024) { + puts("# 476 sock_flush : busy looping ?"); + abort(); + } +} + + +volatile int fd_current_pos = 0; +volatile int fd_filesize = 0; + + +ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { + int busy = 0; + int rcv = -1; + sock_flush(); +/* + while (access(PGS_OUT, F_OK) != 0) { + if (!(++busy % 555111)) { + printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); + } + if (busy>1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcv1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcv + + + + +
emscripten
@@ -175,7 +239,7 @@ - const text_codec = new TextDecoder() + const text_codec = new TextDecoder(); //{ decode : new TextDecoder().decode , encode : new TextEncoder("utf-8").encode } const sixel_prefix = String.fromCharCode(27)+"Pq" function b_utf8(s) { @@ -191,13 +255,69 @@ var data = null switch (test_step) { case 0: - data = "SHOW client_encoding;"; + data = "SHOW client_encoding;CREATE EXTENSION IF NOT EXISTS plpgsql;" break case 1: - data = "SELECT now();" + data = "SELECT now(), current_database(), session_user, current_user;CREATE EXTENSION IF NOT EXISTS vector;" + break + + case 2: + data = `CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + number INT + ); + INSERT INTO test (number) VALUES (42); +`; + + break + +/* + case 2: + data = "SELECT now(), current_database(), session_user, current_user;CREATE EXTENSION IF NOT EXISTS postgis;" + break + + + case 3: + data = "SELECT now(), PostGIS_Version();" + break + + case 4: + data = "CREATE EXTENSION IF NOT EXISTS arrays;" + break + + + case 5: + data = "CHECKPOINT;SELECT pg_terminate_backend(42);"; break + + + case 2: + data = "CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434';" + break + + case 3: + data = "CREATE DATABASE test_user WITH OWNER = test_user;" + break + + case 4: + data = "CREATE SCHEMA test_user;" + break + + case 5: + data = `CREATE TABLE IF NOT EXISTS test_user.test ( + id SERIAL PRIMARY KEY, + number INT + ); + INSERT INTO test_user.test (number) VALUES (42); + ALTER TABLE test_user.test OWNER TO test_user; +`; + case 6: + data = "SELECT now(), current_database(), session_user, current_user;" + break +*/ +/* case 2: //data = "CREATE EXTENSION vector;" data = "LISTEN template1;" @@ -206,15 +326,28 @@ case 3: // data = "SELECT * FROM pg_extension;" data = "NOTIFY template1, 'hello';" + break + case 4: + data = "CREATE OR REPLACE FUNCTION test_func() RETURNS TEXT AS $$ BEGIN RETURN 'test'; END; $$ LANGUAGE plpgsql;" break + + case 5: + data = "SELECT test_func();" + break + + case 6: + data = "CREATE EXTENSION pg_stat_statements;" + break +*/ + } if (data!=null) { test_step++ console.log("SQL:", data) vm.readline(data+"\n\n") - setTimeout(test_drive, 1500); + setTimeout(test_drive, 800); } else { console.log("SQL:End") } @@ -233,6 +366,7 @@ function fs_callback(err) { if (!err) { +/* if (vm.FS.filesystems.PGFS && vm.FS.filesystems.PGFS.load_package) { vm.FS.filesystems.PGFS.load_package("vector","vector.tar.gz") if (vm.FS.analyzePath(PGDATA+"/PG_VERSION").exists) { @@ -243,6 +377,7 @@ console.warn("@@@@ no db @@@@"); } } +*/ setTimeout(delayed_init, 0) } else console.error("fs callback : FS error", err) @@ -271,15 +406,87 @@ } } */ + console.error("TODO: load ext from pg_extensions") await Module.load_package("vector", "vector.tar.gz") + // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") + + //await Module.load_package("postgis", "postgis-3.tar.gz") + //await Module.load_package("arrays", "arrays.tar.gz") - vm._setenv(vm.stringToUTF8OnStack("REPL"),vm.stringToUTF8OnStack("Y"),1) - vm._pg_initdb() - vm._pg_repl_raf() + //await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") - setTimeout(test_drive, 2400) + // vm._setenv(vm.stringToUTF8OnStack("REPL"),vm.stringToUTF8OnStack("Y"),1) + + + console.log("XXXXXX CMA ADDR REPLACEMENT ( for -O0 ) XXXXXXXXXXXX", vm._pg_getport() ); + + const idb = vm._pg_initdb() + + if (!idb) { + console.error("FATAL: INITDB failed to return value"); + return + } + + if (idb & 0b0001) { + console.error("INITDB failed"); + return + } + + if (idb & 0b0010) { + console.log(" #1 initdb was called"); + if (idb & 0b0100 ) { + console.log(" #2 found db") + + if (idb & (0b0100|0b1000)) { + console.log(" #3 found db+user : switch user") + // switch role + // vm.readline("SET ROLE test_user;"); + } + console.error("Invalid user ?"); + } else { + console.warn(" TODO: create db+user here / callback / throw ") +/* + vm.readline(` +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + number INT + ); + INSERT INTO test (number) VALUES (42); + ALTER TABLE test OWNER TO test_user; +SET ROLE test_user; +`); +*/ + } + + +/* + vm.PGDATA = PGDATA + await pglite_conf(vm,{"shared_preload_libraries" : ["pg_stat_statements"]}, {}) + +compute_query_id = on +pg_stat_statements.max = 10000 +pg_stat_statements.track = all +*/ + + vm.PGDATA = PGDATA + await pglite_conf(vm, { + "shared_preload_libraries" : ["pg_stat_statements"], + "compute_query_id" : "on", + "pg_stat_statements.max" : "10000", + "pg_stat_statements.track": "all", + }) + } + + + + +//vm._pg_repl_raf() + + setTimeout(test_drive, 1500) } // [] @@ -349,6 +556,7 @@ function copyFrom(filename, is_program) { console.warn("copyFrom:", filename, is_program); } + function copyTo(filename, is_program) { console.warn("copyTo:", filename, is_program); } @@ -368,8 +576,8 @@ load_pg_extension: load_pg_extension, load_package: load_package, - - arguments : ["PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", "REPL=N"], +// "PGDATABASE=test_user", "PGUSER=test_user" + arguments : ["--single", "postgres", "--", "PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", "PGUSER=postgres", "REPL=N"], config : { cdn : "https://pygame-web.github.io/archives/0.9/", @@ -447,12 +655,34 @@ }, readline : function(line) { - const ud = { "type" : "stdin", "data" : line } - if (window.worker) { - window.worker.postMessage({ target: 'custom', userData: ud }); + const os = vm.os + if (line.startsWith('?')){ + setTimeout(async()=> { await os.exec("pg_dump", "--help") } , 0 ) + return + } + if (line.startsWith('v')){ + setTimeout(async()=> { await os.exec("pg_dump", "--version") } , 0 ) + return + } + if (line.startsWith('!')){ + setTimeout(async()=> { await os.exec("pg_dump", "-U", "postgres", "--inserts", "-j", "1", "-v", "-c", "-C", "-f", "/tmp/out.sql", "--disable-dollar-quoting", "postgres") } , 0 ) + return + } + + if (line.startsWith('pg_dump ')) { + console.log("RL:", line) + } else { - window.Module.postMessage(ud); + + const ud = { "type" : "stdin", "data" : line } + if (window.worker) { + window.worker.postMessage({ target: 'custom', userData: ud }); + } else { + window.Module.postMessage(ud); + } + vm._interactive_one() } + } }; @@ -468,7 +698,6 @@ }; try { - console.log(initModule) globalThis.window.is_es6 = true } catch (x) { globalThis.window.is_es6 = false @@ -485,17 +714,58 @@ document.head.appendChild(jsloader) } - jsimport("tinytar.min.js") + //jsimport("tinytar.min.js") window.vm = Module const { WasmTerminal } = await /**/ import("./vtx.js") - Module.vt = new WasmTerminal("repl",132,32,12) + Module.vt = new WasmTerminal("repl", 200, 52, 10) + + Module.bc = new BroadcastChannel("pglite") + + const pg_lck = "/tmp/pglite/base/.s.PGSQL.5432.lck.in" + const pg_in = "/tmp/pglite/base/.s.PGSQL.5432.in" + const pg_out = "/tmp/pglite/base/.s.PGSQL.5432.out" + + function send_pglite_as_socketfile(data) { + FS.writeFile(pg_lck, event.data) + FS.rename(pg_lck, pg_in) + Module._interactive_one() + const fstat = FS.stat(pg_out) + console.log("pgreply", fstat.size) + } + + Module.bc.onmessage = (event) => { + const FS = Module.FS + console.warn("BC(srv):", event); + send_pglite_as_socketfile(event.data) + /* CMA + Module._interactive_write(event.data.byteLength) + Module.HEAPU8.set(event.data, Module.cma_port) + */ + + /* + Module._interactive_one() + const fstat = FS.stat(pg_out) + console.log("pgreply", fstat.size) + */ + + var stream = FS.open(pg_out, 'r'); + var buf = new Uint8Array(fstat.size); + FS.read(stream, buf, 0, fstat.size, 0); + FS.close(stream); + FS.unlink(pg_out) + Module.bc.postMessage(buf) + }; + + + #if MODULARIZE - if (is_es6) + if (is_es6) { window.Module = await initModule(Module) - else + Module.os = await async_wasi_init(Module.FS, null, null) + } else #endif { window.Module = Module @@ -503,12 +773,303 @@ console.log("main:", Module ) } + } window.addEventListener("load", boot ) +#if MODULARIZE + + +#endif + + From 34cd61666e6a5136fa6d923e091f0180745f397e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:24:05 +0000 Subject: [PATCH 2/4] Publish new versions (#422) Co-authored-by: github-actions[bot] --- .changeset/angry-scissors-grin.md | 5 ----- .changeset/cold-bears-return.md | 5 ----- .changeset/gentle-olives-hope.md | 5 ----- .changeset/modern-lemons-live.md | 5 ----- .changeset/poor-zebras-cheer.md | 5 ----- .changeset/popular-cheetahs-invite.md | 5 ----- .changeset/swift-news-move.md | 5 ----- .changeset/thin-trains-scream.md | 6 ------ .changeset/unlucky-ties-admire.md | 5 ----- packages/benchmark/CHANGELOG.md | 13 +++++++++++++ packages/benchmark/package.json | 2 +- packages/pglite-react/CHANGELOG.md | 14 ++++++++++++++ packages/pglite-react/package.json | 2 +- packages/pglite-repl/CHANGELOG.md | 14 ++++++++++++++ packages/pglite-repl/package.json | 2 +- packages/pglite-sync/CHANGELOG.md | 14 ++++++++++++++ packages/pglite-sync/package.json | 2 +- packages/pglite-tools/CHANGELOG.md | 7 +++++++ packages/pglite-tools/package.json | 2 +- packages/pglite-vue/CHANGELOG.md | 13 +++++++++++++ packages/pglite-vue/package.json | 2 +- packages/pglite/CHANGELOG.md | 12 ++++++++++++ packages/pglite/package.json | 2 +- 23 files changed, 94 insertions(+), 53 deletions(-) delete mode 100644 .changeset/angry-scissors-grin.md delete mode 100644 .changeset/cold-bears-return.md delete mode 100644 .changeset/gentle-olives-hope.md delete mode 100644 .changeset/modern-lemons-live.md delete mode 100644 .changeset/poor-zebras-cheer.md delete mode 100644 .changeset/popular-cheetahs-invite.md delete mode 100644 .changeset/swift-news-move.md delete mode 100644 .changeset/thin-trains-scream.md delete mode 100644 .changeset/unlucky-ties-admire.md create mode 100644 packages/pglite-tools/CHANGELOG.md diff --git a/.changeset/angry-scissors-grin.md b/.changeset/angry-scissors-grin.md deleted file mode 100644 index 6f1e19ce..00000000 --- a/.changeset/angry-scissors-grin.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -New `runExclusive` method on PGlite that allows you to hold an exclusive lock on the database, for use with `execProtocol*` methods diff --git a/.changeset/cold-bears-return.md b/.changeset/cold-bears-return.md deleted file mode 100644 index cfaec503..00000000 --- a/.changeset/cold-bears-return.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -A new `execProtocolRawSync` method that can execute a postgres wire protocol synchronously diff --git a/.changeset/gentle-olives-hope.md b/.changeset/gentle-olives-hope.md deleted file mode 100644 index de64cab8..00000000 --- a/.changeset/gentle-olives-hope.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -Make pglite compatible with @jest-environment: node diff --git a/.changeset/modern-lemons-live.md b/.changeset/modern-lemons-live.md deleted file mode 100644 index ec6b2c54..00000000 --- a/.changeset/modern-lemons-live.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -Bump Emscripten to 3.1.72 diff --git a/.changeset/poor-zebras-cheer.md b/.changeset/poor-zebras-cheer.md deleted file mode 100644 index bbedc883..00000000 --- a/.changeset/poor-zebras-cheer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite-sync': patch ---- - -Bump the supported version of the ElectricSQL sync server to the latest version diff --git a/.changeset/popular-cheetahs-invite.md b/.changeset/popular-cheetahs-invite.md deleted file mode 100644 index d71c005f..00000000 --- a/.changeset/popular-cheetahs-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite-tools': patch ---- - -Alpha version of pg_dump support in the browser and Node using a WASM build of pg_dump diff --git a/.changeset/swift-news-move.md b/.changeset/swift-news-move.md deleted file mode 100644 index 38d75330..00000000 --- a/.changeset/swift-news-move.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -Fix a bug with pipelining prepared statements. diff --git a/.changeset/thin-trains-scream.md b/.changeset/thin-trains-scream.md deleted file mode 100644 index a7dad149..00000000 --- a/.changeset/thin-trains-scream.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@electric-sql/pglite-react': patch -'@electric-sql/pglite': patch ---- - -Add a `offset` and `limit` option to live queries, when used it will return the total count for the query along with efficient updating of the offset. This works well with windowed or virtualised scrolling components. diff --git a/.changeset/unlucky-ties-admire.md b/.changeset/unlucky-ties-admire.md deleted file mode 100644 index 2d26a748..00000000 --- a/.changeset/unlucky-ties-admire.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite': patch ---- - -Use PG "WITH RECURSIVE" to traverse the live query dependencies diff --git a/packages/benchmark/CHANGELOG.md b/packages/benchmark/CHANGELOG.md index f21acf54..e5b1e7f6 100644 --- a/packages/benchmark/CHANGELOG.md +++ b/packages/benchmark/CHANGELOG.md @@ -1,5 +1,18 @@ # benchmark +## 0.0.14 + +### Patch Changes + +- Updated dependencies [6547374] +- Updated dependencies [6547374] +- Updated dependencies [df5c290] +- Updated dependencies [1784d04] +- Updated dependencies [ae36974] +- Updated dependencies [75f9f6d] +- Updated dependencies [ce212cf] + - @electric-sql/pglite@0.2.14 + ## 0.0.13 ### Patch Changes diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 7d84ced2..e2bfcd16 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -1,6 +1,6 @@ { "name": "benchmark", - "version": "0.0.13", + "version": "0.0.14", "description": "", "main": "index.js", "type": "module", diff --git a/packages/pglite-react/CHANGELOG.md b/packages/pglite-react/CHANGELOG.md index 4e34faaf..5877ef85 100644 --- a/packages/pglite-react/CHANGELOG.md +++ b/packages/pglite-react/CHANGELOG.md @@ -1,5 +1,19 @@ # @electric-sql/pglite-react +## 0.2.14 + +### Patch Changes + +- 75f9f6d: Add a `offset` and `limit` option to live queries, when used it will return the total count for the query along with efficient updating of the offset. This works well with windowed or virtualised scrolling components. +- Updated dependencies [6547374] +- Updated dependencies [6547374] +- Updated dependencies [df5c290] +- Updated dependencies [1784d04] +- Updated dependencies [ae36974] +- Updated dependencies [75f9f6d] +- Updated dependencies [ce212cf] + - @electric-sql/pglite@0.2.14 + ## 0.2.13 ### Patch Changes diff --git a/packages/pglite-react/package.json b/packages/pglite-react/package.json index ea7d518d..43ef468d 100644 --- a/packages/pglite-react/package.json +++ b/packages/pglite-react/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-react", - "version": "0.2.13", + "version": "0.2.14", "description": "Hooks for using PGlite", "type": "module", "private": false, diff --git a/packages/pglite-repl/CHANGELOG.md b/packages/pglite-repl/CHANGELOG.md index 3c1d3f6d..2effd3dd 100644 --- a/packages/pglite-repl/CHANGELOG.md +++ b/packages/pglite-repl/CHANGELOG.md @@ -1,5 +1,19 @@ # @electric-sql/pglite-repl +## 0.2.14 + +### Patch Changes + +- Updated dependencies [6547374] +- Updated dependencies [6547374] +- Updated dependencies [df5c290] +- Updated dependencies [1784d04] +- Updated dependencies [ae36974] +- Updated dependencies [75f9f6d] +- Updated dependencies [ce212cf] + - @electric-sql/pglite@0.2.14 + - @electric-sql/pglite-react@0.2.14 + ## 0.2.13 ### Patch Changes diff --git a/packages/pglite-repl/package.json b/packages/pglite-repl/package.json index b72a0d6d..69dd839a 100644 --- a/packages/pglite-repl/package.json +++ b/packages/pglite-repl/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-repl", - "version": "0.2.13", + "version": "0.2.14", "author": "Electric DB Limited", "homepage": "https://pglite.dev", "license": "Apache-2.0", diff --git a/packages/pglite-sync/CHANGELOG.md b/packages/pglite-sync/CHANGELOG.md index 3dffde82..55cd8ba1 100644 --- a/packages/pglite-sync/CHANGELOG.md +++ b/packages/pglite-sync/CHANGELOG.md @@ -1,5 +1,19 @@ # @electric-sql/pglite-sync +## 0.2.16 + +### Patch Changes + +- d1dd12b: Bump the supported version of the ElectricSQL sync server to the latest version +- Updated dependencies [6547374] +- Updated dependencies [6547374] +- Updated dependencies [df5c290] +- Updated dependencies [1784d04] +- Updated dependencies [ae36974] +- Updated dependencies [75f9f6d] +- Updated dependencies [ce212cf] + - @electric-sql/pglite@0.2.14 + ## 0.2.14 ### Patch Changes diff --git a/packages/pglite-sync/package.json b/packages/pglite-sync/package.json index 44b922b0..af232ca9 100644 --- a/packages/pglite-sync/package.json +++ b/packages/pglite-sync/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-sync", - "version": "0.2.15", + "version": "0.2.16", "description": "ElectricSQL Sync for PGlite", "type": "module", "private": false, diff --git a/packages/pglite-tools/CHANGELOG.md b/packages/pglite-tools/CHANGELOG.md new file mode 100644 index 00000000..dd82982f --- /dev/null +++ b/packages/pglite-tools/CHANGELOG.md @@ -0,0 +1,7 @@ +# @electric-sql/pglite-tools + +## 0.2.1 + +### Patch Changes + +- 6547374: Alpha version of pg_dump support in the browser and Node using a WASM build of pg_dump diff --git a/packages/pglite-tools/package.json b/packages/pglite-tools/package.json index 46e9d8dd..234233d2 100644 --- a/packages/pglite-tools/package.json +++ b/packages/pglite-tools/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-tools", - "version": "0.2.0", + "version": "0.2.1", "description": "Tools for working with PGlite databases", "author": "Electric DB Limited", "homepage": "https://pglite.dev", diff --git a/packages/pglite-vue/CHANGELOG.md b/packages/pglite-vue/CHANGELOG.md index bf584e20..d263ca55 100644 --- a/packages/pglite-vue/CHANGELOG.md +++ b/packages/pglite-vue/CHANGELOG.md @@ -1,5 +1,18 @@ # @electric-sql/pglite-vue +## 0.2.14 + +### Patch Changes + +- Updated dependencies [6547374] +- Updated dependencies [6547374] +- Updated dependencies [df5c290] +- Updated dependencies [1784d04] +- Updated dependencies [ae36974] +- Updated dependencies [75f9f6d] +- Updated dependencies [ce212cf] + - @electric-sql/pglite@0.2.14 + ## 0.2.13 ### Patch Changes diff --git a/packages/pglite-vue/package.json b/packages/pglite-vue/package.json index 26ab91ae..b7e6fbeb 100644 --- a/packages/pglite-vue/package.json +++ b/packages/pglite-vue/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-vue", - "version": "0.2.13", + "version": "0.2.14", "description": "Vue hooks for using PGlite", "type": "module", "private": false, diff --git a/packages/pglite/CHANGELOG.md b/packages/pglite/CHANGELOG.md index c5cbb2ef..db84d384 100644 --- a/packages/pglite/CHANGELOG.md +++ b/packages/pglite/CHANGELOG.md @@ -1,5 +1,17 @@ # @electric-sql/pglite +## 0.2.14 + +### Patch Changes + +- 6547374: New `runExclusive` method on PGlite that allows you to hold an exclusive lock on the database, for use with `execProtocol*` methods +- 6547374: A new `execProtocolRawSync` method that can execute a postgres wire protocol synchronously +- df5c290: Make pglite compatible with @jest-environment: node +- 1784d04: Bump Emscripten to 3.1.72 +- ae36974: Fix a bug with pipelining prepared statements. +- 75f9f6d: Add a `offset` and `limit` option to live queries, when used it will return the total count for the query along with efficient updating of the offset. This works well with windowed or virtualised scrolling components. +- ce212cf: Use PG "WITH RECURSIVE" to traverse the live query dependencies + ## 0.2.13 ### Patch Changes diff --git a/packages/pglite/package.json b/packages/pglite/package.json index a3189d40..c3a96f83 100644 --- a/packages/pglite/package.json +++ b/packages/pglite/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite", - "version": "0.2.13", + "version": "0.2.14", "private": false, "publishConfig": { "access": "public" From 17c9875fa38be1a9a0c6a5480b5ca9596284a96b Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 27 Nov 2024 11:11:30 +0000 Subject: [PATCH 3/4] fix(pglite/tools): add node imports to the package.json browser excludes (#444) --- .changeset/fifty-beers-swim.md | 5 +++++ packages/pglite-tools/package.json | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 .changeset/fifty-beers-swim.md diff --git a/.changeset/fifty-beers-swim.md b/.changeset/fifty-beers-swim.md new file mode 100644 index 00000000..5a105cc8 --- /dev/null +++ b/.changeset/fifty-beers-swim.md @@ -0,0 +1,5 @@ +--- +'@electric-sql/pglite-tools': patch +--- + +add node imports to the package.json browser excludes diff --git a/packages/pglite-tools/package.json b/packages/pglite-tools/package.json index 234233d2..b5010c28 100644 --- a/packages/pglite-tools/package.json +++ b/packages/pglite-tools/package.json @@ -35,6 +35,10 @@ "stylecheck": "pnpm lint && prettier --check ./src ./tests", "test": "vitest" }, + "browser": { + "fs": false, + "fs/promises": false + }, "devDependencies": { "@electric-sql/pglite": "workspace:*", "@types/emscripten": "^1.39.13", From bfddaaa46c507f18c9a12ee621e637b56a7fd709 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 11:14:15 +0000 Subject: [PATCH 4/4] Publish new versions (#445) Co-authored-by: github-actions[bot] --- .changeset/fifty-beers-swim.md | 5 ----- packages/pglite-tools/CHANGELOG.md | 6 ++++++ packages/pglite-tools/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/fifty-beers-swim.md diff --git a/.changeset/fifty-beers-swim.md b/.changeset/fifty-beers-swim.md deleted file mode 100644 index 5a105cc8..00000000 --- a/.changeset/fifty-beers-swim.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@electric-sql/pglite-tools': patch ---- - -add node imports to the package.json browser excludes diff --git a/packages/pglite-tools/CHANGELOG.md b/packages/pglite-tools/CHANGELOG.md index dd82982f..57b0996b 100644 --- a/packages/pglite-tools/CHANGELOG.md +++ b/packages/pglite-tools/CHANGELOG.md @@ -1,5 +1,11 @@ # @electric-sql/pglite-tools +## 0.2.2 + +### Patch Changes + +- 17c9875: add node imports to the package.json browser excludes + ## 0.2.1 ### Patch Changes diff --git a/packages/pglite-tools/package.json b/packages/pglite-tools/package.json index b5010c28..861522b3 100644 --- a/packages/pglite-tools/package.json +++ b/packages/pglite-tools/package.json @@ -1,6 +1,6 @@ { "name": "@electric-sql/pglite-tools", - "version": "0.2.1", + "version": "0.2.2", "description": "Tools for working with PGlite databases", "author": "Electric DB Limited", "homepage": "https://pglite.dev",