From f5cf341b085602afccfdb75c7c19ba1417493c17 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Apr 2024 11:15:51 +0300 Subject: [PATCH 1/4] feat: update API to 7.3 --- .devcontainer/add_erlang_bindings.patch | 12 ++++++------ .devcontainer/docker-compose.yaml | 4 ++-- .github/workflows/ci.yml | 8 ++++---- .github/workflows/docker.yml | 2 +- scripts/fetch_fdb_cli.sh | 2 +- src/erlfdb.app.src | 2 +- src/erlfdb_nif.erl | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.devcontainer/add_erlang_bindings.patch b/.devcontainer/add_erlang_bindings.patch index 0f4b921..56acf74 100644 --- a/.devcontainer/add_erlang_bindings.patch +++ b/.devcontainer/add_erlang_bindings.patch @@ -1,5 +1,5 @@ diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py -index 17d06cf..1f6e1e3 100644 +index bfe569e27..9694766d1 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -22,7 +22,7 @@ import math @@ -7,19 +7,19 @@ index 17d06cf..1f6e1e3 100644 import os -sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', 'python')] -+#sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', 'python')] ++# sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', 'python')] import util - + from fdb import LATEST_API_VERSION diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py -index 70e1f81..85dcde5 100644 +index 30a0b72fa..ae77c9507 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py -@@ -58,6 +58,7 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % ( +@@ -60,6 +60,7 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % ( # We could set min_api_version lower on some of these if the testers were updated to support them testers = { -+ 'erlang': Tester('erlang', '/usr/src/erlfdb/test/tester.es', 2040, 710, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), ++ 'erlang': Tester('erlang', '/usr/src/erlfdb/test/tester.es', 2040, 730, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True), 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION), diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index 2c6b331..1df3ad7 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -7,7 +7,7 @@ services: ERLANG_VERSION: "24" # This should always match the value in fdb.image - FDB_VERSION: "7.1.43" + FDB_VERSION: "7.3.35" environment: # This needs to match the name of the FoundationDB service below @@ -28,4 +28,4 @@ services: network_mode: service:fdb fdb: - image: foundationdb/foundationdb:7.1.43 + image: foundationdb/foundationdb:7.3.35 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0da5cf7..8400f25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: otp-version: ['24', '25'] - fdb-version: ['7.1.43'] + fdb-version: ['7.3.35'] runs-on: ubuntu-22.04 env: FDB_VERSION: ${{ matrix.fdb-version }} @@ -47,12 +47,12 @@ jobs: strategy: matrix: test-name: [api, directory, directory_hca, tuple] - api-version: [710] + api-version: [730] container: - image: ghcr.io/emqx/couchdb-erlfdb:erlang-25-fdb-7.1.43 + image: ghcr.io/emqx/couchdb-erlfdb:erlang-25-fdb-7.3.35 services: foundationdb: - image: foundationdb/foundationdb:7.1.43 + image: foundationdb/foundationdb:7.3.35 env: REBAR_PROFILE: devcontainer # Set to 1 for verbose rebar3 logging diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8ee4d06..a534620 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: otp-version: ['24', '25'] - fdb-version: ['7.1.43'] + fdb-version: ['7.3.35'] steps: - uses: actions/checkout@v4 diff --git a/scripts/fetch_fdb_cli.sh b/scripts/fetch_fdb_cli.sh index 67daa6b..1f34ae8 100755 --- a/scripts/fetch_fdb_cli.sh +++ b/scripts/fetch_fdb_cli.sh @@ -13,7 +13,7 @@ if [ "$#" -ne 3 ]; then usage fi -FDB_VERSION="7.1.43" +FDB_VERSION="7.3.35" TYPE="${1}" OUTPUT="${2}" diff --git a/src/erlfdb.app.src b/src/erlfdb.app.src index 1ddad44..9636f5d 100644 --- a/src/erlfdb.app.src +++ b/src/erlfdb.app.src @@ -18,7 +18,7 @@ {maintainers, ["Paul J. Davis"]}, {links, [{"GitHub", "https://github.com/emqx/couchdb-erlfdb"}]}, {env, [ - {api_version, 710}, + {api_version, 730}, {network_options, []} ]} ]}. diff --git a/src/erlfdb_nif.erl b/src/erlfdb_nif.erl index 1d97554..d723c26 100644 --- a/src/erlfdb_nif.erl +++ b/src/erlfdb_nif.erl @@ -66,7 +66,7 @@ error_predicate/2 ]). --define(DEFAULT_API_VERSION, 710). +-define(DEFAULT_API_VERSION, 730). -type error() :: {erlfdb_error, Code :: integer()}. -type future() :: {erlfdb_future, reference(), reference()}. From 5371f2f22eb6e6d201f34287d63f309db1f2f2b2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Apr 2024 13:06:09 +0300 Subject: [PATCH 2/4] feat: enable tenants in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8400f25..ed035b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: shell: bash run: bash /usr/local/bin/create_cluster_file.bash - name: Initialize FDB database - run: fdbcli --exec "configure new single ssd" + run: fdbcli --exec "configure new single ssd tenant_mode=optional_experimental" - name: Execute unit tests run: rebar3 eunit - name: Execute binding test From 275cc06d181a12fb11c38c9da2a6ded1bdd55b7b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Apr 2024 14:57:40 +0300 Subject: [PATCH 3/4] feat(7.3): add _TENANT command support in tester --- test/tester.es | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/test/tester.es b/test/tester.es index 734042d..64e429e 100755 --- a/test/tester.es +++ b/test/tester.es @@ -24,6 +24,7 @@ index, is_db, is_snapshot, + is_tenant, last_version, pids, @@ -228,6 +229,9 @@ stack_pop_tuples(St, Count) -> get_transaction(TxName) -> get({'$erlfdb_tx', TxName}). +tenant_or_db(#st{db = Db, tenant = undefined}) -> Db; +tenant_or_db(#st{tenant = Tenant}) -> Tenant. + new_transaction(#st{db = Db, tenant = Tenant}, TxName) -> Src = case Tenant of @@ -321,6 +325,7 @@ init_run_loop(Db, Prefix) -> index = 0, is_db = undefined, is_snapshot = undefined, + is_tenant = undefined, last_version = 0, pids = [], @@ -362,31 +367,39 @@ run_loop(#st{} = St) -> IsDb = has_suffix(Op, <<"_DATABASE">>), IsSS = has_suffix(Op, <<"_SNAPSHOT">>), + IsTenant = has_suffix(Op, <<"_TENANT">>), IsDir = has_prefix(Op, <<"DIRECTORY_">>), OpName = if - not (IsDb or IsSS) -> - Op; - true -> - % strip off _DATABASE/_SNAPSHOT - binary:part(Op, {0, size(Op) - 9}) + IsDb -> binary:part(Op, {0, size(Op) - 9}); + IsSS -> binary:part(Op, {0, size(Op) - 9}); + IsTenant -> binary:part(Op, {0, size(Op) - 7}); + true -> Op end, TxObj = - case {IsDb, IsSS} of - {true, false} -> - Db; - {false, false} -> - get_transaction(TxName); - {false, true} -> - erlfdb:snapshot(get_transaction(TxName)) + if + IsDb -> Db; + IsSS -> erlfdb:snapshot(get_transaction(TxName)); + IsTenant -> tenant_or_db(St); + true -> get_transaction(TxName) end, + % case {IsDb, IsSS} of + % {true, false} -> + % Db; + % {false, false} -> + % get_transaction(TxName); + % {false, true} -> + + % end, + PreSt = St#st{ op_tuple = OpTuple, is_db = IsDb, is_snapshot = IsSS, + is_tenant = IsTenant, is_directory_op = IsDir }, From e7e743a811aa7781c5870bc7e79baf0fd959d4cd Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Apr 2024 15:37:32 +0300 Subject: [PATCH 4/4] feat(7.3): add list_tenants command --- src/erlfdb_tenant.erl | 20 +++++++++++++++++++- test/tester.es | 6 ++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/erlfdb_tenant.erl b/src/erlfdb_tenant.erl index d088ba1..85fd903 100644 --- a/src/erlfdb_tenant.erl +++ b/src/erlfdb_tenant.erl @@ -12,7 +12,7 @@ -module(erlfdb_tenant). --export([create_tenant/2, open_tenant/2, delete_tenant/2]). +-export([create_tenant/2, open_tenant/2, delete_tenant/2, list_tenants/4]). -define(IS_DB, {erlfdb_database, _}). -define(TENANT_MAP_PREFIX, <<16#FF, 16#FF, "/management/tenant_map/">>). @@ -50,8 +50,26 @@ delete_tenant(Tx, Tenant) -> clear(Tx, Key) end. +list_tenants(?IS_DB = Db, From, To, Limit) -> + transactional(Db, fun(Tx) -> + list_tenants(Tx, From, To, Limit) + end); +list_tenants(Tx, From, To, Limit) -> + FullFrom = tenant_key(From), + FullTo = tenant_key(To), + lists:map( + fun({K, V}) -> + {unprefix_tenant_key(K), V} + end, + erlfdb:range(Tx, FullFrom, FullTo, [{limit, Limit}]) + ). + check_tenant_existence(Tx, Key) -> wait(get(Tx, Key)). tenant_key(Tenant) -> <>. + +unprefix_tenant_key(Key) -> + binary:part(Key, byte_size(?TENANT_MAP_PREFIX), byte_size(Key) - byte_size(?TENANT_MAP_PREFIX)). + diff --git a/test/tester.es b/test/tester.es index 64e429e..7ab11a8 100755 --- a/test/tester.es +++ b/test/tester.es @@ -794,6 +794,12 @@ execute(_TxObj, #st{db = Db} = St, <<"TENANT_SET_ACTIVE">>) -> St#st{tenant = Result}; execute(_TxObj, St, <<"TENANT_CLEAR_ACTIVE">>) -> St#st{tenant = undefined}; +execute(_TxObj, #st{db = Db} = St, <<"TENANT_LIST">>) -> + [Start, End, Limit] = stack_pop(St, 3), + TenantsKV = erlfdb_tenant:list_tenants(Db, Start, End, Limit), + {Tenants, _} = lists:unzip(TenantsKV), %% TODO: verify value json? + stack_push(St, erlfdb_tuple:pack(list_to_tuple(Tenants))), + St; execute(_TxObj, _St, UnknownOp) -> erlang:error({unknown_op, UnknownOp}).