From 5e233dbb45f391c70c9a3d5bdb220a997d55dca4 Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Wed, 21 Oct 2020 11:06:38 +0200 Subject: [PATCH 1/3] * enable write.wait_for_active_shards, by enabling "write.wait_for_active_shards = 1" writes are possible also when table health is not 100% * use variable to set values for wait_for_active_shards --- docs/manuals/admin/configuration.md | 1 + src/translators/crate.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/manuals/admin/configuration.md b/docs/manuals/admin/configuration.md index f1ff49f9..fa36d9bd 100644 --- a/docs/manuals/admin/configuration.md +++ b/docs/manuals/admin/configuration.md @@ -22,6 +22,7 @@ To configure QuantumLeap you can use the following environment variables: | `DEFAULT_CACHE_TTL`| Time to live of metadata cache, default: 60 (seconds) | | | `QL_CONFIG` | Pathname for tenant configuration | | `QL_DEFAULT_DB` | Default backend: `timescale` or `crate` | +| `CRATE_WAIT_ACTIV_SHARDS` | Specifies the number of shard copies that need to be active for write operations to proceed. Default `1`. See related [crate documentation](https://crate.io/docs/crate/reference/en/4.3/sql/statements/create-table.html#write-wait-for-active-shards). | | `USE_FLASK` | `True` or `False` to use flask server (only for Dev) or gunicorn. Default to `False` | | `LOGLEVEL` | Define the log level for all services (`DEBUG`, `INFO`, `WARNING` , `ERROR`) | diff --git a/src/translators/crate.py b/src/translators/crate.py index c55f8461..f50e9672 100644 --- a/src/translators/crate.py +++ b/src/translators/crate.py @@ -69,6 +69,7 @@ def setup(self): # we need to think if we want to cache this information # and save few msec for evey API call self.db_version = self.get_db_version() + self.active_shards = EnvReader(log=logging.getLogger(__name__).debug).read(StrVar('CRATE_WAIT_ACTIV_SHARDS', '1')) major = int(self.db_version.split('.')[0]) if major <= 2: @@ -170,8 +171,9 @@ def _create_data_table(self, table_name, table, fiware_service): columns = ', '.join('"{}" {}'.format(cn.lower(), ct) for cn, ct in table.items()) stmt = "create table if not exists {} ({}) with " \ - "(number_of_replicas = '2-all', " \ - "column_policy = 'strict')".format(table_name, columns) + "(\"number_of_replicas\" = '2-all', " \ + "\"column_policy\" = 'strict', " \ + "\"write.wait_for_active_shards\" = '{}')".format(table_name, columns, self.active_shards) self.cursor.execute(stmt) def _update_data_table(self, table_name, new_columns, fiware_service): @@ -188,7 +190,9 @@ def _should_insert_original_entities(self, def _create_metadata_table(self): stmt = "create table if not exists {} " \ "(table_name string primary key, entity_attrs object) " \ - "with (number_of_replicas = '2-all', column_policy = 'dynamic')" + "with (" \ + "number_of_replicas = '2-all', " \ + "column_policy = 'dynamic')" op = stmt.format(METADATA_TABLE_NAME) self.cursor.execute(op) From 1197ec9a1a844ef1ffac4dfe7dcf0530d740e22e Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Fri, 20 Nov 2020 11:19:57 +0100 Subject: [PATCH 2/3] fix line line lenght --- src/translators/crate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/translators/crate.py b/src/translators/crate.py index f50e9672..db44ca25 100644 --- a/src/translators/crate.py +++ b/src/translators/crate.py @@ -69,7 +69,8 @@ def setup(self): # we need to think if we want to cache this information # and save few msec for evey API call self.db_version = self.get_db_version() - self.active_shards = EnvReader(log=logging.getLogger(__name__).debug).read(StrVar('CRATE_WAIT_ACTIV_SHARDS', '1')) + self.active_shards = EnvReader(log=logging.getLogger(__name__).debug)\ + .read(StrVar('CRATE_WAIT_ACTIV_SHARDS', '1')) major = int(self.db_version.split('.')[0]) if major <= 2: @@ -173,7 +174,8 @@ def _create_data_table(self, table_name, table, fiware_service): stmt = "create table if not exists {} ({}) with " \ "(\"number_of_replicas\" = '2-all', " \ "\"column_policy\" = 'strict', " \ - "\"write.wait_for_active_shards\" = '{}')".format(table_name, columns, self.active_shards) + "\"write.wait_for_active_shards\" = '{}'" \ + ")".format(table_name, columns, self.active_shards) self.cursor.execute(stmt) def _update_data_table(self, table_name, new_columns, fiware_service): From 8e86a3f116f9e9b39e72463d5fa6abe0b5ebfa5c Mon Sep 17 00:00:00 2001 From: "Federico M. Facca" Date: Fri, 20 Nov 2020 16:03:41 +0100 Subject: [PATCH 3/3] fix variable naming --- docs/manuals/admin/configuration.md | 2 +- src/translators/crate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/manuals/admin/configuration.md b/docs/manuals/admin/configuration.md index fa36d9bd..2cced5af 100644 --- a/docs/manuals/admin/configuration.md +++ b/docs/manuals/admin/configuration.md @@ -22,7 +22,7 @@ To configure QuantumLeap you can use the following environment variables: | `DEFAULT_CACHE_TTL`| Time to live of metadata cache, default: 60 (seconds) | | | `QL_CONFIG` | Pathname for tenant configuration | | `QL_DEFAULT_DB` | Default backend: `timescale` or `crate` | -| `CRATE_WAIT_ACTIV_SHARDS` | Specifies the number of shard copies that need to be active for write operations to proceed. Default `1`. See related [crate documentation](https://crate.io/docs/crate/reference/en/4.3/sql/statements/create-table.html#write-wait-for-active-shards). | +| `CRATE_WAIT_ACTIVE_SHARDS` | Specifies the number of shard copies that need to be active for write operations to proceed. Default `1`. See related [crate documentation](https://crate.io/docs/crate/reference/en/4.3/sql/statements/create-table.html#write-wait-for-active-shards). | | `USE_FLASK` | `True` or `False` to use flask server (only for Dev) or gunicorn. Default to `False` | | `LOGLEVEL` | Define the log level for all services (`DEBUG`, `INFO`, `WARNING` , `ERROR`) | diff --git a/src/translators/crate.py b/src/translators/crate.py index db44ca25..eaa53037 100644 --- a/src/translators/crate.py +++ b/src/translators/crate.py @@ -70,7 +70,7 @@ def setup(self): # and save few msec for evey API call self.db_version = self.get_db_version() self.active_shards = EnvReader(log=logging.getLogger(__name__).debug)\ - .read(StrVar('CRATE_WAIT_ACTIV_SHARDS', '1')) + .read(StrVar('CRATE_WAIT_ACTIVE_SHARDS', '1')) major = int(self.db_version.split('.')[0]) if major <= 2: