From 29dd2029c0bc1b5047689faf34d8ebde7a5f5f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Tue, 3 Dec 2024 12:32:40 +0100 Subject: [PATCH 1/4] docs: add sample for read-only transactions Adds a sample and documentation for read-only transactions. Fixes #493 --- README.rst | 13 +- samples/noxfile.py | 5 + samples/read_only_transaction_sample.py | 64 ++++++++++ test/mockserver_tests/read_only_model.py | 33 +++++ .../test_read_only_transaction.py | 117 ++++++++++++++++++ 5 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 samples/read_only_transaction_sample.py create mode 100644 test/mockserver_tests/read_only_model.py create mode 100644 test/mockserver_tests/test_read_only_transaction.py diff --git a/README.rst b/README.rst index 5c39e139..852bdc5c 100644 --- a/README.rst +++ b/README.rst @@ -344,8 +344,9 @@ ReadOnly transactions ~~~~~~~~~~~~~~~~~~~~~ By default, transactions produced by a Spanner connection are in -ReadWrite mode. However, some applications require an ability to grant -ReadOnly access to users/methods; for these cases Spanner dialect +ReadWrite mode. However, workloads that only read data perform better +if they use read-only transactions, as Spanner does not need to take +locks for the data that is read; for these cases, the Spanner dialect supports the ``read_only`` execution option, which switches a connection into ReadOnly mode: @@ -354,11 +355,13 @@ into ReadOnly mode: with engine.connect().execution_options(read_only=True) as connection: connection.execute(select(["*"], from_obj=table)).fetchall() -Note that execution options are applied lazily - on the ``execute()`` -method call, right before it. +See the `Read-only transaction sample +`__ +for a concrete example. ReadOnly/ReadWrite mode of a connection can't be changed while a -transaction is in progress - first you must commit or rollback it. +transaction is in progress - you must commit or rollback the current +transaction before changing the mode. Stale reads ~~~~~~~~~~~ diff --git a/samples/noxfile.py b/samples/noxfile.py index b103fd77..5011e971 100644 --- a/samples/noxfile.py +++ b/samples/noxfile.py @@ -57,6 +57,11 @@ def transaction(session): _sample(session) +@nox.session() +def read_only_transaction(session): + _sample(session) + + @nox.session() def _all_samples(session): _sample(session) diff --git a/samples/read_only_transaction_sample.py b/samples/read_only_transaction_sample.py new file mode 100644 index 00000000..35ef84e7 --- /dev/null +++ b/samples/read_only_transaction_sample.py @@ -0,0 +1,64 @@ +# Copyright 2024 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from sqlalchemy import create_engine, Engine +from sqlalchemy.orm import Session + +from sample_helper import run_sample +from model import Singer, Concert, Venue + + +# Shows how to execute a read-only transaction on Spanner using SQLAlchemy. +def read_only_transaction_sample(): + engine = create_engine( + "spanner:///projects/sample-project/" + "instances/sample-instance/" + "databases/sample-database", + echo=True, + ) + # First insert a few test rows that can be queried in a read-only transaction. + insert_test_data(engine) + + # Create a session that uses a read-only transaction. + # Read-only transactions do not take locks, and are therefore preferred + # above read/write transactions for workloads that only read data on Spanner. + with Session(engine.execution_options(read_only=True)) as session: + print("Singers ordered by last name") + singers = session.query(Singer).order_by(Singer.last_name).all() + for singer in singers: + print("Singer: ", singer.full_name) + + print() + print("Singers ordered by first name") + singers = session.query(Singer).order_by(Singer.first_name).all() + for singer in singers: + print("Singer: ", singer.full_name) + + +def insert_test_data(engine: Engine): + with Session(engine) as session: + session.add_all( + [ + Singer(id=str(uuid.uuid4()), first_name="John", last_name="Doe"), + Singer(id=str(uuid.uuid4()), first_name="Jane", last_name="Doe"), + ] + ) + session.commit() + + +if __name__ == "__main__": + run_sample(read_only_transaction_sample) diff --git a/test/mockserver_tests/read_only_model.py b/test/mockserver_tests/read_only_model.py new file mode 100644 index 00000000..b76cdd3f --- /dev/null +++ b/test/mockserver_tests/read_only_model.py @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy import String, BigInteger, Sequence, TextClause +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column + + +class Base(DeclarativeBase): + pass + + +class Singer(Base): + __tablename__ = "singers" + id: Mapped[int] = mapped_column( + BigInteger, + Sequence("singer_id"), + server_default=TextClause("GET_NEXT_SEQUENCE_VALUE(SEQUENCE singer_id)"), + primary_key=True, + ) + name: Mapped[str] = mapped_column(String) diff --git a/test/mockserver_tests/test_read_only_transaction.py b/test/mockserver_tests/test_read_only_transaction.py new file mode 100644 index 00000000..83597b70 --- /dev/null +++ b/test/mockserver_tests/test_read_only_transaction.py @@ -0,0 +1,117 @@ +# Copyright 2024 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session +from sqlalchemy.testing import eq_, is_instance_of +from google.cloud.spanner_v1 import ( + FixedSizePool, + BatchCreateSessionsRequest, + ExecuteSqlRequest, + GetSessionRequest, + BeginTransactionRequest, + TransactionOptions, +) +from test.mockserver_tests.mock_server_test_base import MockServerTestBase +from test.mockserver_tests.mock_server_test_base import add_result +import google.cloud.spanner_v1.types.type as spanner_type +import google.cloud.spanner_v1.types.result_set as result_set + + +class TestReadOnlyTransaction(MockServerTestBase): + def test_read_only_transaction(self): + from test.mockserver_tests.read_only_model import Singer + + add_singer_query_result("SELECT singers.id, singers.name \n" + "FROM singers") + engine = create_engine( + "spanner:///projects/p/instances/i/databases/d", + echo=True, + connect_args={"client": self.client, "pool": FixedSizePool(size=10)}, + ) + + with Session(engine.execution_options(read_only=True)) as session: + # Execute two queries in a read-only transaction. + session.scalars(select(Singer)).all() + session.scalars(select(Singer)).all() + + # Verify the requests that we got. + requests = self.spanner_service.requests + eq_(5, len(requests)) + is_instance_of(requests[0], BatchCreateSessionsRequest) + # We should get rid of this extra round-trip for GetSession.... + is_instance_of(requests[1], GetSessionRequest) + is_instance_of(requests[2], BeginTransactionRequest) + is_instance_of(requests[3], ExecuteSqlRequest) + is_instance_of(requests[4], ExecuteSqlRequest) + # Verify that the transaction is a read-only transaction. + begin_request: BeginTransactionRequest = requests[2] + eq_( + TransactionOptions( + dict( + read_only=TransactionOptions.ReadOnly( + dict( + strong=True, + return_read_timestamp=True, + ) + ) + ) + ), + begin_request.options, + ) + + +def add_singer_query_result(sql: str): + result = result_set.ResultSet( + dict( + metadata=result_set.ResultSetMetadata( + dict( + row_type=spanner_type.StructType( + dict( + fields=[ + spanner_type.StructType.Field( + dict( + name="singers_id", + type=spanner_type.Type( + dict(code=spanner_type.TypeCode.INT64) + ), + ) + ), + spanner_type.StructType.Field( + dict( + name="singers_name", + type=spanner_type.Type( + dict(code=spanner_type.TypeCode.STRING) + ), + ) + ), + ] + ) + ) + ) + ), + ) + ) + result.rows.extend( + [ + ( + "1", + "Jane Doe", + ), + ( + "2", + "John Doe", + ), + ] + ) + add_result(sql, result) From fbba4d21b8cba4837d30fae1366044746b05b83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Wed, 4 Dec 2024 14:42:12 +0100 Subject: [PATCH 2/4] chore: run two read-only transactions in test --- .../test_read_only_transaction.py | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/test/mockserver_tests/test_read_only_transaction.py b/test/mockserver_tests/test_read_only_transaction.py index 83597b70..b8e8e170 100644 --- a/test/mockserver_tests/test_read_only_transaction.py +++ b/test/mockserver_tests/test_read_only_transaction.py @@ -40,35 +40,41 @@ def test_read_only_transaction(self): connect_args={"client": self.client, "pool": FixedSizePool(size=10)}, ) - with Session(engine.execution_options(read_only=True)) as session: - # Execute two queries in a read-only transaction. - session.scalars(select(Singer)).all() - session.scalars(select(Singer)).all() + for i in range(2): + with Session(engine.execution_options(read_only=True)) as session: + # Execute two queries in a read-only transaction. + session.scalars(select(Singer)).all() + session.scalars(select(Singer)).all() # Verify the requests that we got. requests = self.spanner_service.requests - eq_(5, len(requests)) + eq_(9, len(requests)) is_instance_of(requests[0], BatchCreateSessionsRequest) # We should get rid of this extra round-trip for GetSession.... is_instance_of(requests[1], GetSessionRequest) is_instance_of(requests[2], BeginTransactionRequest) is_instance_of(requests[3], ExecuteSqlRequest) is_instance_of(requests[4], ExecuteSqlRequest) + is_instance_of(requests[5], GetSessionRequest) + is_instance_of(requests[6], BeginTransactionRequest) + is_instance_of(requests[7], ExecuteSqlRequest) + is_instance_of(requests[8], ExecuteSqlRequest) # Verify that the transaction is a read-only transaction. - begin_request: BeginTransactionRequest = requests[2] - eq_( - TransactionOptions( - dict( - read_only=TransactionOptions.ReadOnly( - dict( - strong=True, - return_read_timestamp=True, + for index in [2, 6]: + begin_request: BeginTransactionRequest = requests[index] + eq_( + TransactionOptions( + dict( + read_only=TransactionOptions.ReadOnly( + dict( + strong=True, + return_read_timestamp=True, + ) ) ) - ) - ), - begin_request.options, - ) + ), + begin_request.options, + ) def add_singer_query_result(sql: str): From b51482d07ecc2e78af7d5fc37559b954f60a2aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Fri, 6 Dec 2024 13:06:20 +0100 Subject: [PATCH 3/4] chore: remove GetSession requests --- .../test_read_only_transaction.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/test/mockserver_tests/test_read_only_transaction.py b/test/mockserver_tests/test_read_only_transaction.py index b8e8e170..6f9d916b 100644 --- a/test/mockserver_tests/test_read_only_transaction.py +++ b/test/mockserver_tests/test_read_only_transaction.py @@ -48,19 +48,16 @@ def test_read_only_transaction(self): # Verify the requests that we got. requests = self.spanner_service.requests - eq_(9, len(requests)) + eq_(7, len(requests)) is_instance_of(requests[0], BatchCreateSessionsRequest) - # We should get rid of this extra round-trip for GetSession.... - is_instance_of(requests[1], GetSessionRequest) - is_instance_of(requests[2], BeginTransactionRequest) + is_instance_of(requests[1], BeginTransactionRequest) + is_instance_of(requests[2], ExecuteSqlRequest) is_instance_of(requests[3], ExecuteSqlRequest) - is_instance_of(requests[4], ExecuteSqlRequest) - is_instance_of(requests[5], GetSessionRequest) - is_instance_of(requests[6], BeginTransactionRequest) - is_instance_of(requests[7], ExecuteSqlRequest) - is_instance_of(requests[8], ExecuteSqlRequest) + is_instance_of(requests[4], BeginTransactionRequest) + is_instance_of(requests[5], ExecuteSqlRequest) + is_instance_of(requests[6], ExecuteSqlRequest) # Verify that the transaction is a read-only transaction. - for index in [2, 6]: + for index in [1, 4]: begin_request: BeginTransactionRequest = requests[index] eq_( TransactionOptions( From dc4e3010bece0a91a3fb2fdb0b4ca846a5894e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Fri, 6 Dec 2024 13:08:49 +0100 Subject: [PATCH 4/4] chore: remove unused import --- test/mockserver_tests/test_read_only_transaction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/mockserver_tests/test_read_only_transaction.py b/test/mockserver_tests/test_read_only_transaction.py index 6f9d916b..18abf69f 100644 --- a/test/mockserver_tests/test_read_only_transaction.py +++ b/test/mockserver_tests/test_read_only_transaction.py @@ -19,7 +19,6 @@ FixedSizePool, BatchCreateSessionsRequest, ExecuteSqlRequest, - GetSessionRequest, BeginTransactionRequest, TransactionOptions, )