From 981b86c5d78d813284a7c148e27caff3714df30a Mon Sep 17 00:00:00 2001 From: Nicholas Hilton <32165552+NickHilton@users.noreply.github.com> Date: Wed, 21 Apr 2021 10:08:47 -0700 Subject: [PATCH 01/11] BUG: 'bigserial' dtype should not be a cast type - Adds 'bigserial' to list of dtypes which should not be cast in updates --- querybuilder/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/querybuilder/query.py b/querybuilder/query.py index e521c0b..45001ce 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -13,6 +13,7 @@ from querybuilder.helpers import set_value_for_keypath from querybuilder.tables import TableFactory, ModelTable, QueryTable +SERIAL_DTYPES = ['serial', 'bigserial'] class Join(object): """ @@ -1170,7 +1171,7 @@ def get_update_sql(self, rows): db_type = field_object.db_type(self.connection) # Don't cast the pk - if db_type == 'serial': + if db_type in SERIAL_DTYPES: placeholders.append('%s') else: # Cast the placeholder to the data type From dad9d03c2fede815b4a888e29d046298516f1516 Mon Sep 17 00:00:00 2001 From: Nicholas Hilton <32165552+NickHilton@users.noreply.github.com> Date: Wed, 21 Apr 2021 10:26:09 -0700 Subject: [PATCH 02/11] Update query.py --- querybuilder/query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/querybuilder/query.py b/querybuilder/query.py index 45001ce..4ed07bd 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -15,6 +15,7 @@ SERIAL_DTYPES = ['serial', 'bigserial'] + class Join(object): """ Represents the JOIN clauses of a Query. The join can be of any join type. From 81bb4708714e01400a8acf573c9096c0f67014af Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Wed, 21 Apr 2021 13:35:09 -0400 Subject: [PATCH 03/11] version bump and note --- CONTRIBUTORS | 1 + docs/release_notes.rst | 4 ++++ querybuilder/version.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index cd75269..d5c8391 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,3 +3,4 @@ Micah Hausler (micah.hausler@ambition.com) Andrew Plummer (https://github.com/plumdog) Jure Žvelc (https://github.com/jzvelc) Timothy J Laurent (https://github.com/timothyjlaurent) +NickHilton (https://github.com/NickHilton) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index d9383b6..c3fdf12 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,10 @@ Release Notes ============= +v2.0.1 +------ +* BUG: 'bigserial' dtype should not be a cast type - NickHilton + v2.0.0 ------ * Add support Django 3.0, 3.1 diff --git a/querybuilder/version.py b/querybuilder/version.py index afced14..3f39079 100644 --- a/querybuilder/version.py +++ b/querybuilder/version.py @@ -1 +1 @@ -__version__ = '2.0.0' +__version__ = '2.0.1' From 51312e9ccbb5b507b5681ba235f32c03c1b898cd Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 15:32:17 -0400 Subject: [PATCH 04/11] manifest and contributor --- CONTRIBUTORS | 1 + MANIFEST.in | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d5c8391..6e275fd 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -4,3 +4,4 @@ Andrew Plummer (https://github.com/plumdog) Jure Žvelc (https://github.com/jzvelc) Timothy J Laurent (https://github.com/timothyjlaurent) NickHilton (https://github.com/NickHilton) +John Vandenberg (https://github.com/jayvdb) diff --git a/MANIFEST.in b/MANIFEST.in index ddffd89..17fc0c4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include README.rst include LICENSE recursive-include requirements * +include *.py +recursive-include docs *.py From 57a32a09ad107289a3232daf3fb34f0bde6c3306 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 15:43:05 -0400 Subject: [PATCH 05/11] debugging 4.1 --- .github/workflows/tests.yml | 39 +++++++++++++++--------------- querybuilder/query.py | 4 +++ querybuilder/tests/update_tests.py | 14 +++++------ 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f057c67..f38e5f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,32 +12,33 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9'] + python: ['3.9'] +# python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: - - 'Django~=2.2.0' - - 'Django~=3.0.0' - - 'Django~=3.1.0' - - 'Django~=3.2.0' - - 'Django~=4.0.0' +# - 'Django~=2.2.0' +# - 'Django~=3.0.0' +# - 'Django~=3.1.0' +# - 'Django~=3.2.0' +# - 'Django~=4.0.0' - 'Django~=4.1.0' experimental: [false] - include: - - python: '3.9' - django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' - experimental: true - # NOTE this job will appear to pass even when it fails because of - # `continue-on-error: true`. Github Actions apparently does not - # have this feature, similar to Travis' allow-failure, yet. - # https://github.com/actions/toolkit/issues/399 - exclude: - - python: '3.7' - django: 'Django~=4.0.0' - - python: '3.7' - django: 'Django~=4.1.0' +# include: +# - python: '3.9' +# django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' +# experimental: true +# # NOTE this job will appear to pass even when it fails because of +# # `continue-on-error: true`. Github Actions apparently does not +# # have this feature, similar to Travis' allow-failure, yet. +# # https://github.com/actions/toolkit/issues/399 +# exclude: +# - python: '3.7' +# django: 'Django~=4.0.0' +# - python: '3.7' +# django: 'Django~=4.1.0' services: postgres: image: postgres:latest diff --git a/querybuilder/query.py b/querybuilder/query.py index f3da935..3600d98 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -1171,6 +1171,10 @@ def get_update_sql(self, rows): field_object = self.tables[0].model._meta.get_field(field_names[field_index]) db_type = field_object.db_type(self.connection) + print('---------') + print('---------') + print('---------') + print('this is casting', db_type) # Don't cast the pk if db_type in SERIAL_DTYPES: placeholders.append('%s') diff --git a/querybuilder/tests/update_tests.py b/querybuilder/tests/update_tests.py index c655d18..f27e50a 100644 --- a/querybuilder/tests/update_tests.py +++ b/querybuilder/tests/update_tests.py @@ -18,9 +18,9 @@ def setUp(self): self.logger.start_logging() # Starting on Django 4, the id field adds ::integer automatically - self.integer_cast_string = '' - if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: - self.integer_cast_string = '::integer' + # self.integer_cast_string = '' + # if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: + # self.integer_cast_string = '::integer' def test_update_single_row(self): query = Query().from_table( @@ -46,7 +46,7 @@ def test_update_single_row(self): 'SET user_id = new_values.user_id, ' 'first_name = new_values.first_name, ' 'last_name = new_values.last_name ' - f'FROM (VALUES (%s{self.integer_cast_string}, %s::integer, %s::varchar(64), %s::varchar(64))) ' + f'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64))) ' 'AS new_values (id, user_id, first_name, last_name) ' 'WHERE querybuilder_tests_account.id = new_values.id' ) @@ -66,7 +66,7 @@ def test_update_single_row(self): "SET user_id = new_values.user_id, " "first_name = new_values.first_name, " "last_name = new_values.last_name " - f"FROM (VALUES (1{self.integer_cast_string}, 1::integer, " + f"FROM (VALUES (1, 1::integer, " "'Test''s'::varchar(64), '\"User\"'::varchar(64))) " "AS new_values (id, user_id, first_name, last_name) " "WHERE querybuilder_tests_account.id = new_values.id" @@ -122,7 +122,7 @@ def test_update_multiple_rows(self): 'SET user_id = new_values.user_id, ' 'first_name = new_values.first_name, ' 'last_name = new_values.last_name ' - f'FROM (VALUES (%s{self.integer_cast_string}, %s::integer, %s::varchar(64), %s::varchar(64)), ' + f'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64)), ' '(%s, %s, %s, %s)) ' 'AS new_values (id, user_id, first_name, last_name) ' 'WHERE querybuilder_tests_account.id = new_values.id' @@ -146,7 +146,7 @@ def test_update_multiple_rows(self): "SET user_id = new_values.user_id, " "first_name = new_values.first_name, " "last_name = new_values.last_name " - f"FROM (VALUES (1{self.integer_cast_string}, 1::integer, 'Test'::varchar(64), 'User'::varchar(64)), " + f"FROM (VALUES (1, 1::integer, 'Test'::varchar(64), 'User'::varchar(64)), " "(2, 2, 'Test2', 'User2')) " "AS new_values (id, user_id, first_name, last_name) " "WHERE querybuilder_tests_account.id = new_values.id" From fe4d596f1c86077507c08c8f9209e1228301a666 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 15:54:30 -0400 Subject: [PATCH 06/11] include 4 to see update query --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f38e5f3..d2fac73 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: # - 'Django~=3.0.0' # - 'Django~=3.1.0' # - 'Django~=3.2.0' -# - 'Django~=4.0.0' + - 'Django~=4.0.0' - 'Django~=4.1.0' experimental: [false] # include: From c0f702c5137f113b9b4f0ebcc180afd0c7073a25 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 16:35:54 -0400 Subject: [PATCH 07/11] possibly handle 4.1 casting --- querybuilder/query.py | 20 ++++++++++++++------ querybuilder/tests/update_tests.py | 6 ------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/querybuilder/query.py b/querybuilder/query.py index 3600d98..ea5a59d 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -1,5 +1,6 @@ from copy import deepcopy +from django import VERSION from django.db import connection as default_django_connection from django.db.models import Q, AutoField from django.db.models.query import QuerySet @@ -1119,6 +1120,17 @@ def get_insert_sql(self, rows): return self.sql, sql_args + def should_not_cast_value(self, db_type): + """ + In Django 4.1 on PostgreSQL, AutoField, BigAutoField, and SmallAutoField are now created as identity + columns rather than serial columns with sequences. + """ + if db_type in SERIAL_DTYPES: + return True + if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: + return True + return False + def get_update_sql(self, rows): """ Returns SQL UPDATE for rows ``rows`` @@ -1171,12 +1183,8 @@ def get_update_sql(self, rows): field_object = self.tables[0].model._meta.get_field(field_names[field_index]) db_type = field_object.db_type(self.connection) - print('---------') - print('---------') - print('---------') - print('this is casting', db_type) - # Don't cast the pk - if db_type in SERIAL_DTYPES: + # Don't cast serial types + if self.should_not_cast_value(db_type): placeholders.append('%s') else: # Cast the placeholder to the data type diff --git a/querybuilder/tests/update_tests.py b/querybuilder/tests/update_tests.py index f27e50a..4104282 100644 --- a/querybuilder/tests/update_tests.py +++ b/querybuilder/tests/update_tests.py @@ -1,6 +1,5 @@ import json -from django import VERSION from django.test.utils import override_settings from django_dynamic_fixture import G @@ -17,11 +16,6 @@ def setUp(self): self.logger = Logger() self.logger.start_logging() - # Starting on Django 4, the id field adds ::integer automatically - # self.integer_cast_string = '' - # if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: - # self.integer_cast_string = '::integer' - def test_update_single_row(self): query = Query().from_table( table=Account, From ee7901cbf2c40667e899aa72427f9c142ab74df8 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 16:40:56 -0400 Subject: [PATCH 08/11] don't use fstring --- querybuilder/tests/update_tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/querybuilder/tests/update_tests.py b/querybuilder/tests/update_tests.py index 4104282..ec02ada 100644 --- a/querybuilder/tests/update_tests.py +++ b/querybuilder/tests/update_tests.py @@ -40,7 +40,7 @@ def test_update_single_row(self): 'SET user_id = new_values.user_id, ' 'first_name = new_values.first_name, ' 'last_name = new_values.last_name ' - f'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64))) ' + 'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64))) ' 'AS new_values (id, user_id, first_name, last_name) ' 'WHERE querybuilder_tests_account.id = new_values.id' ) @@ -60,8 +60,7 @@ def test_update_single_row(self): "SET user_id = new_values.user_id, " "first_name = new_values.first_name, " "last_name = new_values.last_name " - f"FROM (VALUES (1, 1::integer, " - "'Test''s'::varchar(64), '\"User\"'::varchar(64))) " + "FROM (VALUES (1, 1::integer, 'Test''s'::varchar(64), '\"User\"'::varchar(64))) " "AS new_values (id, user_id, first_name, last_name) " "WHERE querybuilder_tests_account.id = new_values.id" ) @@ -116,7 +115,7 @@ def test_update_multiple_rows(self): 'SET user_id = new_values.user_id, ' 'first_name = new_values.first_name, ' 'last_name = new_values.last_name ' - f'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64)), ' + 'FROM (VALUES (%s, %s::integer, %s::varchar(64), %s::varchar(64)), ' '(%s, %s, %s, %s)) ' 'AS new_values (id, user_id, first_name, last_name) ' 'WHERE querybuilder_tests_account.id = new_values.id' @@ -140,7 +139,7 @@ def test_update_multiple_rows(self): "SET user_id = new_values.user_id, " "first_name = new_values.first_name, " "last_name = new_values.last_name " - f"FROM (VALUES (1, 1::integer, 'Test'::varchar(64), 'User'::varchar(64)), " + "FROM (VALUES (1, 1::integer, 'Test'::varchar(64), 'User'::varchar(64)), " "(2, 2, 'Test2', 'User2')) " "AS new_values (id, user_id, first_name, last_name) " "WHERE querybuilder_tests_account.id = new_values.id" From b081fd08e4ceb46cf03e972b58771e89f73992d1 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 17:51:27 -0400 Subject: [PATCH 09/11] better logic for 4.1 --- querybuilder/query.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/querybuilder/query.py b/querybuilder/query.py index ea5a59d..1b75b79 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -1120,15 +1120,17 @@ def get_insert_sql(self, rows): return self.sql, sql_args - def should_not_cast_value(self, db_type): + def should_not_cast_value(self, field_object): """ In Django 4.1 on PostgreSQL, AutoField, BigAutoField, and SmallAutoField are now created as identity columns rather than serial columns with sequences. """ + db_type = field_object.db_type(self.connection) if db_type in SERIAL_DTYPES: return True if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: - return True + if getattr(field_object, 'primary_key', None) and getattr(field_object, 'serialized', None) is False: + return True return False def get_update_sql(self, rows): @@ -1184,7 +1186,7 @@ def get_update_sql(self, rows): db_type = field_object.db_type(self.connection) # Don't cast serial types - if self.should_not_cast_value(db_type): + if self.should_not_cast_value(field_object): placeholders.append('%s') else: # Cast the placeholder to the data type From de71eeba2f84f8b1b53b861a6431eab4b587cee7 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 17:56:15 -0400 Subject: [PATCH 10/11] fix name --- querybuilder/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/querybuilder/query.py b/querybuilder/query.py index 1b75b79..7dfb681 100644 --- a/querybuilder/query.py +++ b/querybuilder/query.py @@ -1129,7 +1129,7 @@ def should_not_cast_value(self, field_object): if db_type in SERIAL_DTYPES: return True if (VERSION[0] == 4 and VERSION[1] >= 1) or VERSION[0] >= 5: - if getattr(field_object, 'primary_key', None) and getattr(field_object, 'serialized', None) is False: + if getattr(field_object, 'primary_key', None) and getattr(field_object, 'serialize', None) is False: return True return False From 924b32572c5b09d89ffd1ec593ff2aa4695b2e47 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 22 Aug 2022 17:59:54 -0400 Subject: [PATCH 11/11] test all versions --- .github/workflows/tests.yml | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d2fac73..f057c67 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,33 +12,32 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.9'] -# python: ['3.7', '3.8', '3.9'] + python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: -# - 'Django~=2.2.0' -# - 'Django~=3.0.0' -# - 'Django~=3.1.0' -# - 'Django~=3.2.0' + - 'Django~=2.2.0' + - 'Django~=3.0.0' + - 'Django~=3.1.0' + - 'Django~=3.2.0' - 'Django~=4.0.0' - 'Django~=4.1.0' experimental: [false] -# include: -# - python: '3.9' -# django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' -# experimental: true -# # NOTE this job will appear to pass even when it fails because of -# # `continue-on-error: true`. Github Actions apparently does not -# # have this feature, similar to Travis' allow-failure, yet. -# # https://github.com/actions/toolkit/issues/399 -# exclude: -# - python: '3.7' -# django: 'Django~=4.0.0' -# - python: '3.7' -# django: 'Django~=4.1.0' + include: + - python: '3.9' + django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' + experimental: true + # NOTE this job will appear to pass even when it fails because of + # `continue-on-error: true`. Github Actions apparently does not + # have this feature, similar to Travis' allow-failure, yet. + # https://github.com/actions/toolkit/issues/399 + exclude: + - python: '3.7' + django: 'Django~=4.0.0' + - python: '3.7' + django: 'Django~=4.1.0' services: postgres: image: postgres:latest