Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

psycopg3 compatibility #126

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 76 additions & 72 deletions querybuilder/cursor.py
Original file line number Diff line number Diff line change
@@ -1,77 +1,81 @@
import contextlib
import json
from psycopg2.extras import register_default_jsonb
from psycopg2._json import JSONB_OID


def jsonify_cursor(django_cursor, enabled=True):
"""
Adjust an already existing cursor to ensure it will return structured types (list or dict)
from jsonb columns instead of strings. Django 3.1.1+ returns strings for raw queries.
https://code.djangoproject.com/ticket/31956
https://code.djangoproject.com/ticket/31973
https://www.psycopg.org/docs/extras.html#psycopg2.extras.register_default_jsonb
"""

# The thing that is returned by connection.cursor() is (normally) a Django object
# of type CursorWrapper that itself has the "real" cursor as a property called cursor.
# However, it could be a CursorDebugWrapper instead, or it could be an outer wrapper
# wrapping one of those. For example django-debug-toolbar wraps CursorDebugWrapper in
# a NormalCursorWrapper. The django-db-readonly package wraps the Django CursorWrapper
# in a ReadOnlyCursorWrapper. I'm not sure if they ever nest multiple levels. I tried
# looping with `while isinstance(inner_cursor, CursorWrapper)`, but it seems that the
# outer wrapper is not necessarily a subclass of the Django wrapper. My next best option
# is to make the assumption that we need to get to the last property called `cursor`,
# basically assuming that any wrapper is going to have a property called `cursor`
# that is the real cursor or the next-level wrapper.
# Another option might be to check the class of inner_cursor to see if it is the real
# database cursor. That would require importing more django libraries, and probably
# having to handle some changes in those libraries over different versions.

# This register_default_jsonb functionality in psycopg2 does not itself have a "deregister"
# capability. So to deregister, we pass in a different value for the loads method; in this
# case just the str() built-in, which just returns the value passed in. Note that passing
# None for loads does NOT do a deregister; it uses the default value, which as it turns out
# is json.loads anyway!
loads_func = json.loads if enabled else str

# We expect that there is always at least one wrapper, but we might as well handle
# the possibility that we get passed the inner cursor.
inner_cursor = django_cursor

while hasattr(inner_cursor, 'cursor'):
inner_cursor = inner_cursor.cursor

# Hopefully we have the right thing now, but try/catch so we can get a little better info
# if it is not. Another option might be an isinstance, or another function that tests the cursor?
try:
register_default_jsonb(conn_or_curs=inner_cursor, loads=loads_func)
except TypeError as e:
raise Exception(f'jsonify_cursor: conn_or_curs was actually a {type(inner_cursor)}: {e}')


def dejsonify_cursor(django_cursor):
"""
Re-adjust a cursor that was "jsonified" so it no longer performs the json.loads().
"""
jsonify_cursor(django_cursor, enabled=False)


@contextlib.contextmanager
def json_cursor(django_database_connection):
"""
Cast json fields into their specific types to account for django bugs
https://code.djangoproject.com/ticket/31956
https://code.djangoproject.com/ticket/31973
https://www.psycopg.org/docs/extras.html#psycopg2.extras.register_default_jsonb
"""
with django_database_connection.cursor() as cursor:
jsonify_cursor(cursor)
yield cursor
# This should really not be necessary, because the cursor context manager will
# be closing the cursor on __exit__ anyway. But just in case.
dejsonify_cursor(cursor)

# from psycopg2.extras import register_default_jsonb
# from psycopg2._json import JSONB_OID

# constant for jsonb column type in postgressql - setting explicitly instead of pulling
# from psycopg2 in order to reduce reliance on it (so we can move towards psycopg3)
JSONB_OID = 3802


# def jsonify_cursor(django_cursor, enabled=True):
# """
# Adjust an already existing cursor to ensure it will return structured types (list or dict)
# from jsonb columns instead of strings. Django 3.1.1+ returns strings for raw queries.
# https://code.djangoproject.com/ticket/31956
# https://code.djangoproject.com/ticket/31973
# https://www.psycopg.org/docs/extras.html#psycopg2.extras.register_default_jsonb
# """
#
# # The thing that is returned by connection.cursor() is (normally) a Django object
# # of type CursorWrapper that itself has the "real" cursor as a property called cursor.
# # However, it could be a CursorDebugWrapper instead, or it could be an outer wrapper
# # wrapping one of those. For example django-debug-toolbar wraps CursorDebugWrapper in
# # a NormalCursorWrapper. The django-db-readonly package wraps the Django CursorWrapper
# # in a ReadOnlyCursorWrapper. I'm not sure if they ever nest multiple levels. I tried
# # looping with `while isinstance(inner_cursor, CursorWrapper)`, but it seems that the
# # outer wrapper is not necessarily a subclass of the Django wrapper. My next best option
# # is to make the assumption that we need to get to the last property called `cursor`,
# # basically assuming that any wrapper is going to have a property called `cursor`
# # that is the real cursor or the next-level wrapper.
# # Another option might be to check the class of inner_cursor to see if it is the real
# # database cursor. That would require importing more django libraries, and probably
# # having to handle some changes in those libraries over different versions.
#
# # This register_default_jsonb functionality in psycopg2 does not itself have a "deregister"
# # capability. So to deregister, we pass in a different value for the loads method; in this
# # case just the str() built-in, which just returns the value passed in. Note that passing
# # None for loads does NOT do a deregister; it uses the default value, which as it turns out
# # is json.loads anyway!
# loads_func = json.loads if enabled else str
#
# # We expect that there is always at least one wrapper, but we might as well handle
# # the possibility that we get passed the inner cursor.
# inner_cursor = django_cursor
#
# while hasattr(inner_cursor, 'cursor'):
# inner_cursor = inner_cursor.cursor
#
# # Hopefully we have the right thing now, but try/catch so we can get a little better info
# # if it is not. Another option might be an isinstance, or another function that tests the cursor?
# try:
# register_default_jsonb(conn_or_curs=inner_cursor, loads=loads_func)
# except TypeError as e:
# raise Exception(f'jsonify_cursor: conn_or_curs was actually a {type(inner_cursor)}: {e}')
#
#
# def dejsonify_cursor(django_cursor):
# """
# Re-adjust a cursor that was "jsonified" so it no longer performs the json.loads().
# """
# jsonify_cursor(django_cursor, enabled=False)
#
#
# @contextlib.contextmanager
# def json_cursor(django_database_connection):
# """
# Cast json fields into their specific types to account for django bugs
# https://code.djangoproject.com/ticket/31956
# https://code.djangoproject.com/ticket/31973
# https://www.psycopg.org/docs/extras.html#psycopg2.extras.register_default_jsonb
# """
# with django_database_connection.cursor() as cursor:
# jsonify_cursor(cursor)
# yield cursor
# # This should really not be necessary, because the cursor context manager will
# # be closing the cursor on __exit__ anyway. But just in case.
# dejsonify_cursor(cursor)
#

def json_fetch_all_as_dict(cursor):
"""
Expand Down
15 changes: 0 additions & 15 deletions querybuilder/tests/cursor_tests.py

This file was deleted.

2 changes: 1 addition & 1 deletion querybuilder/version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '3.1.0'
__version__ = '3.2.0'