diff --git a/polars_db_schema/README.rst b/polars_db_schema/README.rst new file mode 100644 index 000000000..7eb218475 --- /dev/null +++ b/polars_db_schema/README.rst @@ -0,0 +1,88 @@ +================ +Polars Db Schema +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:0e6c26c886f0fc18c585b743314981ec58966dd913b8a469530e62e313327de2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github + :target: https://github.com/OCA/server-backend/tree/18.0/polars_db_schema + :alt: OCA/server-backend +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_schema + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-backend&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Introspect external database + +.. image:: https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png + +Use case: you want discover an external database extracting data from +only relevant tables and columns + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Contributors +------------ + +- Akretion + + - David BEAL david.beal@akretion.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-backend `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/polars_db_schema/__init__.py b/polars_db_schema/__init__.py new file mode 100644 index 000000000..0650744f6 --- /dev/null +++ b/polars_db_schema/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/polars_db_schema/__manifest__.py b/polars_db_schema/__manifest__.py new file mode 100644 index 000000000..2e63b2f9f --- /dev/null +++ b/polars_db_schema/__manifest__.py @@ -0,0 +1,21 @@ +{ + "name": "Polars Db Schema", + "version": "18.0.1.0.0", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/server-backend", + "license": "AGPL-3", + "depends": [ + "polars_db_process", + ], + "external_dependencies": {"python": []}, + "data": [ + "security/ir.model.access.xml", + "views/db_config.xml", + "views/db_table.xml", + "views/db_type.xml", + "data/db_type.xml", + ], + "demo": [], + "installable": True, +} diff --git a/polars_db_schema/data/db_type.xml b/polars_db_schema/data/db_type.xml new file mode 100644 index 000000000..b8bd3898d --- /dev/null +++ b/polars_db_schema/data/db_type.xml @@ -0,0 +1,44 @@ + + + SQL Server + mssql + Binary + +-- This query order tables by rows count +SELECT sOBJ.name AS 'name', SUM(sPTN.Rows) AS row_count +FROM + sys.objects AS sOBJ + INNER JOIN sys.partitions AS sPTN + ON sOBJ.object_id = sPTN.object_id +WHERE + sOBJ.type = 'U' + AND sOBJ.is_ms_shipped = 0x0 + AND index_id < 2 -- 0:Heap, 1:Clustered +GROUP BY + sOBJ.schema_id, sOBJ.name +ORDER BY row_count desc + + + + + Sqlite + sqlite + BLOB + +-- This query order tables by rows count +SELECT tbl, stat FROM sqlite_stat1 + + + + + PostgreSQL + postgresql + bytea + +-- This query order tables by rows count +SELECT relname AS name, n_live_tup AS row_count + FROM pg_stat_user_tables +ORDER BY row_count DESC; + + + diff --git a/polars_db_schema/models/__init__.py b/polars_db_schema/models/__init__.py new file mode 100644 index 000000000..0ee0fe9c7 --- /dev/null +++ b/polars_db_schema/models/__init__.py @@ -0,0 +1,4 @@ +from . import db_config +from . import db_type +from . import db_table +from . import db_source diff --git a/polars_db_schema/models/db_config.py b/polars_db_schema/models/db_config.py new file mode 100644 index 000000000..8e721d795 --- /dev/null +++ b/polars_db_schema/models/db_config.py @@ -0,0 +1,46 @@ +import polars as pl + +from odoo import fields, models + + +class DbConfig(models.Model): + _inherit = "db.config" + + db_type_id = fields.Many2one(comodel_name="db.type") + db_table_ids = fields.One2many(comodel_name="db.table", inverse_name="db_config_id") + row_count_query = fields.Text(related="db_type_id.row_count_query") + + def get_db_metadata(self): + self.ensure_one() + if self.row_count_query: + self._read_sql("SELECT 1") + df = self._read_sql(self.row_count_query) + if self.db_type_id.code == "sqlite": + # https://docs.pola.rs/user-guide/expressions/user-defined-functions/#processing-individual-values-with-map_elements + df = ( + df.with_columns( + pl.col("stat").map_elements(sqlite, return_dtype=pl.Int32) + ) + # rename columns + .rename({"tbl": "name", "stat": "row_count"}) + # stat columns store extra info leading to duplicate lines, + # then make it unique + .unique(maintain_order=True) + ) + df = df.filter(pl.col("row_count") > 0).with_columns( + # add m2o foreign key + db_config_id=pl.lit(self.id) + ) + df = self._filter_df(df) + self.env["db.table"].search([("db_config_id", "=", self.id)]).unlink() + self.env["db.table"].create(df.to_dicts()) + + def _filter_df(self, df): + "You may want ignore some tables: inherit me" + return df + + +def sqlite(value): + "Extract row_count info from 'stat' column" + values = value.split(" ") + return values and int(values[0]) or int(value) diff --git a/polars_db_schema/models/db_source.py b/polars_db_schema/models/db_source.py new file mode 100644 index 000000000..9b83b133f --- /dev/null +++ b/polars_db_schema/models/db_source.py @@ -0,0 +1,17 @@ +from odoo import models + + +class DfSource(models.Model): + _inherit = "df.source" + + # def _get_test_file_paths(self): + # res = super()._get_test_file_paths() + # res.update( + # { + # "polars_db_schema": { + # "relative_path": "tests/files", + # "xmlid": "migr.contact", + # } + # } + # ) + # return res diff --git a/polars_db_schema/models/db_table.py b/polars_db_schema/models/db_table.py new file mode 100644 index 000000000..fec0a4771 --- /dev/null +++ b/polars_db_schema/models/db_table.py @@ -0,0 +1,91 @@ +import base64 +import io + +import connectorx as cx +import polars as pl + +from odoo import _, exceptions, fields, models + + +class DbTable(models.Model): + _name = "db.table" + _description = "Access to database tables" + _order = "row_count DESC" + + name = fields.Char(required=True, help="Name of the table") + row_count = fields.Integer(required=True, help="Number of rows contained in table") + xlsx = fields.Binary(string="File", attachment=False, readonly=True) + db_config_id = fields.Many2one(comodel_name="db.config") + filename = fields.Char() + sql = fields.Text( + string="Significant Columns", help="Columns with variable data over rows" + ) + uniques = fields.Text( + string="Unique Values", + help="Columns with the same value over rows.\n" + "It could be useless to extract data from these columns,\n" + "because they're probably unused by the application", + ) + + def get_metadata_info(self): + self.ensure_one() + query = f"SELECT * FROM {self.name}" + connexion = self.db_config_id._get_connexion() + df = cx.read_sql(connexion, query, return_type="polars") + excluded_types = self.db_config_id.db_type_id.excluded_types.split("\n") + cols = [x[0] for x in df.schema.items() if str(x[1]) not in excluded_types] + new_cols = [] + uniques = {} + for col in cols: + # TODO improve it + # Some database have dirty column names: :-( + conditions = [x for x in (" ", "*", "-") if x in col] + if any(conditions): + continue + query = f"SELECT distinct {col} FROM self" + res = df.sql(query) + if len(res) > 1: + new_cols.append(col) + else: + # column has the same value whatever row + uniques[col] = res.to_series()[0] + self.uniques = f"{uniques}" + if new_cols: + self.sql = f"SELECT {', '. join(new_cols)}\nFROM {self.name};\n" + + # WARNING Thread virtual real time limit (151/120s) reached. + # Dumping stacktrace of limit exceeding threads before reloading + + def get_spreadsheet(self): + self.ensure_one() + if not self.sql: + self.get_metadata_info() + if not self.sql: + raise exceptions.ValidationError( + _( + "There is no column with varaiable data in this table: " + "check Uniques Values column" + ) + ) + df = cx.read_sql( + self.db_config_id._get_connexion(), self.sql, return_type="polars" + ) + excel_stream = io.BytesIO() + vals = {"workbook": excel_stream} + vals.update(self.get_spreadsheet_settings()) + df.write_excel(**vals) + excel_stream.seek(0) + self.filename = f"{self.name}.xlsx" + self.xlsx = base64.encodebytes(excel_stream.read()) + + def get_spreadsheet_settings(self): + return { + "position": "A1", + "table_style": "Table Style Light 16", + "dtype_formats": {pl.Date: "dd/mm/yyyy"}, + "float_precision": 6, + "header_format": {"bold": True, "font_color": "#702963"}, + "freeze_panes": "A2", + "autofit": True, + } diff --git a/polars_db_schema/models/db_type.py b/polars_db_schema/models/db_type.py new file mode 100644 index 000000000..4967dc704 --- /dev/null +++ b/polars_db_schema/models/db_type.py @@ -0,0 +1,15 @@ +from odoo import fields, models + + +class DbType(models.Model): + _name = "db.type" + _description = "Constant parameters about database" + + name = fields.Char(required=True) + code = fields.Char(required=True) + row_count_query = fields.Text( + required=True, help="SQL code to find how many records contains each table" + ) + excluded_types = fields.Text( + help="Column types to ignore for better introspection (set 1 data by line)" + ) diff --git a/polars_db_schema/pyproject.toml b/polars_db_schema/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/polars_db_schema/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/polars_db_schema/readme/CONTRIBUTORS.md b/polars_db_schema/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..dbb2754b0 --- /dev/null +++ b/polars_db_schema/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Akretion + - David BEAL diff --git a/polars_db_schema/readme/DESCRIPTION.md b/polars_db_schema/readme/DESCRIPTION.md new file mode 100644 index 000000000..2832ea239 --- /dev/null +++ b/polars_db_schema/readme/DESCRIPTION.md @@ -0,0 +1,5 @@ +Introspect external database + +.. image:: ../static/description/figure1.png + +Use case: you want discover an external database extracting data from only relevant tables and columns diff --git a/polars_db_schema/security/ir.model.access.xml b/polars_db_schema/security/ir.model.access.xml new file mode 100644 index 000000000..cd5d12469 --- /dev/null +++ b/polars_db_schema/security/ir.model.access.xml @@ -0,0 +1,20 @@ + + + Database Table + + + + + + + + + Database Type + + + + + + + + diff --git a/polars_db_schema/static/description/figure1.png b/polars_db_schema/static/description/figure1.png new file mode 100644 index 000000000..c180b6135 Binary files /dev/null and b/polars_db_schema/static/description/figure1.png differ diff --git a/polars_db_schema/static/description/index.html b/polars_db_schema/static/description/index.html new file mode 100644 index 000000000..6cdf1aac1 --- /dev/null +++ b/polars_db_schema/static/description/index.html @@ -0,0 +1,435 @@ + + + + + +Polars Db Schema + + + +
+

Polars Db Schema

+ + +

Alpha License: AGPL-3 OCA/server-backend Translate me on Weblate Try me on Runboat

+

Introspect external database

+https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png +

Use case: you want discover an external database extracting data from +only relevant tables and columns

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/server-backend project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/polars_db_schema/views/db_config.xml b/polars_db_schema/views/db_config.xml new file mode 100644 index 000000000..e6536188b --- /dev/null +++ b/polars_db_schema/views/db_config.xml @@ -0,0 +1,25 @@ + + + db.config + + + +