diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py index d3941e7add0fd0..9cd485f6d9a4b1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py @@ -156,7 +156,7 @@ class MSSQLDataJob: entity: Union[StoredProcedure, JobStep] type: str = "dataJob" source: str = "mssql" - external_url: str = "" + external_url: Optional[str] = None description: Optional[str] = None status: Optional[str] = None incoming: List[str] = field(default_factory=list) @@ -228,7 +228,7 @@ class MSSQLDataFlow: entity: Union[MSSQLJob, MSSQLProceduresContainer] type: str = "dataFlow" source: str = "mssql" - external_url: str = "" + external_url: Optional[str] = None flow_properties: Dict[str, str] = field(default_factory=dict) def add_property( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index a2338f14196d77..2d0c4dc9f6f6f9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -11,6 +11,7 @@ from sqlalchemy.engine.reflection import Inspector from sqlalchemy.exc import ProgrammingError, ResourceClosedError +import datahub.metadata.schema_classes as models from datahub.configuration.common import AllowDenyPattern from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext @@ -49,21 +50,15 @@ make_sqlalchemy_uri, ) from datahub.ingestion.source.sql.sql_report import SQLSourceReport -from datahub.metadata.schema_classes import ( - BooleanTypeClass, - NumberTypeClass, - StringTypeClass, - UnionTypeClass, -) from datahub.utilities.file_backed_collections import FileBackedList logger: logging.Logger = logging.getLogger(__name__) -register_custom_type(sqlalchemy.dialects.mssql.BIT, BooleanTypeClass) -register_custom_type(sqlalchemy.dialects.mssql.MONEY, NumberTypeClass) -register_custom_type(sqlalchemy.dialects.mssql.SMALLMONEY, NumberTypeClass) -register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, UnionTypeClass) -register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, StringTypeClass) +register_custom_type(sqlalchemy.dialects.mssql.BIT, models.BooleanTypeClass) +register_custom_type(sqlalchemy.dialects.mssql.MONEY, models.NumberTypeClass) +register_custom_type(sqlalchemy.dialects.mssql.SMALLMONEY, models.NumberTypeClass) +register_custom_type(sqlalchemy.dialects.mssql.SQL_VARIANT, models.UnionTypeClass) +register_custom_type(sqlalchemy.dialects.mssql.UNIQUEIDENTIFIER, models.StringTypeClass) class SQLServerConfig(BasicSQLAlchemyConfig): @@ -651,6 +646,26 @@ def construct_job_workunits( entityUrn=data_job.urn, aspect=data_job.as_datajob_input_output_aspect, ).as_workunit() + + if ( + self.config.include_stored_procedures_code + and isinstance(data_job.entity, StoredProcedure) + and data_job.entity.code is not None + ): + yield MetadataChangeProposalWrapper( + entityUrn=data_job.urn, + aspect=models.DataTransformLogicClass( + transforms=[ + models.DataTransformClass( + queryStatement=models.QueryStatementClass( + value=data_job.entity.code, + language=models.QueryLanguageClass.SQL, + ), + ) + ] + ), + ).as_workunit() + # TODO: Add SubType when it appear def construct_flow_workunits( diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 0d9386dcda0cdb..0b478e13b86d2a 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -95,7 +95,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "Weekly Demo Data Backup" } }, @@ -113,17 +112,16 @@ "aspect": { "json": { "customProperties": { - "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", + "job_id": "f5a6c120-500a-4300-9b21-0c3225af1f80", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-26 12:03:35.420000", - "date_modified": "2024-12-26 12:03:35.590000", + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, - "externalUrl": "", "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" @@ -2259,7 +2257,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "DemoData.Foo.stored_procedures" } }, @@ -2282,10 +2279,9 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-26 12:03:35.230000", - "date_modified": "2024-12-26 12:03:35.230000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2298,6 +2294,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", @@ -2310,10 +2329,9 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-12-26 12:03:35.237000", - "date_modified": "2024-12-26 12:03:35.237000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.NewProc", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2326,6 +2344,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -4928,7 +4969,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214618898, + "time": 1735588784503, "actor": "urn:li:corpuser:_ingestion" } } @@ -5051,7 +5092,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214618906, + "time": 1735588784511, "actor": "urn:li:corpuser:_ingestion" } } diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 07098f0161fc3d..c1748ff13ac93c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -95,7 +95,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "Weekly Demo Data Backup" } }, @@ -113,17 +112,16 @@ "aspect": { "json": { "customProperties": { - "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", + "job_id": "f5a6c120-500a-4300-9b21-0c3225af1f80", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-26 12:03:35.420000", - "date_modified": "2024-12-26 12:03:35.590000", + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, - "externalUrl": "", "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" @@ -2259,7 +2257,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "DemoData.Foo.stored_procedures" } }, @@ -2282,10 +2279,9 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-26 12:03:35.230000", - "date_modified": "2024-12-26 12:03:35.230000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2298,6 +2294,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2675,7 +2694,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214621644, + "time": 1735588789629, "actor": "urn:li:corpuser:_ingestion" } } diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index bf30448469c309..cbfc374decab2f 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -102,7 +102,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "Weekly Demo Data Backup" } }, @@ -137,17 +136,16 @@ "aspect": { "json": { "customProperties": { - "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", + "job_id": "f5a6c120-500a-4300-9b21-0c3225af1f80", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-26 12:03:35.420000", - "date_modified": "2024-12-26 12:03:35.590000", + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, - "externalUrl": "", "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" @@ -2492,7 +2490,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "DemoData.Foo.stored_procedures" } }, @@ -2532,10 +2529,9 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-26 12:03:35.230000", - "date_modified": "2024-12-26 12:03:35.230000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2565,6 +2561,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),NewProc)", @@ -2577,10 +2596,9 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-12-26 12:03:35.237000", - "date_modified": "2024-12-26 12:03:35.237000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.NewProc", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2610,6 +2628,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,my-instance.DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:5631370915311469374ef3cb5f0ebbf0", @@ -3005,7 +3046,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214620908, + "time": 1735588787786, "actor": "urn:li:corpuser:_ingestion" } } diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index ff27989d71de1b..70f5784d6d0b18 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -95,7 +95,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "Weekly Demo Data Backup" } }, @@ -113,17 +112,16 @@ "aspect": { "json": { "customProperties": { - "job_id": "a06cfdca-b65e-42de-8db2-8c21c183c5dd", + "job_id": "f5a6c120-500a-4300-9b21-0c3225af1f80", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-12-26 12:03:35.420000", - "date_modified": "2024-12-26 12:03:35.590000", + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, - "externalUrl": "", "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" @@ -2259,7 +2257,6 @@ "aspect": { "json": { "customProperties": {}, - "externalUrl": "", "name": "DemoData.Foo.stored_procedures" } }, @@ -2282,10 +2279,9 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-12-26 12:03:35.230000", - "date_modified": "2024-12-26 12:03:35.230000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2298,6 +2294,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", @@ -2310,10 +2329,9 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-12-26 12:03:35.237000", - "date_modified": "2024-12-26 12:03:35.237000" + "date_created": "2024-12-30 19:59:24.690000", + "date_modified": "2024-12-30 19:59:24.690000" }, - "externalUrl": "", "name": "DemoData.Foo.NewProc", "type": { "string": "MSSQL_STORED_PROCEDURE" @@ -2326,6 +2344,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "dataTransformLogic", + "aspect": { + "json": { + "transforms": [ + { + "queryStatement": { + "value": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "language": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -4978,7 +5019,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214622805, + "time": 1735588791954, "actor": "urn:li:corpuser:_ingestion" } } @@ -5125,7 +5166,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1735214622810, + "time": 1735588791966, "actor": "urn:li:corpuser:_ingestion" } }