OMOP schema is captured as a dacpac

README.md

@@ -76,7 +76,7 @@ To import the vocabulary and synthetic data manually, be sure to comment out the
 
 ### Loading Clinical Data 
 
-[Staging Tables SQL Script](./sql/OMOP_CDM_sql_server_staging_ddl_indexes.sql) is currently executed in the Terraform to aid in loading clinical data into the OMOP CDM database, where, new clinical data can be added to the target staging table and eventually migrated to the permanent clinical table using a SQL Stored Procedure or an alternative approach. You can comment out this import in the Terraform `main.tf` script (lines 154-156) if this isn't desired. 
+[Staging Tables SQL Script](./sql/OMOP_CDM_sql_server_staging_ddl_indexes.sql) is currently executed in the Terraform to aid in loading clinical data into the OMOP CDM database, where, new clinical data can be added to the target staging table and eventually migrated to the permanent clinical table using a [SQL Stored Procedure](./sql/SPROC_move_data_to_permanent_tables.sql) or an alternative approach. You can comment out this import in the Terraform `main.tf` script (lines 154-156) if this isn't desired. 
 
 ## Using Achilles
 

sql/OMOP_CDM_sql_server_staging_ddl.sql

@@ -62,12 +62,4 @@ Standardized derived elements
 
 SELECT TOP 0 * INTO dbo.staging_condition_era FROM dbo.condition_era;
 
-SELECT TOP 0 * INTO dbo.staging_drug_era FROM dbo.drug_era;
-
-
-/************************
-
-Helper Table to move data from staging tables to permanent tables
-
-************************/
-CREATE TABLE watermark_copy_staging_tables (table_name VARCHAR(MAX))
+SELECT TOP 0 * INTO dbo.staging_drug_era FROM dbo.drug_era;

sql/SPROC_move_data_to_permanent_tables.sql

@@ -0,0 +1,111 @@
+CREATE PROCEDURE move_data_to_permanent_tables @watermark_table NVARCHAR(100)
+AS
+    -- SET XACT_ABORT ON will cause the transaction to be uncommittable  
+    -- when the constraint violation occurs.   
+    SET XACT_ABORT ON;
+    BEGIN TRANSACTION move_data_to_permanent_tables
+    BEGIN TRY
+        DECLARE @current_table_name VARCHAR(MAX);
+        DECLARE @staging_table_name VARCHAR(MAX);
+        DECLARE @log_date AS NVARCHAR(50);
+        DECLARE @log_message NVARCHAR(512);
+        SET @current_table_name = ''
+
+        -- The use of RAISERROR with the NOWAIT option shouldn't actually be emitting an exception as you might expect.  It's used
+        -- to immediately emmit log/console messages.  For additional details, see
+        -- https://www.mssqltips.com/sqlservertip/1660/using-the-nowait-option-with-the-sql-server-raiserror-statement/
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is starting execution'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is removing all indexes and constraints on OMOP tables'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+
+        EXEC dbo.remove_indexes_constraints;
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables removed all indexes and constraints on OMOP tables'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+
+        DECLARE @cursor_sql VARCHAR(MAX)
+        SET @cursor_sql = 'DECLARE omop_tables_cursor CURSOR FOR SELECT table_name FROM ' + @watermark_table
+        EXEC(@cursor_sql)
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is moving data for tables in ' + @watermark_table
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        -- https://www.mssqltips.com/sqlservertip/1599/cursor-in-sql-server/
+        OPEN omop_tables_cursor
+        FETCH NEXT FROM omop_tables_cursor INTO @current_table_name
+
+        WHILE @@FETCH_STATUS = 0
+
+        BEGIN  
+        DECLARE @truncate_sql VARCHAR(MAX)
+        DECLARE @switch_sql VARCHAR(MAX)
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is truncating table ' + @current_table_name
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        SET @truncate_sql = 'TRUNCATE TABLE ' + @current_table_name
+        SET @staging_table_name = 'staging_' + @current_table_name
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is moving data from staging to ' + @current_table_name
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        SET @switch_sql = 'ALTER TABLE ' + @staging_table_name + ' SWITCH TO ' + @current_table_name
+        EXEC (@truncate_sql)
+        EXEC (@switch_sql)
+        FETCH NEXT FROM omop_tables_cursor INTO @current_table_name;
+        END 
+        CLOSE omop_tables_cursor;  
+        DEALLOCATE omop_tables_cursor; 
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables moved all data from staging for tables in ' + @watermark_table
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is applying all indexes and constraints on OMOP tables'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        EXEC dbo.add_indexes_constraints;
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables applied all indexes and constraints on OMOP tables'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        COMMIT TRANSACTION move_data_to_permanent_tables;
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables is ending execution'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+        RETURN 1; 
+    END TRY
+    BEGIN CATCH
+
+        SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+        SET @log_message = @log_date + ' move_data_to_permanent_tables entered catch block'
+        RAISERROR (@log_message, 0, 1) WITH NOWAIT
+
+        IF (SELECT CURSOR_STATUS('global','omop_tables_cursor')) >= -1
+        BEGIN
+            DEALLOCATE omop_tables_cursor;
+        END
+
+        -- Transaction uncommittable
+        IF (XACT_STATE()) = -1
+        BEGIN
+            SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+            SET @log_message = @log_date + ' move_data_to_permanent_tables is rolling back transaction'
+            RAISERROR (@log_message, 0, 1) WITH NOWAIT
+            ROLLBACK TRANSACTION move_data_to_permanent_tables;
+            THROW;
+        END
+        -- Transaction committable
+        IF (XACT_STATE()) = 1
+        BEGIN
+            SET @log_date = CONVERT(NVARCHAR(50),GETDATE(),121);
+            SET @log_message = @log_date + ' move_data_to_permanent_tables is committing transaction'
+            RAISERROR (@log_message, 0, 1) WITH NOWAIT
+            COMMIT TRANSACTION move_data_to_permanent_tables;
+        END
+
+    END CATCH;
+
+GO