Skip to content

Commit

Permalink
Now dm+d sources are updated separately
Browse files Browse the repository at this point in the history
  • Loading branch information
Timur committed Feb 3, 2021
1 parent f56b3a2 commit 5fba558
Show file tree
Hide file tree
Showing 10 changed files with 335 additions and 191 deletions.
13 changes: 3 additions & 10 deletions SNOMED/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,6 @@ Rename files to sct2_Concept_Full_GB_DE.txt, sct2_Description_Full-en-GB_DE.txt,
- der2_cRefset_AssociationFull_GBxxxxxxx_YYYYMMDD.txt from SnomedCT_UKDrugRF2_Production_YYYYMMDDTzzzzzz\Full\Refset\Content
Rename to der2_cRefset_AssociationFull_INT.txt, der2_cRefset_AssociationFull_UK.txt, der2_cRefset_AssociationFull_US.txt and der2_cRefset_AssociationFull_GB_DE.txt


11. Add DM+D: Download nhsbsa_dmd_X.X.X_xxxxxxxxxxxxxx.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/6/subpack/24/releases
12. Extract f_ampp2_xxxxxxx.xml, f_amp2_xxxxxxx.xml, f_vmpp2_xxxxxxx.xml, f_vmp2_xxxxxxx.xml, f_lookup2_xxxxxxx.xml, f_vtm2_xxxxxxx.xml and f_ingredient2_xxxxxxx.xml
13. Rename to f_ampp2.xml, f_amp2.xml, f_vmpp2.xml, f_vmp2.xml, f_lookup2.xml, f_vtm2.xml and f_ingredient2.xml
14. Download nhsbsa_dmdbonus_X.X.X_YYYYMMDDXXXXXX.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/6/subpack/25/releases
15. Extract weekXXYYYY-rX_X-BNF.zip/f_bnf1_XXXXXXX.xml and rename him to dmdbonus.xml

16. Run in devv5 (with fresh vocabulary date and version): SELECT sources.load_input_tables('SNOMED',TO_DATE('20180131','YYYYMMDD'),'Snomed Release 20180131');
17. Run load_stage.sql
18. Run generic_update: devv5.GenericUpdate();
11. Run in devv5 (with fresh vocabulary date and version): SELECT sources.load_input_tables('SNOMED',TO_DATE('20180131','YYYYMMDD'),'Snomed Release 20180131');
12. Run load_stage.sql
13. Run generic_update: devv5.GenericUpdate();
28 changes: 28 additions & 0 deletions dmd/create_source_tables.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/**************************************************************************
* Copyright 2016 Observational Health Data Sciences and Informatics (OHDSI)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Authors: Timur Vakhitov
* Date: 2021
**************************************************************************/

DROP TABLE IF EXISTS SOURCES.F_LOOKUP2,SOURCES.F_INGREDIENT2,SOURCES.F_VTM2,SOURCES.F_VMP2,SOURCES.F_AMP2,SOURCES.F_VMPP2,SOURCES.F_AMPP2,SOURCES.DMDBONUS;
CREATE TABLE SOURCES.F_LOOKUP2 (xmlfield XML, vocabulary_date DATE, vocabulary_version VARCHAR (200));
CREATE TABLE SOURCES.F_INGREDIENT2 (xmlfield XML);
CREATE TABLE SOURCES.F_VTM2 (xmlfield XML);
CREATE TABLE SOURCES.F_VMP2 (xmlfield XML);
CREATE TABLE SOURCES.F_VMPP2 (xmlfield XML);
CREATE TABLE SOURCES.F_AMP2 (xmlfield XML);
CREATE TABLE SOURCES.F_AMPP2 (xmlfield XML);
CREATE TABLE SOURCES.DMDBONUS (xmlfield XML);
4 changes: 2 additions & 2 deletions dmd/load_stage.sql
Original file line number Diff line number Diff line change
Expand Up @@ -1282,8 +1282,8 @@ DO $_$
BEGIN
PERFORM VOCABULARY_PACK.SetLatestUpdate(
pVocabularyName => 'dm+d',
pVocabularyDate => TO_DATE ('20201207', 'yyyymmdd'),
pVocabularyVersion => 'dm+d Version 12.0.0 20201207',
pVocabularyDate => (SELECT vocabulary_date FROM sources.f_lookup2 LIMIT 1),
pVocabularyVersion => (SELECT vocabulary_version FROM sources.f_lookup2 LIMIT 1),
pVocabularyDevSchema => 'DEV_DMD'
);
PERFORM VOCABULARY_PACK.SetLatestUpdate(
Expand Down
19 changes: 14 additions & 5 deletions dmd/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,22 @@ Prerequisites:
- Updated and released versions of RxNorm, RxNorm Extension and SNOMED vocabularies

Update algorithm:
1. Run additional_DDL.sql
2. Run load_stage.sql;

1. Run create_source_tables.sql
2. Download nhsbsa_dmd_X.X.X_xxxxxxxxxxxxxx.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/6/subpack/24/releases
3. Extract f_ampp2_xxxxxxx.xml, f_amp2_xxxxxxx.xml, f_vmpp2_xxxxxxx.xml, f_vmp2_xxxxxxx.xml, f_lookup2_xxxxxxx.xml, f_vtm2_xxxxxxx.xml and f_ingredient2_xxxxxxx.xml
4. Rename to f_ampp2.xml, f_amp2.xml, f_vmpp2.xml, f_vmp2.xml, f_lookup2.xml, f_vtm2.xml and f_ingredient2.xml
5. Download nhsbsa_dmdbonus_X.X.X_YYYYMMDDXXXXXX.zip from https://isd.digital.nhs.uk/trud3/user/authenticated/group/0/pack/6/subpack/25/releases
6. Extract weekXXYYYY-rX_X-BNF.zip/f_bnf1_XXXXXXX.xml and rename to dmdbonus.xml
7. Run in devv5 (with fresh vocabulary date and version): SELECT sources.load_input_tables('DMD',TO_DATE('20210201','YYYYMMDD'),'dm+d Version 2.0.0 20210201');

8. Run additional_DDL.sql
9. Run load_stage.sql;
- Update latest_update to current version
- Commented portions include SELECTs needed to update manual tables
3. Run build_RxE.sql from ../working
4. Run postprocessing.sql
5. Run devv5.generic_update() function
10. Run build_RxE.sql from ../working
11. Run postprocessing.sql
12. Run devv5.generic_update() function

R_TO_C_ALL contents which is sufficient for December 2020 1st week release is available under:
https://drive.google.com/drive/u/2/folders/1-_gcibtoQwhqnqAtf1bd02djLEicBr9w
23 changes: 13 additions & 10 deletions working/packages/load_input_tables/load_input_tables.sql
Original file line number Diff line number Diff line change
Expand Up @@ -485,16 +485,6 @@ begin
analyze sources.sct2_concept_full_merged;
analyze sources.sct2_desc_full_merged;
analyze sources.sct2_rela_full_merged;
--load XML sources
truncate table sources.f_lookup2, sources.f_ingredient2, sources.f_vtm2, sources.f_vmp2, sources.f_vmpp2, sources.f_amp2, sources.f_ampp2, sources.dmdbonus;
execute 'COPY sources.f_lookup2 FROM '''||pVocabularyPath||'f_lookup2.xml'' delimiter E''\b''';
execute 'COPY sources.f_ingredient2 FROM '''||pVocabularyPath||'f_ingredient2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vtm2 FROM '''||pVocabularyPath||'f_vtm2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vmp2 FROM '''||pVocabularyPath||'f_vmp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vmpp2 FROM '''||pVocabularyPath||'f_vmpp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_amp2 FROM '''||pVocabularyPath||'f_amp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_ampp2 FROM '''||pVocabularyPath||'f_ampp2.xml'' delimiter E''\b''';
execute 'COPY sources.dmdbonus FROM '''||pVocabularyPath||'dmdbonus.xml'' delimiter E''\b''';
--loading der2_sRefset_SimpleMapFull_INT
truncate table sources.der2_srefset_simplemapfull_int;
execute 'COPY sources.der2_srefset_simplemapfull_int FROM '''||pVocabularyPath||'der2_sRefset_SimpleMapFull_INT.txt'' delimiter E''\t'' csv quote E''\b'' HEADER';
Expand Down Expand Up @@ -778,6 +768,19 @@ begin
execute 'COPY sources.hemonc_crs FROM '''||pVocabularyPath||'concept_relationship_stage.tab'' delimiter E''\t'' csv quote ''"'' FORCE NULL concept_id_1,concept_id_2,concept_code_1,concept_code_2,vocabulary_id_1,vocabulary_id_2,relationship_id HEADER';
execute 'COPY sources.hemonc_css FROM '''||pVocabularyPath||'concept_synonym_stage.tab'' delimiter E''\t'' csv quote ''"'' FORCE NULL synonym_concept_id,synonym_name,synonym_concept_code,synonym_vocabulary_id,language_concept_id HEADER';
update sources.hemonc_cs set vocabulary_date=COALESCE(pVocabularyDate,current_date), vocabulary_version=COALESCE(pVocabularyVersion,pVocabularyID||' '||current_date);
when 'DMD' then
truncate table sources.f_lookup2, sources.f_ingredient2, sources.f_vtm2, sources.f_vmp2, sources.f_vmpp2, sources.f_amp2, sources.f_ampp2, sources.dmdbonus;
execute 'COPY sources.f_lookup2 (xmlfield) FROM '''||pVocabularyPath||'f_lookup2.xml'' delimiter E''\b''';
execute 'COPY sources.f_ingredient2 FROM '''||pVocabularyPath||'f_ingredient2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vtm2 FROM '''||pVocabularyPath||'f_vtm2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vmp2 FROM '''||pVocabularyPath||'f_vmp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_vmpp2 FROM '''||pVocabularyPath||'f_vmpp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_amp2 FROM '''||pVocabularyPath||'f_amp2.xml'' delimiter E''\b''';
execute 'COPY sources.f_ampp2 FROM '''||pVocabularyPath||'f_ampp2.xml'' delimiter E''\b''';
execute 'COPY sources.dmdbonus FROM '''||pVocabularyPath||'dmdbonus.xml'' delimiter E''\b''';
update sources.f_lookup2 set vocabulary_date=COALESCE(pVocabularyDate,current_date), vocabulary_version=COALESCE(pVocabularyVersion,pVocabularyID||' '||current_date);
when 'DM+D' then
RAISE EXCEPTION 'Use ''DMD'' instead of %', pVocabularyID;
else
RAISE EXCEPTION 'Vocabulary with id=% not found', pVocabularyID;
end case;
Expand Down
65 changes: 65 additions & 0 deletions working/packages/vocabulary_download/bash_functions_dmd.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
--DMD part
DO $_$
DECLARE
z text;
BEGIN
z:=$FUNCTIONBODY$
CREATE OR REPLACE FUNCTION vocabulary_download.get_dmd_prepare_dmd (iPath text, iFilename text)
RETURNS void AS
$BODY$#!/bin/bash
#set permissions=775 by default
umask 002 && \
cd "$1/work" && \
unzip -oqj "$2" "f_ampp2_*.xml" -d . && \
unzip -oqj "$2" "f_amp2_*.xml" -d . && \
unzip -oqj "$2" "f_vmpp2_*.xml" -d . && \
unzip -oqj "$2" "f_vmp2_*.xml" -d . && \
unzip -oqj "$2" "f_lookup2_*.xml" -d . && \
unzip -oqj "$2" "f_vtm2_*.xml" -d . && \
unzip -oqj "$2" "f_ingredient2_*.xml" -d .

#move result to original folder
cd "$1"
rm -f *.xml
mv work/f_ampp2_*.xml "f_ampp2.xml" && \
mv work/f_amp2_*.xml "f_amp2.xml" && \
mv work/f_vmpp2_*.xml "f_vmpp2.xml" && \
mv work/f_vmp2_*.xml "f_vmp2.xml" && \
mv work/f_lookup2_*.xml "f_lookup2.xml" && \
mv work/f_vtm2_*.xml "f_vtm2.xml" && \
mv work/f_ingredient2_*.xml "f_ingredient2.xml"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
$FUNCTIONBODY$;
--convert CRLF to LF for bash
EXECUTE REPLACE(z,E'\r','');
END $_$;

--DMD bonus part
DO $_$
DECLARE
z text;
BEGIN
z:=$FUNCTIONBODY$
CREATE OR REPLACE FUNCTION vocabulary_download.get_dmd_prepare_dmdbonus (iPath text, iFilename text)
RETURNS void AS
$BODY$#!/bin/bash
#set permissions=775 by default
umask 002 && \
cd "$1/work" && \
unzip -oqj "$2" "*.zip" -d . && \
unzip -oqj "week*BNF.zip" "f_bnf1_*.xml" -d . && \
rm -f week*BNF.zip

#move result to original folder
cd "$1"
rm -f dmdbonus.xml
mv work/f_bnf1_*.xml "dmdbonus.xml"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
$FUNCTIONBODY$;
--convert CRLF to LF for bash
EXECUTE REPLACE(z,E'\r','');
END $_$;
67 changes: 1 addition & 66 deletions working/packages/vocabulary_download/bash_functions_snomed.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
20200519 added extraction of der2_cRefset_LanguageFull.txt
20200511 added extraction of der2_ssRefset_ModuleDependencyFull*.txt
20201117 added extraction of der2_iisssccRefset_ExtendedMapFull*.txt
20210202 dm+d removed
*/

--INT part
Expand Down Expand Up @@ -152,70 +153,4 @@ BEGIN
$FUNCTIONBODY$;
--convert CRLF to LF for bash
EXECUTE REPLACE(z,E'\r','');
END $_$;

--DMD part
DO $_$
DECLARE
z text;
BEGIN
z:=$FUNCTIONBODY$
CREATE OR REPLACE FUNCTION vocabulary_download.get_snomed_prepare_dmd (iPath text, iFilename text)
RETURNS void AS
$BODY$#!/bin/bash
#set permissions=775 by default
umask 002 && \
cd "$1/work" && \
unzip -oqj "$2" "f_ampp2_*.xml" -d . && \
unzip -oqj "$2" "f_amp2_*.xml" -d . && \
unzip -oqj "$2" "f_vmpp2_*.xml" -d . && \
unzip -oqj "$2" "f_vmp2_*.xml" -d . && \
unzip -oqj "$2" "f_lookup2_*.xml" -d . && \
unzip -oqj "$2" "f_vtm2_*.xml" -d . && \
unzip -oqj "$2" "f_ingredient2_*.xml" -d .

#move result to original folder
cd "$1"
rm -f *.xml
mv work/f_ampp2_*.xml "f_ampp2.xml" && \
mv work/f_amp2_*.xml "f_amp2.xml" && \
mv work/f_vmpp2_*.xml "f_vmpp2.xml" && \
mv work/f_vmp2_*.xml "f_vmp2.xml" && \
mv work/f_lookup2_*.xml "f_lookup2.xml" && \
mv work/f_vtm2_*.xml "f_vtm2.xml" && \
mv work/f_ingredient2_*.xml "f_ingredient2.xml"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
$FUNCTIONBODY$;
--convert CRLF to LF for bash
EXECUTE REPLACE(z,E'\r','');
END $_$;

--DMD bonus part
DO $_$
DECLARE
z text;
BEGIN
z:=$FUNCTIONBODY$
CREATE OR REPLACE FUNCTION vocabulary_download.get_snomed_prepare_dmdbonus (iPath text, iFilename text)
RETURNS void AS
$BODY$#!/bin/bash
#set permissions=775 by default
umask 002 && \
cd "$1/work" && \
unzip -oqj "$2" "*.zip" -d . && \
unzip -oqj "week*BNF.zip" "f_bnf1_*.xml" -d . && \
rm -f week*BNF.zip

#move result to original folder
cd "$1"
rm -f dmdbonus.xml
mv work/f_bnf1_*.xml "dmdbonus.xml"
$BODY$
LANGUAGE 'plsh'
SECURITY DEFINER;
$FUNCTIONBODY$;
--convert CRLF to LF for bash
EXECUTE REPLACE(z,E'\r','');
END $_$;
Loading

0 comments on commit 5fba558

Please sign in to comment.