From 1dc34f337823471f46940fb87cbe6b3dcc51c89f Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 30 Jun 2023 15:22:28 +0200 Subject: [PATCH 01/39] Typo in documentation --- doc/src/ingest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/ingest.rst b/doc/src/ingest.rst index b867f4e1..10817fbc 100644 --- a/doc/src/ingest.rst +++ b/doc/src/ingest.rst @@ -88,7 +88,7 @@ might then look like:: raise RuntimeError("invalid ingest file") from e # Create the datasets. In a real production script, you'd copy the - # content of the datafile to IDS storage at the same time: + # content of the datafiles to IDS storage at the same time: for ds in datasets: ds.create() From 054960bf894129162179b7dfd36de4b014193518 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 16 Oct 2023 17:12:11 +0200 Subject: [PATCH 02/39] Add a rule giving CRU permission on Sample to ingest to the example data --- doc/examples/icatdump-4.10.xml | 95 +++++++++-------- doc/examples/icatdump-4.10.yaml | 94 ++++++++-------- doc/examples/icatdump-4.4.xml | 95 +++++++++-------- doc/examples/icatdump-4.4.yaml | 94 ++++++++-------- doc/examples/icatdump-4.7.xml | 95 +++++++++-------- doc/examples/icatdump-4.7.yaml | 94 ++++++++-------- doc/examples/icatdump-5.0.xml | 153 ++++++++++++++------------- doc/examples/icatdump-5.0.yaml | 152 +++++++++++++------------- doc/examples/init-icat.py | 2 +- tests/data/legacy-icatdump-4.10.xml | 95 +++++++++-------- tests/data/legacy-icatdump-4.10.yaml | 94 ++++++++-------- tests/data/legacy-icatdump-4.4.xml | 95 +++++++++-------- tests/data/legacy-icatdump-4.4.yaml | 94 ++++++++-------- tests/data/legacy-icatdump-4.7.xml | 95 +++++++++-------- tests/data/legacy-icatdump-4.7.yaml | 94 ++++++++-------- tests/data/ref-icatdump-5.0.xml | 97 +++++++++-------- tests/data/ref-icatdump-5.0.yaml | 96 +++++++++-------- tests/data/summary-4 | 2 +- tests/data/summary-5 | 2 +- tests/test_06_query.py | 4 +- 20 files changed, 857 insertions(+), 785 deletions(-) diff --git a/doc/examples/icatdump-4.10.xml b/doc/examples/icatdump-4.10.xml index 336b8431..3afbaff1 100644 --- a/doc/examples/icatdump-4.10.xml +++ b/doc/examples/icatdump-4.10.xml @@ -1,7 +1,7 @@ - 2022-12-21T15:08:20+00:00 + 2023-10-16T14:48:15+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.10.0 icatdump (python-icat 1.0.0) @@ -431,226 +431,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/doc/examples/icatdump-4.10.yaml b/doc/examples/icatdump-4.10.yaml index a57f707a..b051ca22 100644 --- a/doc/examples/icatdump-4.10.yaml +++ b/doc/examples/icatdump-4.10.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Wed, 21 Dec 2022 15:08:23 +0000 +# Date: Mon, 16 Oct 2023 14:48:11 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.10.0 # Generator: icatdump (python-icat 1.0.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/doc/examples/icatdump-4.4.xml b/doc/examples/icatdump-4.4.xml index 933aa5ef..e0563413 100644 --- a/doc/examples/icatdump-4.4.xml +++ b/doc/examples/icatdump-4.4.xml @@ -1,7 +1,7 @@ - 2022-12-21T14:56:55+00:00 + 2023-10-16T14:20:21+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.4.0 icatdump (python-icat 1.0.0) @@ -403,226 +403,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/doc/examples/icatdump-4.4.yaml b/doc/examples/icatdump-4.4.yaml index 49c19739..6ced4485 100644 --- a/doc/examples/icatdump-4.4.yaml +++ b/doc/examples/icatdump-4.4.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Wed, 21 Dec 2022 14:57:08 +0000 +# Date: Mon, 16 Oct 2023 14:20:09 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.4.0 # Generator: icatdump (python-icat 1.0.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/doc/examples/icatdump-4.7.xml b/doc/examples/icatdump-4.7.xml index a81ac848..55484db3 100644 --- a/doc/examples/icatdump-4.7.xml +++ b/doc/examples/icatdump-4.7.xml @@ -1,7 +1,7 @@ - 2022-12-21T15:02:37+00:00 + 2023-10-16T14:34:50+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.7.0 icatdump (python-icat 1.0.0) @@ -414,226 +414,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/doc/examples/icatdump-4.7.yaml b/doc/examples/icatdump-4.7.yaml index 9bed459a..c1525829 100644 --- a/doc/examples/icatdump-4.7.yaml +++ b/doc/examples/icatdump-4.7.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Wed, 21 Dec 2022 15:02:40 +0000 +# Date: Mon, 16 Oct 2023 14:34:39 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.7.0 # Generator: icatdump (python-icat 1.0.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/doc/examples/icatdump-5.0.xml b/doc/examples/icatdump-5.0.xml index 6dd08266..3bcc44e6 100644 --- a/doc/examples/icatdump-5.0.xml +++ b/doc/examples/icatdump-5.0.xml @@ -1,9 +1,9 @@ - 2022-12-21T15:14:23+00:00 + 2023-10-16T15:00:34+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl - 5.0.0 + 5.0.1 icatdump (python-icat 1.0.0) @@ -529,366 +529,371 @@ + CRU + Sample + + + R Shift - + CRUD Affiliation - + CRUD DataPublication - + CRUD DataPublicationDate - + CRUD DataPublicationFunding - + CRUD DataPublicationUser - + CRUD FundingReference - + CRUD RelatedItem - + R SELECT o FROM DataPublication o - + R SELECT o FROM Datafile o JOIN o.dataCollectionDatafiles AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R SELECT o FROM Datafile o JOIN o.dataset AS ds JOIN ds.dataCollectionDatasets AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R SELECT o FROM Datafile o JOIN o.dataset AS ds JOIN ds.investigation AS i JOIN i.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R SELECT o FROM Dataset o JOIN o.dataCollectionDatasets AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R SELECT o FROM Dataset o JOIN o.investigation AS i JOIN i.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R SELECT o FROM Investigation o JOIN o.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - + R Affiliation - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionInvestigation - + R DataCollectionParameter - + R DataPublication - + R DataPublicationDate - + R DataPublicationFunding - + R DataPublicationUser - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetInstrument - + R DatasetParameter - + R DatasetTechnique - + R FundingReference - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationFacilityCycle - + R InvestigationFunding - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R RelatedItem - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD FundingReference - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationFunding - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/doc/examples/icatdump-5.0.yaml b/doc/examples/icatdump-5.0.yaml index b9533b34..484d0b1f 100644 --- a/doc/examples/icatdump-5.0.yaml +++ b/doc/examples/icatdump-5.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 -# Date: Wed, 21 Dec 2022 15:14:26 +0000 +# Date: Mon, 16 Oct 2023 15:00:30 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl -# ICAT-API: 5.0.0 +# ICAT-API: 5.0.1 # Generator: icatdump (python-icat 1.0.0) --- grouping: @@ -557,304 +557,308 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000086: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000087: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000087: + Rule_00000088: crudFlags: CRUD grouping: Grouping_name-publisher what: Affiliation - Rule_00000088: + Rule_00000089: crudFlags: CRUD grouping: Grouping_name-publisher what: DataPublication - Rule_00000089: + Rule_00000090: crudFlags: CRUD grouping: Grouping_name-publisher what: DataPublicationDate - Rule_00000090: + Rule_00000091: crudFlags: CRUD grouping: Grouping_name-publisher what: DataPublicationFunding - Rule_00000091: + Rule_00000092: crudFlags: CRUD grouping: Grouping_name-publisher what: DataPublicationUser - Rule_00000092: + Rule_00000093: crudFlags: CRUD grouping: Grouping_name-publisher what: FundingReference - Rule_00000093: + Rule_00000094: crudFlags: CRUD grouping: Grouping_name-publisher what: RelatedItem - Rule_00000094: + Rule_00000095: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM DataPublication o - Rule_00000095: + Rule_00000096: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Datafile o JOIN o.dataCollectionDatafiles AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000096: + Rule_00000097: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Datafile o JOIN o.dataset AS ds JOIN ds.dataCollectionDatasets AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000097: + Rule_00000098: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Datafile o JOIN o.dataset AS ds JOIN ds.investigation AS i JOIN i.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000098: + Rule_00000099: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Dataset o JOIN o.dataCollectionDatasets AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000099: + Rule_00000100: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Dataset o JOIN o.investigation AS i JOIN i.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000100: + Rule_00000101: crudFlags: R grouping: Grouping_name-pubreader what: SELECT o FROM Investigation o JOIN o.dataCollectionInvestigations AS s1 JOIN s1.dataCollection AS s2 JOIN s2.dataPublications AS s3 WHERE s3.id IS NOT NULL - Rule_00000101: + Rule_00000102: crudFlags: R grouping: Grouping_name-rall what: Affiliation - Rule_00000102: + Rule_00000103: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000103: + Rule_00000104: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000104: + Rule_00000105: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000105: + Rule_00000106: crudFlags: R grouping: Grouping_name-rall what: DataCollectionInvestigation - Rule_00000106: + Rule_00000107: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000107: + Rule_00000108: crudFlags: R grouping: Grouping_name-rall what: DataPublication - Rule_00000108: + Rule_00000109: crudFlags: R grouping: Grouping_name-rall what: DataPublicationDate - Rule_00000109: + Rule_00000110: crudFlags: R grouping: Grouping_name-rall what: DataPublicationFunding - Rule_00000110: + Rule_00000111: crudFlags: R grouping: Grouping_name-rall what: DataPublicationUser - Rule_00000111: + Rule_00000112: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000112: + Rule_00000113: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000113: + Rule_00000114: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000114: + Rule_00000115: crudFlags: R grouping: Grouping_name-rall what: DatasetInstrument - Rule_00000115: + Rule_00000116: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000116: + Rule_00000117: crudFlags: R grouping: Grouping_name-rall what: DatasetTechnique - Rule_00000117: + Rule_00000118: crudFlags: R grouping: Grouping_name-rall what: FundingReference - Rule_00000118: + Rule_00000119: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000119: + Rule_00000120: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000120: + Rule_00000121: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000121: + Rule_00000122: crudFlags: R grouping: Grouping_name-rall what: InvestigationFacilityCycle - Rule_00000122: + Rule_00000123: crudFlags: R grouping: Grouping_name-rall what: InvestigationFunding - Rule_00000123: + Rule_00000124: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000124: + Rule_00000125: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000125: + Rule_00000126: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000126: + Rule_00000127: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000127: + Rule_00000128: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000128: + Rule_00000129: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000129: + Rule_00000130: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000130: + Rule_00000131: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000131: + Rule_00000132: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000132: + Rule_00000133: crudFlags: R grouping: Grouping_name-rall what: RelatedItem - Rule_00000133: + Rule_00000134: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000134: + Rule_00000135: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000135: + Rule_00000136: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000136: + Rule_00000137: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000137: + Rule_00000138: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000138: + Rule_00000139: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000139: + Rule_00000140: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000140: + Rule_00000141: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000141: + Rule_00000142: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000142: + Rule_00000143: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000143: + Rule_00000144: crudFlags: CRUD grouping: Grouping_name-useroffice what: FundingReference - Rule_00000144: + Rule_00000145: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000145: + Rule_00000146: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000146: + Rule_00000147: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000147: + Rule_00000148: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationFunding - Rule_00000148: + Rule_00000149: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000149: + Rule_00000150: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000150: + Rule_00000151: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000151: + Rule_00000152: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000152: + Rule_00000153: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000153: + Rule_00000154: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000154: + Rule_00000155: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000155: + Rule_00000156: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000156: + Rule_00000157: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000157: + Rule_00000158: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000158: + Rule_00000159: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/doc/examples/init-icat.py b/doc/examples/init-icat.py index f6189ba3..ef0017ac 100755 --- a/doc/examples/init-icat.py +++ b/doc/examples/init-icat.py @@ -131,7 +131,7 @@ def getUser(client, attrs): ingest = client.createUser("simple/dataingest", fullName="Data Ingester") ingestgroup = client.createGroup("ingest", [ ingest ]) client.createRules("R", [ "Investigation", "Shift" ], ingestgroup) -ingest_cru_classes = [ "Dataset", "Datafile", +ingest_cru_classes = [ "Sample", "Dataset", "Datafile", "DatasetParameter", "DatafileParameter" ] if "datasetInstrument" in client.typemap: ingest_cru_classes.append("DatasetInstrument") diff --git a/tests/data/legacy-icatdump-4.10.xml b/tests/data/legacy-icatdump-4.10.xml index 24b310e9..df2dbc0e 100644 --- a/tests/data/legacy-icatdump-4.10.xml +++ b/tests/data/legacy-icatdump-4.10.xml @@ -1,7 +1,7 @@ - 2022-12-04T19:31:45+00:00 + 2023-10-16T14:48:15+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.10 icatdump (python-icat 0.21.0) @@ -431,226 +431,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/tests/data/legacy-icatdump-4.10.yaml b/tests/data/legacy-icatdump-4.10.yaml index f38d4dbf..e45589d6 100644 --- a/tests/data/legacy-icatdump-4.10.yaml +++ b/tests/data/legacy-icatdump-4.10.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Sun, 04 Dec 2022 19:31:43 +0000 +# Date: Mon, 16 Oct 2023 14:48:11 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.10 # Generator: icatdump (python-icat 0.21.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/tests/data/legacy-icatdump-4.4.xml b/tests/data/legacy-icatdump-4.4.xml index b7b40b1f..e77e717e 100644 --- a/tests/data/legacy-icatdump-4.4.xml +++ b/tests/data/legacy-icatdump-4.4.xml @@ -1,7 +1,7 @@ - 2022-12-04T19:02:46+00:00 + 2023-10-16T14:20:21+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.4 icatdump (python-icat 0.21.0) @@ -403,226 +403,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/tests/data/legacy-icatdump-4.4.yaml b/tests/data/legacy-icatdump-4.4.yaml index fc926450..9f51fc85 100644 --- a/tests/data/legacy-icatdump-4.4.yaml +++ b/tests/data/legacy-icatdump-4.4.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Sun, 04 Dec 2022 19:02:35 +0000 +# Date: Mon, 16 Oct 2023 14:20:09 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.4 # Generator: icatdump (python-icat 0.21.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/tests/data/legacy-icatdump-4.7.xml b/tests/data/legacy-icatdump-4.7.xml index fc89f2aa..15cd6f96 100644 --- a/tests/data/legacy-icatdump-4.7.xml +++ b/tests/data/legacy-icatdump-4.7.xml @@ -1,7 +1,7 @@ - 2022-12-04T19:18:32+00:00 + 2023-10-16T14:34:50+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl 4.7 icatdump (python-icat 0.21.0) @@ -414,226 +414,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/tests/data/legacy-icatdump-4.7.yaml b/tests/data/legacy-icatdump-4.7.yaml index 634c477f..f7ffa6f1 100644 --- a/tests/data/legacy-icatdump-4.7.yaml +++ b/tests/data/legacy-icatdump-4.7.yaml @@ -1,5 +1,5 @@ %YAML 1.1 -# Date: Sun, 04 Dec 2022 19:18:29 +0000 +# Date: Mon, 16 Oct 2023 14:34:39 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl # ICAT-API: 4.7 # Generator: icatdump (python-icat 0.21.0) @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/tests/data/ref-icatdump-5.0.xml b/tests/data/ref-icatdump-5.0.xml index 52ee96aa..9f3fc5c5 100644 --- a/tests/data/ref-icatdump-5.0.xml +++ b/tests/data/ref-icatdump-5.0.xml @@ -1,9 +1,9 @@ - 2022-12-21T15:21:54+00:00 + 2023-10-16T15:00:34+00:00 https://icat.example.com:8181/ICATService/ICAT?wsdl - 5.0.0 + 5.0.1 icatdump (python-icat 1.0.0) @@ -431,226 +431,231 @@ + CRU + Sample + + + R Shift - + R DataCollection - + R DataCollectionDatafile - + R DataCollectionDataset - + R DataCollectionParameter - + R Datafile - + R DatafileParameter - + R Dataset - + R DatasetParameter - + R Grouping - + R InstrumentScientist - + R Investigation - + R InvestigationGroup - + R InvestigationInstrument - + R InvestigationParameter - + R InvestigationUser - + R Job - + R Keyword - + R PublicStep - + R Publication - + R RelatedDatafile - + R Rule - + R Sample - + R SampleParameter - + R Shift - + R Study - + R StudyInvestigation - + R UserGroup - + RU Sample - + UD SampleType - + CRUD FacilityCycle - + CRUD Grouping - + CRUD InstrumentScientist - + CRUD Investigation - + CRUD InvestigationGroup - + CRUD InvestigationInstrument - + CRUD InvestigationParameter - + CRUD InvestigationUser - + CRUD Keyword - + CRUD Publication - + CRUD Shift - + CRUD Study - + CRUD StudyInvestigation - + CRUD User - + CRUD UserGroup diff --git a/tests/data/ref-icatdump-5.0.yaml b/tests/data/ref-icatdump-5.0.yaml index 1ea8fe6e..25b4b92d 100644 --- a/tests/data/ref-icatdump-5.0.yaml +++ b/tests/data/ref-icatdump-5.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 -# Date: Wed, 21 Dec 2022 15:22:05 +0000 +# Date: Mon, 16 Oct 2023 15:00:30 +0000 # Service: https://icat.example.com:8181/ICATService/ICAT?wsdl -# ICAT-API: 5.0.0 +# ICAT-API: 5.0.1 # Generator: icatdump (python-icat 1.0.0) --- grouping: @@ -421,182 +421,186 @@ rule: grouping: Grouping_name-ingest what: Investigation Rule_00000066: + crudFlags: CRU + grouping: Grouping_name-ingest + what: Sample + Rule_00000067: crudFlags: R grouping: Grouping_name-ingest what: Shift - Rule_00000067: + Rule_00000068: crudFlags: R grouping: Grouping_name-rall what: DataCollection - Rule_00000068: + Rule_00000069: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDatafile - Rule_00000069: + Rule_00000070: crudFlags: R grouping: Grouping_name-rall what: DataCollectionDataset - Rule_00000070: + Rule_00000071: crudFlags: R grouping: Grouping_name-rall what: DataCollectionParameter - Rule_00000071: + Rule_00000072: crudFlags: R grouping: Grouping_name-rall what: Datafile - Rule_00000072: + Rule_00000073: crudFlags: R grouping: Grouping_name-rall what: DatafileParameter - Rule_00000073: + Rule_00000074: crudFlags: R grouping: Grouping_name-rall what: Dataset - Rule_00000074: + Rule_00000075: crudFlags: R grouping: Grouping_name-rall what: DatasetParameter - Rule_00000075: + Rule_00000076: crudFlags: R grouping: Grouping_name-rall what: Grouping - Rule_00000076: + Rule_00000077: crudFlags: R grouping: Grouping_name-rall what: InstrumentScientist - Rule_00000077: + Rule_00000078: crudFlags: R grouping: Grouping_name-rall what: Investigation - Rule_00000078: + Rule_00000079: crudFlags: R grouping: Grouping_name-rall what: InvestigationGroup - Rule_00000079: + Rule_00000080: crudFlags: R grouping: Grouping_name-rall what: InvestigationInstrument - Rule_00000080: + Rule_00000081: crudFlags: R grouping: Grouping_name-rall what: InvestigationParameter - Rule_00000081: + Rule_00000082: crudFlags: R grouping: Grouping_name-rall what: InvestigationUser - Rule_00000082: + Rule_00000083: crudFlags: R grouping: Grouping_name-rall what: Job - Rule_00000083: + Rule_00000084: crudFlags: R grouping: Grouping_name-rall what: Keyword - Rule_00000084: + Rule_00000085: crudFlags: R grouping: Grouping_name-rall what: PublicStep - Rule_00000085: + Rule_00000086: crudFlags: R grouping: Grouping_name-rall what: Publication - Rule_00000086: + Rule_00000087: crudFlags: R grouping: Grouping_name-rall what: RelatedDatafile - Rule_00000087: + Rule_00000088: crudFlags: R grouping: Grouping_name-rall what: Rule - Rule_00000088: + Rule_00000089: crudFlags: R grouping: Grouping_name-rall what: Sample - Rule_00000089: + Rule_00000090: crudFlags: R grouping: Grouping_name-rall what: SampleParameter - Rule_00000090: + Rule_00000091: crudFlags: R grouping: Grouping_name-rall what: Shift - Rule_00000091: + Rule_00000092: crudFlags: R grouping: Grouping_name-rall what: Study - Rule_00000092: + Rule_00000093: crudFlags: R grouping: Grouping_name-rall what: StudyInvestigation - Rule_00000093: + Rule_00000094: crudFlags: R grouping: Grouping_name-rall what: UserGroup - Rule_00000094: + Rule_00000095: crudFlags: RU grouping: Grouping_name-scientific=5Fstaff what: Sample - Rule_00000095: + Rule_00000096: crudFlags: UD grouping: Grouping_name-scientific=5Fstaff what: SampleType - Rule_00000096: + Rule_00000097: crudFlags: CRUD grouping: Grouping_name-useroffice what: FacilityCycle - Rule_00000097: + Rule_00000098: crudFlags: CRUD grouping: Grouping_name-useroffice what: Grouping - Rule_00000098: + Rule_00000099: crudFlags: CRUD grouping: Grouping_name-useroffice what: InstrumentScientist - Rule_00000099: + Rule_00000100: crudFlags: CRUD grouping: Grouping_name-useroffice what: Investigation - Rule_00000100: + Rule_00000101: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationGroup - Rule_00000101: + Rule_00000102: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationInstrument - Rule_00000102: + Rule_00000103: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationParameter - Rule_00000103: + Rule_00000104: crudFlags: CRUD grouping: Grouping_name-useroffice what: InvestigationUser - Rule_00000104: + Rule_00000105: crudFlags: CRUD grouping: Grouping_name-useroffice what: Keyword - Rule_00000105: + Rule_00000106: crudFlags: CRUD grouping: Grouping_name-useroffice what: Publication - Rule_00000106: + Rule_00000107: crudFlags: CRUD grouping: Grouping_name-useroffice what: Shift - Rule_00000107: + Rule_00000108: crudFlags: CRUD grouping: Grouping_name-useroffice what: Study - Rule_00000108: + Rule_00000109: crudFlags: CRUD grouping: Grouping_name-useroffice what: StudyInvestigation - Rule_00000109: + Rule_00000110: crudFlags: CRUD grouping: Grouping_name-useroffice what: User - Rule_00000110: + Rule_00000111: crudFlags: CRUD grouping: Grouping_name-useroffice what: UserGroup diff --git a/tests/data/summary-4 b/tests/data/summary-4 index 7f7108f2..bb82042e 100644 --- a/tests/data/summary-4 +++ b/tests/data/summary-4 @@ -31,7 +31,7 @@ PermissibleStringValue : 6 PublicStep : 24 Publication : 1 RelatedDatafile : 1 -Rule : 110 +Rule : 111 Sample : 3 SampleParameter : 2 SampleType : 3 diff --git a/tests/data/summary-5 b/tests/data/summary-5 index 2c469095..97cc2126 100644 --- a/tests/data/summary-5 +++ b/tests/data/summary-5 @@ -44,7 +44,7 @@ PublicStep : 37 Publication : 1 RelatedDatafile : 1 RelatedItem : 1 -Rule : 158 +Rule : 159 Sample : 3 SampleParameter : 2 SampleType : 3 diff --git a/tests/test_06_query.py b/tests/test_06_query.py index 2679b4de..cc76c200 100644 --- a/tests/test_06_query.py +++ b/tests/test_06_query.py @@ -35,8 +35,8 @@ def client(setupicat): # The the actual number of rules in the test data differs with the # ICAT version. have_icat_5 = 0 if icat_version < "5.0" else 1 -all_rules = 110 + 48*have_icat_5 -grp_rules = 50 + 30*have_icat_5 +all_rules = 111 + 48*have_icat_5 +grp_rules = 51 + 30*have_icat_5 @pytest.mark.dependency(name='get_investigation') def test_query_simple(client): From 684b49b1dcb2de06c2cd8b5fe8a63989cdcf229f Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 16 Oct 2023 17:31:51 +0200 Subject: [PATCH 03/39] Typo in comment --- tests/test_04_client_cleanup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_04_client_cleanup.py b/tests/test_04_client_cleanup.py index 2e7198f9..0cd7f5ae 100644 --- a/tests/test_04_client_cleanup.py +++ b/tests/test_04_client_cleanup.py @@ -78,7 +78,7 @@ def test_client_delete(registerClient): client, conf = getConfig(confSection="acord") client.login(conf.auth, conf.credentials) assert len(SessionRegisterClient.Sessions) == 1 - # Keep a wek reference to the client to be able to observe when it + # Keep a weak reference to the client to be able to observe when it # has been garbage collected. r = weakref.ref(client) assert r() is client @@ -96,7 +96,7 @@ def test_client_garbage_collect(registerClient): client, conf = getConfig(confSection="acord") client.login(conf.auth, conf.credentials) assert len(SessionRegisterClient.Sessions) == 1 - # Keep a wek reference to the client to be able to observe when it + # Keep a weak reference to the client to be able to observe when it # has been garbage collected. r = weakref.ref(client) assert r() is client From d7e472a8f692d3a0a78310dc09b968f8e3af4925 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 09:59:31 +0200 Subject: [PATCH 04/39] Add a fixture to test_06_icatingest.py capable to cleanup an arbitrary list of objects that may have been created during a test --- tests/test_06_icatingest.py | 51 ++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index fe397fa7..9c3e1088 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -27,7 +27,35 @@ def cmdargs(setupicat): return conf.cmdargs + ["-f", "XML"] @pytest.fixture(scope="function") -def dataset(client): +def cleanup_list(client): + """Delete some objects (that may or may not have been) created during + a test + """ + obj_list = [] + yield obj_list + for obj in obj_list: + try: + obj = client.searchMatching(obj) + except icat.SearchResultError: + # obj not found, maybe the test failed, nothing to do on + # this object then. + pass + else: + if obj.BeanName == "Dataset": + # obj is a dataset. If any related datafile has been + # uploaded (i.e. the location is not NULL), need to + # delete it from IDS first. Any other related + # datafile or dataset parameter will be deleted + # automatically with the dataset by cascading in the + # ICAT server. + query = Query(client, "Datafile", + conditions={"dataset.id": "= %d" % obj.id, + "location": "IS NOT NULL"}) + client.deleteData(client.search(query)) + client.delete(obj) + +@pytest.fixture(scope="function") +def dataset(client, cleanup_list): """A dataset to be used in the test. The dataset is not created by the fixture, it is assumed that the @@ -39,25 +67,8 @@ def dataset(client): dataset = client.new("Dataset", name="e208343", complete=False, investigation=inv, type=dstype) - yield dataset - try: - ds = client.searchMatching(dataset) - dataset.id = ds.id - except icat.SearchResultError: - # Dataset not found, maybe the test failed, nothing to - # clean up then. - pass - else: - # If any datafile has been uploaded (i.e. the location is - # not NULL), need to delete it from IDS first. Any other - # datafile or dataset parameter will be deleted - # automatically with the dataset by cascading in the ICAT - # server. - query = Query(client, "Datafile", - conditions={"dataset.id": "= %d" % dataset.id, - "location": "IS NOT NULL"}) - client.deleteData(client.search(query)) - client.delete(dataset) + cleanup_list.append(dataset) + return dataset # Test datafiles to be created by test_ingest_datafiles: From 63792a2e2f1054eb130c6fccecab2c2650bf55a4 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 11:03:21 +0200 Subject: [PATCH 05/39] Add an icatingest test featuring datasets with relations to samples --- doc/examples/ingest-sample-ds.xml | 51 +++++++++++++++++++++++++++++++ setup.py | 3 +- tests/test_06_icatingest.py | 42 +++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 doc/examples/ingest-sample-ds.xml diff --git a/doc/examples/ingest-sample-ds.xml b/doc/examples/ingest-sample-ds.xml new file mode 100644 index 00000000..439adfd5 --- /dev/null +++ b/doc/examples/ingest-sample-ds.xml @@ -0,0 +1,51 @@ + + + + + 2023-10-17T07:33:36Z + manual edit + + + + + + + false + 2012-07-30T01:10:08+00:00 + e209001 + 2012-07-26T15:44:24+00:00 + + + + + + false + 2012-08-06T01:10:08+00:00 + e209002 + 2012-08-02T05:30:00+00:00 + + + + + + false + 2012-07-16T14:30:17+00:00 + e209003 + 2012-07-16T11:42:05+00:00 + + + + + + false + 2012-07-31T22:52:23+00:00 + e209004 + 2012-07-31T20:20:37+00:00 + + + + + diff --git a/setup.py b/setup.py index e6ef9d4e..66fcf4f5 100755 --- a/setup.py +++ b/setup.py @@ -124,7 +124,8 @@ def copy_test_data(self): files = [] files += [ os.path.join("doc", "examples", f) for f in ["example_data.yaml", - "ingest-datafiles.xml", "ingest-ds-params.xml"] ] + "ingest-datafiles.xml", "ingest-ds-params.xml", + "ingest-sample-ds.xml"] ] files += [ os.path.join("doc", "examples", "icatdump-%s.%s" % (ver, ext)) for ver in ("4.4", "4.7", "4.10", "5.0") diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index 9c3e1088..0fff22ce 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -13,6 +13,7 @@ # Test input ds_params = str(gettestdata("ingest-ds-params.xml")) datafiles = str(gettestdata("ingest-datafiles.xml")) +sample_ds = str(gettestdata("ingest-sample-ds.xml")) @pytest.fixture(scope="module") def client(setupicat): @@ -357,3 +358,44 @@ def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs): assert df.checksum == f.crc32 if f.mtime: assert df.datafileModTime == f.mtime + + +def test_ingest_dataset_samples(client, cleanup_list, cmdargs): + """Ingest some datasets that are releated to samples. + """ + ds_sample_map = { + "e209001": "ab3465", + "e209002": "ab3465", + "e209003": "ab3466", + "e209004": None, + } + inv = client.assertedSearch("Investigation [name='10100601-ST']")[0] + query = Query(client, "SampleType", conditions={"name": "= 'NiMnGa'"}) + sample_type = client.assertedSearch(query)[0] + ds_type = client.assertedSearch("DatasetType [name='raw']")[0] + for name in ds_sample_map.keys(): + # the datasets are supposed to be created by the imgest file + dataset = client.new("Dataset", + name=name, complete=False, + investigation=inv, type=ds_type) + cleanup_list.append(dataset) + for name in {v for v in ds_sample_map.values() if v}: + # the samples are referenced from the ingest file, but are are + # assumed to already exist, so we need to create them here. + sample = client.new("Sample", + name=name, investigation=inv, type=sample_type) + sample.create() + cleanup_list.append(sample) + args = cmdargs + ["-i", sample_ds] + callscript("icatingest.py", args) + for ds_name, sample_name in ds_sample_map.items(): + query = Query(client, "Dataset", conditions={ + "investigation.id": "= %d" % inv.id, + "name": "= '%s'" % ds_name, + }, includes=["sample"]) + ds = client.assertedSearch(query)[0] + if sample_name: + assert ds.sample + assert ds.sample.name == sample_name + else: + assert not ds.sample From 6269ab97ddf3bf50a35f7e228db1da097f17cee9 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 11:23:41 +0200 Subject: [PATCH 06/39] Change the test added in 63792a2 to use the file syntax to be implemented in #122. Obviously this is an xfail for now. --- doc/examples/ingest-sample-ds.xml | 6 ++---- tests/test_06_icatingest.py | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/examples/ingest-sample-ds.xml b/doc/examples/ingest-sample-ds.xml index 439adfd5..f02d31c5 100644 --- a/doc/examples/ingest-sample-ds.xml +++ b/doc/examples/ingest-sample-ds.xml @@ -8,10 +8,8 @@ - - + + false 2012-07-30T01:10:08+00:00 diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index 0fff22ce..824014e6 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -360,6 +360,7 @@ def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs): assert df.datafileModTime == f.mtime +@pytest.mark.xfail(raises=CalledProcessError, reason="Issue #122") def test_ingest_dataset_samples(client, cleanup_list, cmdargs): """Ingest some datasets that are releated to samples. """ From 02e89b18ed338ccfcafc6159cf441bad217ee120 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 12:06:52 +0200 Subject: [PATCH 07/39] Implemented referencing related objects by reference key in object references in XML ICAT data file format, ref #122 --- icat/dumpfile_xml.py | 10 +++++++++- tests/test_06_icatingest.py | 1 - 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/icat/dumpfile_xml.py b/icat/dumpfile_xml.py index 1e4c1052..db676b13 100644 --- a/icat/dumpfile_xml.py +++ b/icat/dumpfile_xml.py @@ -62,7 +62,15 @@ def _searchByReference(self, element, objtype, objindex): else: # object is referenced by attributes. attrs = set(element.keys()) - {'id'} - conditions = { a: "= '%s'" % element.get(a) for a in attrs } + conditions = dict() + for attr in attrs: + if attr.endswith(".ref"): + ref = element.get(attr) + robj = self.client.searchUniqueKey(ref, objindex) + attr = "%s.id" % attr[:-4] + conditions[attr] = "= %d" % robj.id + else: + conditions[attr] = "= '%s'" % element.get(attr) query = Query(self.client, objtype, conditions=conditions) return self.client.assertedSearch(query)[0] diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index 824014e6..0fff22ce 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -360,7 +360,6 @@ def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs): assert df.datafileModTime == f.mtime -@pytest.mark.xfail(raises=CalledProcessError, reason="Issue #122") def test_ingest_dataset_samples(client, cleanup_list, cmdargs): """Ingest some datasets that are releated to samples. """ From eb0d7f2956b3a1b9e1bae6ea9d56c7309aaa44c2 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 14:15:32 +0200 Subject: [PATCH 08/39] Update the icatdata XML Schema Definitions to include the new option to reference related objects --- doc/icatdata-4.10.xsd | 28 +++++++++++++++++++++------- doc/icatdata-4.3.xsd | 28 +++++++++++++++++++++------- doc/icatdata-4.4.xsd | 28 +++++++++++++++++++++------- doc/icatdata-4.7.xsd | 28 +++++++++++++++++++++------- doc/icatdata-5.0.xsd | 33 ++++++++++++++++++++++++++------- 5 files changed, 110 insertions(+), 35 deletions(-) diff --git a/doc/icatdata-4.10.xsd b/doc/icatdata-4.10.xsd index 5716464c..5c71d4ea 100644 --- a/doc/icatdata-4.10.xsd +++ b/doc/icatdata-4.10.xsd @@ -169,13 +169,16 @@ reference the target object in other reference objects. The target object may either be referenced by reference key or by - object attributes. In the former case, the key is given in the - ref attribute and other attributes SHOULD NOT be present. The - reference key may either have been defined earlier in the id - attribute of an other entity object or reference in the same data - element or it must the unique key as returned by the - getUniqueKey() method of the target object. In the latter case, - the attribute values must uniquely define the target object. + object attributes. The former case is implemeted by adding a + special attribute "ref" to hold the key. In this case, other + attributes SHOULD NOT be present. The reference key may either + have been defined earlier in the id attribute of another entity + object or reference in the same data element or it must the + unique key as returned by the getUniqueKey() method of the target + object. When referencing the target object by attributes, the + attribute values must uniquely define the target object. In this + case, related objects may in turn be referenced either by + reference or by attributes accordingly. Note that in the case of referencing the target object by attribute values, this schema is somewhat too restricted: in @@ -200,6 +203,7 @@ + @@ -218,6 +222,7 @@ + @@ -232,6 +237,7 @@ + @@ -242,6 +248,7 @@ + @@ -254,6 +261,7 @@ + @@ -279,6 +287,7 @@ + @@ -289,6 +298,7 @@ + @@ -300,6 +310,7 @@ + @@ -309,6 +320,7 @@ + @@ -320,6 +332,7 @@ + @@ -332,6 +345,7 @@ + diff --git a/doc/icatdata-4.3.xsd b/doc/icatdata-4.3.xsd index 89aea0c6..072a6bda 100644 --- a/doc/icatdata-4.3.xsd +++ b/doc/icatdata-4.3.xsd @@ -165,13 +165,16 @@ reference the target object in other reference objects. The target object may either be referenced by reference key or by - object attributes. In the former case, the key is given in the - ref attribute and other attributes SHOULD NOT be present. The - reference key may either have been defined earlier in the id - attribute of an other entity object or reference in the same data - element or it must the unique key as returned by the - getUniqueKey() method of the target object. In the latter case, - the attribute values must uniquely define the target object. + object attributes. The former case is implemeted by adding a + special attribute "ref" to hold the key. In this case, other + attributes SHOULD NOT be present. The reference key may either + have been defined earlier in the id attribute of another entity + object or reference in the same data element or it must the + unique key as returned by the getUniqueKey() method of the target + object. When referencing the target object by attributes, the + attribute values must uniquely define the target object. In this + case, related objects may in turn be referenced either by + reference or by attributes accordingly. Note that in the case of referencing the target object by attribute values, this schema is somewhat too restricted: in @@ -196,6 +199,7 @@ + @@ -206,6 +210,7 @@ + @@ -220,6 +225,7 @@ + @@ -230,6 +236,7 @@ + @@ -242,6 +249,7 @@ + @@ -267,6 +275,7 @@ + @@ -276,6 +285,7 @@ + @@ -287,6 +297,7 @@ + @@ -296,6 +307,7 @@ + @@ -306,6 +318,7 @@ + @@ -317,6 +330,7 @@ + diff --git a/doc/icatdata-4.4.xsd b/doc/icatdata-4.4.xsd index a769c44e..22952b48 100644 --- a/doc/icatdata-4.4.xsd +++ b/doc/icatdata-4.4.xsd @@ -165,13 +165,16 @@ reference the target object in other reference objects. The target object may either be referenced by reference key or by - object attributes. In the former case, the key is given in the - ref attribute and other attributes SHOULD NOT be present. The - reference key may either have been defined earlier in the id - attribute of an other entity object or reference in the same data - element or it must the unique key as returned by the - getUniqueKey() method of the target object. In the latter case, - the attribute values must uniquely define the target object. + object attributes. The former case is implemeted by adding a + special attribute "ref" to hold the key. In this case, other + attributes SHOULD NOT be present. The reference key may either + have been defined earlier in the id attribute of another entity + object or reference in the same data element or it must the + unique key as returned by the getUniqueKey() method of the target + object. When referencing the target object by attributes, the + attribute values must uniquely define the target object. In this + case, related objects may in turn be referenced either by + reference or by attributes accordingly. Note that in the case of referencing the target object by attribute values, this schema is somewhat too restricted: in @@ -196,6 +199,7 @@ + @@ -206,6 +210,7 @@ + @@ -220,6 +225,7 @@ + @@ -230,6 +236,7 @@ + @@ -242,6 +249,7 @@ + @@ -267,6 +275,7 @@ + @@ -276,6 +285,7 @@ + @@ -287,6 +297,7 @@ + @@ -296,6 +307,7 @@ + @@ -306,6 +318,7 @@ + @@ -317,6 +330,7 @@ + diff --git a/doc/icatdata-4.7.xsd b/doc/icatdata-4.7.xsd index ac2abe3c..50e850ba 100644 --- a/doc/icatdata-4.7.xsd +++ b/doc/icatdata-4.7.xsd @@ -167,13 +167,16 @@ reference the target object in other reference objects. The target object may either be referenced by reference key or by - object attributes. In the former case, the key is given in the - ref attribute and other attributes SHOULD NOT be present. The - reference key may either have been defined earlier in the id - attribute of an other entity object or reference in the same data - element or it must the unique key as returned by the - getUniqueKey() method of the target object. In the latter case, - the attribute values must uniquely define the target object. + object attributes. The former case is implemeted by adding a + special attribute "ref" to hold the key. In this case, other + attributes SHOULD NOT be present. The reference key may either + have been defined earlier in the id attribute of another entity + object or reference in the same data element or it must the + unique key as returned by the getUniqueKey() method of the target + object. When referencing the target object by attributes, the + attribute values must uniquely define the target object. In this + case, related objects may in turn be referenced either by + reference or by attributes accordingly. Note that in the case of referencing the target object by attribute values, this schema is somewhat too restricted: in @@ -198,6 +201,7 @@ + @@ -216,6 +220,7 @@ + @@ -230,6 +235,7 @@ + @@ -240,6 +246,7 @@ + @@ -252,6 +259,7 @@ + @@ -277,6 +285,7 @@ + @@ -286,6 +295,7 @@ + @@ -297,6 +307,7 @@ + @@ -306,6 +317,7 @@ + @@ -316,6 +328,7 @@ + @@ -327,6 +340,7 @@ + diff --git a/doc/icatdata-5.0.xsd b/doc/icatdata-5.0.xsd index 4215a5c0..03ad331b 100644 --- a/doc/icatdata-5.0.xsd +++ b/doc/icatdata-5.0.xsd @@ -214,13 +214,16 @@ reference the target object in other reference objects. The target object may either be referenced by reference key or by - object attributes. In the former case, the key is given in the - ref attribute and other attributes SHOULD NOT be present. The - reference key may either have been defined earlier in the id - attribute of an other entity object or reference in the same data - element or it must the unique key as returned by the - getUniqueKey() method of the target object. In the latter case, - the attribute values must uniquely define the target object. + object attributes. The former case is implemeted by adding a + special attribute "ref" to hold the key. In this case, other + attributes SHOULD NOT be present. The reference key may either + have been defined earlier in the id attribute of another entity + object or reference in the same data element or it must the + unique key as returned by the getUniqueKey() method of the target + object. When referencing the target object by attributes, the + attribute values must uniquely define the target object. In this + case, related objects may in turn be referenced either by + reference or by attributes accordingly. Note that in the case of referencing the target object by attribute values, this schema is somewhat too restricted: in @@ -244,6 +247,7 @@ + @@ -262,6 +266,7 @@ + @@ -271,6 +276,7 @@ + @@ -280,8 +286,10 @@ + + @@ -291,6 +299,7 @@ + @@ -305,6 +314,7 @@ + @@ -315,6 +325,7 @@ + @@ -327,6 +338,7 @@ + @@ -344,6 +356,7 @@ + @@ -371,6 +384,7 @@ + @@ -381,6 +395,7 @@ + @@ -392,6 +407,7 @@ + @@ -401,6 +417,7 @@ + @@ -412,6 +429,7 @@ + @@ -424,6 +442,7 @@ + From e7cd4f614e9acb3b3f0f729a4f6a5e881d6cf748 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 14:31:52 +0200 Subject: [PATCH 09/39] Update changelog --- CHANGES.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5be9c3b9..53495001 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,19 @@ Changelog ========= +1.2.0 (not yet released) +~~~~~~~~~~~~~~~~~~~~~~~~ + +New features +------------ + ++ `#122`_, `#133`_: Allow referencing related objects by reference key + in object references in XML ICAT data file format. + +.. _#122: https://github.com/icatproject/python-icat/issues/122 +.. _#133: https://github.com/icatproject/python-icat/pull/133 + + 1.1.0 (2023-06-30) ~~~~~~~~~~~~~~~~~~ From 58392e973236373934ff50c128d802467bd99fad Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 17 Oct 2023 14:47:57 +0200 Subject: [PATCH 10/39] Set meaningful pytest parameter ids for test_ingest_duplicate_check_types --- tests/test_06_icatingest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index 0fff22ce..a7a88411 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -289,11 +289,11 @@ def test_ingest_duplicate_overwrite(client, dataset, cmdargs): """ @pytest.mark.parametrize("inputdata", [ - ingest_data_string, - ingest_data_int, - ingest_data_boolean, - ingest_data_float, - ingest_data_date, + pytest.param(ingest_data_string, id="ingest_data_string"), + pytest.param(ingest_data_int, id="ingest_data_int"), + pytest.param(ingest_data_boolean, id="ingest_data_boolean"), + pytest.param(ingest_data_float, id="ingest_data_float"), + pytest.param(ingest_data_date, id="ingest_data_date"), ]) def test_ingest_duplicate_check_types(tmpdirsec, dataset, cmdargs, inputdata): """Ingest with a collision of a duplicate object. From f13ce2159c4201c5d8d572d28541e1479e130861 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 08:07:20 +0200 Subject: [PATCH 11/39] Force downgrade jinja2 < 3.1 in rtd build --- .rtd-require | 1 + 1 file changed, 1 insertion(+) diff --git a/.rtd-require b/.rtd-require index c3329cc4..2de815cd 100644 --- a/.rtd-require +++ b/.rtd-require @@ -4,5 +4,6 @@ packaging setuptools setuptools_scm suds +jinja2<3.1 sphinx>=2,<3 sphinx-rtd-theme>=0.5,<1 From dc8cd2044004fcce124ed897b3a23d354818e9af Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 09:35:49 +0200 Subject: [PATCH 12/39] Add a test for Issue #131 --- tests/test_03_getversion.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_03_getversion.py b/tests/test_03_getversion.py index 4c332d7e..b96ae5d2 100644 --- a/tests/test_03_getversion.py +++ b/tests/test_03_getversion.py @@ -4,6 +4,7 @@ import pytest import icat import icat.config +import conftest from conftest import getConfig @@ -22,6 +23,18 @@ def test_get_icat_version(): print("\nConnect to %s\nICAT version %s\n" % (conf.url, client.apiversion)) +@pytest.mark.xfail(conftest.ids_version > '1.99', reason="Issue #131") +def test_ids_version_calls(): + """Test that client.ids.getApiVersion() and client.ids.version() yield + coherent results. Ref. #131. + """ + + client, conf = getConfig(needlogin=False, ids="mandatory") + ids_apiversion = client.ids.getApiVersion() + ids_verion = client.ids.version() + assert ids_apiversion == ids_verion["version"] + + def test_get_ids_version(): """Query the version from the test IDS server. From 733a1781bac984c693fe1f0f732f2d698a63c808 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 10:14:14 +0200 Subject: [PATCH 13/39] Review client.ids.getApiVersion() and client.ids.version() and make sure they will yield correct results also for ids.server 2.0 and newer --- icat/ids.py | 39 +++++++++++++++++++++---------------- tests/test_03_getversion.py | 1 - 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/icat/ids.py b/icat/ids.py index 1d715a67..8df49d4a 100644 --- a/icat/ids.py +++ b/icat/ids.py @@ -218,17 +218,19 @@ def ping(self): if result != "IdsOK": raise IDSResponseError("unexpected response to ping: %s" % result) - def getApiVersion(self): - """Get the version of the IDS server. + def _get_legacy_version(self): + """Try to figure out the ids.server version in legacy cases. + + .. note:: + This method will yield a **wrong** result for ids.server + 2.0 and newer. It should only be called when the ^version^ + call is not available. - Note: the `getApiVersion` call has been added in IDS server - version 1.3.0. For older servers, try to guess the server - version from features visible in the API. Obviously this - cannot always be accurate as we cannot distinguish server - version with no visible API changes. In particular, versions - older then 1.2.0 will always reported as 1.0.0. Nevertheless, - the result of the guess should be fair enough for most use - cases. + + The `getApiVersion` call has been added in ids.server version + 1.3.0. In version 1.8.0 the newer `version` call has been + added, deprecating `getApiVersion` at the same time. In + version 2.0.0 `getApiVersion` has definitely been removed. """ try: req = IDSRequest(self.url + "getApiVersion") @@ -252,14 +254,17 @@ def getApiVersion(self): # No way to distinguish 1.1.0, 1.0.1, and 1.0.0, report as 1.0.0. return "1.0.0" - def version(self): + def getApiVersion(self): """Get the version of the IDS server. - Note: the `version` call has been added in IDS server version - 1.8.0, deprecating `getApiVersion` at the same time. For - older servers, we fall back to `getApiVersion` to emulate this - call. Note furthermore that `version` returns a dict, while - `getApiVersion` returns the plain version number as a string. + The `getApiVersion` call used to be present ids.version from + version 1.3.0 to 1.12.*. Emulate it using the newer + `version` call. + """ + return self.version()["version"] + + def version(self): + """Get the version of the IDS server. """ try: req = IDSRequest(self.url + "version") @@ -267,7 +272,7 @@ def version(self): return json.loads(result) except (HTTPError, IDSError) as err: try: - return {"version": self.getApiVersion()} + return {"version": self._get_legacy_version()} except: raise err diff --git a/tests/test_03_getversion.py b/tests/test_03_getversion.py index b96ae5d2..112253e2 100644 --- a/tests/test_03_getversion.py +++ b/tests/test_03_getversion.py @@ -23,7 +23,6 @@ def test_get_icat_version(): print("\nConnect to %s\nICAT version %s\n" % (conf.url, client.apiversion)) -@pytest.mark.xfail(conftest.ids_version > '1.99', reason="Issue #131") def test_ids_version_calls(): """Test that client.ids.getApiVersion() and client.ids.version() yield coherent results. Ref. #131. From cb01336c5815e97b3ca84ca6580ebae439de7e48 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 10:37:26 +0200 Subject: [PATCH 14/39] Update changelog --- CHANGES.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 53495001..e28f3faa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,8 +11,16 @@ New features + `#122`_, `#133`_: Allow referencing related objects by reference key in object references in XML ICAT data file format. +Bug fixes and minor changes +--------------------------- + ++ `#131`_, `#135`_: Fix :meth:`icat.ids.IDSClient.getApiVersion` to + yield correct results for ids.server 2.0.0 and newer. + .. _#122: https://github.com/icatproject/python-icat/issues/122 +.. _#131: https://github.com/icatproject/python-icat/issues/131 .. _#133: https://github.com/icatproject/python-icat/pull/133 +.. _#135: https://github.com/icatproject/python-icat/pull/135 1.1.0 (2023-06-30) From ddc96b9d0736761bd5c239198a7c15b8e39a869c Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 10:58:09 +0200 Subject: [PATCH 15/39] Set all object attributes of class Client to some value before doing anything that may fail --- icat/client.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/icat/client.py b/icat/client.py index b7c3415f..144b24a2 100644 --- a/icat/client.py +++ b/icat/client.py @@ -134,9 +134,16 @@ def __init__(self, url, idsurl=None, self.kwargs['caPath'] = caPath self.kwargs['sslContext'] = sslContext self.kwargs['proxy'] = proxy - idsurl = _complete_url(idsurl, default_path="/ids") + self.apiversion = None + self.entityInfoCache = {} + self.typemap = None + self.ids = None + self.sessionId = None + self.autoLogout = True + self._schedule_auto_refresh("never") + if sslContext: self.sslContext = sslContext else: @@ -151,15 +158,10 @@ def __init__(self, url, idsurl=None, if self.apiversion < '4.3.0': warn(ClientVersionWarning(self.apiversion, "too old")) - self.entityInfoCache = {} self.typemap = getTypeMap(self) - self.ids = None - self.sessionId = None - self.autoLogout = True if idsurl: self.add_ids(idsurl) - self._schedule_auto_refresh("never") self.Register[id(self)] = self def __del__(self): From ec2c5059bc31bf5c9e8d8fd044dc0183c90ad86b Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Wed, 18 Oct 2023 11:19:31 +0200 Subject: [PATCH 16/39] Update changelog --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e28f3faa..d26910a0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,10 +17,15 @@ Bug fixes and minor changes + `#131`_, `#135`_: Fix :meth:`icat.ids.IDSClient.getApiVersion` to yield correct results for ids.server 2.0.0 and newer. ++ `#132`_, `#136`_: Fix a spurious :exc:`AttributeError` on cleanup + after connecting to an invalid url. + .. _#122: https://github.com/icatproject/python-icat/issues/122 .. _#131: https://github.com/icatproject/python-icat/issues/131 +.. _#132: https://github.com/icatproject/python-icat/issues/132 .. _#133: https://github.com/icatproject/python-icat/pull/133 .. _#135: https://github.com/icatproject/python-icat/pull/135 +.. _#136: https://github.com/icatproject/python-icat/pull/136 1.1.0 (2023-06-30) From eaa25e7048ad98c8688f10d9891a6f3766d7416d Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Thu, 19 Oct 2023 09:58:08 +0200 Subject: [PATCH 17/39] Set more meaningful pytest parameter ids --- tests/test_05_dump.py | 57 +++++++++------ tests/test_05_dumpfile.py | 57 +++++++++------ tests/test_06_client.py | 125 +++++++++++++++++++++------------ tests/test_07_dataselection.py | 24 +++++-- 4 files changed, 172 insertions(+), 91 deletions(-) diff --git a/tests/test_05_dump.py b/tests/test_05_dump.py index fd196694..cea26410 100644 --- a/tests/test_05_dump.py +++ b/tests/test_05_dump.py @@ -66,27 +66,42 @@ def depends(request, other_tests): # to verify that object relations are kept intact after an icatdump / # icatingest cycle. queries = [ - ("Datafile.name <-> Dataset <-> Investigation [name='10100601-ST']", - ['e208339.dat', 'e208339.nxs', 'e208341.dat', 'e208341.nxs']), - ("SELECT p.numericValue FROM DatasetParameter p " - "JOIN p.dataset AS ds JOIN ds.investigation AS i JOIN p.type AS t " - "WHERE i.name = '10100601-ST' AND ds.name = 'e208339' " - "AND t.name = 'Magnetic field'", - [7.3]), - ("SELECT ds.name FROM Dataset ds " - "JOIN ds.dataCollectionDatasets AS dcds " - "JOIN dcds.dataCollection AS dc JOIN dc.jobsAsOutput AS j " - "WHERE j.id IS NOT NULL", - ["e208947"]), - ("SELECT df.name FROM Datafile df " - "JOIN df.dataCollectionDatafiles AS dcdf " - "JOIN dcdf.dataCollection AS dc JOIN dc.jobsAsInput AS j " - "WHERE j.id IS NOT NULL", - ["e208945.nxs"]), - ("SELECT COUNT(dc) FROM DataCollection dc " - "JOIN dc.dataCollectionDatasets AS dcds JOIN dcds.dataset AS ds " - "WHERE ds.name = 'e201215'", - [1]), + pytest.param( + "Datafile.name <-> Dataset <-> Investigation [name='10100601-ST']", + ['e208339.dat', 'e208339.nxs', 'e208341.dat', 'e208341.nxs'], + id="df.name" + ), + pytest.param( + "SELECT p.numericValue FROM DatasetParameter p " + "JOIN p.dataset AS ds JOIN ds.investigation AS i JOIN p.type AS t " + "WHERE i.name = '10100601-ST' AND ds.name = 'e208339' " + "AND t.name = 'Magnetic field'", + [7.3], + id="param.name" + ), + pytest.param( + "SELECT ds.name FROM Dataset ds " + "JOIN ds.dataCollectionDatasets AS dcds " + "JOIN dcds.dataCollection AS dc JOIN dc.jobsAsOutput AS j " + "WHERE j.id IS NOT NULL", + ["e208947"], + id="jobout_ds.name" + ), + pytest.param( + "SELECT df.name FROM Datafile df " + "JOIN df.dataCollectionDatafiles AS dcdf " + "JOIN dcdf.dataCollection AS dc JOIN dc.jobsAsInput AS j " + "WHERE j.id IS NOT NULL", + ["e208945.nxs"], + id="jobin_df.name" + ), + pytest.param( + "SELECT COUNT(dc) FROM DataCollection dc " + "JOIN dc.dataCollectionDatasets AS dcds JOIN dcds.dataset AS ds " + "WHERE ds.name = 'e201215'", + [1], + id="dc.count" + ), ] @pytest.fixture(scope="module") diff --git a/tests/test_05_dumpfile.py b/tests/test_05_dumpfile.py index 0c736411..539201df 100644 --- a/tests/test_05_dumpfile.py +++ b/tests/test_05_dumpfile.py @@ -55,27 +55,42 @@ def depends(request, other_tests): # to verify that object relations are kept intact after an icatdump / # icatingest cycle. queries = [ - ("Datafile.name <-> Dataset <-> Investigation [name='10100601-ST']", - ['e208339.dat', 'e208339.nxs', 'e208341.dat', 'e208341.nxs']), - ("SELECT p.numericValue FROM DatasetParameter p " - "JOIN p.dataset AS ds JOIN ds.investigation AS i JOIN p.type AS t " - "WHERE i.name = '10100601-ST' AND ds.name = 'e208339' " - "AND t.name = 'Magnetic field'", - [7.3]), - ("SELECT ds.name FROM Dataset ds " - "JOIN ds.dataCollectionDatasets AS dcds " - "JOIN dcds.dataCollection AS dc JOIN dc.jobsAsOutput AS j " - "WHERE j.id IS NOT NULL", - ["e208947"]), - ("SELECT df.name FROM Datafile df " - "JOIN df.dataCollectionDatafiles AS dcdf " - "JOIN dcdf.dataCollection AS dc JOIN dc.jobsAsInput AS j " - "WHERE j.id IS NOT NULL", - ["e208945.nxs"]), - ("SELECT COUNT(dc) FROM DataCollection dc " - "JOIN dc.dataCollectionDatasets AS dcds JOIN dcds.dataset AS ds " - "WHERE ds.name = 'e201215'", - [1]), + pytest.param( + "Datafile.name <-> Dataset <-> Investigation [name='10100601-ST']", + ['e208339.dat', 'e208339.nxs', 'e208341.dat', 'e208341.nxs'], + id="df.name" + ), + pytest.param( + "SELECT p.numericValue FROM DatasetParameter p " + "JOIN p.dataset AS ds JOIN ds.investigation AS i JOIN p.type AS t " + "WHERE i.name = '10100601-ST' AND ds.name = 'e208339' " + "AND t.name = 'Magnetic field'", + [7.3], + id="param.name" + ), + pytest.param( + "SELECT ds.name FROM Dataset ds " + "JOIN ds.dataCollectionDatasets AS dcds " + "JOIN dcds.dataCollection AS dc JOIN dc.jobsAsOutput AS j " + "WHERE j.id IS NOT NULL", + ["e208947"], + id="jobout_ds.name" + ), + pytest.param( + "SELECT df.name FROM Datafile df " + "JOIN df.dataCollectionDatafiles AS dcdf " + "JOIN dcdf.dataCollection AS dc JOIN dc.jobsAsInput AS j " + "WHERE j.id IS NOT NULL", + ["e208945.nxs"], + id="jobin_df.name" + ), + pytest.param( + "SELECT COUNT(dc) FROM DataCollection dc " + "JOIN dc.dataCollectionDatasets AS dcds JOIN dcds.dataset AS ds " + "WHERE ds.name = 'e201215'", + [1], + id="dc.count" + ), ] # ======== function equivalents to icatdump and icatingest =========== diff --git a/tests/test_06_client.py b/tests/test_06_client.py index aa4d45c5..b36cefb1 100644 --- a/tests/test_06_client.py +++ b/tests/test_06_client.py @@ -76,20 +76,26 @@ def test_new_obj_none(client): cest = datetime.timezone(datetime.timedelta(hours=2)) @pytest.mark.parametrize(("query", "result"), [ - ("SELECT o.name, o.title, o.startDate FROM Investigation o", - [("08100122-EF", "Durol single crystal", - datetime.datetime(2008, 3, 13, 11, 39, 42, tzinfo=cet)), - ("10100601-ST", "Ni-Mn-Ga flat cone", - datetime.datetime(2010, 9, 30, 12, 27, 24, tzinfo=cest)), - ("12100409-ST", "NiO SC OF1 JUH HHL", - datetime.datetime(2012, 7, 26, 17, 44, 24, tzinfo=cest))]), - ("SELECT i.name, ds.name FROM Dataset ds JOIN ds.investigation AS i " - "WHERE i.startDate < '2011-01-01'", - [("08100122-EF", "e201215"), - ("08100122-EF", "e201216"), - ("10100601-ST", "e208339"), - ("10100601-ST", "e208341"), - ("10100601-ST", "e208342")]), + pytest.param( + "SELECT o.name, o.title, o.startDate FROM Investigation o", + [("08100122-EF", "Durol single crystal", + datetime.datetime(2008, 3, 13, 11, 39, 42, tzinfo=cet)), + ("10100601-ST", "Ni-Mn-Ga flat cone", + datetime.datetime(2010, 9, 30, 12, 27, 24, tzinfo=cest)), + ("12100409-ST", "NiO SC OF1 JUH HHL", + datetime.datetime(2012, 7, 26, 17, 44, 24, tzinfo=cest))], + id="inv_attrs" + ), + pytest.param( + "SELECT i.name, ds.name FROM Dataset ds JOIN ds.investigation AS i " + "WHERE i.startDate < '2011-01-01'", + [("08100122-EF", "e201215"), + ("08100122-EF", "e201216"), + ("10100601-ST", "e208339"), + ("10100601-ST", "e208341"), + ("10100601-ST", "e208342")], + id="inv_ds_name" + ), ]) def test_search_mulitple_fields(client, query, result): """Search for mutliple fields. @@ -179,21 +185,31 @@ def test_assertedSearch_unique_mulitple_fields(client): # query strings using JPQL style syntax, Query objects. For each # type, try both, simple and complex queries. @pytest.mark.parametrize(("query",), [ - ("User",), - ("User <-> UserGroup <-> Grouping <-> " - "InvestigationGroup [role='writer'] <-> " - "Investigation [name='08100122-EF']",), - ("SELECT u FROM User u",), - ("SELECT u FROM User u " - "JOIN u.userGroups AS ug JOIN ug.grouping AS g " - "JOIN g.investigationGroups AS ig JOIN ig.investigation AS i " - "WHERE i.name ='08100122-EF' AND ig.role = 'writer' " - "ORDER BY u.name",), - (lambda client: Query(client, "User"),), - (lambda client: Query(client, "User", order=True, conditions={ - "userGroups.grouping.investigationGroups.role": "= 'writer'", - "userGroups.grouping.investigationGroups.investigation.name": "= '08100122-EF'" - }),), + pytest.param("User", id="legacy_user"), + pytest.param( + "User <-> UserGroup <-> Grouping <-> " + "InvestigationGroup [role='writer'] <-> " + "Investigation [name='08100122-EF']", + id="legacy_writer" + ), + pytest.param("SELECT u FROM User u", id="jpql_user"), + pytest.param( + "SELECT u FROM User u " + "JOIN u.userGroups AS ug JOIN ug.grouping AS g " + "JOIN g.investigationGroups AS ig JOIN ig.investigation AS i " + "WHERE i.name ='08100122-EF' AND ig.role = 'writer' " + "ORDER BY u.name", + id="jpql_writer" + ), + pytest.param(lambda client: Query(client, "User"), id="query_user"), + pytest.param( + lambda client: Query(client, "User", order=True, conditions={ + "userGroups.grouping.investigationGroups.role": "= 'writer'", + "userGroups.grouping.investigationGroups.investigation.name": + "= '08100122-EF'" + }), + id="query_writer" + ), ]) def test_searchChunked_simple(client, query): """A simple search with searchChunked(). @@ -250,11 +266,17 @@ def test_searchChunked_limit(client, skip, count, chunksize): assert objs == users[skip:skip+count] @pytest.mark.parametrize(("query",), [ - ("User [name LIKE 'j%']",), - ("SELECT u FROM User u WHERE u.name LIKE 'j%' ORDER BY u.name",), - (lambda client: Query(client, "User", order=True, conditions={ - "name": "LIKE 'j%'", - }),), + pytest.param("User [name LIKE 'j%']", id="legacy"), + pytest.param( + "SELECT u FROM User u WHERE u.name LIKE 'j%' ORDER BY u.name", + id="jpql" + ), + pytest.param( + lambda client: Query(client, "User", order=True, conditions={ + "name": "LIKE 'j%'", + }), + id="query" + ), ]) def test_searchChunked_percent(client, query): """Search with searchChunked() with a percent character in the query. @@ -269,8 +291,14 @@ def test_searchChunked_percent(client, query): assert objs == users @pytest.mark.parametrize(("query",), [ - ("SELECT o FROM Investigation o WHERE o.id = %d",), - ("SELECT o FROM Investigation o WHERE o.id in (%d)",), + pytest.param( + "SELECT o FROM Investigation o WHERE o.id = %d", + id="id_equal" + ), + pytest.param( + "SELECT o FROM Investigation o WHERE o.id in (%d)", + id="id_in" + ), ]) def test_searchChunked_id(client, query): """Search by id with searchChunked(). @@ -350,9 +378,13 @@ def test_searchChunked_mulitple_fields(client): # ==================== test searchUniqueKey() ====================== @pytest.mark.parametrize(("key", "attrs"), [ - ("Facility_name-ESNF", {"name": "ESNF"}), - ("Investigation_facility-(name-ESNF)_name-12100409=2DST_visitId-1=2E1=2DP", - {"name": "12100409-ST", "visitId": "1.1-P"}), + pytest.param("Facility_name-ESNF", {"name": "ESNF"}, id="facility"), + pytest.param( + "Investigation_facility-(name-ESNF)" + "_name-12100409=2DST_visitId-1=2E1=2DP", + {"name": "12100409-ST", "visitId": "1.1-P"}, + id="investigation" + ), ]) def test_searchUniqueKey_simple(client, key, attrs): """Search a few objects by their unique key. @@ -364,10 +396,17 @@ def test_searchUniqueKey_simple(client, key, attrs): assert getattr(obj, k) == attrs[k] @pytest.mark.parametrize(("key", "relobjs"), [ - ("Dataset_investigation-(facility-(name-ESNF)_name-12100409=2DST_visitId-1=2E1=2DP)_name-e208945", [ - ("investigation", "Investigation_facility-(name-ESNF)_name-12100409=2DST_visitId-1=2E1=2DP"), - ("investigation.facility", "Facility_name-ESNF"), - ]), + pytest.param( + "Dataset_investigation-(facility-(name-ESNF)" + "_name-12100409=2DST_visitId-1=2E1=2DP)_name-e208945", + [ + ("investigation", + "Investigation_facility-(name-ESNF)" + "_name-12100409=2DST_visitId-1=2E1=2DP"), + ("investigation.facility", "Facility_name-ESNF"), + ], + id="dataset" + ), ]) def test_searchUniqueKey_objindex(client, key, relobjs): """Test caching of objects in the objindex. diff --git a/tests/test_07_dataselection.py b/tests/test_07_dataselection.py index 48c9b3c2..e341493e 100644 --- a/tests/test_07_dataselection.py +++ b/tests/test_07_dataselection.py @@ -22,12 +22,24 @@ def client(setupicat): ([42], [47,11], [6,666,66]), ] param_queries = [ - ("Investigation [name = '10100601-ST']"), - ("Dataset <-> Investigation [name = '10100601-ST']"), - ("Datafile <-> Dataset <-> Investigation [name = '10100601-ST']"), - ("SELECT dc FROM DataCollection dc " - "INCLUDE dc.dataCollectionDatafiles AS dcdf, dcdf.datafile, " - "dc.dataCollectionDatasets AS dcds, dcds.dataset"), + pytest.param( + "Investigation [name = '10100601-ST']", + id="investigations" + ), + pytest.param( + "Dataset <-> Investigation [name = '10100601-ST']", + id="datasets" + ), + pytest.param( + "Datafile <-> Dataset <-> Investigation [name = '10100601-ST']", + id="datafiles" + ), + pytest.param( + "SELECT dc FROM DataCollection dc " + "INCLUDE dc.dataCollectionDatafiles AS dcdf, dcdf.datafile, " + "dc.dataCollectionDatasets AS dcds, dcds.dataset", + id="dataCollections" + ), ] def get_obj_ids(objs): From fe6a826639ec69600db4633f9da841b41b3ea91b Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Thu, 19 Oct 2023 12:00:48 +0200 Subject: [PATCH 18/39] Normalize pytest skip messages --- tests/test_03_login.py | 2 +- tests/test_06_exception.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_03_login.py b/tests/test_03_login.py index ad7fe6f6..5bc50ac9 100644 --- a/tests/test_03_login.py +++ b/tests/test_03_login.py @@ -64,7 +64,7 @@ def test_login_ids_shorturl(): # clients. client, conf = getConfig() if not client.ids: - pytest.skip("no IDS") + pytest.skip("no IDS configured") try: icatURL = client.ids.getIcatUrl() except icat.VersionMethodError: diff --git a/tests/test_06_exception.py b/tests/test_06_exception.py index a420b526..c9d93490 100644 --- a/tests/test_06_exception.py +++ b/tests/test_06_exception.py @@ -119,7 +119,7 @@ def test_ids_exception(client, errcond): """ # Same comment as in test_icat_exception() applies. if not client.ids: - pytest.skip("No IDS configured.") + pytest.skip("no IDS configured") with pytest.raises(icat.IDSError) as einfo: errcond(client) err = einfo.value From e9193a47b24f8fdbd1ac7f9e0fb982fbf6eefd2a Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Thu, 19 Oct 2023 12:48:18 +0200 Subject: [PATCH 19/39] Avoid needless skipping of tests in test_06_icatingest.py if IDS is not available, only skip tests that actually need IDS --- tests/test_06_icatingest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index a7a88411..7ab155d2 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -17,14 +17,14 @@ @pytest.fixture(scope="module") def client(setupicat): - client, conf = getConfig(confSection="acord", ids="mandatory") + client, conf = getConfig(confSection="acord", ids="optional") client.login(conf.auth, conf.credentials) return client @pytest.fixture(scope="module") def cmdargs(setupicat): require_dumpfile_backend("XML") - _, conf = getConfig(confSection="acord", ids="mandatory") + _, conf = getConfig(confSection="acord", ids="optional") return conf.cmdargs + ["-f", "XML"] @pytest.fixture(scope="function") @@ -42,7 +42,7 @@ def cleanup_list(client): # this object then. pass else: - if obj.BeanName == "Dataset": + if client.ids and obj.BeanName == "Dataset": # obj is a dataset. If any related datafile has been # uploaded (i.e. the location is not NULL), need to # delete it from IDS first. Any other related @@ -322,7 +322,7 @@ def test_ingest_datafiles(tmpdirsec, client, dataset, cmdargs): dummyfiles = [ f['dfname'] for f in testdatafiles ] args = cmdargs + ["-i", datafiles] callscript("icatingest.py", args) - # Verify that the datafiles have been uploaded. + # Verify that the datafiles have been created but not uploaded. dataset = client.searchMatching(dataset) for fname in dummyfiles: query = Query(client, "Datafile", conditions={ @@ -340,6 +340,8 @@ def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs): icatingest will not create the datafiles as objects in the ICAT, but upload the files to IDS instead. """ + if not client.ids: + pytest.skip("no IDS configured") dummyfiles = [ DummyDatafile(tmpdirsec, f['dfname'], f['size'], f['mtime']) for f in testdatafiles ] args = cmdargs + ["-i", datafiles, "--upload-datafiles", From cb8a4bc5c699719eff52f4dbcee37eea0c598d6d Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Thu, 19 Oct 2023 14:56:31 +0200 Subject: [PATCH 20/39] Add a helper fixture cleanup_objs to conftest.py --- tests/conftest.py | 33 +++++++++++++++++++++++++++ tests/test_06_icatingest.py | 38 +++++--------------------------- tests/test_06_ingest.py | 8 +++---- tests/test_07_entity_validate.py | 6 ++--- 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 65157838..25c01dcb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ import icat import icat.config import icat.dumpfile +from icat.query import Query try: import icat.dumpfile_xml except ImportError: @@ -240,6 +241,38 @@ def setupicat(standardCmdArgs): args = standardCmdArgs + ["-f", "YAML", "-i", str(testcontent)] callscript("icatingest.py", args) +@pytest.fixture(scope="session") +def rootclient(setupicat): + client, conf = getConfig(confSection="root", ids="optional") + client.login(conf.auth, conf.credentials) + return client + +@pytest.fixture(scope="function") +def cleanup_objs(rootclient): + """Delete some objects (that may or may not have been) created during + a test + """ + obj_list = [] + yield obj_list + for obj in obj_list: + try: + if not obj.id: + obj = rootclient.searchMatching(obj) + except icat.SearchResultError: + # obj not found, maybe the test failed, nothing to do on + # this object then. + pass + else: + if rootclient.ids and obj.BeanName == "Dataset": + # obj is a dataset. If any related datafile has been + # uploaded (i.e. the location is not NULL), need to + # delete it from IDS first. + query = Query(rootclient, "Datafile", + conditions={"dataset.id": "= %d" % obj.id, + "location": "IS NOT NULL"}) + rootclient.deleteData(rootclient.search(query)) + rootclient.delete(obj) + # ============================= hooks ================================ def pytest_report_header(config): diff --git a/tests/test_06_icatingest.py b/tests/test_06_icatingest.py index 7ab155d2..be7e138a 100644 --- a/tests/test_06_icatingest.py +++ b/tests/test_06_icatingest.py @@ -28,35 +28,7 @@ def cmdargs(setupicat): return conf.cmdargs + ["-f", "XML"] @pytest.fixture(scope="function") -def cleanup_list(client): - """Delete some objects (that may or may not have been) created during - a test - """ - obj_list = [] - yield obj_list - for obj in obj_list: - try: - obj = client.searchMatching(obj) - except icat.SearchResultError: - # obj not found, maybe the test failed, nothing to do on - # this object then. - pass - else: - if client.ids and obj.BeanName == "Dataset": - # obj is a dataset. If any related datafile has been - # uploaded (i.e. the location is not NULL), need to - # delete it from IDS first. Any other related - # datafile or dataset parameter will be deleted - # automatically with the dataset by cascading in the - # ICAT server. - query = Query(client, "Datafile", - conditions={"dataset.id": "= %d" % obj.id, - "location": "IS NOT NULL"}) - client.deleteData(client.search(query)) - client.delete(obj) - -@pytest.fixture(scope="function") -def dataset(client, cleanup_list): +def dataset(client, cleanup_objs): """A dataset to be used in the test. The dataset is not created by the fixture, it is assumed that the @@ -68,7 +40,7 @@ def dataset(client, cleanup_list): dataset = client.new("Dataset", name="e208343", complete=False, investigation=inv, type=dstype) - cleanup_list.append(dataset) + cleanup_objs.append(dataset) return dataset @@ -362,7 +334,7 @@ def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs): assert df.datafileModTime == f.mtime -def test_ingest_dataset_samples(client, cleanup_list, cmdargs): +def test_ingest_dataset_samples(client, cleanup_objs, cmdargs): """Ingest some datasets that are releated to samples. """ ds_sample_map = { @@ -380,14 +352,14 @@ def test_ingest_dataset_samples(client, cleanup_list, cmdargs): dataset = client.new("Dataset", name=name, complete=False, investigation=inv, type=ds_type) - cleanup_list.append(dataset) + cleanup_objs.append(dataset) for name in {v for v in ds_sample_map.values() if v}: # the samples are referenced from the ingest file, but are are # assumed to already exist, so we need to create them here. sample = client.new("Sample", name=name, investigation=inv, type=sample_type) sample.create() - cleanup_list.append(sample) + cleanup_objs.append(sample) args = cmdargs + ["-i", sample_ds] callscript("icatingest.py", args) for ds_name, sample_name in ds_sample_map.items(): diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index a4dd6fe3..9fd1de63 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -19,19 +19,17 @@ def client(setupicat): return client @pytest.fixture(scope="function") -def investigation(client): +def investigation(client, cleanup_objs): query = Query(client, "Investigation", conditions={ "name": "= '12100409-ST'", }) inv = client.assertedSearch(query)[0] yield inv - rootclient, rootconf = getConfig(confSection="root", ids=False) - rootclient.login(rootconf.auth, rootconf.credentials) - query = Query(rootclient, "Dataset", conditions={ + query = Query(client, "Dataset", conditions={ "investigation.id": "= %d" % inv.id, "name": "LIKE 'testingest_%'", }) - rootclient.deleteMany(client.search(query)) + cleanup_objs.extend(client.search(query)) @pytest.fixture(scope="function") def schemadir(monkeypatch): diff --git a/tests/test_07_entity_validate.py b/tests/test_07_entity_validate.py index 0e9ed815..17c12f27 100644 --- a/tests/test_07_entity_validate.py +++ b/tests/test_07_entity_validate.py @@ -14,7 +14,7 @@ def client(setupicat): return client @pytest.fixture(scope="function") -def dataset(client): +def dataset(client, cleanup_objs): """Create a temporary Dataset for the tests. """ inv = client.assertedSearch("Investigation [name='08100122-EF']")[0] @@ -23,8 +23,8 @@ def dataset(client): name="test_07_entity_validate", complete=False, investigation=inv, type=dstype) dataset.create() - yield dataset - client.delete(dataset) + cleanup_objs.append(dataset) + return dataset def validate_param(self): From 0c90a0c110177cf805b8f8fbf985f679e133811d Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Thu, 19 Oct 2023 17:14:28 +0200 Subject: [PATCH 21/39] Update changelog --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d26910a0..31a4f370 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,12 +20,16 @@ Bug fixes and minor changes + `#132`_, `#136`_: Fix a spurious :exc:`AttributeError` on cleanup after connecting to an invalid url. ++ `#130`_, `#137`_: Review test suite. + .. _#122: https://github.com/icatproject/python-icat/issues/122 +.. _#130: https://github.com/icatproject/python-icat/issues/130 .. _#131: https://github.com/icatproject/python-icat/issues/131 .. _#132: https://github.com/icatproject/python-icat/issues/132 .. _#133: https://github.com/icatproject/python-icat/pull/133 .. _#135: https://github.com/icatproject/python-icat/pull/135 .. _#136: https://github.com/icatproject/python-icat/pull/136 +.. _#137: https://github.com/icatproject/python-icat/pull/137 1.1.0 (2023-06-30) From e8777d92ff8fd4f43717c6fefe73b195abffd0e4 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 20 Oct 2023 11:48:25 +0200 Subject: [PATCH 22/39] Create some samples to be referenced in ingest files in test_06_ingest.py --- tests/test_06_ingest.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index 9fd1de63..53c2ded5 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -12,18 +12,38 @@ from conftest import getConfig, gettestdata, icat_version, testdatadir +def get_test_investigation(client): + query = Query(client, "Investigation", conditions={ + "name": "= '12100409-ST'", + }) + return client.assertedSearch(query)[0] + @pytest.fixture(scope="module") def client(setupicat): client, conf = getConfig(confSection="ingest", ids=False) client.login(conf.auth, conf.credentials) return client +@pytest.fixture(scope="module") +def samples(rootclient): + """Create some samples that are referenced in some of the ingest files. + """ + query = Query(rootclient, "SampleType", conditions={ + "name": "= 'Nickel(II) oxide SC'" + }) + st = rootclient.assertedSearch(query)[0] + inv = get_test_investigation(rootclient) + samples = [] + for n in ("ab3465", "ab3466"): + s = rootclient.new("Sample", name=n, type=st, investigation=inv) + s.create() + samples.append(s) + yield samples + rootclient.deleteMany(samples) + @pytest.fixture(scope="function") def investigation(client, cleanup_objs): - query = Query(client, "Investigation", conditions={ - "name": "= '12100409-ST'", - }) - inv = client.assertedSearch(query)[0] + inv = get_test_investigation(client) yield inv query = Query(client, "Dataset", conditions={ "investigation.id": "= %d" % inv.id, @@ -240,7 +260,7 @@ def schemadir(monkeypatch): @pytest.mark.parametrize("case", [ pytest.param(c, id=c.metadata.name, marks=c.marks) for c in cases ]) -def test_ingest(client, investigation, schemadir, case): +def test_ingest(client, investigation, samples, schemadir, case): datasets = [] for name in case.data: datasets.append(client.new("Dataset", name=name)) From 1c9cbf707673f4a5a0ad71a5caf1bbd25170f9cf Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 20 Oct 2023 13:43:42 +0200 Subject: [PATCH 23/39] Add a test for Issue #125 --- doc/examples/metadata-sample.xml | 36 ++++++++++++++++++ tests/test_06_ingest.py | 63 ++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 doc/examples/metadata-sample.xml diff --git a/doc/examples/metadata-sample.xml b/doc/examples/metadata-sample.xml new file mode 100644 index 00000000..5f815e7c --- /dev/null +++ b/doc/examples/metadata-sample.xml @@ -0,0 +1,36 @@ + + + + 2023-10-20T12:15:26+02:00 + metadata-writer 0.28 + + + + testingest_sample_1 + ab3465 at 2.7 K + 2020-09-30T18:02:17+02:00 + 2020-09-30T20:18:36+02:00 + + + + testingest_sample_2 + ab3465 at 5.1 K + 2020-09-30T20:29:19+02:00 + 2020-09-30T21:23:49+02:00 + + + + testingest_sample_3 + ab3466 at 2.7 K + 2020-09-30T21:35:16+02:00 + 2020-09-30T23:04:27+02:00 + + + + testingest_sample_4 + reference + 2020-09-30T23:04:31+02:00 + 2020-10-01T01:26:07+02:00 + + + diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index 53c2ded5..ee114699 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -256,6 +256,69 @@ def schemadir(monkeypatch): reason="Need ICAT schema 5.0 or newer"), ), ), + Case( + data = [ "testingest_sample_1", "testingest_sample_2", + "testingest_sample_3", "testingest_sample_4" ], + metadata = gettestdata("metadata-sample.xml"), + checks = { + "testingest_sample_1": [ + ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", + "ab3465 at 2.7 K"), + ("SELECT ds.startDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 18, 2, 17, tzinfo=cest)), + ("SELECT ds.endDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 20, 18, 36, tzinfo=cest)), + (("SELECT COUNT(s) FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + 1), + (("SELECT s.name FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + "ab3465"), + ], + "testingest_sample_2": [ + ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", + "ab3465 at 5.1 K"), + ("SELECT ds.startDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 20, 29, 19, tzinfo=cest)), + ("SELECT ds.endDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 21, 23, 49, tzinfo=cest)), + (("SELECT COUNT(s) FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + 1), + (("SELECT s.name FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + "ab3465"), + ], + "testingest_sample_3": [ + ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", + "ab3466 at 2.7 K"), + ("SELECT ds.startDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 21, 35, 16, tzinfo=cest)), + ("SELECT ds.endDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 23, 4, 27, tzinfo=cest)), + (("SELECT COUNT(s) FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + 1), + (("SELECT s.name FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + "ab3466"), + ], + "testingest_sample_4": [ + ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", + "reference"), + ("SELECT ds.startDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 9, 30, 23, 4, 31, tzinfo=cest)), + ("SELECT ds.endDate FROM Dataset ds WHERE ds.id = %d", + datetime.datetime(2020, 10, 1, 1, 26, 7, tzinfo=cest)), + (("SELECT COUNT(s) FROM Sample s JOIN s.datasets AS ds " + "WHERE ds.id = %d"), + 0), + ], + }, + marks = ( + pytest.mark.xfail(reason="Issue #125"), + ), + ), ] @pytest.mark.parametrize("case", [ pytest.param(c, id=c.metadata.name, marks=c.marks) for c in cases From a93cc4627c0f1594d2fe06efca562bfdb7eb8092 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 20 Oct 2023 19:06:25 +0200 Subject: [PATCH 24/39] Simplify ingest-10.xsd --- etc/ingest-10.xsd | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/etc/ingest-10.xsd b/etc/ingest-10.xsd index c600957b..c4b6072e 100644 --- a/etc/ingest-10.xsd +++ b/etc/ingest-10.xsd @@ -73,7 +73,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -108,12 +108,7 @@ - - - - - - + From 1cfed18c651abc28f148a94e7ac89c69a913e241 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 20 Oct 2023 19:10:18 +0200 Subject: [PATCH 25/39] Add ingest-11.xsd --- etc/ingest-11.xsd | 137 ++++++++++++++++++++++++++++++++++++++++++++++ icat/ingest.py | 1 + setup.py | 2 +- 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 etc/ingest-11.xsd diff --git a/etc/ingest-11.xsd b/etc/ingest-11.xsd new file mode 100644 index 00000000..7b77f7d3 --- /dev/null +++ b/etc/ingest-11.xsd @@ -0,0 +1,137 @@ + + + + + + + Schema definition for ingest files to ICAT. + Version 1.1. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/icat/ingest.py b/icat/ingest.py index 7e09f66a..8e62008a 100644 --- a/icat/ingest.py +++ b/icat/ingest.py @@ -43,6 +43,7 @@ class IngestReader(icat.dumpfile_xml.XMLDumpFileReader): """ XSD_Map = { ('icatingest', '1.0'): "ingest-10.xsd", + ('icatingest', '1.1'): "ingest-11.xsd", } """A mapping to select the XSD file to use. Keys are pairs of root element name and version attribute, the values are the diff --git a/setup.py b/setup.py index 66fcf4f5..14871f70 100755 --- a/setup.py +++ b/setup.py @@ -132,7 +132,7 @@ def copy_test_data(self): for ext in ("xml", "yaml") ] files += glob(os.path.join("doc", "examples", "metadata-*.xml")) files += [ os.path.join("etc", f) - for f in ["ingest-10.xsd", "ingest.xslt"] ] + for f in ["ingest-10.xsd", "ingest-11.xsd", "ingest.xslt"] ] for f in files: dest = os.path.join(destdir, os.path.basename(f)) self.copy_file(f, dest, preserve_mode=False) From 72576a3867870cf7361b34b418e1904d01f598a8 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Fri, 20 Oct 2023 19:06:25 +0200 Subject: [PATCH 26/39] Simplify ingest-10.xsd --- etc/ingest-10.xsd | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/etc/ingest-10.xsd b/etc/ingest-10.xsd index c600957b..c4b6072e 100644 --- a/etc/ingest-10.xsd +++ b/etc/ingest-10.xsd @@ -73,7 +73,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -108,12 +108,7 @@ - - - - - - + From 587aca56b483580fdd391696f84f5ff165fedfde Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 14:56:31 +0100 Subject: [PATCH 27/39] Add a test for Issue #138 --- setup.py | 1 + tests/test_06_ingest.py | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66fcf4f5..2b87d8a9 100755 --- a/setup.py +++ b/setup.py @@ -130,6 +130,7 @@ def copy_test_data(self): "icatdump-%s.%s" % (ver, ext)) for ver in ("4.4", "4.7", "4.10", "5.0") for ext in ("xml", "yaml") ] + files += glob(os.path.join("doc", "icatdata-*.xsd")) files += glob(os.path.join("doc", "examples", "metadata-*.xml")) files += [ os.path.join("etc", f) for f in ["ingest-10.xsd", "ingest.xslt"] ] diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index 9fd1de63..480e2890 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -5,6 +5,7 @@ import datetime import pytest pytest.importorskip("lxml") +from lxml import etree import icat import icat.config from icat.ingest import IngestReader @@ -38,13 +39,14 @@ def schemadir(monkeypatch): cet = datetime.timezone(datetime.timedelta(hours=1)) cest = datetime.timezone(datetime.timedelta(hours=2)) -Case = namedtuple('Case', ['data', 'metadata', 'checks', 'marks']) +Case = namedtuple('Case', ['data', 'metadata', 'schema', 'checks', 'marks']) # Try out different variants for the metadata input file cases = [ Case( data = ["testingest_inl_1", "testingest_inl_2"], metadata = gettestdata("metadata-4.4-inl.xml"), + schema = gettestdata("icatdata-4.4.xsd"), checks = { "testingest_inl_1": [ ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", @@ -84,6 +86,7 @@ def schemadir(monkeypatch): Case( data = ["testingest_inl5_1", "testingest_inl5_2"], metadata = gettestdata("metadata-5.0-inl.xml"), + schema = gettestdata("icatdata-5.0.xsd"), checks = { "testingest_inl5_1": [ ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", @@ -142,6 +145,7 @@ def schemadir(monkeypatch): Case( data = ["testingest_sep_1", "testingest_sep_2"], metadata = gettestdata("metadata-4.4-sep.xml"), + schema = gettestdata("icatdata-4.4.xsd"), checks = { "testingest_sep_1": [ ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", @@ -181,6 +185,7 @@ def schemadir(monkeypatch): Case( data = ["testingest_sep5_1", "testingest_sep5_2"], metadata = gettestdata("metadata-5.0-sep.xml"), + schema = gettestdata("icatdata-5.0.xsd"), checks = { "testingest_sep5_1": [ ("SELECT ds.description FROM Dataset ds WHERE ds.id = %d", @@ -237,6 +242,23 @@ def schemadir(monkeypatch): ), ), ] + +@pytest.mark.xfail(reason="Issue #138") +@pytest.mark.parametrize("case", [ + pytest.param(c, id=c.metadata.name, marks=c.marks) for c in cases +]) +def test_ingest_schema(client, investigation, schemadir, case): + """Check that the ingest data after transformation is valid according + to icatdata schema. + """ + datasets = [] + for name in case.data: + datasets.append(client.new("Dataset", name=name)) + reader = IngestReader(client, case.metadata, investigation) + with case.schema.open("rb") as f: + schema = etree.XMLSchema(etree.parse(f)) + assert schema.validate(reader.infile) + @pytest.mark.parametrize("case", [ pytest.param(c, id=c.metadata.name, marks=c.marks) for c in cases ]) @@ -263,6 +285,7 @@ def test_ingest(client, investigation, schemadir, case): Case( data = ["e208339"], metadata = gettestdata("metadata-5.0-badref.xml"), + schema = gettestdata("icatdata-5.0.xsd"), checks = {}, marks = ( pytest.mark.skipif(icat_version < "5.0", From 71d185068f9c30eac0a55b9cad7f4de32c826d48 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 15:03:05 +0100 Subject: [PATCH 28/39] Fix the order of the type subelement in dataset --- etc/ingest.xslt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ingest.xslt b/etc/ingest.xslt index e220ad88..bd09bf13 100644 --- a/etc/ingest.xslt +++ b/etc/ingest.xslt @@ -21,12 +21,13 @@ - false + false + @@ -36,7 +37,6 @@ - From 231bc6acd041d3855866327b1ccaabe282231006 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 15:11:57 +0100 Subject: [PATCH 29/39] Fix the order of datasetTechnique and datasetInstrument elements in ingest file format 1.0 --- doc/examples/metadata-5.0-sep.xml | 16 ++++++++-------- etc/ingest-10.xsd | 12 ++++++------ tests/test_06_ingest.py | 1 - 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/doc/examples/metadata-5.0-sep.xml b/doc/examples/metadata-5.0-sep.xml index 419bb696..c026d790 100644 --- a/doc/examples/metadata-5.0-sep.xml +++ b/doc/examples/metadata-5.0-sep.xml @@ -17,14 +17,6 @@ 2022-02-03T17:13:10+01:00 2022-02-03T18:45:27+01:00 - - - - - - - - @@ -33,6 +25,14 @@ + + + + + + + + neutron diff --git a/etc/ingest-10.xsd b/etc/ingest-10.xsd index c4b6072e..0f54398d 100644 --- a/etc/ingest-10.xsd +++ b/etc/ingest-10.xsd @@ -30,10 +30,10 @@ - + @@ -68,23 +68,23 @@ - + - + - + - + diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index 480e2890..4ea1b880 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -243,7 +243,6 @@ def schemadir(monkeypatch): ), ] -@pytest.mark.xfail(reason="Issue #138") @pytest.mark.parametrize("case", [ pytest.param(c, id=c.metadata.name, marks=c.marks) for c in cases ]) From 68c991a1492105d2c697eeab5d0603ee4e3df4c4 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 15:14:25 +0100 Subject: [PATCH 30/39] Update documentation --- doc/src/ingest.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/ingest.rst b/doc/src/ingest.rst index 10817fbc..6a1fe479 100644 --- a/doc/src/ingest.rst +++ b/doc/src/ingest.rst @@ -30,8 +30,8 @@ below. The input accepted by :class:`~icat.ingest.IngestReader` consists of one or more ``Dataset`` objects that all need to relate to the same -``Investigation`` and any number of related ``DatasetInstrument``, -``DatasetTechnique``, and ``DatasetParameter`` objects. The +``Investigation`` and any number of related ``DatasetTechnique``, +``DatasetInstrument``, and ``DatasetParameter`` objects. The ``Investigation`` must exist beforehand in ICAT. The relation from the ``Dataset`` objects to the ``Investigation`` will be set by :class:`~icat.ingest.IngestReader` accordingly. (Actually, the XSLT @@ -44,7 +44,7 @@ and ``Datafile`` objects in ICAT at the same time. But the attributes of the datasets will be read from the input file and set in the ``Dataset`` objects by :class:`~icat.ingest.IngestReader`. :class:`~icat.ingest.IngestReader` will also create the related -``DatasetInstrument``, ``DatasetTechnique`` and ``DatasetParameter`` +``DatasetTechnique``, ``DatasetInstrument`` and ``DatasetParameter`` objects read from the input file in ICAT. .. autoclass:: icat.ingest.IngestReader From c923a85dd5605a9d05002bdd1ee05589cd102f65 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 15:55:28 +0100 Subject: [PATCH 31/39] Update changelog --- CHANGES.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 31a4f370..ccfbf54a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,17 @@ New features + `#122`_, `#133`_: Allow referencing related objects by reference key in object references in XML ICAT data file format. +Incompatible changes +-------------------- + ++ `#138`_, `#139`_: Fix the input that :mod:`icat.ingest` generates on + the fly to be valid according to the ICAT data file schema. This + also affects the input that the module accepts: the order of + subelements of `data` need to be changed such that + `datasetTechnique` comes before `datasetInstrument`. + + Note that :mod:`icat.ingest` has been declared experimental for now. + Bug fixes and minor changes --------------------------- @@ -30,6 +41,8 @@ Bug fixes and minor changes .. _#135: https://github.com/icatproject/python-icat/pull/135 .. _#136: https://github.com/icatproject/python-icat/pull/136 .. _#137: https://github.com/icatproject/python-icat/pull/137 +.. _#138: https://github.com/icatproject/python-icat/issues/138 +.. _#139: https://github.com/icatproject/python-icat/pull/139 1.1.0 (2023-06-30) From bd3d71b81e4fe9d53ba9fd3d95483de93303e528 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 17:23:37 +0100 Subject: [PATCH 32/39] Simplify ingest.xslt --- etc/ingest.xslt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/etc/ingest.xslt b/etc/ingest.xslt index bd09bf13..797b8398 100644 --- a/etc/ingest.xslt +++ b/etc/ingest.xslt @@ -28,15 +28,9 @@ - - - - - - - - - + + + From eb984457d2cfd8d80137c8ce0e3d63ea746b4312 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 17:29:35 +0100 Subject: [PATCH 33/39] Add copying of sample references in datasets in ingest.xslt --- etc/ingest.xslt | 8 ++++++++ tests/test_06_ingest.py | 4 +--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/etc/ingest.xslt b/etc/ingest.xslt index 797b8398..6e1e5cee 100644 --- a/etc/ingest.xslt +++ b/etc/ingest.xslt @@ -27,6 +27,7 @@ + @@ -34,6 +35,13 @@ + + + _Investigation + + + + diff --git a/tests/test_06_ingest.py b/tests/test_06_ingest.py index c23c427b..03a84de4 100644 --- a/tests/test_06_ingest.py +++ b/tests/test_06_ingest.py @@ -321,9 +321,7 @@ def schemadir(monkeypatch): 0), ], }, - marks = ( - pytest.mark.xfail(reason="Issue #125"), - ), + marks = (), ), ] From 97aa9b70566bbed5fbcd4bb61650735eaf270517 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 17:50:11 +0100 Subject: [PATCH 34/39] Update documentation --- doc/src/ingest.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/src/ingest.rst b/doc/src/ingest.rst index 6a1fe479..7c5cb502 100644 --- a/doc/src/ingest.rst +++ b/doc/src/ingest.rst @@ -47,6 +47,14 @@ of the datasets will be read from the input file and set in the ``DatasetTechnique``, ``DatasetInstrument`` and ``DatasetParameter`` objects read from the input file in ICAT. +Using ingest file format 1.1, ``Dataset`` objects may also include a +reference to a ``Sample``. That ``Sample`` objects needs to exist +beforehand and needs to be related to the same ``Investigation`` as +the ``Dataset``. + +.. versionchanged: 1.2.0 + add version 1.1 of the ingest file format, including references to samples + .. autoclass:: icat.ingest.IngestReader :members: :show-inheritance: From b91df391d2ff66dd46491078998eac0b595a4723 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 17:59:51 +0100 Subject: [PATCH 35/39] Update changelog --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index ccfbf54a..beb48ac9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,9 @@ Changelog New features ------------ ++ `#125`_, `#140`_: Add support to link datasets with samples in + :mod:`icat.ingest`. + + `#122`_, `#133`_: Allow referencing related objects by reference key in object references in XML ICAT data file format. @@ -34,6 +37,7 @@ Bug fixes and minor changes + `#130`_, `#137`_: Review test suite. .. _#122: https://github.com/icatproject/python-icat/issues/122 +.. _#125: https://github.com/icatproject/python-icat/issues/125 .. _#130: https://github.com/icatproject/python-icat/issues/130 .. _#131: https://github.com/icatproject/python-icat/issues/131 .. _#132: https://github.com/icatproject/python-icat/issues/132 @@ -43,6 +47,7 @@ Bug fixes and minor changes .. _#137: https://github.com/icatproject/python-icat/pull/137 .. _#138: https://github.com/icatproject/python-icat/issues/138 .. _#139: https://github.com/icatproject/python-icat/pull/139 +.. _#140: https://github.com/icatproject/python-icat/pull/140 1.1.0 (2023-06-30) From 0f446179f5dc5c7f363d88ee63ac7327b09804d1 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 18:05:12 +0100 Subject: [PATCH 36/39] Typo in documentation --- doc/src/ingest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/ingest.rst b/doc/src/ingest.rst index 7c5cb502..4fed8b7e 100644 --- a/doc/src/ingest.rst +++ b/doc/src/ingest.rst @@ -52,7 +52,7 @@ reference to a ``Sample``. That ``Sample`` objects needs to exist beforehand and needs to be related to the same ``Investigation`` as the ``Dataset``. -.. versionchanged: 1.2.0 +.. versionchanged:: 1.2.0 add version 1.1 of the ingest file format, including references to samples .. autoclass:: icat.ingest.IngestReader From c2be11ea21033d5d8ab7c496abd75a93cbd5296b Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 22:24:19 +0100 Subject: [PATCH 37/39] Add Download to Project-URLs in package metadata --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0a63560a..d1b8f904 100755 --- a/setup.py +++ b/setup.py @@ -210,6 +210,7 @@ def run(self): project_urls = dict( Documentation="https://python-icat.readthedocs.io/", Source="https://github.com/icatproject/python-icat/", + Download="https://github.com/icatproject/python-icat/releases/latest", Changes="https://python-icat.readthedocs.io/en/latest/changelog.html", ), packages = ["icat"], From 1f4030bb8b0a87529c1d10fbe5220cd1b5ebeb44 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Mon, 30 Oct 2023 22:40:24 +0100 Subject: [PATCH 38/39] Minor tweaks in CITATION.cff --- CITATION.cff | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index bf89ce2b..2a72aa6b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,7 +3,6 @@ cff-version: 1.2.0 title: python-icat – Python interface to ICAT and IDS -message: Please cite this software using these metadata. type: software authors: - given-names: Rolf @@ -13,22 +12,21 @@ authors: Helmholtz-Zentrum Berlin für Materialien und Energie orcid: 'https://orcid.org/0000-0002-1266-3819' - - name: "The ICAT project" - website: https://icatproject.org/ + - name: The ICAT project + website: 'https://icatproject.org/' repository-code: 'https://github.com/icatproject/python-icat' -repository-artifact: 'https://pypi.org/project/python-icat/' +repository-artifact: 'https://github.com/icatproject/python-icat/releases/latest' abstract: >- - The ICAT server is a metadata catalogue to support - Large Facility experimental data, linking all - aspects of the research chain from proposal through - to publication. It provides SOAP and RESTful web - service interfaces to an underlying database. + The ICAT server is a metadata catalogue to support Large + Facility experimental data, linking all aspects of the + research chain from proposal through to publication. It + provides SOAP and RESTful web service interfaces to an + underlying database. - python-icat is a Python package that provides a - collection of modules for writing programs that - access an ICAT service using the SOAP interface. It - is based on Suds and extends it with ICAT specific - features. + python-icat is a Python package that provides a collection + of modules for writing programs that access an ICAT + service using the SOAP interface. It is based on Suds and + extends it with ICAT specific features. keywords: - python - metadata From ced13e0ace223c09a933dfce73d262d5c8c733f1 Mon Sep 17 00:00:00 2001 From: Rolf Krahl Date: Tue, 31 Oct 2023 06:55:49 +0100 Subject: [PATCH 39/39] Prepare release 1.2.0 --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index beb48ac9..7a21a6cd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,8 +2,8 @@ Changelog ========= -1.2.0 (not yet released) -~~~~~~~~~~~~~~~~~~~~~~~~ +1.2.0 (2023-10-31) +~~~~~~~~~~~~~~~~~~ New features ------------