Merge pull request #930 from christian-monch/basic-metalad

add a metalad description

.appveyor.yml

@@ -26,7 +26,7 @@ shallow_clone: false
 environment:
   DTS: datalad_next
   APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
-  INSTALL_SYSPKGS: python3-virtualenv graphicsmagick-imagemagick-compat moreutils jq
+  INSTALL_SYSPKGS: python3-virtualenv graphicsmagick-imagemagick-compat moreutils jq uidmap
   INSTALL_GITANNEX: git-annex -m snapshot
 
 

docs/beyond_basics/_examples/DL-101-181-101

@@ -0,0 +1,13 @@
+$ datalad clone https://github.com/datalad-handbook/cozy-screensavers.git
+$ cd cozy-screensavers
+[INFO] Attempting a clone into /home/me/beyond_basics/meta/cozy-screensavers
+[INFO] Attempting to clone from https://github.com/datalad-handbook/cozy-screensavers.git to /home/me/beyond_basics/meta/cozy-screensavers
+[INFO] Start enumerating objects
+[INFO] Start counting objects
+[INFO] Start compressing objects
+[INFO] Start receiving objects
+[INFO] Start resolving deltas
+[INFO] Completed clone attempts for Dataset(/home/me/beyond_basics/meta/cozy-screensavers)
+[INFO] Remote origin not usable by git-annex; setting annex-ignore
+[INFO] https://github.com/datalad-handbook/cozy-screensavers.git/config download failed: Not Found
+install(ok): /home/me/beyond_basics/meta/cozy-screensavers (dataset)

docs/beyond_basics/_examples/DL-101-181-102

@@ -0,0 +1,5 @@
+$ tree
+.
+└── zen.jpg -> .git/annex/objects/PV/X5/MD5E-s5217743--ad8f6e7832f1752fe480051bcd06c719.jpg/MD5E-s5217743--ad8f6e7832f1752fe480051bcd06c719.jpg
+
+0 directories, 1 file

docs/beyond_basics/_examples/DL-101-181-103

@@ -0,0 +1,2 @@
+$ datalad get zen.jpg
+get(ok): zen.jpg (file) [from web...]

docs/beyond_basics/_examples/DL-101-181-104

@@ -0,0 +1,17 @@
+cat << EOT > metadata-zen.json
+{
+      "type": "file",
+      "path": "zen.png",
+      "dataset_id": "2d540a9d-2ef7-4b5f-8931-7c92f483f0c7",
+      "dataset_version": "19f2d98d758116d099d467260a5a71082b2c6a29",
+      "extractor_name": "Picture2Words",
+      "extractor_version": "0.1",
+      "extraction_parameter": {},
+      "extraction_time": 1675113291.1464975,
+      "agent_name": "Overworked CTO",
+      "agent_email": "[email protected]",
+      "extracted_metadata": {
+      "description": "A lake with waterlilies in a front of snow covered mountains"
+      }
+}
+EOT

docs/beyond_basics/_examples/DL-101-181-105

@@ -0,0 +1,2 @@
+$ datalad meta-add -d . - < metadata-zen.json
+meta_add(ok): /home/me/beyond_basics/meta/cozy-screensavers/zen.png (file) [added file metadata to /home/me/beyond_basics/meta/cozy-screensavers]

docs/beyond_basics/_examples/DL-101-181-106

@@ -0,0 +1,2 @@
+$ datalad meta-dump -d . .:zen.png
+{"type": "file", "path": "zen.png", "dataset_id": "2d540a9d-2ef7-4b5f-8931-7c92f483f0c7", "dataset_version": "19f2d98d758116d099d467260a5a71082b2c6a29", "extraction_time": 1675113291.1464975, "agent_name": "Overworked CTO", "agent_email": "[email protected]", "extractor_name": "Picture2Words", "extractor_version": "0.1", "extraction_parameter": {}, "extracted_metadata": {"description": "A lake with waterlilies in a front of snow covered mountains"}}

docs/beyond_basics/_examples/DL-101-181-107

@@ -0,0 +1,24 @@
+$ datalad -f json_pp meta-dump -d . .:zen.png
+{
+  "action": "meta_dump",
+  "backend": "git",
+  "metadata": {
+    "agent_email": "[email protected]",
+    "agent_name": "Overworked CTO",
+    "dataset_id": "2d540a9d-2ef7-4b5f-8931-7c92f483f0c7",
+    "dataset_version": "19f2d98d758116d099d467260a5a71082b2c6a29",
+    "extracted_metadata": {
+      "description": "A lake with waterlilies in a front of snow covered mountains"
+    },
+    "extraction_parameter": {},
+    "extraction_time": 1675113291.1464975,
+    "extractor_name": "Picture2Words",
+    "extractor_version": "0.1",
+    "path": "zen.png",
+    "type": "file"
+  },
+  "metadata_source": ".",
+  "path": "/home/me/beyond_basics/meta/cozy-screensavers/zen.png",
+  "status": "ok",
+  "type": "dataset"
+}

docs/beyond_basics/_examples/DL-101-181-108

@@ -0,0 +1,49 @@
+$ datalad meta-extract -d . metalad_core | jq
+[INFO] Start core metadata extraction from Dataset(/home/me/beyond_basics/meta/cozy-screensavers)
+[INFO] Extracted core metadata from /home/me/beyond_basics/meta/cozy-screensavers
+[INFO] Finished core metadata extraction from Dataset(/home/me/beyond_basics/meta/cozy-screensavers)
+{
+  "type": "dataset",
+  "dataset_id": "2d540a9d-2ef7-4b5f-8931-7c92f483f0c7",
+  "dataset_version": "19f2d98d758116d099d467260a5a71082b2c6a29",
+  "extractor_name": "metalad_core",
+  "extractor_version": "1",
+  "extraction_parameter": {},
+  "extraction_time": 1676301065.0952392,
+  "agent_name": "Elena Piscopia",
+  "agent_email": "[email protected]",
+  "extracted_metadata": {
+    "@context": {
+      "@vocab": "http://schema.org/",
+      "datalad": "http://dx.datalad.org/"
+    },
+    "@graph": [
+      {
+        "@id": "d3765bf6e3a68497b42584fa0774695e",
+        "@type": "agent",
+        "name": "Adina Wagner",
+        "email": "[email protected]"
+      },
+      {
+        "@id": "19f2d98d758116d099d467260a5a71082b2c6a29",
+        "identifier": "2d540a9d-2ef7-4b5f-8931-7c92f483f0c7",
+        "@type": "Dataset",
+        "version": "0-2-g19f2d98",
+        "dateCreated": "2023-02-13T07:48:15+01:00",
+        "dateModified": "2023-02-13T07:55:38+01:00",
+        "hasContributor": {
+          "@id": "d3765bf6e3a68497b42584fa0774695e"
+        },
+        "distribution": [
+          {
+            "@id": "b8938507-77c0-4479-84a6-cbeb41214ca6"
+          },
+          {
+            "name": "origin",
+            "url": "https://github.com/datalad-handbook/cozy-screensavers.git"
+          }
+        ]
+      }
+    ]
+  }
+}

docs/beyond_basics/_examples/DL-101-181-110

@@ -0,0 +1,3 @@
+$ datalad create metadata-assimilation
+$ cd metadata-assimilation
+create(ok): /home/me/beyond_basics/meta/metadata-assimilation (dataset)

docs/beyond_basics/_examples/DL-101-181-111

@@ -0,0 +1,7 @@
+$ git fetch \
+   "https://github.com/christian-monch/metadata-test.git" \
+   "refs/datalad/*:refs/datalad/*"
+From https://github.com/christian-monch/metadata-test
+ * [new ref]         refs/datalad/dataset-tree-version-list -> refs/datalad/dataset-tree-version-list
+ * [new ref]         refs/datalad/dataset-uuid-set -> refs/datalad/dataset-uuid-set
+ * [new ref]         refs/datalad/object-references-2.0 -> refs/datalad/object-references-2.0

docs/beyond_basics/_examples/DL-101-181-113

@@ -0,0 +1,14 @@
+$ tree .git/refs
+.git/refs
+├── annex
+│   └── last-index
+├── datalad
+│   ├── dataset-tree-version-list
+│   ├── dataset-uuid-set
+│   └── object-references-2.0
+├── heads
+│   ├── git-annex
+│   └── master
+└── tags
+
+4 directories, 6 files

docs/beyond_basics/_examples/DL-101-181-114

@@ -0,0 +1,2 @@
+$ cat .git/refs/datalad/dataset-tree-version-list
+92dcce5f8634ec4f872addefad0cd7a2e4372a24

docs/beyond_basics/_examples/DL-101-181-115

@@ -0,0 +1,17 @@
+
+$ git show 92dcce5f8634ec4f872addefad0cd7a2e4372a24 | jq
+[
+  {
+    "primary_data_version": "b3f02438b210c2bfbd4f0da79f2791c599c8856e",
+    "time_stamp": "1675090457.6913724",
+    "path": "",
+    "dataset_tree": {
+      "@": {
+        "type": "Reference",
+        "version": "2.0"
+      },
+      "class_name": "MTreeNode",
+      "location": "65c896c86f0e920439bca908eb11c98c9a706606"
+    }
+  }
+]

docs/beyond_basics/_examples/DL-101-181-116

@@ -0,0 +1,28 @@
+
+$ git ls-tree 65c896c86f0e920439bca908eb11c98c9a706606
+040000 tree d1ad9bfa56f5aa25a1d28caf13db719b9e710d28	study-100
+040000 tree fc901487449e7336f7d4df9cd7d5f9724ce8a76a	study-101
+040000 tree 3f20a64361a4d653f19135c21c3ce56e96309498	study-102
+040000 tree 694104a8cfb957de8cbae8b1f6758449d27b2636	study-103
+040000 tree 160c40d21d42717695e4eda5a7b9c8b40fff4d4c	study-104
+040000 tree f71ebc620b33d6f520babbc7527ec4b7fa2e18e8	study-106
+040000 tree bb30110dd1bb07ee16bf2c47cf64f66b04f46f30	study-107
+040000 tree afc5401106b3a1e5f2b57733a84ad536fec6fcf6	study-108
+040000 tree cdd07567699ccdfdda8ecab1533201d738eaee09	study-109
+040000 tree 333dcec283b2f8e417a215819668c792666b0a36	study-115
+040000 tree 8afc5c03297f9006d682805803c3b02860c8a56b	study-44
+040000 tree 6ef944e65a270b98276d421fc8b65f3898d013dc	study-47
+040000 tree 0551814bc3be223c7901d5ceb005253cd28faec4	study-74
+040000 tree c20e3858c5c1e1e31994337a5c2f8fd344dc5f84	study-82
+040000 tree 9b70616ae5f23bc69132962a5622ce48729d9ef5	study-83
+040000 tree 9cad80e81e6f2ad88c05b29728b04158370f6061	study-84
+040000 tree 0e6fa14639d0a0717b1971341d28ea1ae790bf5a	study-85
+040000 tree 6308ff19e4abbe25d1e14fe76cddbd413d9521b2	study-90
+040000 tree 8cb5725b38463777d9e72ccdda9614884e2f4e79	study-91
+040000 tree bd485bc9d532d33f6a2c88d0e63cc21fb7753ad8	study-92
+040000 tree 32b389b5143669a0cb07fad657fad21408eecea5	study-93
+040000 tree c0ddb3e6cf209554f50ef41c374546fcbfb2cdca	study-95
+040000 tree cc1dc24ef2282d7fa8d4b902d579e7275c484e6f	study-96
+040000 tree 33cf358d5ea8b17f6d80f201cc94e49866d4797e	study-97
+040000 tree 125fbd2318d75f9b431988833c78481650adecbf	study-98
+040000 tree f61a7ddb02d162321d196239fd2d5e03af528230	study-99

docs/beyond_basics/_examples/DL-101-181-119

@@ -0,0 +1,68 @@
+$ datalad meta-dump -r | jq '.extracted_metadata["@graph"][3]["@list"][].familyName' | sort | uniq
+jq: error (at <stdin>:16): Cannot iterate over null (null)
+jq: error (at <stdin>:22): Cannot iterate over null (null)
+jq: error (at <stdin>:23): Cannot iterate over null (null)
+jq: error (at <stdin>:24): Cannot iterate over null (null)
+"Agor"
+"Akolkol"
+"Akolmormal"
+"Amodkosgoru"
+"Apali"
+"Apalsul"
+"Apir"
+"Aselmoro"
+"Ekos"
+"Ekoslud"
+"Emalkosseli"
+"Emalselkosu"
+"Esanmodmadu"
+"Esel"
+"Eselmodi"
+"Gos"
+"Gosa"
+"Gose"
+"Igorsele"
+"Ilud"
+"Imadrig"
+"Imolpala"
+"Imolsel"
+"Irigludkolo"
+"Lad"
+"Ladi"
+"Ladlud"
+"Ladmol"
+"Lado"
+"Modmadkol"
+"Modmod"
+"Modpiru"
+"Modsul"
+"Mora"
+"Mori"
+"Morkosu"
+"Omadsulgor"
+"Omelgorladi"
+"Omol"
+"Osel"
+"Pal"
+"Pir"
+"Pirpirludu"
+"Pirsul"
+"Pulmelkol"
+"Pulpula"
+"Rigi"
+"Rigmorsan"
+"Sanmolmor"
+"Selkolu"
+"Selsulmor"
+"Sul"
+"Sulmod"
+"Ugore"
+"Ugorsan"
+"Ugos"
+"Uludpirkoli"
+"Umalpalo"
+"Umelmola"
+"Upalo"
+"Upalpal"
+"Upirmor"
+"Uselmale"

docs/beyond_basics/_examples/DL-101-181-120

@@ -0,0 +1,4 @@
+$ datalad meta-dump \
+   -d  https://github.com/christian-monch/metadata-test.git \
+   ./study-100
+{"type": "dataset", "root_dataset_id": "<unknown>", "root_dataset_version": "b3f02438b210c2bfbd4f0da79f2791c599c8856e", "dataset_path": "study-100", "dataset_id": "52142b84-dc76-11ea-98c5-7cdd908c7490", "dataset_version": "244a8ad43b00622989ae7f0d2b59c80697dadb80", "extraction_time": 1675090457.6464868, "agent_name": "Mad Isanpire", "agent_email": "[email protected]", "extractor_name": "metalad_studyminimeta", "extractor_version": "0.1", "extraction_parameter": {}, "extracted_metadata": {"@context": {"@vocab": "http://schema.org/"}, "@graph": [{"@id": "#study", "@type": "CreativeWork", "name": "Camilleri18_eMDN", "keywords": ["neural correlates"], "accountablePerson": "[email protected]", "contributor": [{"@id": "[email protected]"}]}, {"@id": "https://schema.datalad.org/datalad_dataset#52142b84-dc76-11ea-98c5-7cdd908c7490", "@type": "Dataset", "version": "244a8ad43b00622989ae7f0d2b59c80697dadb80", "name": "Dataset_omorpallud_69", "url": "https://example.com/users/omorpallud/dataset_69", "author": [{"@id": "[email protected]"}], "description": "<this is an autogenerated description for dataset 52142b84-dc76-11ea-98c5-7cdd908c7490, since no description was provided by the author, and because google rich-results requires the description-property in schmema.org/Dataset metadatatypes>"}, {"@id": "#publicationList", "@list": [{"@id": "#publication[0]", "@type": "ScholarlyArticle", "headline": "multiple-demand Definition extended network and", "datePublished": 2171, "sameAs": "10.100/99.abc-70", "pagination": "1549-1564", "author": [{"@id": "[email protected]"}, {"@id": "[email protected]"}, {"@id": "[email protected]"}, {"@id": "[email protected]"}, {"@id": "[email protected]"}, {"@id": "[email protected]"}, {"@id": "[email protected]"}], "publication": {"@id": "https://schema.datalad.org/publication_event#Neuroimage", "@type": "PublicationEvent", "name": "Neuroimage"}, "isPartOf": {"@id": "#volume(165)", "@type": "PublicationVolume", "volumeNumber": 165}}]}, {"@id": "#personList", "@list": [{"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Upir Pulpula", "givenName": "Upir", "familyName": "Pulpula"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Ipalmad Selkolu", "givenName": "Ipalmad", "familyName": "Selkolu"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Kosselo Umalpalo", "givenName": "Kosselo", "familyName": "Umalpalo"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Dr. phil. Sansul Omelgorladi", "givenName": "Sansul", "familyName": "Omelgorladi", "honorificSuffix": "Dr. phil."}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Imolpir Mora", "givenName": "Imolpir", "familyName": "Mora"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Melmodmad Imolsel", "givenName": "Melmodmad", "familyName": "Imolsel"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Epalmollud Irigludkolo", "givenName": "Epalmollud", "familyName": "Irigludkolo"}, {"@id": "https://schema.datalad.org/person#[email protected]", "@type": "Person", "email": "[email protected]", "name": "Goso Pirpirludu", "givenName": "Goso", "familyName": "Pirpirludu"}]}]}}