diff --git a/.gitignore b/.gitignore index 7d065aca061..afeb206a241 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,8 @@ package-lock.json .env /nbproject/ -junit.xml - -/src/mirador-viewer/config.local.js +# import data python module +python_data_import/debug.log.txt +python_data_import/logs.txt +python_data_import/date.txt +*/__pycache__/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..16fbb4e8d5d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "python_data_import/lib"] + path = python_data_import/lib + url = https://github.com/dataquest-dev/dspace-blackbox-testing.git diff --git a/python_data_import/import/data/license_definitions.json b/python_data_import/import/data/license_definitions.json new file mode 100644 index 00000000000..b3fe1f90506 --- /dev/null +++ b/python_data_import/import/data/license_definitions.json @@ -0,0 +1,218 @@ +[ + { + "name": "GNU General Public Licence, version 3", + "definition": "http://opensource.org/licenses/GPL-3.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "GNU General Public License, version 2", + "definition": "http://www.gnu.org/licenses/gpl-2.0.html", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "The MIT License (MIT)", + "definition": "http://opensource.org/licenses/mit-license.php", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Artistic License 2.0", + "definition": "http://opensource.org/licenses/Artistic-2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Artistic License (Perl) 1.0", + "definition": "http://opensource.org/licenses/Artistic-Perl-1.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Attribution-NonCommercial-NoDerivs 3.0 Unported (CC BY-NC-ND 3.0)", + "definition": "http://creativecommons.org/licenses/by-nc-nd/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "BSD 2-Clause 'Simplified' or 'FreeBSD' license", + "definition": "http://opensource.org/licenses/BSD-2-Clause", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "BSD 3-Clause 'New' or 'Revised' license", + "definition": "http://opensource.org/licenses/BSD-3-Clause", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Attribution-NonCommercial 3.0 Unported (CC BY-NC 3.0)", + "definition": "http://creativecommons.org/licenses/by-nc/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Attribution-NonCommercial-ShareAlike 3.0 Unported (CC BY-NC-SA 3.0)", + "definition": "http://creativecommons.org/licenses/by-nc-sa/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Attribution-NoDerivs 3.0 Unported (CC BY-ND 3.0)", + "definition": "http://creativecommons.org/licenses/by-nd/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)", + "definition": "http://creativecommons.org/licenses/by-sa/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution 3.0 Unported (CC BY 3.0)", + "definition": "http://creativecommons.org/licenses/by/3.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "PDTSL", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-pdtsl", + "ePersonId": 1, + "labelId": 3, + "confirmation": 2, + "requiredInfo": "" + }, + { + "name": "HamleDT 1.0 Licence Agreement", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-hamledt", + "ePersonId": 1, + "labelId": 3, + "confirmation": 2, + "requiredInfo": "SEND_TOKEN, NAME, ADDRESS, COUNTRY, EXTRA_EMAIL" + }, + { + "name": "HamleDT 2.0 Licence Agreement", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-hamledt-2.0", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "SEND_TOKEN, NAME, ADDRESS, COUNTRY, EXTRA_EMAIL" + }, + { + "name": "Czech National Corpus (Shuffled Corpus Data)", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-cnc", + "ePersonId": 1, + "labelId": 2, + "confirmation": 1, + "requiredInfo": "" + }, + { + "name": "CC-BY-NC-SA + LDC99T42", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-pcedt2", + "ePersonId": 1, + "labelId": 3, + "confirmation": 1, + "requiredInfo": "" + }, + { + "name": "PDT 2.0 License", + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-pdt2", + "ePersonId": 1, + "labelId": 2, + "confirmation": 1, + "requiredInfo": "" + }, + { + "name": "CC0-No Rights Reserved", + "definition": "http://creativecommons.org/publicdomain/zero/1.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Apache License 2.0", + "definition": "http://opensource.org/licenses/Apache-2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution 4.0 International (CC BY 4.0)", + "definition": "http://creativecommons.org/licenses/by/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", + "definition": "http://creativecommons.org/licenses/by-sa/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution-NoDerivatives 4.0 International (CC BY-ND 4.0)", + "definition": "http://creativecommons.org/licenses/by-nd/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)", + "definition": "http://creativecommons.org/licenses/by-nc/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)", + "definition": "http://creativecommons.org/licenses/by-nc-sa/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "name": "Creative Commons - Attribution-NonCommercial-NoDerivatives 4.0 International (CC BY-NC-ND 4.0)", + "definition": "http://creativecommons.org/licenses/by-nc-nd/4.0/", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + } +] diff --git a/python_data_import/import/data/license_definitions_v2.json b/python_data_import/import/data/license_definitions_v2.json new file mode 100644 index 00000000000..f39b8c5a8bd --- /dev/null +++ b/python_data_import/import/data/license_definitions_v2.json @@ -0,0 +1,604 @@ +[ + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-2.10", + "name":"Licence Universal Dependencies v2.10", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-unisegs-1.0", + "name": "Universal Segmentations 1.0 License Terms", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-corefud-0.2", + "name": "Licence CorefUD v0.2", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-2.9", + "name": "Licence Universal Dependencies v2.9", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UDer-1.1", + "name": "Universal Derivations v1.1 License Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-2.8", + "name": "Licence Universal Dependencies v2.8", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-corefud-0.1", + "name": "Licence CorefUD v0.1", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/deep-sequoia-licence", + "name": "Deep Sequoia Licence", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-2.7", + "name": "Licence Universal Dependencies v2.7", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-mwe-1.2-raw", + "name": "PARSEME Shared Task Raw Corpus Data (v. 1.2) Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-mwe-1.2", + "name": "PARSEME Shared Task Data (v. 1.2) Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UDer-1.0", + "name": "Universal Derivations v1.0 License Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-2.6", + "name": "Licence Universal Dependencies v2.6", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.5", + "name": "Licence Universal Dependencies v2.5", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UDer-0.5", + "name": "Universal Derivations v0.5 License Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.4", + "name": "Licence Universal Dependencies v2.4", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-mwe-literal", + "name": "License agreement for The Multilingual corpus of literal occurrences of multiword expressions", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.3", + "name": "Licence Universal Dependencies v2.3", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-mwe-1.1", + "name": "PARSEME Shared Task Data (v. 1.1) Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.2", + "name": "Licence Universal Dependencies v2.2", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.1", + "name": "Licence Universal Dependencies v2.1", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-mwe-1.0", + "name": "PARSEME Shared Task Data (v. 1.0) Agreement", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-2.0", + "name": "Licence Universal Dependencies v2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-1.4", + "name": "Licence Universal Dependencies v1.4", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-1.3", + "name": "Licence Universal Dependencies v1.3", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-TAUS_QT21", + "name": "AGREEMENT ON THE USE OF DATA IN QT21 APE Task", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-TAUS_QT21", + "name": "AGREEMENT ON THE USE OF DATA IN QT21", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-1.2", + "name": "Licence Universal Dependencies v1.2", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-hamledt-3.0", + "name": "HamleDT 3.0 License Terms", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/licence-UD-1.1", + "name": "Licence Universal Dependencies v1.1", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by-nc-nd/4.0/", + "name": "Creative Commons - Attribution-NonCommercial-NoDerivatives 4.0 International (CC BY-NC-ND 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by-nc-sa/4.0/", + "name": "Creative Commons - Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by-nc/4.0/", + "name":"Creative Commons - Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by-nd/4.0/", + "name":"Creative Commons - Attribution-NoDerivatives 4.0 International (CC BY-ND 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by-sa/4.0/", + "name":"Creative Commons - Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/licenses/by/4.0/", + "name": "Creative Commons - Attribution 4.0 International (CC BY 4.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-ud-1.0", + "name":"Universal Dependencies 1.0 License Set", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://creativecommons.org/publicdomain/mark/1.0/", + "name": "Public Domain Mark (PD)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opendatacommons.org/licenses/pddl/summary/", + "name":"Open Data Commons Public Domain Dedication and License (PDDL)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opendatacommons.org/licenses/odbl/summary/", + "name":"Open Data Commons Open Database License (ODbL)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opendatacommons.org/licenses/by/summary/", + "name":"Open Data Commons Attribution License (ODC-By)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/MPL-2.0", + "name":"Mozilla Public License 2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/LGPL-3.0", + "name": "GNU Library or Lesser General Public License 3.0 (LGPL-3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/LGPL-2.1", + "name": "GNU Library or Lesser General Public License 2.1 or later (LGPL-2.1)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/LGPL-2.1", + "name": "GNU Library or Lesser General Public License 2.1 (LGPL-2.1)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/GPL-2.0", + "name":"GNU General Public License 2 or later (GPL-2.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/EPL-1.0", + "name":"Eclipse Public License 1.0 (EPL-1.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/CDDL-1.0", + "name": "Common Development and Distribution License (CDDL-1.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/AGPL-3.0", + "name": "Affero General Public License 3 (AGPL-3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://www.affero.org/oagpl.html", + "name":"Affero General Public License 1 (AGPL-1.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/Apache-2.0", + "name": "Apache License 2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/Artistic-2.0", + "name": "Artistic License 2.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/Artistic-Perl-1.0", + "name": "Artistic License (Perl) 1.0", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "http://opensource.org/licenses/GPL-3.0", + "name": "GNU General Public Licence, version 3", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://opensource.org/licenses/BSD-2-Clause", + "name":"BSD 2-Clause Simplified or FreeBSD license", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://opensource.org/licenses/BSD-3-Clause", + "name": "BSD 3-Clause New or Revised license", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/publicdomain/zero/1.0/", + "name": "Public Domain Dedication (CC Zero)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://opensource.org/licenses/mit-license.php", + "name": "The MIT License (MIT)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by/3.0/", + "name": "Creative Commons - Attribution 3.0 Unported (CC BY 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by-sa/3.0/", + "name": "Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by-nd/3.0/", + "name": "Attribution-NoDerivs 3.0 Unported (CC BY-ND 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by-nc-nd/3.0/", + "name": "Attribution-NonCommercial-NoDerivs 3.0 Unported (CC BY-NC-ND 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://www.gnu.org/licenses/gpl-2.0.html", + "name": "GNU General Public License, version 2", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by-nc-sa/3.0/", + "name": "Attribution-NonCommercial-ShareAlike 3.0 Unported (CC BY-NC-SA 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"http://creativecommons.org/licenses/by-nc/3.0/", + "name": "Attribution-NonCommercial 3.0 Unported (CC BY-NC 3.0)", + "ePersonId": 1, + "labelId": 1, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-lb", + "name": "Dictionary of Medieval Latin in the Czech Lands - digital version 2.2 License Agreement", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-cnc-data", + "name": "License Agreement for Czech National Corpus Data", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-NLPC-WeC", + "name": "NLP Centre Web Corpus License", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/licence-hamledt-2.0", + "name": "HamleD 2.0 Licence Agreement", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/licence-hamledt", + "name": "HamleD 1.0 Licence Agreement", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-cnc", + "name": "Czech National Corpus (Shuffled Corpus Data)", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-pdt2", + "name": "PDT 2.0 License", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition": "https://lindat.mff.cuni.cz/repository/xmlui/page/license-pcedt2", + "name": "CC-BY-NC-SA + LDC99T42", + "ePersonId": 1, + "labelId": 3, + "confirmation": 0, + "requiredInfo": "" + }, + { + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/licence-pdtsl", + "name": "PDTSL", + "ePersonId": 1, + "labelId": 2, + "confirmation": 0, + "requiredInfo": "" + }, + { + "id": 68, + "definition":"https://lindat.mff.cuni.cz/repository/xmlui/page/license-PAWS", + "name": "PAWS License", + "ePersonId": 1, + "labelId": 3, + "confirmation": 0, + "requiredInfo": "" + } + ] + \ No newline at end of file diff --git a/python_data_import/import/data/license_labels.json b/python_data_import/import/data/license_labels.json new file mode 100644 index 00000000000..bc687e33e80 --- /dev/null +++ b/python_data_import/import/data/license_labels.json @@ -0,0 +1,98 @@ +[ + { + "id": 1, + "label": "PUB", + "title": "Publicly Available", + "extended": false + }, + { + "id": 2, + "label": "ACA", + "title": "Academic Use", + "extended": false + }, + { + "id": 3, + "label": "RES", + "title": "Restricted Use", + "extended": false + }, + { + "id": 4, + "label": "CC", + "title": "Distributed under Creative Commons", + "extended": true + }, + { + "id": 5, + "label": "BY", + "title": "Attribution Required", + "extended": true + }, + { + "id": 6, + "label": "SA", + "title": "Share Alike", + "extended": true + }, + { + "id": 7, + "label": "NC", + "title": "Noncommercial", + "extended": true + }, + { + "id": 8, + "label": "ND", + "title": "No Derivative Works", + "extended": true + }, + { + "id": 9, + "label": "Inf", + "title": "Inform Before Use", + "extended": true + }, + { + "id": 10, + "label": "ReD", + "title": "Redeposit Modified", + "extended": true + }, + { + "id": 11, + "label": "ZERO", + "title": "No Copyright", + "extended": true + }, + { + "id": 12, + "label": "GPLv3", + "title": "GNU General Public License, version 3.0", + "extended": true + }, + { + "id": 13, + "label": "GPLv2", + "title": "GNU General Public License, version 2.0", + "extended": true + }, + { + "id": 14, + "label": "BSD", + "title": "BSD", + "extended": true + }, + { + "id": 15, + "label": "MIT", + "title": "The MIT License", + "extended": true + }, + { + "id": 16, + "label": "OSI", + "title": "The Open Source Initiative", + "extended": true + } +] \ No newline at end of file diff --git a/python_data_import/import_initial_data.py b/python_data_import/import_initial_data.py new file mode 100644 index 00000000000..f848ad10fc5 --- /dev/null +++ b/python_data_import/import_initial_data.py @@ -0,0 +1,15 @@ +import sys +sys.path.insert(1, 'lib') +from support import logs + +orig = logs.write_to_console + +logs.write_to_console = True + +import import_license_labels_2_db +import import_licenses_2_db + +import_license_labels_2_db.import_license_labels() +import_licenses_2_db.import_licenses() + +logs.write_to_console = orig diff --git a/python_data_import/import_license_labels_2_db.py b/python_data_import/import_license_labels_2_db.py new file mode 100644 index 00000000000..b4b77ddc0b3 --- /dev/null +++ b/python_data_import/import_license_labels_2_db.py @@ -0,0 +1,41 @@ +import json + +import const +from support.dspace_proxy import rest_proxy +from support.item_checking import import_license_label +from support.logs import log, Severity + + +def import_license_labels(): + log('Going to import license labels.') + # Opening JSON file + with open('import/data/license_labels.json') as json_file: + licenseLabelsJson = json.load(json_file) + lic_labels = {} + lic_respo = rest_proxy.d.api_get(const.API_URL + '/core/clarinlicenselabels?page=0&size=2000').json() + if const.EMBEDDED in lic_respo: + license_labels = lic_respo["_embedded"]["clarinlicenselabels"] + for lic in license_labels: + if lic["label"] in lic_labels: + log("DUPLICATE LABELS FOUND ON WEBSITE!!", Severity.WARN) + lic_labels[lic["label"]] = lic + + for licenseLabel in licenseLabelsJson: + if licenseLabel["label"] in lic_labels: + log(f"License label {licenseLabel['title']} was already imported; skipping.") + all_good = True + check_attrs = ["id", "title", "extended"] + original = licenseLabel + installed = lic_labels[licenseLabel["label"]] + for attr in check_attrs: + if original[attr] != installed[attr]: + log(f"bad value of {attr} for {licenseLabel['label']}: original {original[attr]};" + f" found on server: {installed[attr]}.", Severity.WARN) + all_good = False + if not all_good: + log("incorrectly imported icense label " + str(licenseLabel), Severity.WARN) + else: + import_license_label(licenseLabel["id"], licenseLabel["label"], licenseLabel["title"], licenseLabel["extended"]) + log(f'License label: {licenseLabel} imported!') + + diff --git a/python_data_import/import_licenses_2_db.py b/python_data_import/import_licenses_2_db.py new file mode 100644 index 00000000000..d3a897ec2a9 --- /dev/null +++ b/python_data_import/import_licenses_2_db.py @@ -0,0 +1,26 @@ +import json + +import const +from support.dspace_proxy import rest_proxy +from support.item_checking import import_license +from support.logs import log + + +def import_licenses(): + log('Going to import licenses.') + # Opening JSON file + with open('import/data/license_definitions_v2.json') as json_file: + license_definitions = json.load(json_file) + lic_def = [] + lic_respo = rest_proxy.d.api_get(const.API_URL + '/core/clarinlicenses?page=0&size=2000').json() + if const.EMBEDDED in lic_respo: + licenses = lic_respo["_embedded"]["clarinlicenses"] + for lic in licenses: + lic_def.append(lic["definition"]) + for lic in license_definitions: + if lic["definition"] in lic_def: + log(lic["definition"] + " was already imported; skipping.") + continue + else: + import_license(lic["name"], lic["definition"], lic["labelId"], lic["confirmation"], lic["requiredInfo"]) + diff --git a/python_data_import/lib b/python_data_import/lib new file mode 160000 index 00000000000..a22159ab082 --- /dev/null +++ b/python_data_import/lib @@ -0,0 +1 @@ +Subproject commit a22159ab0829e8b0fd55ad43377fbadfc44ce29d