datastreams: funders celery task using ror-http (#331)

invenio_vocabularies/services/tasks.py

@@ -11,6 +11,7 @@
 from flask import current_app
 
 from ..datastreams.factories import DataStreamFactory
+from ..factories import get_vocabulary_config
 
 
 @shared_task(ignore_result=True)
@@ -26,3 +27,32 @@ def process_datastream(config):
         if result.errors:
             for err in result.errors:
                 current_app.logger.error(err)
+
+
+@shared_task()
+def import_funders():
+    """Import the funders vocabulary.
+
+    Only new records are imported.
+    Existing records are not updated.
+    """
+    vc = get_vocabulary_config("funders")
+    config = vc.get_config()
+
+    # When importing funders via a Celery task, make sure that we are automatically downloading the ROR file,
+    # instead of relying on a local file on the file system.
+    if config["readers"][0]["type"] == "ror-http":
+        readers_config_with_ror_http = config["readers"]
+    else:
+        readers_config_with_ror_http = [{"type": "ror-http"}] + config["readers"]
+
+    ds = DataStreamFactory.create(
+        readers_config=readers_config_with_ror_http,
+        transformers_config=config.get("transformers"),
+        writers_config=config["writers"],
+    )
+
+    for result in ds.process():
+        if result.errors:
+            for err in result.errors:
+                current_app.logger.error(err)

setup.cfg

@@ -98,6 +98,8 @@ invenio_assets.webpack =
     invenio_vocabularies = invenio_vocabularies.webpack:theme
 invenio_i18n.translations =
     invenio_vocabularies = invenio_vocabularies
+invenio_celery.tasks =
+    invenio_vocabularies_services = invenio_vocabularies.services.tasks
 
 
 [build_sphinx]