Release of version 1.0.7 (#147)

* Upgrade to rapydo 0.6.6

* SDC: Introduced error 4044 - Parameter download_path is empty

* SDC: importmanager is now https

* Working on recursive pid cache

* Added workers-only mode

docs/deploy/startup.md

@@ -5,7 +5,7 @@
 To clone the working code:
 
 ```bash
-$ VERSION=1.0.6 \
+$ VERSION=1.0.7 \
     && git clone https://github.com/EUDAT-B2STAGE/http-api.git \
     && cd http-api \
     && git checkout $VERSION
@@ -51,7 +51,7 @@ $ rapydo init
 If you wish to __**manually upgrade**__:
 
 ```bash
-VERSION="0.6.5"
+VERSION="0.6.6"
 git checkout $VERSION
 
 # supposely the rapydo framework has been updated, so you need to check:

docs/quick_start.md

@@ -24,7 +24,7 @@ cd latest
 ################
 # install the corrensponding rapydo framework version
 sudo pip3 install rapydo-controller
-rapydo --project b2stage install --git 0.6.5
+rapydo --project b2stage install --git 0.6.6
 
 # build and run
 rapydo --project b2stage init

projects/b2stage/project_configuration.yaml

@@ -5,8 +5,8 @@
 project:
   title: EUDAT-B2STAGE HTTP-API server
   description: Processing data in EUDAT domains
-  version: 1.0.6
-  rapydo: 0.6.5
+  version: 1.0.7
+  rapydo: 0.6.6
 
 tags:
   eudat: all endpoints associated to EUDAT services
@@ -111,38 +111,42 @@ releases:
   '0.6.0':
     type: RC1
     rapydo: 0.5.3
-    status: released
+    status: discontinued
   '0.6.1':
     type: RC2
     rapydo: 0.5.4
-    status: released
+    status: discontinued
   '1.0.0':
     type: stable
     rapydo: 0.6.0
-    status: released
+    status: discontinued
   '1.0.1':
     type: patch
     rapydo: 0.6.1
-    status: released
+    status: discontinued
   '1.0.2':
     type: patch
     rapydo: 0.6.1
-    status: released
+    status: discontinued
   '1.0.3':
     type: stable
     rapydo: 0.6.2
-    status: released
+    status: discontinued
   '1.0.4':
     type: stable
     rapydo: 0.6.3
-    status: released
+    status: discontinued
   '1.0.5':
     type: stable
     rapydo: 0.6.4
     status: released
   '1.0.6':
-    type: development
+    type: stable
     rapydo: 0.6.5
+    status: released
+  '1.0.7':
+    type: development
+    rapydo: 0.6.6
     status: development
 
 controller:

projects/b2stage/requirements.txt

@@ -1,2 +1,2 @@
-git+https://github.com/rapydo/[email protected].5
-git+https://github.com/rapydo/[email protected].5
+git+https://github.com/rapydo/[email protected].6
+git+https://github.com/rapydo/[email protected].6

projects/seadata/backend/apis/commons/seadatacloud.py

@@ -64,6 +64,7 @@ class ErrorCodes(object):
     MISSING_DOWNLOAD_PATH_PARAM = ("4041", "Parameter download_path is missing")
     UNABLE_TO_CREATE_ZIP_FILE = ("4042", "Unable to create merged zip file")
     INVALID_ZIP_SPLIT_OUTPUT = ("4043", "Unable to retrieve results from zip split")
+    EMPTY_DOWNLOAD_PATH_PARAM = ("4044", "Parameter download_path is empty")
 
 
 class Metadata(object):
@@ -109,7 +110,7 @@ def post(self, payload, backdoor=False, edmo_code=None):
                 "The following json should be sent to ImportManagerAPI, " +
                 "but you enabled the backdoor")
             log.info(payload)
-            return True
+            return False
 
         from restapi.confs import PRODUCTION
         if not PRODUCTION:
@@ -122,19 +123,23 @@ def post(self, payload, backdoor=False, edmo_code=None):
         import requests
         # print("TEST", self._uri)
         r = requests.post(self._uri, json=payload)
+        log.info("POST external IM API, status=%s, uri=%s", r.status_code, self._uri)
 
         from utilities import htmlcodes as hcodes
         if r.status_code != hcodes.HTTP_OK_BASIC:
             log.error(
                 "CDI: failed to call external APIs (status: %s, uri: %s)",
-                (r.status_code, self._uri))
+                r.status_code, self._uri)
             return False
         else:
             log.info(
                 "CDI: called POST on external APIs (status: %s, uri: %s)",
-                (r.status_code, self._uri))
+                r.status_code, self._uri)
             return True
 
+        log.warning("Unknown external APIs status")
+        return False
+
 
 # NOTE this function is outside the previous class, and self is passed as parameter
 def seadata_pid(self, pid):

projects/seadata/backend/tasks/seadata.py

@@ -136,6 +136,11 @@ def download_batch(self, batch_path, local_path, myjson):
                 ErrorCodes.MISSING_DOWNLOAD_PATH_PARAM,
                 myjson, backdoor, self
             )
+        if download_path == '':
+            return notify_error(
+                ErrorCodes.EMPTY_DOWNLOAD_PATH_PARAM,
+                myjson, backdoor, self
+            )
 
         file_count = params.get("data_file_count")
         if file_count is None:
@@ -199,6 +204,13 @@ def download_batch(self, batch_path, local_path, myjson):
                 myjson, backdoor, self,
                 subject=download_url
             )
+        except requests.exceptions.MissingSchema as e:
+            log.error(str(e))
+            return notify_error(
+                ErrorCodes.UNREACHABLE_DOWNLOAD_PATH,
+                myjson, backdoor, self,
+                subject=download_url
+            )
 
         if r.status_code != 200:
 
@@ -316,7 +328,8 @@ def download_batch(self, batch_path, local_path, myjson):
         log.info("Copied: %s", irods_batch_file)
 
         request_edmo_code = myjson.get('edmo_code')
-        ext_api.post(myjson, backdoor=backdoor, edmo_code=request_edmo_code)
+        ret = ext_api.post(myjson, backdoor=backdoor, edmo_code=request_edmo_code)
+        log.info('CDI IM CALL = %s', ret)
         return "COMPLETED"
 
 
@@ -449,7 +462,8 @@ def move_to_production_task(self, batch_id, irods_path, myjson):
             myjson[key] = value
         if len(errors) > 0:
             myjson['errors'] = errors
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
 
         out = {
             'total': total, 'step': counter,
@@ -708,7 +722,6 @@ def unrestricted_order(self, order_id, order_path, zip_file_name, myjson):
         #########################
         # NOTE: should I close the iRODS session ?
         #########################
-        pass
         # imain.prc
 
         ##################
@@ -735,7 +748,8 @@ def unrestricted_order(self, order_id, order_path, zip_file_name, myjson):
             myjson['errors'] = errors
         myjson[reqkey] = self.request.id
         # log.pp(myjson)
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
 
         ##################
         out = {
@@ -787,6 +801,11 @@ def download_restricted_order(self, order_id, order_path, myjson):
                 ErrorCodes.MISSING_DOWNLOAD_PATH_PARAM,
                 myjson, backdoor, self
             )
+        if download_path == '':
+            return notify_error(
+                ErrorCodes.EMPTY_DOWNLOAD_PATH_PARAM,
+                myjson, backdoor, self
+            )
 
         # NAME OF FINAL ZIP
         filename = params.get('zipfile_name')
@@ -876,6 +895,13 @@ def download_restricted_order(self, order_id, order_path, myjson):
                 myjson, backdoor, self,
                 subject=download_url
             )
+        except requests.exceptions.MissingSchema as e:
+            log.error(str(e))
+            return notify_error(
+                ErrorCodes.UNREACHABLE_DOWNLOAD_PATH,
+                myjson, backdoor, self,
+                subject=download_url
+            )
 
         if r.status_code != 200:
 
@@ -1039,7 +1065,7 @@ def download_restricted_order(self, order_id, order_path, myjson):
             log.info("Creating a backup copy of final zip")
             backup_zip = final_zip + ".bak"
             if imain.is_dataobject(backup_zip):
-                log.info("%s already exists, removing previous backup")
+                log.info("%s already exists, removing previous backup", backup_zip)
                 imain.remove(backup_zip)
             imain.move(final_zip, backup_zip)
 
@@ -1133,7 +1159,8 @@ def download_restricted_order(self, order_id, order_path, myjson):
 
         if len(errors) > 0:
             myjson['errors'] = errors
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
         return "COMPLETED"
 
         # 0 - avoid concurrent execution, introduce a cache like:
@@ -1214,7 +1241,8 @@ def delete_orders(self, orders_path, local_orders_path, myjson):
 
         if len(errors) > 0:
             myjson['errors'] = errors
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
         return "COMPLETED"
 
 
@@ -1280,10 +1308,24 @@ def delete_batches(self, batches_path, local_batches_path, myjson):
 
         if len(errors) > 0:
             myjson['errors'] = errors
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
         return "COMPLETED"
 
 
+def recursive_list_files(imain, irods_path):
+
+    data = []
+    for current in imain.list(irods_path):
+        ifile = path.join(irods_path, current, return_str=True)
+        if imain.is_dataobject(ifile):
+            data.append(ifile)
+        else:
+            data.extend(recursive_list_files(imain, ifile))
+
+    return data
+
+
 @celery_app.task(bind=True)
 @send_errors_by_email
 def cache_batch_pids(self, irods_path):
@@ -1300,8 +1342,14 @@ def cache_batch_pids(self, irods_path):
             'errors': 0,
         }
 
-        for current in imain.list(irods_path):
-            ifile = path.join(irods_path, current, return_str=True)
+        data = recursive_list_files(imain, irods_path)
+        log.info("Found %s files", len(data))
+
+        # for current in imain.list(irods_path):
+        #     ifile = path.join(irods_path, current, return_str=True)
+
+        for ifile in data:
+
             stats['total'] += 1
 
             pid = r.get(ifile)
@@ -1406,6 +1454,7 @@ def list_resources(self, batch_path, order_path, myjson):
         for n in orders:
             myjson[param_key]['orders'].append(n)
 
-        ext_api.post(myjson, backdoor=backdoor)
+        ret = ext_api.post(myjson, backdoor=backdoor)
+        log.info('CDI IM CALL = %s', ret)
 
         return "COMPLETED"

projects/seadata/builds/celery/Dockerfile

@@ -1,4 +1,4 @@
-FROM rapydo/backendirods:0.6.5
+FROM rapydo/backendirods:0.6.6
 MAINTAINER "Mattia D'Antonio <[email protected]>"
 
 RUN apt-get update \