Implement uploading <project_uuid>.hazmapper to tapis. (#54)

* Implement uploading <project_uuid>.hazmapper to tapis.

* Improve project save behavior.

* Fix tests.

* Add tests.

* Add deleting agave files.

* Add link parameter to observable projects.

* Fix tests for project deletion.

* Add migrations for project system information.

* Save to json rather than empty file.

* Fix tests for file suffix.

* Remove projectid from tile server tests.

* Fix tile server test.

* Change test route for tile server.

* file_suffix to file_name.

* Combine link and export functions and make file deletion a task to
avoid 504.

* Refactor project export function.

* Fix tests for project export.

* Fix observable projects for new export.

* Retry.

* Correct None condition for system_path.

* Remove exception for updating tile servers.

* Remove observable project export.

* Export project for observable projects.

* Remove system_name.

* Add migration and remove export route from public projects.

geoapi/migrations/versions/751a1f2e8b5a_.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 751a1f2e8b5a
+Revises: 9d043dc43f64
+Create Date: 2021-04-30 21:09:16.630406
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '751a1f2e8b5a'
+down_revision = '9d043dc43f64'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('projects', sa.Column('system_file', sa.String(), nullable=True))
+    op.add_column('projects', sa.Column('system_id', sa.String(), nullable=True))
+    op.add_column('projects', sa.Column('system_name', sa.String(), nullable=True))
+    op.add_column('projects', sa.Column('system_path', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('projects', 'system_path')
+    op.drop_column('projects', 'system_name')
+    op.drop_column('projects', 'system_id')
+    op.drop_column('projects', 'system_file')
+    # ### end Alembic commands ###

geoapi/migrations/versions/a1f677535e21_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: a1f677535e21
+Revises: 751a1f2e8b5a
+Create Date: 2021-05-24 16:42:16.631669
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a1f677535e21'
+down_revision = '751a1f2e8b5a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('projects', 'system_name')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('projects', sa.Column('system_name', sa.VARCHAR(), autoincrement=False, nullable=True))
+    # ### end Alembic commands ###

geoapi/models/project.py

@@ -22,6 +22,9 @@ class Project(Base):
     id = Column(Integer, primary_key=True)
     uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False)
     tenant_id = Column(String, nullable=False)
+    system_id = Column(String, nullable=True)
+    system_path = Column(String, nullable=True)
+    system_file = Column(String, nullable=True)
     name = Column(String, nullable=False)
     description = Column(String)
     public = Column(Boolean, default=False)

geoapi/routes/projects.py

@@ -58,7 +58,10 @@
     'name': fields.String(required=True),
     'description': fields.String(required=False),
     'public': fields.Boolean(required=False),
-    'uuid': fields.String()
+    'uuid': fields.String(),
+    'system_file': fields.String(),
+    'system_id': fields.String(),
+    'system_path': fields.String()
 })
 
 user = api.model('User', {
@@ -124,6 +127,12 @@
     'path': fields.String(required=True)
 })
 
+tapis_save_file = api.model('TapisSaveFile', {
+    'system_id': fields.String(required=True),
+    'path': fields.String(required=True),
+    'file_name': fields.String(required=True)
+})
+
 tapis_files_import = api.model('TapisFileImport', {
     'files': fields.List(fields.Nested(tapis_file), required=True)
 })
@@ -209,7 +218,7 @@ def delete(self, projectId: int):
         u = request.current_user
         logger.info("Delete project:{} for user:{}".format(projectId,
                                                            u.username))
-        return ProjectsService.delete(projectId)
+        return ProjectsService.delete(u, projectId)
 
     @api.doc(id="updateProject",
              description="Update metadata about a project")
@@ -223,6 +232,19 @@ def put(self, projectId: int):
                                       data=api.payload)
 
 
+@api.route('/<int:projectId>/export/')
+class ExportProject(Resource):
+    @project_permissions
+    @api.expect(tapis_save_file)
+    @api.doc(id="exportProject",
+             description='Save a project file to tapis')
+    @api.marshal_with(project)
+    def put(self, projectId):
+        u = request.current_user
+        logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload))
+        return ProjectsService.export(u, api.payload, False, projectId)
+
+
 @api.route('/<int:projectId>/users/')
 class ProjectUsersResource(Resource):
 
@@ -334,7 +356,7 @@ def post(self, projectId: int, featureId: int):
 
 
 @api.route('/<int:projectId>/features/<int:featureId>/styles/')
-class ProjectFeaturePropertiesResource(Resource):
+class ProjectFeatureStylesResource(Resource):
 
     @api.doc(id="updateFeatureStyles",
              description="Update the styles of a feature. This will replace any styles"
@@ -633,8 +655,7 @@ def put(self, projectId: int):
         logger.info("Update project:{} for user:{}".format(projectId,
                                                            u.username))
 
-        ts = FeaturesService.updateTileServers(projectId=projectId,
-                                               dataList=api.payload)
+        ts = FeaturesService.updateTileServers(dataList=api.payload)
         return ts
 
 
@@ -647,7 +668,7 @@ class ProjectTileServerResource(Resource):
     def delete(self, projectId: int, tileServerId: int) -> str:
         logger.info("Delete tile server:{} in project:{} for user:{}".format(
             tileServerId, projectId, request.current_user.username))
-        FeaturesService.deleteTileServer(projectId, tileServerId)
+        FeaturesService.deleteTileServer(tileServerId)
         return "Tile Server {id} deleted".format(id=tileServerId)
 
     @api.doc(id="updateTileServer",
@@ -659,6 +680,5 @@ def put(self, projectId: int, tileServerId: int):
         logger.info("Update project:{} for user:{}".format(projectId,
                                                            u.username))
 
-        return FeaturesService.updateTileServer(projectId=projectId,
-                                                tileServerId=tileServerId,
+        return FeaturesService.updateTileServer(tileServerId=tileServerId,
                                                 data=api.payload)

geoapi/services/features.py

@@ -543,21 +543,21 @@ def getTileServers(projectId: int) -> List[TileServer]:
         return tile_servers
 
     @staticmethod
-    def deleteTileServer(projectId: int, tileServerId: int) -> None:
+    def deleteTileServer(tileServerId: int) -> None:
         ts = db_session.query(TileServer).get(tileServerId)
         db_session.delete(ts)
         db_session.commit()
 
     @staticmethod
-    def updateTileServer(projectId: int, tileServerId: int, data: dict):
+    def updateTileServer(tileServerId: int, data: dict):
         ts = db_session.query(TileServer).get(tileServerId)
         for key, value in data.items():
             setattr(ts, key, value)
         db_session.commit()
         return ts
 
     @staticmethod
-    def updateTileServers(projectId: int, dataList: List[dict]):
+    def updateTileServers(dataList: List[dict]):
         ret_list = []
         for tsv in dataList:
             ts = db_session.query(TileServer).get(int(tsv['id']))

geoapi/services/projects.py

@@ -10,7 +10,7 @@
 from geoapi.services.users import UserService
 from geoapi.utils.agave import AgaveUtils, get_system_users
 from geoapi.utils.assets import get_project_asset_dir
-from geoapi.tasks.external_data import import_from_agave
+from geoapi.tasks.external_data import import_from_agave, delete_agave_file
 from geoapi.log import logging
 from geoapi.exceptions import ApiException, ObservableProjectAlreadyExists
 
@@ -30,7 +30,6 @@ def create(data: dict, user: User) -> Project:
         :param user: User
         :return: Project
         """
-
         project = Project(**data)
         project.tenant_id = user.tenant_id
         project.users.append(user)
@@ -55,9 +54,11 @@ def createRapidProject(data: dict, user: User) -> Project:
         system = AgaveUtils(user.jwt).systemsGet(systemId)
         proj = Project(
             name=name,
-            description=system["description"],
-            tenant_id=user.tenant_id
+            description=system['description'],
+            tenant_id=user.tenant_id,
+            system_id=systemId
         )
+
         obs = ObservableDataProject(
             system_id=systemId,
             path=path
@@ -80,6 +81,65 @@ def createRapidProject(data: dict, user: User) -> Project:
             raise ObservableProjectAlreadyExists("'{}' project already exists".format(name))
         import_from_agave.apply_async(args=[obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id])
 
+        ProjectsService.export(user,
+                               {'system_id': systemId,
+                                'path': folder_name,
+                                'link': True,
+                                'file_name': ''
+                                },
+                               True,
+                               proj.id)
+
+        return proj
+
+    @staticmethod
+    def export(user: User,
+               data: dict,
+               observable: bool,
+               project_id: int) -> Project:
+        """
+        Save a project UUID file to tapis
+        :param user: User
+        :param data: dict
+        :return: None
+        """
+        proj = ProjectsService.get(project_id=project_id)
+
+        # If already has a saved file remove it
+        if proj.system_path is not None:
+            delete_agave_file.apply_async(args=[proj.system_id,
+                                                '{}/{}'.format(proj.system_path,
+                                                               proj.system_file),
+                                                user.id])
+
+        path = data['path']
+
+        if data['file_name'] == '':
+            file_prefix = str(proj.uuid)
+        else:
+            file_prefix = str(data['file_name'])
+
+        file_name = '{}.{}'.format(file_prefix, 'hazmapper')
+
+        # if 'project' not in data['system_id'] and path == '/':
+        if ('project' not in data['system_id'] and path == '/') or observable:
+            path = "/{}/{}".format(user.username, path)
+
+        proj.system_path = path
+        proj.system_file = file_name
+        proj.system_id = data['system_id']
+        db_session.commit()
+
+        file_content = {
+            'uuid': str(proj.uuid)
+        }
+
+        AgaveUtils(user.jwt).postFile(data['system_id'],
+                                      path,
+                                      file_name,
+                                      file_content
+                                      )
+
         return proj
 
     @staticmethod
@@ -236,15 +296,21 @@ def update(projectId: int, data: dict) -> Project:
         return current_project
 
     @staticmethod
-    def delete(projectId: int) -> dict:
+    def delete(user: User, projectId: int) -> dict:
         """
         Delete a project and all its Features and assets
         :param projectId: int
         :return:
         """
         proj = db_session.query(Project).get(projectId)
+
         db_session.delete(proj)
         db_session.commit()
+
+        if proj.system_path is not None:
+            delete_agave_file.apply_async(args=[proj.system_id,
+                                                proj.system_path + '/' + proj.system_file,
+                                                user.id])
         assets_folder = get_project_asset_dir(projectId)
         try:
             shutil.rmtree(assets_folder)

geoapi/tasks/external_data.py

@@ -127,6 +127,12 @@ def _update_point_cloud_task(pointCloudId: int, description: str = None, status:
         raise
 
 
+@app.task(rate_limit="1/s")
+def delete_agave_file(system_id: str, system_path: str, userId: int):
+    user = db_session.query(User).get(userId)
+    AgaveUtils(user.jwt).deleteFile(system_id, system_path)
+
+
 @app.task(rate_limit="1/s")
 def import_point_clouds_from_agave(userId: int, files, pointCloudId: int):
     user = db_session.query(User).get(userId)

geoapi/tests/api_tests/test_feature_service.py

@@ -139,8 +139,7 @@ def test_remove_tile_server(projects_fixture):
     }
 
     tile_server = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
-    FeaturesService.deleteTileServer(projects_fixture.id,
-                                     tile_server.id)
+    FeaturesService.deleteTileServer(tile_server.id)
 
     assert db_session.query(TileServer).count() == 0
 
@@ -158,8 +157,7 @@ def test_update_tile_server(projects_fixture):
         "name": "NewTestName",
     }
 
-    updated_tile_server = FeaturesService.updateTileServer(projectId=projects_fixture.id,
-                                                           tileServerId=1,
+    updated_tile_server = FeaturesService.updateTileServer(tileServerId=1,
                                                            data=updated_data)
     assert updated_tile_server.name == "NewTestName"
 
@@ -177,7 +175,8 @@ def test_update_tile_servers(projects_fixture):
     updated_data = [{"id": resp1.id, "name": "NewTestName1"},
                     {"id": resp2.id, "name": "NewTestName2"}]
 
-    updated_tile_server_list = FeaturesService.updateTileServers(projectId=projects_fixture.id, dataList=updated_data)
+    updated_tile_server_list = FeaturesService.updateTileServers(dataList=updated_data)
+
 
     assert updated_tile_server_list[0].name == "NewTestName1"
     assert updated_tile_server_list[1].name == "NewTestName2"

geoapi/tests/api_tests/test_projects_routes.py

@@ -317,6 +317,7 @@ def test_observable_project(test_client,
                             userdata,
                             get_system_users_mock,
                             agave_utils_with_geojson_file_mock,
+                            projects_fixture,
                             import_from_agave_mock):
     u1 = db_session.query(User).get(1)
     resp = test_client.post(
@@ -367,3 +368,19 @@ def test_update_project_unauthorized_guest(test_client, public_projects_fixture)
         json=data
     )
     assert resp.status_code == 403
+
+
+def test_export_project(test_client,
+                        projects_fixture,
+                        get_system_users_mock,
+                        agave_utils_with_geojson_file_mock):
+    u1 = db_session.query(User).get(1)
+    resp = test_client.put(
+        '/projects/1/export/',
+        json={"system_id": "testSystem",
+              "path": "testPath",
+              "file_name": "testFilename",
+              "link": False},
+        headers={'x-jwt-assertion-test': u1.jwt}
+    )
+    assert resp.status_code == 200

geoapi/tests/api_tests/test_projects_service.py

@@ -79,6 +79,7 @@ def test_get_features_filter_type(projects_fixture,
     project_features = ProjectsService.getFeatures(projects_fixture.id, query)
     assert len(project_features['features']) == 0
 
+
 def test_update_project(projects_fixture):
     data = {
         "name": "new name",

geoapi/tests/conftest.py

@@ -396,7 +396,8 @@ def agave_utils_with_geojson_file_mock(agave_file_listings_mock, geojson_file_fi
         MockAgaveUtils().listing.return_value = agave_file_listings_mock
         MockAgaveUtils().getFile.return_value = geojson_file_fixture
         MockAgaveUtils().systemsGet.return_value = {"id": "testSystem",
-                                                    "description": "System Description"}
+                                                    "description": "System Description",
+                                                    "name": "System Name"}
         yield MockAgaveUtils()