diff --git a/geoapi/migrations/versions/751a1f2e8b5a_.py b/geoapi/migrations/versions/751a1f2e8b5a_.py new file mode 100644 index 00000000..7e012945 --- /dev/null +++ b/geoapi/migrations/versions/751a1f2e8b5a_.py @@ -0,0 +1,34 @@ +"""empty message + +Revision ID: 751a1f2e8b5a +Revises: 9d043dc43f64 +Create Date: 2021-04-30 21:09:16.630406 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '751a1f2e8b5a' +down_revision = '9d043dc43f64' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects', sa.Column('system_file', sa.String(), nullable=True)) + op.add_column('projects', sa.Column('system_id', sa.String(), nullable=True)) + op.add_column('projects', sa.Column('system_name', sa.String(), nullable=True)) + op.add_column('projects', sa.Column('system_path', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('projects', 'system_path') + op.drop_column('projects', 'system_name') + op.drop_column('projects', 'system_id') + op.drop_column('projects', 'system_file') + # ### end Alembic commands ### diff --git a/geoapi/migrations/versions/a1f677535e21_.py b/geoapi/migrations/versions/a1f677535e21_.py new file mode 100644 index 00000000..8f7abeb2 --- /dev/null +++ b/geoapi/migrations/versions/a1f677535e21_.py @@ -0,0 +1,28 @@ +"""empty message + +Revision ID: a1f677535e21 +Revises: 751a1f2e8b5a +Create Date: 2021-05-24 16:42:16.631669 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a1f677535e21' +down_revision = '751a1f2e8b5a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('projects', 'system_name') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects', sa.Column('system_name', sa.VARCHAR(), autoincrement=False, nullable=True)) + # ### end Alembic commands ### diff --git a/geoapi/models/project.py b/geoapi/models/project.py index b58e4d5a..a8aabe9e 100644 --- a/geoapi/models/project.py +++ b/geoapi/models/project.py @@ -22,6 +22,9 @@ class Project(Base): id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False) tenant_id = Column(String, nullable=False) + system_id = Column(String, nullable=True) + system_path = Column(String, nullable=True) + system_file = Column(String, nullable=True) name = Column(String, nullable=False) description = Column(String) public = Column(Boolean, default=False) diff --git a/geoapi/routes/projects.py b/geoapi/routes/projects.py index dca7d3c7..27fc5507 100644 --- a/geoapi/routes/projects.py +++ b/geoapi/routes/projects.py @@ -58,7 +58,10 @@ 'name': fields.String(required=True), 'description': fields.String(required=False), 'public': fields.Boolean(required=False), - 'uuid': fields.String() + 'uuid': fields.String(), + 'system_file': fields.String(), + 'system_id': fields.String(), + 'system_path': fields.String() }) user = api.model('User', { @@ -124,6 +127,12 @@ 'path': fields.String(required=True) }) +tapis_save_file = api.model('TapisSaveFile', { + 'system_id': fields.String(required=True), + 'path': fields.String(required=True), + 'file_name': fields.String(required=True) +}) + tapis_files_import = api.model('TapisFileImport', { 'files': fields.List(fields.Nested(tapis_file), required=True) }) @@ -209,7 +218,7 @@ def delete(self, projectId: int): u = request.current_user logger.info("Delete project:{} for user:{}".format(projectId, u.username)) - return ProjectsService.delete(projectId) + return ProjectsService.delete(u, projectId) @api.doc(id="updateProject", description="Update metadata about a project") @@ -223,6 +232,19 @@ def put(self, projectId: int): data=api.payload) +@api.route('//export/') +class ExportProject(Resource): + @project_permissions + @api.expect(tapis_save_file) + @api.doc(id="exportProject", + description='Save a project file to tapis') + @api.marshal_with(project) + def put(self, projectId): + u = request.current_user + logger.info("Saving project to tapis for user {}: {}".format(u.username, api.payload)) + return ProjectsService.export(u, api.payload, False, projectId) + + @api.route('//users/') class ProjectUsersResource(Resource): @@ -334,7 +356,7 @@ def post(self, projectId: int, featureId: int): @api.route('//features//styles/') -class ProjectFeaturePropertiesResource(Resource): +class ProjectFeatureStylesResource(Resource): @api.doc(id="updateFeatureStyles", description="Update the styles of a feature. This will replace any styles" @@ -633,8 +655,7 @@ def put(self, projectId: int): logger.info("Update project:{} for user:{}".format(projectId, u.username)) - ts = FeaturesService.updateTileServers(projectId=projectId, - dataList=api.payload) + ts = FeaturesService.updateTileServers(dataList=api.payload) return ts @@ -647,7 +668,7 @@ class ProjectTileServerResource(Resource): def delete(self, projectId: int, tileServerId: int) -> str: logger.info("Delete tile server:{} in project:{} for user:{}".format( tileServerId, projectId, request.current_user.username)) - FeaturesService.deleteTileServer(projectId, tileServerId) + FeaturesService.deleteTileServer(tileServerId) return "Tile Server {id} deleted".format(id=tileServerId) @api.doc(id="updateTileServer", @@ -659,6 +680,5 @@ def put(self, projectId: int, tileServerId: int): logger.info("Update project:{} for user:{}".format(projectId, u.username)) - return FeaturesService.updateTileServer(projectId=projectId, - tileServerId=tileServerId, + return FeaturesService.updateTileServer(tileServerId=tileServerId, data=api.payload) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index cd696d77..a2cb2f9b 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -543,13 +543,13 @@ def getTileServers(projectId: int) -> List[TileServer]: return tile_servers @staticmethod - def deleteTileServer(projectId: int, tileServerId: int) -> None: + def deleteTileServer(tileServerId: int) -> None: ts = db_session.query(TileServer).get(tileServerId) db_session.delete(ts) db_session.commit() @staticmethod - def updateTileServer(projectId: int, tileServerId: int, data: dict): + def updateTileServer(tileServerId: int, data: dict): ts = db_session.query(TileServer).get(tileServerId) for key, value in data.items(): setattr(ts, key, value) @@ -557,7 +557,7 @@ def updateTileServer(projectId: int, tileServerId: int, data: dict): return ts @staticmethod - def updateTileServers(projectId: int, dataList: List[dict]): + def updateTileServers(dataList: List[dict]): ret_list = [] for tsv in dataList: ts = db_session.query(TileServer).get(int(tsv['id'])) diff --git a/geoapi/services/projects.py b/geoapi/services/projects.py index c57f5384..ba2ac169 100644 --- a/geoapi/services/projects.py +++ b/geoapi/services/projects.py @@ -10,7 +10,7 @@ from geoapi.services.users import UserService from geoapi.utils.agave import AgaveUtils, get_system_users from geoapi.utils.assets import get_project_asset_dir -from geoapi.tasks.external_data import import_from_agave +from geoapi.tasks.external_data import import_from_agave, delete_agave_file from geoapi.log import logging from geoapi.exceptions import ApiException, ObservableProjectAlreadyExists @@ -30,7 +30,6 @@ def create(data: dict, user: User) -> Project: :param user: User :return: Project """ - project = Project(**data) project.tenant_id = user.tenant_id project.users.append(user) @@ -55,9 +54,11 @@ def createRapidProject(data: dict, user: User) -> Project: system = AgaveUtils(user.jwt).systemsGet(systemId) proj = Project( name=name, - description=system["description"], - tenant_id=user.tenant_id + description=system['description'], + tenant_id=user.tenant_id, + system_id=systemId ) + obs = ObservableDataProject( system_id=systemId, path=path @@ -80,6 +81,65 @@ def createRapidProject(data: dict, user: User) -> Project: raise ObservableProjectAlreadyExists("'{}' project already exists".format(name)) import_from_agave.apply_async(args=[obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id]) + ProjectsService.export(user, + {'system_id': systemId, + 'path': folder_name, + 'link': True, + 'file_name': '' + }, + True, + proj.id) + + return proj + + @staticmethod + def export(user: User, + data: dict, + observable: bool, + project_id: int) -> Project: + """ + Save a project UUID file to tapis + :param user: User + :param data: dict + :return: None + """ + proj = ProjectsService.get(project_id=project_id) + + # If already has a saved file remove it + if proj.system_path is not None: + delete_agave_file.apply_async(args=[proj.system_id, + '{}/{}'.format(proj.system_path, + proj.system_file), + user.id]) + + path = data['path'] + + if data['file_name'] == '': + file_prefix = str(proj.uuid) + else: + file_prefix = str(data['file_name']) + + file_name = '{}.{}'.format(file_prefix, 'hazmapper') + + # if 'project' not in data['system_id'] and path == '/': + if ('project' not in data['system_id'] and path == '/') or observable: + path = "/{}/{}".format(user.username, path) + + proj.system_path = path + proj.system_file = file_name + proj.system_id = data['system_id'] + db_session.commit() + + file_content = { + 'uuid': str(proj.uuid) + } + + AgaveUtils(user.jwt).postFile(data['system_id'], + path, + file_name, + file_content + ) + return proj @staticmethod @@ -236,15 +296,21 @@ def update(projectId: int, data: dict) -> Project: return current_project @staticmethod - def delete(projectId: int) -> dict: + def delete(user: User, projectId: int) -> dict: """ Delete a project and all its Features and assets :param projectId: int :return: """ proj = db_session.query(Project).get(projectId) + db_session.delete(proj) db_session.commit() + + if proj.system_path is not None: + delete_agave_file.apply_async(args=[proj.system_id, + proj.system_path + '/' + proj.system_file, + user.id]) assets_folder = get_project_asset_dir(projectId) try: shutil.rmtree(assets_folder) diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index e07163fb..b92774c1 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -127,6 +127,12 @@ def _update_point_cloud_task(pointCloudId: int, description: str = None, status: raise +@app.task(rate_limit="1/s") +def delete_agave_file(system_id: str, system_path: str, userId: int): + user = db_session.query(User).get(userId) + AgaveUtils(user.jwt).deleteFile(system_id, system_path) + + @app.task(rate_limit="1/s") def import_point_clouds_from_agave(userId: int, files, pointCloudId: int): user = db_session.query(User).get(userId) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 97daf3ce..42f9a8be 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -139,8 +139,7 @@ def test_remove_tile_server(projects_fixture): } tile_server = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data) - FeaturesService.deleteTileServer(projects_fixture.id, - tile_server.id) + FeaturesService.deleteTileServer(tile_server.id) assert db_session.query(TileServer).count() == 0 @@ -158,8 +157,7 @@ def test_update_tile_server(projects_fixture): "name": "NewTestName", } - updated_tile_server = FeaturesService.updateTileServer(projectId=projects_fixture.id, - tileServerId=1, + updated_tile_server = FeaturesService.updateTileServer(tileServerId=1, data=updated_data) assert updated_tile_server.name == "NewTestName" @@ -177,7 +175,8 @@ def test_update_tile_servers(projects_fixture): updated_data = [{"id": resp1.id, "name": "NewTestName1"}, {"id": resp2.id, "name": "NewTestName2"}] - updated_tile_server_list = FeaturesService.updateTileServers(projectId=projects_fixture.id, dataList=updated_data) + updated_tile_server_list = FeaturesService.updateTileServers(dataList=updated_data) + assert updated_tile_server_list[0].name == "NewTestName1" assert updated_tile_server_list[1].name == "NewTestName2" diff --git a/geoapi/tests/api_tests/test_projects_routes.py b/geoapi/tests/api_tests/test_projects_routes.py index 4e5460fc..2cf3cce6 100644 --- a/geoapi/tests/api_tests/test_projects_routes.py +++ b/geoapi/tests/api_tests/test_projects_routes.py @@ -317,6 +317,7 @@ def test_observable_project(test_client, userdata, get_system_users_mock, agave_utils_with_geojson_file_mock, + projects_fixture, import_from_agave_mock): u1 = db_session.query(User).get(1) resp = test_client.post( @@ -367,3 +368,19 @@ def test_update_project_unauthorized_guest(test_client, public_projects_fixture) json=data ) assert resp.status_code == 403 + + +def test_export_project(test_client, + projects_fixture, + get_system_users_mock, + agave_utils_with_geojson_file_mock): + u1 = db_session.query(User).get(1) + resp = test_client.put( + '/projects/1/export/', + json={"system_id": "testSystem", + "path": "testPath", + "file_name": "testFilename", + "link": False}, + headers={'x-jwt-assertion-test': u1.jwt} + ) + assert resp.status_code == 200 diff --git a/geoapi/tests/api_tests/test_projects_service.py b/geoapi/tests/api_tests/test_projects_service.py index 64813c5c..10e8d1d3 100644 --- a/geoapi/tests/api_tests/test_projects_service.py +++ b/geoapi/tests/api_tests/test_projects_service.py @@ -79,6 +79,7 @@ def test_get_features_filter_type(projects_fixture, project_features = ProjectsService.getFeatures(projects_fixture.id, query) assert len(project_features['features']) == 0 + def test_update_project(projects_fixture): data = { "name": "new name", diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 8dc05d06..03e4e460 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -396,7 +396,8 @@ def agave_utils_with_geojson_file_mock(agave_file_listings_mock, geojson_file_fi MockAgaveUtils().listing.return_value = agave_file_listings_mock MockAgaveUtils().getFile.return_value = geojson_file_fixture MockAgaveUtils().systemsGet.return_value = {"id": "testSystem", - "description": "System Description"} + "description": "System Description", + "name": "System Name"} yield MockAgaveUtils() diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 33085163..5d8c79df 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -131,6 +131,57 @@ def getFile(self, systemId: str, path: str) -> IO: logger.error("Could not fetch file ({}/{}): {}".format(systemId, path, e)) raise e + def postFile(self, systemId: str, path: str, file_name: str, file_content: dict) -> None: + """ + Upload a file to agave + :param systemId: str + :param path (directory): str + :param file_name: str + :param file_content: dict + :return: None + """ + url = quote('/files/media/system/{}/{}'.format(systemId, path)) + try: + logger.info("Uploading to " + self.base_url + url) + tmp = NamedTemporaryFile(delete=True, mode="r+") + tmp.name = file_name + json.dump(file_content, tmp) + tmp.flush() + tmp.seek(0) + files = { 'fileToUpload': tmp } + with self.client.post(self.base_url + url, + verify=False, + files=files) as r: + if r.status_code > 400: + tmp.close() + raise ValueError("Could not post file ({}/{}/{}) status_code:{}".format(systemId, + path, + file_name, + r.status_code)) + tmp.close() + except Exception as e: + logger.error("Could not post file ({}/{}/{}): {}".format(systemId, path, file_name, e)) + raise e + + def deleteFile(self, systemId: str, path: str) -> None: + """ + Delete an agave file + :param systemId: str + :param path (directory): str + :return: None + """ + url = quote('/files/media/system/{}/{}'.format(systemId, path)) + try: + logger.info("Deleting " + self.base_url + url) + with self.client.delete(self.base_url + url) as r: + if r.status_code > 400: + raise ValueError("Could not delete file ({}/{}) status_code:{}".format(systemId, + path, + r.status_code)) + except Exception as e: + logger.error("Could not delete file ({}/{}): {}".format(systemId, path, e)) + raise e + def service_account_client(tenant_id): try: