diff --git a/docker-compose.yml b/docker-compose.yml index 0eff9165..c4be583e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,7 +42,7 @@ services: image: mongo:6.0.6 environment: MONGO_APP_DATABASE: phyloviz-web-platform - MONGO_REPLICA_HOST: host.docker.internal + MONGO_REPLICA_HOST: 172.17.0.1 MONGO_REPLICA_PORT: 27018 entrypoint: > diff --git a/scripts/flowviz_setup.sh b/scripts/flowviz_setup.sh old mode 100644 new mode 100755 index 0756ad8c..e61d51b4 --- a/scripts/flowviz_setup.sh +++ b/scripts/flowviz_setup.sh @@ -8,6 +8,14 @@ cd ./FLOWViZ echo "Steps 1 and 2 of the manual setup are going to be executed..." +npm install + +cd client + +npm install + +cd .. + ./setupAirflow.sh echo "Setting up FLOWViZ docker..." @@ -15,4 +23,4 @@ docker compose up -d echo "FLOWViZ docker setup completed!" echo "Steps 1 and 2 of the manual setup were executed successfully!" -echo "Now you still need to execute steps 3 to 6 of the manual setup. (https://github.com/devandrepascoa/FLOWViZ?tab=readme-ov-file#manual-setup)" \ No newline at end of file +echo "Now you still need to execute steps 3 to 6 of the manual setup. (https://github.com/devandrepascoa/FLOWViZ?tab=readme-ov-file#manual-setup)" diff --git a/scripts/phylodb_cleanup.sh b/scripts/phylodb_cleanup.sh old mode 100644 new mode 100755 index 7ad1e9aa..c296e1d2 --- a/scripts/phylodb_cleanup.sh +++ b/scripts/phylodb_cleanup.sh @@ -6,4 +6,4 @@ docker compose down cd .. -rm -rf ./phyloDB/ \ No newline at end of file +rm -rf ./phyloDB/ diff --git a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepository.java b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepository.java index eaff9d62..222f0b11 100644 --- a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepository.java +++ b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepository.java @@ -28,4 +28,11 @@ public interface S3FileRepository { * @return true if the file was deleted successfully, false otherwise */ boolean delete(String url); + + /** + * Get location of the repository. + * + * @return location of the repository + */ + String getLocation(); } diff --git a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepositoryImpl.java b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepositoryImpl.java index e5ad8c7f..6857dbbd 100644 --- a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepositoryImpl.java +++ b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/S3FileRepositoryImpl.java @@ -27,6 +27,7 @@ @Repository public class S3FileRepositoryImpl implements S3FileRepository { + private final String s3Endpoint; private final String bucketName; private final S3AsyncClient s3Client; private final S3TransferManager transferManager; @@ -53,6 +54,7 @@ public S3FileRepositoryImpl( newS3Client.createBucket(r -> r.bucket(bucketName)); + this.s3Endpoint = s3endpoint; this.s3Client = newS3Client; this.bucketName = bucketName; this.transferManager = S3TransferManager.builder().s3Client(newS3Client).build(); @@ -136,4 +138,10 @@ public boolean delete(String url) { return true; } + + @Override + public String getLocation() { + return s3Endpoint + "/" + bucketName; + } + } diff --git a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/isolate_data/repository/IsolateDataS3DataRepository.java b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/isolate_data/repository/IsolateDataS3DataRepository.java index d6105eab..3a1a521c 100644 --- a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/isolate_data/repository/IsolateDataS3DataRepository.java +++ b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/isolate_data/repository/IsolateDataS3DataRepository.java @@ -20,7 +20,7 @@ public IsolateDataDataRepositorySpecificData uploadIsolateData(String projectId, s3FileRepository.upload(url, multipartFile); - return new IsolateDataS3DataRepositorySpecificData(url, multipartFile.getOriginalFilename()); + return new IsolateDataS3DataRepositorySpecificData(s3FileRepository.getLocation() + "/" + url, multipartFile.getOriginalFilename()); } @Override diff --git a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/typing_data/repository/TypingDataS3DataRepository.java b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/typing_data/repository/TypingDataS3DataRepository.java index 2b6a566f..b7e049f9 100644 --- a/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/typing_data/repository/TypingDataS3DataRepository.java +++ b/src/backend/microservices/shared/src/main/java/org/phyloviz/pwp/shared/repository/data/typing_data/repository/TypingDataS3DataRepository.java @@ -21,7 +21,7 @@ public TypingDataDataRepositorySpecificData uploadTypingData(String projectId, S s3FileRepository.upload(url, multipartFile); - return new TypingDataS3DataRepositorySpecificData(url, multipartFile.getOriginalFilename()); + return new TypingDataS3DataRepositorySpecificData(s3FileRepository.getLocation() + "/" + url, multipartFile.getOriginalFilename()); } @Override @@ -49,4 +49,4 @@ public void deleteTypingData(TypingDataDataRepositorySpecificData typingDataData s3FileRepository.delete(repositorySpecificData.getUrl()); } -} \ No newline at end of file +} diff --git a/src/backend/mongo/mongo-init.js b/src/backend/mongo/mongo-init.js index eb8d752f..aa464a38 100644 --- a/src/backend/mongo/mongo-init.js +++ b/src/backend/mongo/mongo-init.js @@ -725,10 +725,10 @@ db['workflow-templates'].insertMany( "taskId": "createMetadata", "tool": "metadata_uploader", "action": { - "command": "--original_url=${url} --url=http://localhost:9444/phyloviz-web-platform/projects/${projectId}/${workflowId} --type=${file_type} --project-id=${projectId} --workflow-id=${workflowId}" + "command": "--original_url=${url} --url=https://s3.eu-west-3.amazonaws.com/phyloviz-web-platform/projects/${projectId}/${workflowId} --type=${file_type} --project-id=${projectId} --workflow-id=${workflowId}" } } ] } ] -) \ No newline at end of file +) diff --git a/src/backend/tools/aws_config b/src/backend/tools/aws_config index f61e1d5a..b2d25395 100644 --- a/src/backend/tools/aws_config +++ b/src/backend/tools/aws_config @@ -1,3 +1,3 @@ [default] -region = custom -endpoint_url = http://host.docker.internal:9444 +region = eu-west-3 +endpoint_url = https://s3.eu-west-3.amazonaws.com diff --git a/src/backend/tools/compute_tree_view/Dockerfile b/src/backend/tools/compute_tree_view/Dockerfile index b2b5a3f8..0bb7d918 100644 --- a/src/backend/tools/compute_tree_view/Dockerfile +++ b/src/backend/tools/compute_tree_view/Dockerfile @@ -9,11 +9,11 @@ COPY compute_tree_view/requirements.txt /app/requirements.txt # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" -ENV PHYLODB_URL="http://host.docker.internal:8081" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" +ENV PHYLODB_URL="http://172.17.0.1:8081" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "compute_tree_view.py"] \ No newline at end of file +ENTRYPOINT ["python", "compute_tree_view.py"] diff --git a/src/backend/tools/downloader/Dockerfile b/src/backend/tools/downloader/Dockerfile index 1d76b883..693de2e3 100644 --- a/src/backend/tools/downloader/Dockerfile +++ b/src/backend/tools/downloader/Dockerfile @@ -17,10 +17,10 @@ COPY aws_credentials /root/.aws/credentials # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "downloader.py"] \ No newline at end of file +ENTRYPOINT ["python", "downloader.py"] diff --git a/src/backend/tools/downloader/downloader.py b/src/backend/tools/downloader/downloader.py index 894f965b..2d0b9ce6 100644 --- a/src/backend/tools/downloader/downloader.py +++ b/src/backend/tools/downloader/downloader.py @@ -124,7 +124,7 @@ def download_s3_object(project_id, dataset_id, resource_id, resource_type, out, parsed_url = urlparse(s3_url) path = parsed_url.path.lstrip('/') bucket_name = path.split('/')[0] - object_key = path.lstrip(f'{bucket_name}') + object_key = path.lstrip(f'{bucket_name}/') print("Retrieving S3 object from bucket: " + bucket_name + " and key: " + object_key) diff --git a/src/backend/tools/isolate_data_indexer/Dockerfile b/src/backend/tools/isolate_data_indexer/Dockerfile index 0afcd828..5eaf2042 100644 --- a/src/backend/tools/isolate_data_indexer/Dockerfile +++ b/src/backend/tools/isolate_data_indexer/Dockerfile @@ -9,11 +9,11 @@ COPY isolate_data_indexer/requirements.txt /app/requirements.txt # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" -ENV PHYLODB_URL="http://host.docker.internal:8081" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" +ENV PHYLODB_URL="http://172.17.0.1:8081" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "isolate_data_indexer.py"] \ No newline at end of file +ENTRYPOINT ["python", "isolate_data_indexer.py"] diff --git a/src/backend/tools/metadata_uploader/Dockerfile b/src/backend/tools/metadata_uploader/Dockerfile index 16cfba15..89c29c86 100644 --- a/src/backend/tools/metadata_uploader/Dockerfile +++ b/src/backend/tools/metadata_uploader/Dockerfile @@ -16,10 +16,10 @@ COPY aws_credentials /root/.aws/credentials # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "metadata_uploader.py"] \ No newline at end of file +ENTRYPOINT ["python", "metadata_uploader.py"] diff --git a/src/backend/tools/rclone/rclone.conf b/src/backend/tools/rclone/rclone.conf index 5bb81541..563784b8 100644 --- a/src/backend/tools/rclone/rclone.conf +++ b/src/backend/tools/rclone/rclone.conf @@ -3,4 +3,4 @@ type = s3 provider = other access_key_id = AKIAIOSFODNN7EXAMPLE secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY -endpoint = http://host.docker.internal:9444 \ No newline at end of file +endpoint = https://s3.eu-west-3.amazonaws.com diff --git a/src/backend/tools/tree_indexer/Dockerfile b/src/backend/tools/tree_indexer/Dockerfile index 5eaa3c9d..cafb29db 100644 --- a/src/backend/tools/tree_indexer/Dockerfile +++ b/src/backend/tools/tree_indexer/Dockerfile @@ -9,11 +9,11 @@ COPY tree_indexer/requirements.txt /app/requirements.txt # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" -ENV PHYLODB_URL="http://host.docker.internal:8081" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" +ENV PHYLODB_URL="http://172.17.0.1:8081" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "tree_indexer.py"] \ No newline at end of file +ENTRYPOINT ["python", "tree_indexer.py"] diff --git a/src/backend/tools/typing_data_indexer/Dockerfile b/src/backend/tools/typing_data_indexer/Dockerfile index 8601c850..8f8e5f82 100644 --- a/src/backend/tools/typing_data_indexer/Dockerfile +++ b/src/backend/tools/typing_data_indexer/Dockerfile @@ -9,11 +9,11 @@ COPY typing_data_indexer/requirements.txt /app/requirements.txt # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" -ENV PHYLODB_URL="http://host.docker.internal:8081" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" +ENV PHYLODB_URL="http://172.17.0.1:8081" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "typing_data_indexer.py"] \ No newline at end of file +ENTRYPOINT ["python", "typing_data_indexer.py"] diff --git a/src/backend/tools/upload_to_registry.sh b/src/backend/tools/upload_to_registry.sh old mode 100644 new mode 100755 diff --git a/src/backend/tools/uploader/Dockerfile b/src/backend/tools/uploader/Dockerfile index e522ac74..41f3e57d 100644 --- a/src/backend/tools/uploader/Dockerfile +++ b/src/backend/tools/uploader/Dockerfile @@ -16,10 +16,10 @@ COPY aws_credentials /root/.aws/credentials # Set the working directory to /app WORKDIR /app -ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true" +ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true" # Install any required packages RUN pip install --trusted-host pypi.python.org -r requirements.txt # Run the script when the container launches -ENTRYPOINT ["python", "uploader.py"] \ No newline at end of file +ENTRYPOINT ["python", "uploader.py"] diff --git a/src/backend/tools/uploader/uploader.py b/src/backend/tools/uploader/uploader.py index 75c18fad..f8bb2404 100644 --- a/src/backend/tools/uploader/uploader.py +++ b/src/backend/tools/uploader/uploader.py @@ -25,7 +25,7 @@ datasets_collection = db['datasets'] -def tree_handler(s3_output_path, project_id, dataset_id, tree_id): +def tree_handler(object_key, project_id, dataset_id, tree_id): trees_collection = db['trees'] # Tree should already be indexed, so its metadata exists @@ -38,7 +38,7 @@ def tree_handler(s3_output_path, project_id, dataset_id, tree_id): }, {'$set': { 'repositorySpecificData.s3': { - 'url': f'http://localhost:9444/{bucket_name}{s3_output_path}' + 'url': f'https://s3.eu-west-3.amazonaws.com/{bucket_name}/{object_key}' } }} ) @@ -46,7 +46,7 @@ def tree_handler(s3_output_path, project_id, dataset_id, tree_id): print(f'File uploaded to S3 and tree metadata updated of tree with treeId: {tree_id}') -def distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id, distance_matrix_id, source_type, +def distance_matrix_handler(object_key, project_id, dataset_id, workflow_id, distance_matrix_id, source_type, function): distance_matrix_collection = db['distance-matrices'] @@ -69,7 +69,7 @@ def distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id, 'source': source, 'repositorySpecificData': { 's3': { - 'url': f'http://localhost:9444/{bucket_name}{s3_output_path}' + 'url': f'https://s3.eu-west-3.amazonaws.com/{bucket_name}/{object_key}' } } } @@ -119,17 +119,18 @@ def upload_file_to_s3(file_path, project_id, dataset_id, workflow_id, resource_i collection_name = get_collection_from_resource_type(resource_type) - s3_output_path = f'/{project_id}/{collection_name}/{resource_id}' + object_key = f'{project_id}/{collection_name}/{resource_id}' + # Upload the file to S3 with open(file_path, 'rb') as file: - s3.upload_fileobj(file, bucket_name, s3_output_path) + s3.upload_fileobj(file, bucket_name, object_key) if resource_type == "tree": tree_id = resource_id - tree_handler(s3_output_path, project_id, dataset_id, tree_id) + tree_handler(object_key, project_id, dataset_id, tree_id) elif resource_type == "distance-matrix": distance_matrix_id = resource_id - distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id, distance_matrix_id, source_type, + distance_matrix_handler(object_key, project_id, dataset_id, workflow_id, distance_matrix_id, source_type, function) else: raise Exception(f'Unknown resource type: {resource_type}') diff --git a/start.sh b/start.sh index 30d02508..13a6f9c3 100755 --- a/start.sh +++ b/start.sh @@ -9,6 +9,6 @@ npm install npm start & docker compose up -d --build -sleep 60 echo "Doing very heavy work..." +sleep 90 docker compose up -d