Skip to content

Commit

Permalink
Bug fixes.
Browse files Browse the repository at this point in the history
  • Loading branch information
devandrepascoa committed Oct 21, 2024
1 parent df8200c commit b18043c
Show file tree
Hide file tree
Showing 21 changed files with 63 additions and 39 deletions.
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ services:
image: mongo:6.0.6
environment:
MONGO_APP_DATABASE: phyloviz-web-platform
MONGO_REPLICA_HOST: host.docker.internal
MONGO_REPLICA_HOST: 172.17.0.1
MONGO_REPLICA_PORT: 27018

entrypoint: >
Expand Down
10 changes: 9 additions & 1 deletion scripts/flowviz_setup.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,19 @@ cd ./FLOWViZ

echo "Steps 1 and 2 of the manual setup are going to be executed..."

npm install

cd client

npm install

cd ..

./setupAirflow.sh

echo "Setting up FLOWViZ docker..."
docker compose up -d
echo "FLOWViZ docker setup completed!"

echo "Steps 1 and 2 of the manual setup were executed successfully!"
echo "Now you still need to execute steps 3 to 6 of the manual setup. (https://github.com/devandrepascoa/FLOWViZ?tab=readme-ov-file#manual-setup)"
echo "Now you still need to execute steps 3 to 6 of the manual setup. (https://github.com/devandrepascoa/FLOWViZ?tab=readme-ov-file#manual-setup)"
2 changes: 1 addition & 1 deletion scripts/phylodb_cleanup.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ docker compose down

cd ..

rm -rf ./phyloDB/
rm -rf ./phyloDB/
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,11 @@ public interface S3FileRepository {
* @return true if the file was deleted successfully, false otherwise
*/
boolean delete(String url);

/**
* Get location of the repository.
*
* @return location of the repository
*/
String getLocation();
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
@Repository
public class S3FileRepositoryImpl implements S3FileRepository {

private final String s3Endpoint;
private final String bucketName;
private final S3AsyncClient s3Client;
private final S3TransferManager transferManager;
Expand All @@ -53,6 +54,7 @@ public S3FileRepositoryImpl(

newS3Client.createBucket(r -> r.bucket(bucketName));

this.s3Endpoint = s3endpoint;
this.s3Client = newS3Client;
this.bucketName = bucketName;
this.transferManager = S3TransferManager.builder().s3Client(newS3Client).build();
Expand Down Expand Up @@ -136,4 +138,10 @@ public boolean delete(String url) {

return true;
}

@Override
public String getLocation() {
return s3Endpoint + "/" + bucketName;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public IsolateDataDataRepositorySpecificData uploadIsolateData(String projectId,

s3FileRepository.upload(url, multipartFile);

return new IsolateDataS3DataRepositorySpecificData(url, multipartFile.getOriginalFilename());
return new IsolateDataS3DataRepositorySpecificData(s3FileRepository.getLocation() + "/" + url, multipartFile.getOriginalFilename());
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public TypingDataDataRepositorySpecificData uploadTypingData(String projectId, S

s3FileRepository.upload(url, multipartFile);

return new TypingDataS3DataRepositorySpecificData(url, multipartFile.getOriginalFilename());
return new TypingDataS3DataRepositorySpecificData(s3FileRepository.getLocation() + "/" + url, multipartFile.getOriginalFilename());
}

@Override
Expand Down Expand Up @@ -49,4 +49,4 @@ public void deleteTypingData(TypingDataDataRepositorySpecificData typingDataData

s3FileRepository.delete(repositorySpecificData.getUrl());
}
}
}
4 changes: 2 additions & 2 deletions src/backend/mongo/mongo-init.js
Original file line number Diff line number Diff line change
Expand Up @@ -725,10 +725,10 @@ db['workflow-templates'].insertMany(
"taskId": "createMetadata",
"tool": "metadata_uploader",
"action": {
"command": "--original_url=${url} --url=http://localhost:9444/phyloviz-web-platform/projects/${projectId}/${workflowId} --type=${file_type} --project-id=${projectId} --workflow-id=${workflowId}"
"command": "--original_url=${url} --url=https://s3.eu-west-3.amazonaws.com/phyloviz-web-platform/projects/${projectId}/${workflowId} --type=${file_type} --project-id=${projectId} --workflow-id=${workflowId}"
}
}
]
}
]
)
)
4 changes: 2 additions & 2 deletions src/backend/tools/aws_config
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[default]
region = custom
endpoint_url = http://host.docker.internal:9444
region = eu-west-3
endpoint_url = https://s3.eu-west-3.amazonaws.com
6 changes: 3 additions & 3 deletions src/backend/tools/compute_tree_view/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ COPY compute_tree_view/requirements.txt /app/requirements.txt
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV PHYLODB_URL="http://host.docker.internal:8081"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"
ENV PHYLODB_URL="http://172.17.0.1:8081"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "compute_tree_view.py"]
ENTRYPOINT ["python", "compute_tree_view.py"]
4 changes: 2 additions & 2 deletions src/backend/tools/downloader/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ COPY aws_credentials /root/.aws/credentials
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "downloader.py"]
ENTRYPOINT ["python", "downloader.py"]
2 changes: 1 addition & 1 deletion src/backend/tools/downloader/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def download_s3_object(project_id, dataset_id, resource_id, resource_type, out,
parsed_url = urlparse(s3_url)
path = parsed_url.path.lstrip('/')
bucket_name = path.split('/')[0]
object_key = path.lstrip(f'{bucket_name}')
object_key = path.lstrip(f'{bucket_name}/')

print("Retrieving S3 object from bucket: " + bucket_name + " and key: " + object_key)

Expand Down
6 changes: 3 additions & 3 deletions src/backend/tools/isolate_data_indexer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ COPY isolate_data_indexer/requirements.txt /app/requirements.txt
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV PHYLODB_URL="http://host.docker.internal:8081"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"
ENV PHYLODB_URL="http://172.17.0.1:8081"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "isolate_data_indexer.py"]
ENTRYPOINT ["python", "isolate_data_indexer.py"]
4 changes: 2 additions & 2 deletions src/backend/tools/metadata_uploader/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ COPY aws_credentials /root/.aws/credentials
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "metadata_uploader.py"]
ENTRYPOINT ["python", "metadata_uploader.py"]
2 changes: 1 addition & 1 deletion src/backend/tools/rclone/rclone.conf
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ type = s3
provider = other
access_key_id = AKIAIOSFODNN7EXAMPLE
secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
endpoint = http://host.docker.internal:9444
endpoint = https://s3.eu-west-3.amazonaws.com
6 changes: 3 additions & 3 deletions src/backend/tools/tree_indexer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ COPY tree_indexer/requirements.txt /app/requirements.txt
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV PHYLODB_URL="http://host.docker.internal:8081"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"
ENV PHYLODB_URL="http://172.17.0.1:8081"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "tree_indexer.py"]
ENTRYPOINT ["python", "tree_indexer.py"]
6 changes: 3 additions & 3 deletions src/backend/tools/typing_data_indexer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ COPY typing_data_indexer/requirements.txt /app/requirements.txt
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV PHYLODB_URL="http://host.docker.internal:8081"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"
ENV PHYLODB_URL="http://172.17.0.1:8081"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "typing_data_indexer.py"]
ENTRYPOINT ["python", "typing_data_indexer.py"]
Empty file modified src/backend/tools/upload_to_registry.sh
100644 → 100755
Empty file.
4 changes: 2 additions & 2 deletions src/backend/tools/uploader/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ COPY aws_credentials /root/.aws/credentials
# Set the working directory to /app
WORKDIR /app

ENV MONGO_URI="mongodb://host.docker.internal:28017/?directConnection=true"
ENV MONGO_URI="mongodb://172.17.0.1:28017/?directConnection=true"

# Install any required packages
RUN pip install --trusted-host pypi.python.org -r requirements.txt

# Run the script when the container launches
ENTRYPOINT ["python", "uploader.py"]
ENTRYPOINT ["python", "uploader.py"]
17 changes: 9 additions & 8 deletions src/backend/tools/uploader/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
datasets_collection = db['datasets']


def tree_handler(s3_output_path, project_id, dataset_id, tree_id):
def tree_handler(object_key, project_id, dataset_id, tree_id):
trees_collection = db['trees']

# Tree should already be indexed, so its metadata exists
Expand All @@ -38,15 +38,15 @@ def tree_handler(s3_output_path, project_id, dataset_id, tree_id):
},
{'$set': {
'repositorySpecificData.s3': {
'url': f'http://localhost:9444/{bucket_name}{s3_output_path}'
'url': f'https://s3.eu-west-3.amazonaws.com/{bucket_name}/{object_key}'
}
}}
)

print(f'File uploaded to S3 and tree metadata updated of tree with treeId: {tree_id}')


def distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id, distance_matrix_id, source_type,
def distance_matrix_handler(object_key, project_id, dataset_id, workflow_id, distance_matrix_id, source_type,
function):
distance_matrix_collection = db['distance-matrices']

Expand All @@ -69,7 +69,7 @@ def distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id,
'source': source,
'repositorySpecificData': {
's3': {
'url': f'http://localhost:9444/{bucket_name}{s3_output_path}'
'url': f'https://s3.eu-west-3.amazonaws.com/{bucket_name}/{object_key}'
}
}
}
Expand Down Expand Up @@ -119,17 +119,18 @@ def upload_file_to_s3(file_path, project_id, dataset_id, workflow_id, resource_i

collection_name = get_collection_from_resource_type(resource_type)

s3_output_path = f'/{project_id}/{collection_name}/{resource_id}'
object_key = f'{project_id}/{collection_name}/{resource_id}'

# Upload the file to S3
with open(file_path, 'rb') as file:
s3.upload_fileobj(file, bucket_name, s3_output_path)
s3.upload_fileobj(file, bucket_name, object_key)

if resource_type == "tree":
tree_id = resource_id
tree_handler(s3_output_path, project_id, dataset_id, tree_id)
tree_handler(object_key, project_id, dataset_id, tree_id)
elif resource_type == "distance-matrix":
distance_matrix_id = resource_id
distance_matrix_handler(s3_output_path, project_id, dataset_id, workflow_id, distance_matrix_id, source_type,
distance_matrix_handler(object_key, project_id, dataset_id, workflow_id, distance_matrix_id, source_type,
function)
else:
raise Exception(f'Unknown resource type: {resource_type}')
Expand Down
2 changes: 1 addition & 1 deletion start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ npm install
npm start &

docker compose up -d --build
sleep 60
echo "Doing very heavy work..."
sleep 90
docker compose up -d

0 comments on commit b18043c

Please sign in to comment.