diff --git a/docker-compose.h2.dev.yaml b/.docker-compose/docker-compose.h2.dev.yaml similarity index 79% rename from docker-compose.h2.dev.yaml rename to .docker-compose/docker-compose.h2.dev.yaml index 81c9c4c..7051730 100644 --- a/docker-compose.h2.dev.yaml +++ b/.docker-compose/docker-compose.h2.dev.yaml @@ -5,7 +5,7 @@ services: fits: image: artourkin/fits-web:main container_name: fits - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -16,9 +16,9 @@ services: rest: container_name: rest build: - context: . - dockerfile: ./Dockerfile - env_file: .env + context: .. + dockerfile: ../Dockerfile + env_file: ../.env networks: - web restart: unless-stopped @@ -33,10 +33,10 @@ services: web: build: - context: . - dockerfile: ./web/Dockerfile + context: .. + dockerfile: ../web/Dockerfile container_name: web - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -47,7 +47,7 @@ services: db-docker: image: oscarfonts/h2 container_name: db-docker - env_file: .env + env_file: ../.env environment: - H2_OPTIONS=-ifNotExists networks: diff --git a/docker-compose.mysql.cluster.yaml b/.docker-compose/docker-compose.mysql.cluster.yaml similarity index 85% rename from docker-compose.mysql.cluster.yaml rename to .docker-compose/docker-compose.mysql.cluster.yaml index 5d0f224..6aae85a 100644 --- a/docker-compose.mysql.cluster.yaml +++ b/.docker-compose/docker-compose.mysql.cluster.yaml @@ -4,10 +4,10 @@ services: fits: build: - context: . - dockerfile: ./fits/Dockerfile + context: .. + dockerfile: ../fits/Dockerfile container_name: fits - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -16,9 +16,9 @@ services: rest: build: - context: . - dockerfile: ./Dockerfile - env_file: .env + context: .. + dockerfile: ../Dockerfile + env_file: ../.env environment: - SPRING_DATASOURCE_URL=jdbc:mysql://mysql-router:6446/fitsinn - DB_SELECTOR=mysql @@ -33,10 +33,10 @@ services: web: build: - context: . - dockerfile: ./web/Dockerfile.dev + context: .. + dockerfile: ../web/Dockerfile.dev container_name: web - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -49,7 +49,7 @@ services: adminer: image: adminer container_name: adminer - env_file: .env + env_file: ../.env restart: unless-stopped networks: - web @@ -59,7 +59,7 @@ services: mysql-server-1: container_name: mysql-server-1 env_file: - - utils/mysql-cluster/mysql-server.env + - ../config/mysql-cluster/mysql-server.env image: mysql/mysql-server:8.0.12 networks: - web @@ -84,7 +84,7 @@ services: mysql-server-2: container_name: mysql-server-2 env_file: - - utils/mysql-cluster/mysql-server.env + - ../config/mysql-cluster/mysql-server.env image: mysql/mysql-server:8.0.12 networks: - web @@ -110,7 +110,7 @@ services: mysql-server-3: container_name: mysql-server-3 env_file: - - utils/mysql-cluster/mysql-server.env + - ../config/mysql-cluster/mysql-server.env image: mysql/mysql-server:8.0.12 networks: - web @@ -135,7 +135,7 @@ services: mysql-shell: container_name: mysql-shell env_file: - - utils/mysql-cluster/mysql-shell.env + - ../config/mysql-cluster/mysql-shell.env image: neumayer/mysql-shell-batch networks: - web @@ -149,7 +149,7 @@ services: mysql-router: container_name: mysql-router env_file: - - utils/mysql-cluster/mysql-router.env + - ../config/mysql-cluster/mysql-router.env image: mysql/mysql-router:8.0 networks: - web @@ -165,9 +165,9 @@ services: nginx: image: nginx container_name: nginx - env_file: .env + env_file: ../.env volumes: - - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf + - ./config/nginx/nginx.conf:/etc/nginx/conf.d/default.conf ports: - 8082:80 networks: diff --git a/docker-compose.mysql.dev.yaml b/.docker-compose/docker-compose.mysql.dev.yaml similarity index 76% rename from docker-compose.mysql.dev.yaml rename to .docker-compose/docker-compose.mysql.dev.yaml index 17c9cc3..dda1c5a 100644 --- a/docker-compose.mysql.dev.yaml +++ b/.docker-compose/docker-compose.mysql.dev.yaml @@ -4,10 +4,10 @@ services: fits: build: - context: . - dockerfile: ./fits/Dockerfile + context: .. + dockerfile: ../fits/Dockerfile container_name: fits - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -16,9 +16,9 @@ services: rest: build: - context: . - dockerfile: ./Dockerfile - env_file: .env + context: .. + dockerfile: ../Dockerfile + env_file: ../.env networks: - web restart: unless-stopped @@ -36,10 +36,10 @@ services: web: build: - context: . - dockerfile: ./web/Dockerfile + context: .. + dockerfile: ../web/Dockerfile container_name: web - env_file: .env + env_file: ../.env networks: - web restart: unless-stopped @@ -49,7 +49,7 @@ services: db-docker: image: mysql:8.0 container_name: db-docker - env_file: .env + env_file: ../.env environment: MYSQL_DATABASE: fitsinn MYSQL_USER: user @@ -65,7 +65,7 @@ services: adminer: image: adminer container_name: adminer - env_file: .env + env_file: ../.env restart: unless-stopped networks: - web @@ -75,9 +75,9 @@ services: nginx: image: nginx container_name: nginx - env_file: .env + env_file: ../.env volumes: - - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf + - ./config/nginx/nginx.conf:/etc/nginx/conf.d/default.conf ports: - 8082:80 networks: diff --git a/utils/clickhouse/config.xml b/config/clickhouse/config.xml similarity index 100% rename from utils/clickhouse/config.xml rename to config/clickhouse/config.xml diff --git a/utils/clickhouse/initdb.sql b/config/clickhouse/initdb.sql similarity index 100% rename from utils/clickhouse/initdb.sql rename to config/clickhouse/initdb.sql diff --git a/utils/clickhouse/users.xml b/config/clickhouse/users.xml similarity index 100% rename from utils/clickhouse/users.xml rename to config/clickhouse/users.xml diff --git a/utils/mysql-cluster/mysql-router.env b/config/mysql-cluster/mysql-router.env similarity index 100% rename from utils/mysql-cluster/mysql-router.env rename to config/mysql-cluster/mysql-router.env diff --git a/utils/mysql-cluster/mysql-server.env b/config/mysql-cluster/mysql-server.env similarity index 100% rename from utils/mysql-cluster/mysql-server.env rename to config/mysql-cluster/mysql-server.env diff --git a/utils/mysql-cluster/mysql-shell.env b/config/mysql-cluster/mysql-shell.env similarity index 100% rename from utils/mysql-cluster/mysql-shell.env rename to config/mysql-cluster/mysql-shell.env diff --git a/utils/mysql-cluster/scripts/db.sql b/config/mysql-cluster/scripts/db.sql similarity index 100% rename from utils/mysql-cluster/scripts/db.sql rename to config/mysql-cluster/scripts/db.sql diff --git a/utils/mysql-cluster/scripts/setupCluster.js b/config/mysql-cluster/scripts/setupCluster.js similarity index 100% rename from utils/mysql-cluster/scripts/setupCluster.js rename to config/mysql-cluster/scripts/setupCluster.js diff --git a/utils/nginx/nginx.conf b/config/nginx/nginx.conf similarity index 100% rename from utils/nginx/nginx.conf rename to config/nginx/nginx.conf diff --git a/docker-compose.clickhouse.dev.yaml b/docker-compose.clickhouse.dev.yaml index df98a57..5b9d14a 100644 --- a/docker-compose.clickhouse.dev.yaml +++ b/docker-compose.clickhouse.dev.yaml @@ -59,7 +59,7 @@ services: image: yandex/clickhouse-server container_name: db-docker-init volumes: - - ./utils/clickhouse:/var/clickhouse + - ./config/clickhouse:/var/clickhouse depends_on: - db-docker networks: @@ -93,7 +93,7 @@ services: container_name: nginx env_file: .env volumes: - - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf + - ./config/nginx/nginx.conf:/etc/nginx/conf.d/default.conf ports: - 8082:80 networks: diff --git a/utils/fileupload.py b/utils/fileupload.py index 291e78f..45393a0 100644 --- a/utils/fileupload.py +++ b/utils/fileupload.py @@ -10,11 +10,10 @@ def upload_chunk(url, chunk_files, chunk_count): } start_time = time.time() - # Upload the chunk + response = requests.post(url, headers=headers, files=chunk_files) end_time = time.time() - # Print the response status code and content print(f"Uploaded {chunk_count} files. Time taken: {(end_time - start_time):.2f} seconds. Status Code: {response.status_code}") #print(response.text) @@ -25,12 +24,10 @@ def upload_files_in_chunks_parallel(url, folder_path, chunk_size=100, num_parall 'accept': '*/*', } - # Initialize variables chunk_count = 0 chunk_files = [] total_duration = 0 - # Traverse through the folder and its subfolders for root, _, filenames in os.walk(folder_path): for filename in filenames: if filename.endswith('.xml'): @@ -38,22 +35,16 @@ def upload_files_in_chunks_parallel(url, folder_path, chunk_size=100, num_parall relative_path = os.path.relpath(file_path, folder_path) chunk_files.append(('files', (relative_path, open(file_path, 'rb'), 'text/xml'))) - # Check if the chunk size is reached if len(chunk_files) == chunk_size: - # Use ThreadPoolExecutor to create parallel requests with ThreadPoolExecutor(max_workers=num_parallel_requests) as executor: - # Submit the upload_chunk function with arguments future = executor.submit(upload_chunk, url, chunk_files, (chunk_count+1)*chunk_size) duration = future.result() - # Accumulate duration for statistics total_duration += duration - # Reset variables for the next chunk chunk_count += 1 chunk_files = [] - # Upload the remaining files if any if chunk_files: with ThreadPoolExecutor(max_workers=num_parallel_requests) as executor: future = executor.submit(upload_chunk, url, chunk_files, chunk_count) @@ -72,13 +63,10 @@ def upload_files_in_chunks_parallel(url, folder_path, chunk_size=100, num_parall chunk_size = int(sys.argv[3]) num_parallel_requests = int(sys.argv[4]) - # Measure the script execution duration start_script_time = time.time() - # Call the function to upload files in chunks in parallel total_duration = upload_files_in_chunks_parallel(upload_url, folder_to_upload, chunk_size, num_parallel_requests) - # Print the script execution duration statistics end_script_time = time.time() script_duration = end_script_time - start_script_time print(f"\nScript Execution Duration: {script_duration:.2f} seconds") diff --git a/web/frontend/src/scenes/faq/index.jsx b/web/frontend/src/scenes/faq/index.jsx index 1c8053f..3950f4d 100644 --- a/web/frontend/src/scenes/faq/index.jsx +++ b/web/frontend/src/scenes/faq/index.jsx @@ -36,7 +36,7 @@ const FAQ = () => { It is a set of activities within Digital Preservation to study digital collections based on characterisation results. Such results - are obtained by proccesing the digital object using characterisation + are obtained by processing the digital object using characterisation tools.