Feature/add clickhouse (#11)

* Add clickhouse * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Stabilizing the code * Clickhouse works. * Clickhouse works. Perf improving * Clickhouse works. Perf improving * Clickhouse works. Perf improving * Fixed queries * Minor fixes * adding features * Added SFD sampling * Adding conf res * Adding conf res * Adding conf res * Adding conf res * Adding conf res * Adding conf res * Adding conf res * Adding conf res --------- Co-authored-by: artur <[email protected]>
datascience · May 7, 2024 · ff72ed3 · ff72ed3
1 parent 4daf252
commit ff72ed3
Show file tree

Hide file tree

Showing 49 changed files with 2,290 additions and 244 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 FROM maven:3.9.0 as builder
 COPY . /app
 WORKDIR /app
-RUN --mount=type=cache,target=/root/.m2  mvn -pl -web clean install -Pdocker
+RUN --mount=type=cache,target=/root/.m2  mvn -pl -web clean install
 
 FROM openjdk:21-jdk-slim
 WORKDIR /app

diff --git a/README.md b/README.md
@@ -41,12 +41,12 @@ docker-compose -f docker-compose.dev.yaml up --build
 
 File uploading using bash:
 ```
-bash fileupload.sh http://localhost:8082 ~/rnd/data/govdocs_fits/govdocs1/000/
+bash ./utils/fileupload.sh http://localhost:8082 ~/rnd/data/govdocs_fits/govdocs1/000/
 ```
 
 File uploading using python (pip package requests in necessary):
 ```
-python fileupload.py http://localhost:8082/multipleupload ~/rnd/data/govdocs_fits/govdocs1/000/ 100 3
+python ./utils/fileupload.py http://localhost:8082/multipleupload ~/rnd/data/govdocs_fits/govdocs1/000/ 100 3
 ```
 
 ## Issues

diff --git a/cassandra/docker-compose.yaml b/cassandra/docker-compose.yaml
diff --git a/.../artur/api_impl/ResolveConflictsImpl.java → ...ur/api_impl/CRH_ResolveConflictsImpl.java b/.../artur/api_impl/ResolveConflictsImpl.java → ...ur/api_impl/CRH_ResolveConflictsImpl.java
@@ -12,16 +12,16 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-public class ResolveConflictsImpl implements ResolveConflicts {
+public class CRH_ResolveConflictsImpl {//implements ResolveConflicts {
 
 
     private CharacterisationResultGateway characterisationResultGateway;
 
-    public ResolveConflictsImpl(CharacterisationResultGateway characterisationResultGateway) {
+    public CRH_ResolveConflictsImpl(CharacterisationResultGateway characterisationResultGateway) {
         this.characterisationResultGateway = characterisationResultGateway;
     }
 
-    @Override
+
     public void run() {
         init();
         System.out.println(sourceWeights);

diff --git a/core/src/main/java/rocks/artur/api_impl/Native_ResolveConflictsImpl.java b/core/src/main/java/rocks/artur/api_impl/Native_ResolveConflictsImpl.java
@@ -0,0 +1,16 @@
+package rocks.artur.api_impl;
+
+import rocks.artur.api.ResolveConflicts;
+import rocks.artur.domain.CharacterisationResultGateway;
+
+public class Native_ResolveConflictsImpl implements ResolveConflicts {
+    private CharacterisationResultGateway characterisationResultGateway;
+
+    public Native_ResolveConflictsImpl(CharacterisationResultGateway characterisationResultGateway) {
+        this.characterisationResultGateway = characterisationResultGateway;
+    }
+    @Override
+    public void run() {
+        characterisationResultGateway.resolveConflictsNative();
+    }
+}
diff --git a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java
@@ -85,4 +85,6 @@ public interface CharacterisationResultGateway {
     double getConflictRate();
 
     void delete(CharacterisationResult characterisationResult);
+
+    void resolveConflictsNative();
 }
diff --git a/docker-compose.clickhouse.dev.yaml b/docker-compose.clickhouse.dev.yaml
@@ -0,0 +1,105 @@
+version: '3'
+
+services:
+
+  fits:
+    build:
+      context: .
+      dockerfile: ./fits/Dockerfile
+    container_name: fits
+    env_file: .env
+    networks:
+      - web
+    restart: unless-stopped
+    ports:
+      - 8081:8080
+
+  rest:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    env_file: .env
+    networks:
+      - web
+    restart: unless-stopped
+    environment:
+      - DB_SELECTOR=clickhouse
+    deploy:
+      replicas: 1
+    ports:
+      - 8092:8080
+    depends_on:
+      - fits
+      - db-docker
+
+  web:
+    build:
+      context: .
+      dockerfile: ./web/Dockerfile
+    container_name: web
+    env_file: .env
+    networks:
+      - web
+    restart: unless-stopped
+    ports:
+      - 8080:3000
+
+  db-docker:
+    image: yandex/clickhouse-server
+    container_name: db-docker
+    networks:
+      - web
+    ports:
+      - 8123:8123
+      - 9000:9000
+      - 9004:9004
+
+
+  db-docker-init:
+    image: yandex/clickhouse-server
+    container_name: db-docker-init
+    volumes:
+      - ./utils/clickhouse:/var/clickhouse
+    depends_on:
+      - db-docker
+    networks:
+      - web
+    entrypoint: [ '/bin/sh', '-c' ]
+    command: |
+      "
+      while ! clickhouse-client --host db-docker -q \"SHOW databases;\"; do
+          echo waiting for clickhouse up
+          sleep 1
+      done
+
+      clickhouse-client --host db-docker --queries-file /var/clickhouse/initdb.sql
+
+      tail -f /dev/null
+      "
+
+
+  adminer:
+    image: adminer
+    container_name: adminer
+    env_file: .env
+    restart: unless-stopped
+    networks:
+      - web
+    ports:
+      - 8090:8080
+
+  nginx:
+    image: nginx
+    container_name: nginx
+    env_file: .env
+    volumes:
+      - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf
+    ports:
+      - 8082:80
+    networks:
+      - web
+    depends_on:
+      - rest
+
+networks:
+  web:
diff --git a/docker-compose.dev-cluster.yaml → docker-compose.mysql.cluster.yaml b/docker-compose.dev-cluster.yaml → docker-compose.mysql.cluster.yaml
@@ -21,6 +21,7 @@ services:
     env_file: .env
     environment:
       - SPRING_DATASOURCE_URL=jdbc:mysql://mysql-router:6446/fitsinn
+      - DB_SELECTOR=mysql
     networks:
       - web
     restart: unless-stopped
@@ -58,7 +59,7 @@ services:
   mysql-server-1:
     container_name: mysql-server-1
     env_file:
-      - mysql-cluster/mysql-server.env
+      - utils/mysql-cluster/mysql-server.env
     image: mysql/mysql-server:8.0.12
     networks:
       - web
@@ -83,7 +84,7 @@ services:
   mysql-server-2:
     container_name: mysql-server-2
     env_file:
-      - mysql-cluster/mysql-server.env
+      - utils/mysql-cluster/mysql-server.env
     image: mysql/mysql-server:8.0.12
     networks:
       - web
@@ -109,7 +110,7 @@ services:
   mysql-server-3:
     container_name: mysql-server-3
     env_file:
-      - mysql-cluster/mysql-server.env
+      - utils/mysql-cluster/mysql-server.env
     image: mysql/mysql-server:8.0.12
     networks:
       - web
@@ -134,7 +135,7 @@ services:
   mysql-shell:
     container_name: mysql-shell
     env_file:
-      - mysql-cluster/mysql-shell.env
+      - utils/mysql-cluster/mysql-shell.env
     image: neumayer/mysql-shell-batch
     networks:
       - web
@@ -148,7 +149,7 @@ services:
   mysql-router:
     container_name: mysql-router
     env_file:
-      - mysql-cluster/mysql-router.env
+      - utils/mysql-cluster/mysql-router.env
     image: mysql/mysql-router:8.0
     networks:
       - web
@@ -166,7 +167,7 @@ services:
     container_name: nginx
     env_file: .env
     volumes:
-      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+      - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf
     ports:
       - 8082:80
     networks:

diff --git a/docker-compose.dev.yaml → docker-compose.mysql.dev.yaml b/docker-compose.dev.yaml → docker-compose.mysql.dev.yaml
@@ -25,8 +25,11 @@ services:
     environment:
       - LOGGING_LEVEL_ORG_HIBERNATE_SQL=DEBUG
       - SPRING_JPA_SHOW_SQL=true
+      - DB_SELECTOR=mysql
     deploy:
       replicas: 1
+    ports:
+      - 8092:8080
     depends_on:
       - fits
       - db-docker
@@ -58,6 +61,7 @@ services:
     ports:
       - 3306:3306
 
+
   adminer:
     image: adminer
     container_name: adminer
@@ -73,7 +77,7 @@ services:
     container_name: nginx
     env_file: .env
     volumes:
-      - ./nginx.conf:/etc/nginx/conf.d/default.conf
+      - ./utils/nginx/nginx.conf:/etc/nginx/conf.d/default.conf
     ports:
       - 8082:80
     networks:

diff --git a/fits-client/src/main/java/rocks/artur/FITSClient/FITSClient.java b/fits-client/src/main/java/rocks/artur/FITSClient/FITSClient.java
@@ -19,6 +19,7 @@
 import rocks.artur.api_impl.utils.ByteFile;
 import rocks.artur.domain.CharacterisationResult;
 import rocks.artur.domain.Property;
+import rocks.artur.domain.ValueType;
 import rocks.artur.utils.JSONToolkit;
 import rocks.artur.utils.STAXToolkit;
 
@@ -33,10 +34,12 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.util.*;
 import java.util.stream.Collectors;
 
 //@ApplicationScoped
@@ -45,6 +48,13 @@ public class FITSClient implements CharacterisationResultProducer {
     List<String> knownProperties = Arrays.stream(FITSPropertyJsonPath.values()).map(Enum::name).collect(Collectors.toList());
     private String FITS_URL = "http://localhost:8888";
 
+
+    static DateTimeFormatter outputFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+    static DateTimeFormatter inputFormatter = new DateTimeFormatterBuilder()
+            .appendPattern("[yyyy:MM:dd HH:mm:ssXXX][yyyy:MM:dd HH:mm:ss][yyyy:MM:dd HH:mmXXX][yyyy-MM-dd HH:mm:ss][yyyy/MM/dd HH:mm:ss]")
+            .toFormatter();
+
     @Override
     public String getVersion(){
 
@@ -123,12 +133,46 @@ public List<CharacterisationResult> processFile(ByteFile file)  {
             }
             result.addAll(extractCharacterisationResultsStax(content));
         } catch (Exception e) {
-            LOG.error("Exception occurred during file processing");
+            LOG.error("Exception occurred during FITS file parsing");
             e.printStackTrace();
         }
+
+        result=this.fixDateTypes(result);
+        return result;
+    }
+
+    private ArrayList<CharacterisationResult> fixDateTypes(ArrayList<CharacterisationResult> result) {
+        result.stream().forEach(item -> {
+            if (item.getValueType().equals(ValueType.TIMESTAMP)){
+                String value = item.getValue();
+                LOG.debug(String.format("Parsing Object: %s", item));
+                if (item.getSource().startsWith("OIS File Information")) {
+                    LocalDateTime parsed =
+                            LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(value)),
+                                    TimeZone.getDefault().toZoneId());
+                    item.setValue(parsed.format(outputFormatter));
+                } else {
+                    LocalDateTime parsed = tryParseLocalDateTime(value, inputFormatter);
+                    if (parsed != null) {
+                        item.setValue(parsed.format(outputFormatter));
+                    } else {
+                        item.setValue(null);
+                    }
+                }
+                LOG.debug(String.format("Parsed Result: %s", item));
+            }
+        });
         return result;
     }
 
+     LocalDateTime tryParseLocalDateTime(String datetimeString, DateTimeFormatter formatter) {
+        try {
+            return LocalDateTime.parse(datetimeString, formatter);
+        } catch (DateTimeParseException e) {
+            return null;
+        }
+    }
+
     List<CharacterisationResult> extractCharacterisationResults(String fitsResultXML) throws JSONException {
         List<CharacterisationResult> results = new ArrayList<>();
         String fitsResultJSON = JSONToolkit.translateXML(fitsResultXML);