From ee2a75d1ff34d042336f546b3310683dd80beb1b Mon Sep 17 00:00:00 2001 From: artur Date: Sun, 22 Oct 2023 22:11:35 +0200 Subject: [PATCH 1/7] Add new REST endpoint to query a list of conflicts for a given file --- core/src/main/java/rocks/artur/api/GetObjects.java | 2 ++ .../main/java/rocks/artur/api_impl/GetObjectsImpl.java | 8 ++++++++ .../artur/domain/CharacterisationResultGateway.java | 7 +++++++ .../jpa/CharacterisationResultGatewayJpaImpl.java | 9 +++++++++ .../table/CustomCharacterisationResultRepository.java | 2 ++ .../jpa/view/CharacterisationResultViewRepository.java | 3 ++- .../main/java/rocks/artur/endpoints/RestService.java | 10 ++++++++++ main/src/test/java/rocks/artur/RestServiceTest.java | 10 ++++++++++ 8 files changed, 50 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/rocks/artur/api/GetObjects.java b/core/src/main/java/rocks/artur/api/GetObjects.java index 9fef483..da9aa94 100644 --- a/core/src/main/java/rocks/artur/api/GetObjects.java +++ b/core/src/main/java/rocks/artur/api/GetObjects.java @@ -12,4 +12,6 @@ public interface GetObjects { List getObjects(FilterCriteria filterCriteria); Iterable getObject(String filePath); + + Iterable getConflictsFromObject(String filePath); } diff --git a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java index 7a35243..8aab5a1 100644 --- a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java +++ b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java @@ -26,4 +26,12 @@ public Iterable getObject(String filePath) { Iterable characterisationResultsByFilepath = characterisationResultGateway.getCharacterisationResultsByFilepath(filePath); return characterisationResultsByFilepath; } + + @Override + public Iterable getConflictsFromObject(String filePath) { + Iterable characterisationResultsByFilepath = characterisationResultGateway.getConflictsByFilepath(filePath); + return characterisationResultsByFilepath; + } + + } diff --git a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java index 6cd4f30..eea2ff8 100644 --- a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java +++ b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java @@ -42,6 +42,13 @@ public interface CharacterisationResultGateway { */ List getCharacterisationResultsByFilepath(String filePath); + /** + * gets a list of characterisation results with conflicts for a given digital object. + * + * @return an iterable of characterisation results. + */ + List getConflictsByFilepath(String filepath); + Map getSizeStatistics(); List getPropertyValueDistribution(Property property, FilterCriteria filter); diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java index f1d2466..fffb0df 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java @@ -11,6 +11,7 @@ import rocks.artur.domain.statistics.PropertyValueStatistic; import rocks.artur.jpa.table.CharacterisationResultJPA; import rocks.artur.jpa.table.CharacterisationResultRepository; +import rocks.artur.jpa.view.CharacterisationResultViewJPA; import rocks.artur.jpa.view.CharacterisationResultViewRepository; import java.util.Comparator; @@ -117,6 +118,14 @@ public List getCharacterisationResultsByFilepath(String return result; } + + public List getConflictsByFilepath(String filepath) { + List allJPAByFilePath = characterisationResultViewRepository.findAllByFilePath(filepath); + List result = allJPAByFilePath.stream().filter(item -> item.getValue().equals("CONFLICT")).map(item -> new CharacterisationResult(Property.valueOf(item.getProperty()), item.getValue(), + ValueType.valueOf(item.getValueType()), null, item.getFilePath())).collect(Collectors.toList()); + return result; + } + @Override public Map getSizeStatistics() { Map result = new HashMap<>(); diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java b/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java index 87e5337..50b6407 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java @@ -5,4 +5,6 @@ public interface CustomCharacterisationResultRepository { void saveFast(List results); + + } diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java index 9f2e9e9..4f3ef3d 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java @@ -2,6 +2,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import rocks.artur.jpa.table.CharacterisationResultJPA; import java.util.List; @@ -13,7 +14,7 @@ public interface CharacterisationResultViewRepository extends JpaRepository getPropertyDistribution(); - + List findAllByFilePath(String filePath); diff --git a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java index 9eba2bb..b3f528b 100644 --- a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java +++ b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java @@ -84,6 +84,16 @@ public Iterable getObject( return objects; } + @RequestMapping(method = RequestMethod.POST, value = "/objectconflicts") + @Consumes(MediaType.APPLICATION_JSON) + public Iterable getConflictsPerObject( + @RequestParam(name = "filepath", required = true) @Parameter(name = "filepath", description = "Filepath of a digital object", example = "/home/user/file1") String filepath) { + Iterable objects = getObjects.getConflictsFromObject(filepath); + return objects; + } + + + @RequestMapping(method = RequestMethod.GET, value = "/statistics") public Map getCollectionStatistics() { diff --git a/main/src/test/java/rocks/artur/RestServiceTest.java b/main/src/test/java/rocks/artur/RestServiceTest.java index 52793f2..d4e6ab0 100644 --- a/main/src/test/java/rocks/artur/RestServiceTest.java +++ b/main/src/test/java/rocks/artur/RestServiceTest.java @@ -154,6 +154,16 @@ void getObjectsTest() { System.out.println("Result: " + str); } + @Test + void getObjectConflictsTest() { + String str = given().port(port).param("filepath","/home/artur/file1") + .when().post("/objectconflicts") + .then() + .statusCode(200).extract().asString(); + System.out.println("Result: " + str); + } + + @Test void getPropertyDistributionWithFilterTest() { String str = given().port(port) From d3feac194be5116e8de426134a57a235f7846af6 Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 12:40:00 +0200 Subject: [PATCH 2/7] Add highlighting of conflicts in the UI --- .../main/java/rocks/artur/api/GetObjects.java | 2 +- .../rocks/artur/api_impl/GetObjectsImpl.java | 4 +-- .../rocks/artur/endpoints/RestService.java | 8 +++--- web/Dockerfile | 4 ++- web/frontend/src/components/Table.jsx | 8 +++++- .../src/scenes/objectDetails/index.jsx | 26 +++++++++++++++++-- 6 files changed, 42 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/rocks/artur/api/GetObjects.java b/core/src/main/java/rocks/artur/api/GetObjects.java index da9aa94..a6732f8 100644 --- a/core/src/main/java/rocks/artur/api/GetObjects.java +++ b/core/src/main/java/rocks/artur/api/GetObjects.java @@ -13,5 +13,5 @@ public interface GetObjects { List getObjects(FilterCriteria filterCriteria); Iterable getObject(String filePath); - Iterable getConflictsFromObject(String filePath); + List getConflictsFromObject(String filePath); } diff --git a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java index 8aab5a1..61d2585 100644 --- a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java +++ b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java @@ -28,8 +28,8 @@ public Iterable getObject(String filePath) { } @Override - public Iterable getConflictsFromObject(String filePath) { - Iterable characterisationResultsByFilepath = characterisationResultGateway.getConflictsByFilepath(filePath); + public List getConflictsFromObject(String filePath) { + List characterisationResultsByFilepath = characterisationResultGateway.getConflictsByFilepath(filePath); return characterisationResultsByFilepath; } diff --git a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java index b3f528b..1002e96 100644 --- a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java +++ b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java @@ -22,6 +22,7 @@ import java.text.ParseException; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") @CrossOrigin @@ -86,10 +87,11 @@ public Iterable getObject( @RequestMapping(method = RequestMethod.POST, value = "/objectconflicts") @Consumes(MediaType.APPLICATION_JSON) - public Iterable getConflictsPerObject( + public List getConflictsPerObject( @RequestParam(name = "filepath", required = true) @Parameter(name = "filepath", description = "Filepath of a digital object", example = "/home/user/file1") String filepath) { - Iterable objects = getObjects.getConflictsFromObject(filepath); - return objects; + List objects = getObjects.getConflictsFromObject(filepath); + List collect = objects.stream().map(item -> item.getProperty()).collect(Collectors.toList()); + return collect; } diff --git a/web/Dockerfile b/web/Dockerfile index d8bc4ba..2f6a9c1 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,9 +1,11 @@ FROM node:14.5.0-stretch-slim -WORKDIR /app + COPY ./web/frontend ./ + +WORKDIR /app RUN npm install RUN npm run build diff --git a/web/frontend/src/components/Table.jsx b/web/frontend/src/components/Table.jsx index 0f7e55a..f7bd6cb 100644 --- a/web/frontend/src/components/Table.jsx +++ b/web/frontend/src/components/Table.jsx @@ -4,7 +4,7 @@ import { tokens } from "../theme"; import { useTheme } from "@mui/material"; -const Table = ({ data, columns, initialState, onRowClick }) => { +const Table = ({ data, columns, initialState, onRowClick, rowFunction }) => { const theme = useTheme(); const colors = tokens(theme.palette.mode); @@ -39,6 +39,10 @@ const Table = ({ data, columns, initialState, onRowClick }) => { "& .MuiDataGrid-toolbarContainer .MuiButton-text": { color: `${colors.grey[100]} !important`, }, + '& .conflict': { + backgroundColor: colors.redAccent[700], + color: colors.greenAccent[300], + }, }} > { columns={columns} components={{ Toolbar: GridToolbar }} initialState={initialState} + + getRowClassName={rowFunction} /> ); diff --git a/web/frontend/src/scenes/objectDetails/index.jsx b/web/frontend/src/scenes/objectDetails/index.jsx index 2cbffe2..6438f6b 100644 --- a/web/frontend/src/scenes/objectDetails/index.jsx +++ b/web/frontend/src/scenes/objectDetails/index.jsx @@ -49,14 +49,30 @@ const ObjectDetails = () => { ); const data = await response.json(); + + const response2 = await fetch( + BACKEND_URL + + "/objectConflicts?" + + new URLSearchParams({ + filepath: state.objectdetails, + }), + requestOptions + ); + const conflictedProps = await response2.json(); + + + let id = 0; var tmp = data.map((stat) => { var res = stat; res.id = id++; + if (conflictedProps.includes(res.property)) { + res.conflict=true + } return res; }); - setData(tmp); + } catch (error) { console.log(error); } @@ -88,6 +104,12 @@ const ObjectDetails = () => { }, ]; + + const rowFunction = (params) => { + return params.row.conflict ? 'conflict': '' ; + } + + return (
{ subtitle={"on: " + state.objectdetails} >
- +
); From b6255e48f5c2651325dee40931de8665ba2071bb Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 13:16:45 +0200 Subject: [PATCH 3/7] Fix dockercompose build --- README.md | 8 ++++++++ docker-compose.dev.yaml | 12 +++++++++--- web/Dockerfile | 3 +-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c2d0833..09aebd8 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Place where your FITS files feel good Content Profiling revisited + +# Local build + +docker-compose -f docker-compose.dev.yaml up --build + + File uploading using bash: bash fileupload.sh http://localhost:8082 ~/rnd/data/govdocs_fits/govdocs1/000/ + + diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 41a360b..2bd986a 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -3,7 +3,9 @@ version: '3' services: fits: - build: ./fits/ + build: + context: . + dockerfile: ./fits/Dockerfile container_name: fits env_file: .env networks: @@ -14,7 +16,9 @@ services: rest: container_name: rest - build: ./ + build: + context: . + dockerfile: ./Dockerfile env_file: .env networks: - web @@ -26,7 +30,9 @@ services: - db-docker web: - build: ./web/ + build: + context: . + dockerfile: ./web/Dockerfile container_name: web env_file: .env networks: diff --git a/web/Dockerfile b/web/Dockerfile index 2f6a9c1..a1b2d87 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -2,10 +2,9 @@ FROM node:14.5.0-stretch-slim - +WORKDIR /app COPY ./web/frontend ./ -WORKDIR /app RUN npm install RUN npm run build From 21d8fb127178d1e600ca7bf8abdac15165ee4a9d Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 13:52:58 +0200 Subject: [PATCH 4/7] Conflict highlightning works --- web/frontend/src/scenes/objectDetails/index.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/frontend/src/scenes/objectDetails/index.jsx b/web/frontend/src/scenes/objectDetails/index.jsx index 6438f6b..fd8e6d4 100644 --- a/web/frontend/src/scenes/objectDetails/index.jsx +++ b/web/frontend/src/scenes/objectDetails/index.jsx @@ -52,7 +52,7 @@ const ObjectDetails = () => { const response2 = await fetch( BACKEND_URL + - "/objectConflicts?" + + "/objectconflicts?" + new URLSearchParams({ filepath: state.objectdetails, }), @@ -71,6 +71,7 @@ const ObjectDetails = () => { } return res; }); + console.log(tmp) setData(tmp); } catch (error) { From 820964a638bd0cefae97ba00bce84f126165971e Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 14:55:18 +0200 Subject: [PATCH 5/7] Add conflict rate --- .../main/java/rocks/artur/api/GetCollectionStatistics.java | 2 +- .../rocks/artur/api_impl/GetCollectionStatisticsImpl.java | 4 ++-- .../rocks/artur/domain/CharacterisationResultGateway.java | 2 ++ .../artur/jpa/CharacterisationResultGatewayJpaImpl.java | 7 +++++++ .../jpa/view/CharacterisationResultViewRepository.java | 3 +++ .../rocks/artur/CharacterisationResultGatewayImplTest.java | 7 +++++++ 6 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java b/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java index e4b43a2..264c37a 100644 --- a/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java +++ b/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java @@ -5,5 +5,5 @@ public interface GetCollectionStatistics { Map getSizeStatistics(); - Double getConflictRate(); + double getConflictRate(); } diff --git a/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java b/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java index 6b1e80c..1e747df 100644 --- a/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java +++ b/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java @@ -20,7 +20,7 @@ public Map getSizeStatistics() { } @Override - public Double getConflictRate() { - return 17.0; + public double getConflictRate() { + return characterisationResultGateway.getConflictRate(); } } diff --git a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java index eea2ff8..5b59141 100644 --- a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java +++ b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java @@ -75,4 +75,6 @@ public interface CharacterisationResultGateway { List getSamples(FilterCriteria filterCriteria, SamplingAlgorithms algorithm, List properties); void addCharacterisationResults(List characterisationResults); + + double getConflictRate(); } diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java index fffb0df..75a6544 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java @@ -209,4 +209,11 @@ public void addCharacterisationResults(List characterisa LOG.debug("saving " + collect); characterisationResultRepository.saveAll(collect); } + + @Override + public double getConflictRate() { + Long totalCount = characterisationResultViewRepository.getTotalCount(); + Long conflictCount = characterisationResultViewRepository.getConflictCount(); + return conflictCount/(double)totalCount; + } } diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java index 4f3ef3d..fec6104 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java @@ -14,6 +14,9 @@ public interface CharacterisationResultViewRepository extends JpaRepository getPropertyDistribution(); + @Query("select count(*) as count from CharacterisationResultViewJPA where value='CONFLICT'") + Long getConflictCount(); + List findAllByFilePath(String filePath); diff --git a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java index 369e774..6e572af 100644 --- a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java +++ b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java @@ -114,4 +114,11 @@ void getSFDSamplesTest() { List samples = characterisationResultGatewaySqlImpl.getSamples(null, SamplingAlgorithms.SELECTIVE_FEATURE_DISTRIBUTION, properties); Assert.assertEquals(2, samples.size()); } + + @Test + void getConflictRateTest() { + + double conflictRate = characterisationResultGatewaySqlImpl.getConflictRate(); + Assert.assertEquals(0.333333333333,conflictRate, 0.01); + } } \ No newline at end of file From 849c0088d379f853ee1b5e8ab52a247d306a1340 Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 14:58:23 +0200 Subject: [PATCH 6/7] Add conflict rate --- .../rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java | 3 +++ .../rocks/artur/CharacterisationResultGatewayImplTest.java | 1 + 2 files changed, 4 insertions(+) diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java index 75a6544..4f4ba84 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java @@ -143,6 +143,9 @@ public Map getSizeStatistics() { Long totalCount = characterisationResultViewRepository.getTotalCount(); result.put("totalCount", totalCount); + Long conflictRate = characterisationResultViewRepository.getConflictCount(); + result.put("conflictRate", conflictRate); + List sizeDistribution = characterisationResultViewRepository.getSizeDistribution(); List collect = sizeDistribution.stream() diff --git a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java index 6e572af..aefecfd 100644 --- a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java +++ b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java @@ -96,6 +96,7 @@ void getCharacterisationResultsByFilepathTest() { void getCollectionStatisticsTest() { Map sizeStatistics = characterisationResultGatewaySqlImpl.getSizeStatistics(); Assert.assertEquals(10047L, Long.valueOf(sizeStatistics.get("totalSize").toString()).longValue()); + System.out.println(sizeStatistics); } From 85829c2afad2e08070b685fc32d010d541678da6 Mon Sep 17 00:00:00 2001 From: artur Date: Mon, 23 Oct 2023 15:00:45 +0200 Subject: [PATCH 7/7] Add conflict rate to UI --- web/frontend/src/scenes/dashboard/index.jsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web/frontend/src/scenes/dashboard/index.jsx b/web/frontend/src/scenes/dashboard/index.jsx index 122066e..5d4c662 100644 --- a/web/frontend/src/scenes/dashboard/index.jsx +++ b/web/frontend/src/scenes/dashboard/index.jsx @@ -19,6 +19,7 @@ const Dashboard = () => { ], minSize: 4, maxSize: 10000, + conflictRate: 0.17, }); useEffect(() => { @@ -99,6 +100,15 @@ const Dashboard = () => { : (sizeStatistics.maxSize / 1024 / 1024).toFixed(2) } /> + +