diff --git a/README.md b/README.md index c2d0833..09aebd8 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Place where your FITS files feel good Content Profiling revisited + +# Local build + +docker-compose -f docker-compose.dev.yaml up --build + + File uploading using bash: bash fileupload.sh http://localhost:8082 ~/rnd/data/govdocs_fits/govdocs1/000/ + + diff --git a/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java b/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java index e4b43a2..264c37a 100644 --- a/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java +++ b/core/src/main/java/rocks/artur/api/GetCollectionStatistics.java @@ -5,5 +5,5 @@ public interface GetCollectionStatistics { Map getSizeStatistics(); - Double getConflictRate(); + double getConflictRate(); } diff --git a/core/src/main/java/rocks/artur/api/GetObjects.java b/core/src/main/java/rocks/artur/api/GetObjects.java index 9fef483..a6732f8 100644 --- a/core/src/main/java/rocks/artur/api/GetObjects.java +++ b/core/src/main/java/rocks/artur/api/GetObjects.java @@ -12,4 +12,6 @@ public interface GetObjects { List getObjects(FilterCriteria filterCriteria); Iterable getObject(String filePath); + + List getConflictsFromObject(String filePath); } diff --git a/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java b/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java index 6b1e80c..1e747df 100644 --- a/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java +++ b/core/src/main/java/rocks/artur/api_impl/GetCollectionStatisticsImpl.java @@ -20,7 +20,7 @@ public Map getSizeStatistics() { } @Override - public Double getConflictRate() { - return 17.0; + public double getConflictRate() { + return characterisationResultGateway.getConflictRate(); } } diff --git a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java index 7a35243..61d2585 100644 --- a/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java +++ b/core/src/main/java/rocks/artur/api_impl/GetObjectsImpl.java @@ -26,4 +26,12 @@ public Iterable getObject(String filePath) { Iterable characterisationResultsByFilepath = characterisationResultGateway.getCharacterisationResultsByFilepath(filePath); return characterisationResultsByFilepath; } + + @Override + public List getConflictsFromObject(String filePath) { + List characterisationResultsByFilepath = characterisationResultGateway.getConflictsByFilepath(filePath); + return characterisationResultsByFilepath; + } + + } diff --git a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java index 6cd4f30..5b59141 100644 --- a/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java +++ b/core/src/main/java/rocks/artur/domain/CharacterisationResultGateway.java @@ -42,6 +42,13 @@ public interface CharacterisationResultGateway { */ List getCharacterisationResultsByFilepath(String filePath); + /** + * gets a list of characterisation results with conflicts for a given digital object. + * + * @return an iterable of characterisation results. + */ + List getConflictsByFilepath(String filepath); + Map getSizeStatistics(); List getPropertyValueDistribution(Property property, FilterCriteria filter); @@ -68,4 +75,6 @@ public interface CharacterisationResultGateway { List getSamples(FilterCriteria filterCriteria, SamplingAlgorithms algorithm, List properties); void addCharacterisationResults(List characterisationResults); + + double getConflictRate(); } diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 41a360b..2bd986a 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -3,7 +3,9 @@ version: '3' services: fits: - build: ./fits/ + build: + context: . + dockerfile: ./fits/Dockerfile container_name: fits env_file: .env networks: @@ -14,7 +16,9 @@ services: rest: container_name: rest - build: ./ + build: + context: . + dockerfile: ./Dockerfile env_file: .env networks: - web @@ -26,7 +30,9 @@ services: - db-docker web: - build: ./web/ + build: + context: . + dockerfile: ./web/Dockerfile container_name: web env_file: .env networks: diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java index f1d2466..4f4ba84 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/CharacterisationResultGatewayJpaImpl.java @@ -11,6 +11,7 @@ import rocks.artur.domain.statistics.PropertyValueStatistic; import rocks.artur.jpa.table.CharacterisationResultJPA; import rocks.artur.jpa.table.CharacterisationResultRepository; +import rocks.artur.jpa.view.CharacterisationResultViewJPA; import rocks.artur.jpa.view.CharacterisationResultViewRepository; import java.util.Comparator; @@ -117,6 +118,14 @@ public List getCharacterisationResultsByFilepath(String return result; } + + public List getConflictsByFilepath(String filepath) { + List allJPAByFilePath = characterisationResultViewRepository.findAllByFilePath(filepath); + List result = allJPAByFilePath.stream().filter(item -> item.getValue().equals("CONFLICT")).map(item -> new CharacterisationResult(Property.valueOf(item.getProperty()), item.getValue(), + ValueType.valueOf(item.getValueType()), null, item.getFilePath())).collect(Collectors.toList()); + return result; + } + @Override public Map getSizeStatistics() { Map result = new HashMap<>(); @@ -134,6 +143,9 @@ public Map getSizeStatistics() { Long totalCount = characterisationResultViewRepository.getTotalCount(); result.put("totalCount", totalCount); + Long conflictRate = characterisationResultViewRepository.getConflictCount(); + result.put("conflictRate", conflictRate); + List sizeDistribution = characterisationResultViewRepository.getSizeDistribution(); List collect = sizeDistribution.stream() @@ -200,4 +212,11 @@ public void addCharacterisationResults(List characterisa LOG.debug("saving " + collect); characterisationResultRepository.saveAll(collect); } + + @Override + public double getConflictRate() { + Long totalCount = characterisationResultViewRepository.getTotalCount(); + Long conflictCount = characterisationResultViewRepository.getConflictCount(); + return conflictCount/(double)totalCount; + } } diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java b/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java index 87e5337..50b6407 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/table/CustomCharacterisationResultRepository.java @@ -5,4 +5,6 @@ public interface CustomCharacterisationResultRepository { void saveFast(List results); + + } diff --git a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java index 9f2e9e9..fec6104 100644 --- a/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java +++ b/infra-persistence/src/main/java/rocks/artur/jpa/view/CharacterisationResultViewRepository.java @@ -2,6 +2,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import rocks.artur.jpa.table.CharacterisationResultJPA; import java.util.List; @@ -13,7 +14,10 @@ public interface CharacterisationResultViewRepository extends JpaRepository getPropertyDistribution(); + @Query("select count(*) as count from CharacterisationResultViewJPA where value='CONFLICT'") + Long getConflictCount(); + List findAllByFilePath(String filePath); diff --git a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java index 9eba2bb..1002e96 100644 --- a/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java +++ b/infra-rest/src/main/java/rocks/artur/endpoints/RestService.java @@ -22,6 +22,7 @@ import java.text.ParseException; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") @CrossOrigin @@ -84,6 +85,17 @@ public Iterable getObject( return objects; } + @RequestMapping(method = RequestMethod.POST, value = "/objectconflicts") + @Consumes(MediaType.APPLICATION_JSON) + public List getConflictsPerObject( + @RequestParam(name = "filepath", required = true) @Parameter(name = "filepath", description = "Filepath of a digital object", example = "/home/user/file1") String filepath) { + List objects = getObjects.getConflictsFromObject(filepath); + List collect = objects.stream().map(item -> item.getProperty()).collect(Collectors.toList()); + return collect; + } + + + @RequestMapping(method = RequestMethod.GET, value = "/statistics") public Map getCollectionStatistics() { diff --git a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java index 369e774..aefecfd 100644 --- a/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java +++ b/main/src/test/java/rocks/artur/CharacterisationResultGatewayImplTest.java @@ -96,6 +96,7 @@ void getCharacterisationResultsByFilepathTest() { void getCollectionStatisticsTest() { Map sizeStatistics = characterisationResultGatewaySqlImpl.getSizeStatistics(); Assert.assertEquals(10047L, Long.valueOf(sizeStatistics.get("totalSize").toString()).longValue()); + System.out.println(sizeStatistics); } @@ -114,4 +115,11 @@ void getSFDSamplesTest() { List samples = characterisationResultGatewaySqlImpl.getSamples(null, SamplingAlgorithms.SELECTIVE_FEATURE_DISTRIBUTION, properties); Assert.assertEquals(2, samples.size()); } + + @Test + void getConflictRateTest() { + + double conflictRate = characterisationResultGatewaySqlImpl.getConflictRate(); + Assert.assertEquals(0.333333333333,conflictRate, 0.01); + } } \ No newline at end of file diff --git a/main/src/test/java/rocks/artur/RestServiceTest.java b/main/src/test/java/rocks/artur/RestServiceTest.java index 52793f2..d4e6ab0 100644 --- a/main/src/test/java/rocks/artur/RestServiceTest.java +++ b/main/src/test/java/rocks/artur/RestServiceTest.java @@ -154,6 +154,16 @@ void getObjectsTest() { System.out.println("Result: " + str); } + @Test + void getObjectConflictsTest() { + String str = given().port(port).param("filepath","/home/artur/file1") + .when().post("/objectconflicts") + .then() + .statusCode(200).extract().asString(); + System.out.println("Result: " + str); + } + + @Test void getPropertyDistributionWithFilterTest() { String str = given().port(port) diff --git a/web/Dockerfile b/web/Dockerfile index d8bc4ba..a1b2d87 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,9 +1,10 @@ FROM node:14.5.0-stretch-slim -WORKDIR /app +WORKDIR /app COPY ./web/frontend ./ + RUN npm install RUN npm run build diff --git a/web/frontend/src/components/Table.jsx b/web/frontend/src/components/Table.jsx index 0f7e55a..f7bd6cb 100644 --- a/web/frontend/src/components/Table.jsx +++ b/web/frontend/src/components/Table.jsx @@ -4,7 +4,7 @@ import { tokens } from "../theme"; import { useTheme } from "@mui/material"; -const Table = ({ data, columns, initialState, onRowClick }) => { +const Table = ({ data, columns, initialState, onRowClick, rowFunction }) => { const theme = useTheme(); const colors = tokens(theme.palette.mode); @@ -39,6 +39,10 @@ const Table = ({ data, columns, initialState, onRowClick }) => { "& .MuiDataGrid-toolbarContainer .MuiButton-text": { color: `${colors.grey[100]} !important`, }, + '& .conflict': { + backgroundColor: colors.redAccent[700], + color: colors.greenAccent[300], + }, }} > { columns={columns} components={{ Toolbar: GridToolbar }} initialState={initialState} + + getRowClassName={rowFunction} /> ); diff --git a/web/frontend/src/scenes/dashboard/index.jsx b/web/frontend/src/scenes/dashboard/index.jsx index 122066e..5d4c662 100644 --- a/web/frontend/src/scenes/dashboard/index.jsx +++ b/web/frontend/src/scenes/dashboard/index.jsx @@ -19,6 +19,7 @@ const Dashboard = () => { ], minSize: 4, maxSize: 10000, + conflictRate: 0.17, }); useEffect(() => { @@ -99,6 +100,15 @@ const Dashboard = () => { : (sizeStatistics.maxSize / 1024 / 1024).toFixed(2) } /> + + diff --git a/web/frontend/src/scenes/objectDetails/index.jsx b/web/frontend/src/scenes/objectDetails/index.jsx index 2cbffe2..fd8e6d4 100644 --- a/web/frontend/src/scenes/objectDetails/index.jsx +++ b/web/frontend/src/scenes/objectDetails/index.jsx @@ -49,14 +49,31 @@ const ObjectDetails = () => { ); const data = await response.json(); + + const response2 = await fetch( + BACKEND_URL + + "/objectconflicts?" + + new URLSearchParams({ + filepath: state.objectdetails, + }), + requestOptions + ); + const conflictedProps = await response2.json(); + + + let id = 0; var tmp = data.map((stat) => { var res = stat; res.id = id++; + if (conflictedProps.includes(res.property)) { + res.conflict=true + } return res; }); - + console.log(tmp) setData(tmp); + } catch (error) { console.log(error); } @@ -88,6 +105,12 @@ const ObjectDetails = () => { }, ]; + + const rowFunction = (params) => { + return params.row.conflict ? 'conflict': '' ; + } + + return (
{ subtitle={"on: " + state.objectdetails} >
- +
);