From e1179d6611a1ce620e4fa0b8a96eea20f3e13b3a Mon Sep 17 00:00:00 2001 From: Jonas Bulcke Date: Tue, 8 Oct 2024 10:10:02 +0200 Subject: [PATCH 1/4] fix: improve bucketisation member reader --- .../fragmentation/FragmentationService.java | 38 +++--- .../entities/UnprocessedView.java | 5 + .../repository/UnprocessedViewRepository.java | 9 ++ .../FragmentationServiceTest.java | 114 +++++++++--------- .../services/MemberMetricsRepository.java | 5 - .../UnprocessedViewPostgresRepository.java | 26 ++++ .../mapper/UnprocessedViewRowMapper.java | 19 +++ .../IngestMetricsPostgresRepository.java | 12 -- .../postgres/batch/MemberItemReader.java | 7 +- .../repository/MemberEntityRepository.java | 4 - .../3_4_1/alter-unprocessed-views-view.sql | 10 ++ .../3_4_1/alter-unprocessed-views-view.xml | 17 +++ .../resources/db/changelog/3_4_1/master.xml | 8 ++ .../main/resources/db/changelog/master.xml | 1 + .../ldes/server/FragmentationSteps.java | 2 +- .../server/LdesServerIntegrationTest.java | 11 +- 16 files changed, 180 insertions(+), 108 deletions(-) create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/UnprocessedView.java create mode 100644 ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/UnprocessedViewRepository.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/UnprocessedViewPostgresRepository.java create mode 100644 ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/UnprocessedViewRowMapper.java create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java index d1bf5c5fc..e404ecc60 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationService.java @@ -1,8 +1,8 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.UnprocessedView; import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.exceptions.FragmentationJobException; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.UnprocessedViewRepository; import org.springframework.batch.core.*; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.job.builder.JobBuilder; @@ -29,34 +29,38 @@ public class FragmentationService { public static final String FRAGMENTATION_JOB = "fragmentation"; public static final String COLLECTION_NAME = "collectionName"; public static final String VIEW_NAME = "viewName"; + public static final String COLLECTION_ID = "collectionId"; + public static final String VIEW_ID = "viewId"; public static final String LDES_SERVER_CREATE_FRAGMENTS_COUNT = "ldes_server_create_fragments_count"; private final JobLauncher jobLauncher; private final JobExplorer jobExplorer; private final JobRepository jobRepository; private final Job bucketiseJob; - private final MemberMetricsRepository memberRepository; + private final UnprocessedViewRepository unprocessedViewRepository; public FragmentationService(@Qualifier(ASYNC_JOB_LAUNCHER) JobLauncher jobLauncher, JobRepository jobRepository, JobExplorer jobExplorer, @Qualifier(BUCKETISATION_STEP) Step bucketiseMembersStep, Step paginationStep, - MemberMetricsRepository memberRepository) { + UnprocessedViewRepository unprocessedViewRepository) { this.jobLauncher = jobLauncher; this.jobExplorer = jobExplorer; this.jobRepository = jobRepository; - this.memberRepository = memberRepository; - this.bucketiseJob = createJob(jobRepository, bucketiseMembersStep, paginationStep); + this.unprocessedViewRepository = unprocessedViewRepository; + this.bucketiseJob = createJob(bucketiseMembersStep, paginationStep); this.cleanupOldJobs(); } @Scheduled(cron = FRAGMENTATION_CRON) public void scheduledJobLauncher() { - memberRepository.getUnprocessedViews() + unprocessedViewRepository.findAll() .parallelStream() .filter(this::noJobsRunning) - .forEach(viewName -> { + .forEach(unprocessedView -> { try { jobLauncher.run(bucketiseJob, new JobParametersBuilder() - .addString(VIEW_NAME, viewName.getViewName()) - .addString(COLLECTION_NAME, viewName.getCollectionName()) + .addLong(VIEW_ID, (long) unprocessedView.viewId()) + .addLong(COLLECTION_ID, (long) unprocessedView.collectionId()) + .addString(VIEW_NAME, unprocessedView.viewName()) + .addString(COLLECTION_NAME, unprocessedView.collectionName()) .addLocalDateTime("triggered", LocalDateTime.now()) .toJobParameters()); } catch (JobInstanceAlreadyCompleteException | JobExecutionAlreadyRunningException | @@ -66,18 +70,22 @@ public void scheduledJobLauncher() { }); } - private boolean noJobsRunning(ViewName viewName) { + private boolean noJobsRunning(UnprocessedView unprocessedView) { return jobExplorer.findRunningJobExecutions(FRAGMENTATION_JOB) .stream() .noneMatch(jobExecution -> { var params = jobExecution.getJobParameters(); - String view = Objects.requireNonNull(params.getString(VIEW_NAME)); - String collection = Objects.requireNonNull(params.getString(COLLECTION_NAME)); - return view.equals(viewName.getViewName()) && collection.equals(viewName.getCollectionName()); + final UnprocessedView fromParams = new UnprocessedView( + Objects.requireNonNull(params.getLong(COLLECTION_ID)).intValue(), + params.getString(COLLECTION_NAME), + Objects.requireNonNull(params.getLong(VIEW_ID)).intValue(), + params.getString(VIEW_NAME) + ); + return Objects.equals(fromParams, unprocessedView); }); } - private Job createJob(JobRepository jobRepository, Step step, Step paginationStep) { + private Job createJob(Step step, Step paginationStep) { return new JobBuilder(FRAGMENTATION_JOB, jobRepository) .start(step) .next(paginationStep) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/UnprocessedView.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/UnprocessedView.java new file mode 100644 index 000000000..fc51e6d94 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/entities/UnprocessedView.java @@ -0,0 +1,5 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities; + +public record UnprocessedView(int collectionId, String collectionName, int viewId, String viewName) { + +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/UnprocessedViewRepository.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/UnprocessedViewRepository.java new file mode 100644 index 000000000..699bd3292 --- /dev/null +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/UnprocessedViewRepository.java @@ -0,0 +1,9 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.UnprocessedView; + +import java.util.List; + +public interface UnprocessedViewRepository { + List findAll(); +} diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java index 35af7951d..bf70b717b 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java +++ b/ldes-fragmentisers/ldes-fragmentisers-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/FragmentationServiceTest.java @@ -1,81 +1,77 @@ package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.ServerMetrics; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.UnprocessedView; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.UnprocessedViewRepository; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.Step; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.batch.test.context.SpringBatchTest; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.batch.core.repository.JobRepository; import java.util.List; +import java.util.Objects; import java.util.Set; import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.FragmentationService.*; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BatchConfiguration.ASYNC_JOB_LAUNCHER; -import static be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.batch.BucketJobDefinitions.BUCKETISATION_STEP; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.tuple; import static org.mockito.Mockito.*; -@RunWith(SpringRunner.class) -@SpringBatchTest -@EnableAutoConfiguration -@ContextConfiguration(classes = {SpringBatchConfiguration.class, FragmentationService.class}) -@TestPropertySource(properties = {"ldes-server.fragmentation-cron=*/1 * * * * *"}) +@ExtendWith(MockitoExtension.class) class FragmentationServiceTest { - @MockBean(name = BUCKETISATION_STEP) - Step bucketStep; - @MockBean(name = "paginationStep") - Step paginationStep; - @MockBean - ServerMetrics serverMetrics; - @MockBean - FragmentationStrategyCollection strategyCollection; - @MockBean - MemberMetricsRepository memberMetricsRepository; - @MockBean(name = ASYNC_JOB_LAUNCHER) - JobLauncher jobLauncher; - @MockBean - JobExplorer jobExplorer; - @Autowired + private static final String COLLECTION = "collection"; + @Mock + private Step bucketStep; + @Mock + private Step paginationStep; + @Mock + private MemberMetricsRepository memberMetricsRepository; + @Mock + private JobLauncher jobLauncher; + @Mock + private JobExplorer jobExplorer; + @Mock + private JobRepository jobRepository; + @Mock + private UnprocessedViewRepository unprocessedViewRepository; + @Captor + private ArgumentCaptor captor; private FragmentationService fragmentationService; + private List unprocessedViews; + + @BeforeEach + void setUp() { + fragmentationService = new FragmentationService(jobLauncher, jobRepository, jobExplorer, bucketStep, paginationStep, unprocessedViewRepository); + unprocessedViews = List.of( + new UnprocessedView(1, COLLECTION, 1, "v1"), + new UnprocessedView(1, COLLECTION, 2, "v2") + ); + } @Test void when_unprocessedViews_then_triggerJobsForEachView() throws Exception { - String collection = "collection"; - - when(memberMetricsRepository.getUnprocessedViews()) - .thenReturn(List.of(new ViewName(collection, "v1"), - new ViewName(collection, "v2"))); + when(unprocessedViewRepository.findAll()).thenReturn(unprocessedViews); fragmentationService.scheduledJobLauncher(); - ArgumentCaptor captor = ArgumentCaptor.forClass(JobParameters.class); verify(jobLauncher, times(2)).run(any(), captor.capture()); assertThat(captor.getAllValues()) - .extracting(obj -> obj.getString(COLLECTION_NAME), obj -> obj.getString(VIEW_NAME)) - .containsExactlyInAnyOrder( - tuple(collection, "v1"), - tuple(collection, "v2") - ); + .map(FragmentationServiceTest::mapParamsToUnprocessedView) + .containsExactlyInAnyOrderElementsOf(unprocessedViews); } @Test void when_noUnprocessedViews_then_triggerNone() { - when(memberMetricsRepository.getUnprocessedViews()).thenReturn(List.of()); + when(unprocessedViewRepository.findAll()).thenReturn(List.of()); fragmentationService.scheduledJobLauncher(); @@ -84,29 +80,33 @@ void when_noUnprocessedViews_then_triggerNone() { @Test void when_unprocessedViews_then_triggerJobsForEachViewThatIsntRunningAlready() throws Exception { - String collection = "collection"; - - when(memberMetricsRepository.getUnprocessedViews()) - .thenReturn(List.of(new ViewName(collection, "v1"), new ViewName(collection, "v2"))); + when(unprocessedViewRepository.findAll()).thenReturn(unprocessedViews); JobExecution jobExecution = mock(JobExecution.class); JobParameters jobParameters = new JobParametersBuilder() + .addLong(VIEW_ID, 1L) + .addLong(COLLECTION_ID, 1L) .addString(VIEW_NAME, "v1") - .addString(COLLECTION_NAME, collection) + .addString(COLLECTION_NAME, COLLECTION) .toJobParameters(); when(jobExecution.getJobParameters()).thenReturn(jobParameters); when(jobExplorer.findRunningJobExecutions(FRAGMENTATION_JOB)).thenReturn(Set.of(jobExecution)); fragmentationService.scheduledJobLauncher(); - ArgumentCaptor captor = ArgumentCaptor.forClass(JobParameters.class); - verify(jobLauncher, times(1)).run(any(), captor.capture()); - assertThat(captor.getAllValues()) - .extracting(obj -> obj.getString(COLLECTION_NAME), obj -> obj.getString(VIEW_NAME)) - .containsExactlyInAnyOrder( - tuple(collection, "v2") - ); + verify(jobLauncher).run(any(), captor.capture()); + assertThat(captor.getValue()) + .extracting(FragmentationServiceTest::mapParamsToUnprocessedView) + .isEqualTo(new UnprocessedView(1, COLLECTION, 2, "v2")); } + private static UnprocessedView mapParamsToUnprocessedView(JobParameters params) { + return new UnprocessedView( + Objects.requireNonNull(params.getLong(COLLECTION_ID)).intValue(), + params.getString(COLLECTION_NAME), + Objects.requireNonNull(params.getLong(VIEW_ID)).intValue(), + params.getString(VIEW_NAME) + ); + } } diff --git a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/services/MemberMetricsRepository.java b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/services/MemberMetricsRepository.java index 8c34521d7..6330204a4 100644 --- a/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/services/MemberMetricsRepository.java +++ b/ldes-server-domain/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/domain/services/MemberMetricsRepository.java @@ -1,10 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.server.domain.services; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; - -import java.util.List; - public interface MemberMetricsRepository { int getTotalCount(String collectionName); - List getUnprocessedViews(); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/UnprocessedViewPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/UnprocessedViewPostgresRepository.java new file mode 100644 index 000000000..4b46b7581 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/UnprocessedViewPostgresRepository.java @@ -0,0 +1,26 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.UnprocessedView; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper.UnprocessedViewRowMapper; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.UnprocessedViewRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Repository +public class UnprocessedViewPostgresRepository implements UnprocessedViewRepository { + public static final String SQL = "select * from unprocessed_views"; + private final JdbcTemplate jdbcTemplate; + + public UnprocessedViewPostgresRepository(JdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + @Transactional(readOnly = true) + public List findAll() { + return jdbcTemplate.query(SQL, new UnprocessedViewRowMapper()); + } +} diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/UnprocessedViewRowMapper.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/UnprocessedViewRowMapper.java new file mode 100644 index 000000000..6096ca629 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/mapper/UnprocessedViewRowMapper.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.postgres.mapper; + +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.entities.UnprocessedView; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UnprocessedViewRowMapper implements RowMapper { + @Override + public UnprocessedView mapRow(ResultSet rs, int rowNum) throws SQLException { + return new UnprocessedView( + rs.getInt("collection_id"), + rs.getString("collection_name"), + rs.getInt("view_id"), + rs.getString("view_name") + ); + } +} diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/IngestMetricsPostgresRepository.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/IngestMetricsPostgresRepository.java index f7645e207..7d843dcad 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/IngestMetricsPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/IngestMetricsPostgresRepository.java @@ -1,13 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.model.ViewName; import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import java.util.List; - @Repository public class IngestMetricsPostgresRepository implements MemberMetricsRepository { private final MemberEntityRepository repository; @@ -21,13 +18,4 @@ public IngestMetricsPostgresRepository(MemberEntityRepository repository) { public int getTotalCount(String collectionName) { return repository.countMemberEntitiesByColl(collectionName); } - - @Override - @Transactional(readOnly = true) - public List getUnprocessedViews() { - return repository.getUnprocessedViews() - .stream() - .map(tuple -> new ViewName(tuple.get(0, String.class), tuple.get(1, String.class))) - .toList(); - } } diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java index a3fe12718..990e560eb 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReader.java @@ -50,12 +50,9 @@ private PostgresPagingQueryProvider memberQuery() { join members m on m.collection_id = c.collection_id """); queryProvider.setWhereClause(""" - c.name = :collectionName and + c.collection_id = :collectionId and m.member_id > ( - COALESCE( - (SELECT max(pm.member_id) FROM page_members pm - WHERE pm.view_id IN (select v.view_id from views v where v.name = :viewName)), - (0)::bigint) + COALESCE((SELECT max(pm.member_id) FROM page_members pm WHERE pm.view_id = :viewId), (0)::bigint) ) """); queryProvider.setSortKeys(sortKeys); diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java index e2483ed55..4dc0abf60 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/repository/MemberEntityRepository.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.entity.MemberEntity; -import jakarta.persistence.Tuple; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; @@ -18,9 +17,6 @@ select count(*) """, nativeQuery = true) int countMemberEntitiesByColl(String collectionName); - @Query(value = "select * from unprocessed_views", nativeQuery = true) - List getUnprocessedViews(); - @Query(value = "SELECT CASE WHEN COUNT(*) > 0 THEN true ELSE false END " + "FROM members " + "WHERE collection_id = :collectionId " + diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.sql new file mode 100644 index 000000000..2790162e5 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.sql @@ -0,0 +1,10 @@ +SELECT c.collection_id, + c.name AS collection_name, + v.view_id, + v.name AS view_name +FROM collections c + JOIN views v ON c.collection_id = v.collection_id + JOIN needs_bucketization nb ON nb.collection_id = c.collection_id AND nb.view_id = v.view_id + JOIN needs_pagination np ON np.collection_id = c.collection_id AND np.view_id = v.view_id +WHERE nb.should_bucketize + OR np.should_paginate; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.xml new file mode 100644 index 000000000..35bee533f --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/alter-unprocessed-views-view.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml new file mode 100644 index 000000000..8903f5f29 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index 3061e20c4..13866f719 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -8,5 +8,6 @@ + \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java index 1b067aa8e..1b0528c9c 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/FragmentationSteps.java @@ -183,7 +183,7 @@ public void thisFragmentContainsOnlyMembersWithSkolemizedIdentifiers(int memberC @Then("I wait until all members are fragmented") public void waitUntilAllMembersAreFragmented() { - await().until(() -> memberMetricsRepository.getUnprocessedViews().isEmpty()); + await().until(() -> unprocessedViewRepository.findAll().isEmpty()); } private static Integer countSkolemizedIds(StmtIterator stmtIterator) { diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java index dc53e37f5..75073aa29 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/LdesServerIntegrationTest.java @@ -1,8 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.server; -import be.vlaanderen.informatievlaanderen.ldes.server.domain.services.MemberMetricsRepository; +import be.vlaanderen.informatievlaanderen.ldes.server.fragmentation.repository.UnprocessedViewRepository; import be.vlaanderen.informatievlaanderen.ldes.server.ingest.postgres.repository.MemberEntityRepository; -import be.vlaanderen.informatievlaanderen.ldes.server.ingest.repositories.MemberRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.PageRelationPostgresRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageEntityRepository; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.repository.PageRelationEntityRepository; @@ -14,7 +13,6 @@ import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.ComponentScan; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.context.ActiveProfiles; @@ -47,12 +45,7 @@ public class LdesServerIntegrationTest { @Autowired JdbcTemplate jdbcTemplate; @Autowired - ApplicationEventPublisher applicationEventPublisher; - - @Autowired - MemberRepository memberRepository; - @Autowired - MemberMetricsRepository memberMetricsRepository; + UnprocessedViewRepository unprocessedViewRepository; @Autowired PageRelationEntityRepository pageRelationEntityRepository; @Autowired From 84942860b8d0783ce6766fe4c5dfd37042ce1207 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke Date: Tue, 8 Oct 2024 10:34:50 +0200 Subject: [PATCH 2/4] ci: prepare release 3.4.1 --- ldes-fragmentisers/ldes-fragmentisers-common/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml | 2 +- ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml | 2 +- .../ldes-fragmentisers-timebased-hierarchical/pom.xml | 2 +- ldes-fragmentisers/pom.xml | 2 +- ldes-server-admin/pom.xml | 2 +- ldes-server-application/pom.xml | 2 +- ldes-server-compaction/pom.xml | 2 +- ldes-server-domain/pom.xml | 2 +- ldes-server-infra-postgres/pom.xml | 2 +- ldes-server-infra-postgres/postgres-admin-repository/pom.xml | 2 +- .../postgres-fragmentation-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-ingest-repository/pom.xml | 2 +- ldes-server-infra-postgres/postgres-liquibase/pom.xml | 2 +- .../postgres-pagination-repository/pom.xml | 2 +- .../postgres-retention-repository/pom.xml | 2 +- ldes-server-instrumentation/pom.xml | 2 +- ldes-server-integration-test/pom.xml | 2 +- ldes-server-pagination/pom.xml | 2 +- ldes-server-port-fetch-rest/pom.xml | 2 +- ldes-server-port-fetch/pom.xml | 2 +- ldes-server-port-ingest-rest/pom.xml | 2 +- ldes-server-port-ingest/pom.xml | 2 +- ldes-server-retention/pom.xml | 2 +- pom.xml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml index 4684d1189..a1267fbca 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-common/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0 + 3.4.1-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml index cb5b76515..b65945a32 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-geospatial/pom.xml @@ -3,7 +3,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0 + 3.4.1-SNAPSHOT 4.0.0 diff --git a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml index de91efc4a..b42c62e5e 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-reference/pom.xml @@ -5,7 +5,7 @@ ldes-fragmentisers be.vlaanderen.informatievlaanderen.vsds - 3.4.0 + 3.4.1-SNAPSHOT 4.0.0 jar diff --git a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml index 9764ab4e1..42851309b 100644 --- a/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml +++ b/ldes-fragmentisers/ldes-fragmentisers-timebased-hierarchical/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-fragmentisers - 3.4.0 + 3.4.1-SNAPSHOT ldes-fragmentisers-timebased-hierarchical diff --git a/ldes-fragmentisers/pom.xml b/ldes-fragmentisers/pom.xml index 472ae52f1..bc8676391 100644 --- a/ldes-fragmentisers/pom.xml +++ b/ldes-fragmentisers/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-fragmentisers diff --git a/ldes-server-admin/pom.xml b/ldes-server-admin/pom.xml index 50edf7308..06a4f5217 100644 --- a/ldes-server-admin/pom.xml +++ b/ldes-server-admin/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT diff --git a/ldes-server-application/pom.xml b/ldes-server-application/pom.xml index 3f46314d2..0f939a6ef 100644 --- a/ldes-server-application/pom.xml +++ b/ldes-server-application/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-application diff --git a/ldes-server-compaction/pom.xml b/ldes-server-compaction/pom.xml index 06060e26e..d27690dfb 100644 --- a/ldes-server-compaction/pom.xml +++ b/ldes-server-compaction/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-compaction diff --git a/ldes-server-domain/pom.xml b/ldes-server-domain/pom.xml index b8a3f0c49..e9c9c7d65 100644 --- a/ldes-server-domain/pom.xml +++ b/ldes-server-domain/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-domain diff --git a/ldes-server-infra-postgres/pom.xml b/ldes-server-infra-postgres/pom.xml index bf589c95d..0c6cc71c6 100644 --- a/ldes-server-infra-postgres/pom.xml +++ b/ldes-server-infra-postgres/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT pom diff --git a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml index f5df0f112..78c5a3747 100644 --- a/ldes-server-infra-postgres/postgres-admin-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-admin-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-admin-repository diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml index 6114ca433..ef0e47911 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-fragmentation-repository diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml index 249e7b287..27160d7f8 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-ingest-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-ingest-repository diff --git a/ldes-server-infra-postgres/postgres-liquibase/pom.xml b/ldes-server-infra-postgres/postgres-liquibase/pom.xml index 01aec83ac..f890b5ed8 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/pom.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-liquibase diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml index 0968f190e..60f6bf45e 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-pagination-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-pagination-repository diff --git a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml index 9ce6e79b7..44d4b4765 100644 --- a/ldes-server-infra-postgres/postgres-retention-repository/pom.xml +++ b/ldes-server-infra-postgres/postgres-retention-repository/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server-infra-postgres - 3.4.0 + 3.4.1-SNAPSHOT postgres-retention-repository diff --git a/ldes-server-instrumentation/pom.xml b/ldes-server-instrumentation/pom.xml index 0366c9ea4..510f7b057 100644 --- a/ldes-server-instrumentation/pom.xml +++ b/ldes-server-instrumentation/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-instrumentation diff --git a/ldes-server-integration-test/pom.xml b/ldes-server-integration-test/pom.xml index 3e1d52f70..6dbdfde4a 100644 --- a/ldes-server-integration-test/pom.xml +++ b/ldes-server-integration-test/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-integration-test diff --git a/ldes-server-pagination/pom.xml b/ldes-server-pagination/pom.xml index 61801f5e5..2e025d18d 100644 --- a/ldes-server-pagination/pom.xml +++ b/ldes-server-pagination/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-pagination diff --git a/ldes-server-port-fetch-rest/pom.xml b/ldes-server-port-fetch-rest/pom.xml index 87a922687..9dd629d84 100644 --- a/ldes-server-port-fetch-rest/pom.xml +++ b/ldes-server-port-fetch-rest/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-port-fetch-rest diff --git a/ldes-server-port-fetch/pom.xml b/ldes-server-port-fetch/pom.xml index 871daaf3e..651edbeb9 100644 --- a/ldes-server-port-fetch/pom.xml +++ b/ldes-server-port-fetch/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-port-fetch diff --git a/ldes-server-port-ingest-rest/pom.xml b/ldes-server-port-ingest-rest/pom.xml index 907187466..d58372d09 100644 --- a/ldes-server-port-ingest-rest/pom.xml +++ b/ldes-server-port-ingest-rest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-port-ingest-rest diff --git a/ldes-server-port-ingest/pom.xml b/ldes-server-port-ingest/pom.xml index 0366e607d..dab0a9b4d 100644 --- a/ldes-server-port-ingest/pom.xml +++ b/ldes-server-port-ingest/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-port-ingest diff --git a/ldes-server-retention/pom.xml b/ldes-server-retention/pom.xml index 9e12a4e61..025325820 100644 --- a/ldes-server-retention/pom.xml +++ b/ldes-server-retention/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT ldes-server-retention diff --git a/pom.xml b/pom.xml index 6737fdf99..0ebcee1a9 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ be.vlaanderen.informatievlaanderen.vsds ldes-server - 3.4.0 + 3.4.1-SNAPSHOT pom From 1e040a25ed3a83c3eb5f157b25ab48eb18d9465f Mon Sep 17 00:00:00 2001 From: Jonas Bulcke Date: Tue, 8 Oct 2024 11:37:07 +0200 Subject: [PATCH 3/4] fix: broken test --- .../ingest/postgres/batch/MemberItemReaderTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java index 3e6091e85..56d01bb8b 100644 --- a/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java +++ b/ldes-server-infra-postgres/postgres-ingest-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/ingest/postgres/batch/MemberItemReaderTest.java @@ -53,6 +53,8 @@ class MemberItemReaderTest { false ); private static final String VIEW_NAME = "by-page"; + private static final long COLLECTION_ID = 1; + private static final long BY_PAGE_ID = 2; @Autowired private JdbcTemplate jdbcTemplate; @@ -83,7 +85,7 @@ void tearDown() { @Test void given_EmptyMembersTable_when_ReadNewMembers_then_ReturnNull() throws Exception { - setupStepScope(COLLECTION_NAME, VIEW_NAME); + setupStepScope(COLLECTION_ID, COLLECTION_NAME, BY_PAGE_ID, VIEW_NAME); final FragmentationMember result = newMemberReader.read(); @@ -94,7 +96,7 @@ void given_EmptyMembersTable_when_ReadNewMembers_then_ReturnNull() throws Except @ValueSource(strings = {"fantasy/non-existing", "mobility-hindrances/fantasy-view", "fantasy/by-page"}) void given_AbsentCollectionsAndViews_when_RefragmentMembers_then_ReturnNull(String viewNameAsString) throws Exception { final ViewName viewName = ViewName.fromString(viewNameAsString); - setupStepScope(viewName.getCollectionName(), viewName.getViewName()); + setupStepScope(10, viewName.getCollectionName(), 10, viewName.getViewName()); final FragmentationMember result = newMemberReader.read(); @@ -104,7 +106,7 @@ void given_AbsentCollectionsAndViews_when_RefragmentMembers_then_ReturnNull(Stri @Test void given_MembersPresentInDb_test_ReadNewMembers() throws Exception { int count = 5; - setupStepScope(COLLECTION_NAME, VIEW_NAME); + setupStepScope(COLLECTION_ID, COLLECTION_NAME, BY_PAGE_ID, VIEW_NAME); insertMembers(count); final List readMembers = new ArrayList<>(); @@ -129,8 +131,10 @@ void given_MembersPresentInDb_test_ReadNewMembers() throws Exception { .isEqualToIgnoringNanos(START_TIME); } - private void setupStepScope(String collectionName, String viewName) { + private void setupStepScope(long collectionId, String collectionName, long viewId, String viewName) { JobParameters jobParameters = new JobParametersBuilder() + .addLong("collectionId", collectionId) + .addLong("viewId", viewId) .addString("collectionName", collectionName) .addString("viewName", viewName) .toJobParameters(); From 5782cc74e8a2d5a1e8c5883f3f5c2462ddf62b8d Mon Sep 17 00:00:00 2001 From: Ranko Orlic Date: Tue, 8 Oct 2024 16:02:55 +0200 Subject: [PATCH 4/4] chore: fix view stats (#1396) --- .../db/changelog/3_4_1/fix_view_stats.sql | 30 +++++++++++++++++++ .../resources/db/changelog/3_4_1/master.xml | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/fix_view_stats.sql diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/fix_view_stats.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/fix_view_stats.sql new file mode 100644 index 000000000..dde53befb --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/fix_view_stats.sql @@ -0,0 +1,30 @@ +-- update current counts +update view_stats vs +set + bucketized_count = (select count(distinct(member_id)) from page_members pm WHERE pm.view_id = vs.view_id), + paginated_count = (select count(distinct(member_id)) from page_members pm WHERE pm.view_id = vs.view_id and pm.page_id is not null) +; + +-- fix function to increase view stats count when member bucketized +create or replace function on_page_member_inserted() returns trigger language plpgsql as $$ +begin + if ((select count(member_id) from page_members + where view_id = NEW.view_id and member_id = NEW.member_id) = 1) then + update view_stats set bucketized_count = bucketized_count + 1 where view_id = NEW.view_id; + end if; + return null; +end +$$; + +-- fix function to increase view stats count when member paginated +create or replace function on_page_member_updating() returns trigger language plpgsql as $$ +begin + if (OLD.page_id is null and + NEW.page_id is not null and + ((select count(member_id) from page_members + where view_id = NEW.view_id and member_id = NEW.member_id) = 1)) then + update view_stats set paginated_count = paginated_count + 1 where view_id = NEW.view_id; + end if; + return NEW; +end +$$; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml index 8903f5f29..ac83c73a5 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_4_1/master.xml @@ -5,4 +5,6 @@ https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + + \ No newline at end of file