Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Trm 32146 populate history #529

Merged
merged 6 commits into from
Jan 10, 2025
Merged

Trm 32146 populate history #529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4070b54
change find entries by release name
supun-ebi Dec 18, 2024
dec425e
Merge branch 'main' into TRM-32146-populate-history
supun-ebi Dec 18, 2024
ae805b0
test fix
supun-ebi Dec 20, 2024
806004b
add date time formatting
supun-ebi Dec 20, 2024
624c1d8
make db props empty
supun-ebi Dec 20, 2024
4aaaced
rename uniProtRelease
supun-ebi Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ...ata-rest/src/main/java/org/uniprot/api/support/data/statistics/entity/AttributeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@

@Data
@Entity
@Table(indexes = @Index(columnList = "attributeName, statistics_category_id", unique = true))
@Table(
name = "attribute_query",
indexes = @Index(columnList = "attributeName, statistics_category_id", unique = true))
public class AttributeQuery {
@Id private Long id;

Expand Down
4 changes: 2 additions & 2 deletions ...rc/main/java/org/uniprot/api/support/data/statistics/entity/UniProtKBStatisticsEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

@Data
@Entity
@Table(name = "uniprotkb_statistics_entry")
@Table(name = "uniprotkb_statistics_entry", indexes = @Index(columnList = "release_name"))
public class UniProtKBStatisticsEntry {
@Id private Long id;
private String attributeName;
Expand All @@ -21,7 +21,7 @@ public class UniProtKBStatisticsEntry {

@ManyToOne
@JoinColumn(name = "release_name")
private UniProtRelease releaseName;
private UniProtRelease uniProtRelease;

supun-ebi marked this conversation as resolved.
Show resolved Hide resolved
@Enumerated(EnumType.STRING)
private EntryType entryType;
Expand Down
13 changes: 9 additions & 4 deletions ...ata-rest/src/main/java/org/uniprot/api/support/data/statistics/entity/UniProtRelease.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,21 @@

import java.util.Date;

import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.*;

import lombok.Data;

@Data
@Entity
@Table(name = "uniprot_release")
public class UniProtRelease {
@Id private String id;
@Id private int id;

@Temporal(TemporalType.DATE)
private Date date;

@Enumerated(EnumType.STRING)
private EntryType entryType;

private String name;
}
4 changes: 2 additions & 2 deletions ...a-rest/src/main/java/org/uniprot/api/support/data/statistics/mapper/StatisticsMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ public StatisticsModuleStatisticsAttribute map(
public StatisticsModuleStatisticsHistory mapHistory(UniProtKBStatisticsEntry entry) {
return StatisticsModuleStatisticsHistoryImpl.builder()
.statisticsType(map(entry.getEntryType()))
.releaseName(entry.getReleaseName().getId())
.releaseDate(entry.getReleaseName().getDate())
.releaseName(entry.getUniProtRelease().getName())
.releaseDate(entry.getUniProtRelease().getDate())
.valueCount(entry.getValueCount())
.entryCount(entry.getEntryCount())
.build();
Expand Down
3 changes: 2 additions & 1 deletion ...ain/java/org/uniprot/api/support/data/statistics/repository/AttributeQueryRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@
@Repository
@Primary
public interface AttributeQueryRepository extends JpaRepository<AttributeQuery, Long> {
Optional<AttributeQuery> findByStatisticsCategoryAndAttributeNameIgnoreCase(StatisticsCategory statisticsCategory, String attributeName);
Optional<AttributeQuery> findByStatisticsCategoryAndAttributeNameIgnoreCase(
supun-ebi marked this conversation as resolved.
Show resolved Hide resolved
StatisticsCategory statisticsCategory, String attributeName);
}
12 changes: 2 additions & 10 deletions ...rg/uniprot/api/support/data/statistics/repository/UniProtKBStatisticsEntryRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,13 @@
@Primary
public interface UniProtKBStatisticsEntryRepository
extends JpaRepository<UniProtKBStatisticsEntry, Long> {
List<UniProtKBStatisticsEntry> findAllByReleaseNameAndEntryType(
UniProtRelease releaseName, EntryType entryType);

List<UniProtKBStatisticsEntry> findAllByReleaseNameAndEntryTypeAndStatisticsCategoryIn(
UniProtRelease uniProtRelease,
EntryType entryType,
Collection<StatisticsCategory> statisticsCategory);

List<UniProtKBStatisticsEntry> findAllByAttributeNameIgnoreCaseAndEntryType(
String attributeName, EntryType entryType);

List<UniProtKBStatisticsEntry> findAllByAttributeNameIgnoreCase(String attributeName);

List<UniProtKBStatisticsEntry> findAllByReleaseName(UniProtRelease releaseName);
List<UniProtKBStatisticsEntry> findAllByUniProtRelease(UniProtRelease uniProtRelease);

supun-ebi marked this conversation as resolved.
Show resolved Hide resolved
List<UniProtKBStatisticsEntry> findAllByReleaseNameAndStatisticsCategoryIn(
List<UniProtKBStatisticsEntry> findAllByUniProtReleaseAndStatisticsCategoryIn(
UniProtRelease releaseName, Collection<StatisticsCategory> statisticsCategory);
}
7 changes: 5 additions & 2 deletions ...ain/java/org/uniprot/api/support/data/statistics/repository/UniProtReleaseRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import org.uniprot.api.support.data.statistics.entity.EntryType;
import org.uniprot.api.support.data.statistics.entity.UniProtRelease;

@Repository
@Primary
public interface UniProtReleaseRepository extends JpaRepository<UniProtRelease, String> {
@Query("SELECT MAX (ur.date) from UniProtRelease ur where ur.id<?1")
public interface UniProtReleaseRepository extends JpaRepository<UniProtRelease, Integer> {
@Query("SELECT MAX (ur.date) from UniProtRelease ur where ur.name<?1")
Optional<Date> findPreviousReleaseDate(String currentRelease);
supun-ebi marked this conversation as resolved.
Show resolved Hide resolved

Optional<UniProtRelease> findByNameAndEntryType(String name, EntryType entryType);
}
76 changes: 43 additions & 33 deletions .../src/main/java/org/uniprot/api/support/data/statistics/service/StatisticsServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,20 +46,9 @@ public StatisticsServiceImpl(
@Override
public List<StatisticsModuleStatisticsCategory> findAllByVersionAndStatisticTypeAndCategoryIn(
String version, String statisticType, Set<String> categories) {
List<UniProtKBStatisticsEntry> entries;
if (categories.isEmpty()) {
entries =
statisticsEntryRepository.findAllByReleaseNameAndEntryType(
getRelease(version),
statisticsMapper.map(getStatisticType(statisticType)));
} else {
entries =
statisticsEntryRepository
.findAllByReleaseNameAndEntryTypeAndStatisticsCategoryIn(
getRelease(version),
statisticsMapper.map(getStatisticType(statisticType)),
getCategories(categories));
}
List<UniProtKBStatisticsEntry> entries =
getAllEntriesByVersionAndStatisticTypeAndCategoryIn(
version, statisticType, categories);
return entries.stream()
.collect(Collectors.groupingBy(UniProtKBStatisticsEntry::getStatisticsCategory))
.entrySet()
Expand All @@ -68,15 +57,6 @@ public List<StatisticsModuleStatisticsCategory> findAllByVersionAndStatisticType
.collect(Collectors.toList());
}

private UniProtRelease getRelease(String version) {
return releaseRepository
.findById(version)
.orElseThrow(
() ->
new IllegalArgumentException(
String.format("Invalid Release Version: %s", version)));
}

@Override
public List<StatisticsModuleStatisticsHistory> findAllByAttributeAndStatisticType(
String attribute, String statisticType) {
Expand All @@ -98,14 +78,15 @@ public List<StatisticsModuleStatisticsHistory> findAllByAttributeAndStatisticTyp
@Override
public Collection<StatisticsModuleStatisticsCategory> findAllByVersionAndCategoryIn(
String version, Set<String> categories) {
List<UniProtKBStatisticsEntry> entries;
if (categories.isEmpty()) {
entries = statisticsEntryRepository.findAllByReleaseName(getRelease(version));
} else {
entries =
statisticsEntryRepository.findAllByReleaseNameAndStatisticsCategoryIn(
getRelease(version), getCategories(categories));
List<UniProtKBStatisticsEntry> entries = new LinkedList<>();

for (StatisticsModuleStatisticsType statisticsType :
StatisticsModuleStatisticsType.values()) {
entries.addAll(
getAllEntriesByVersionAndStatisticTypeAndCategoryIn(
version, statisticsType.name(), categories));
}

return entries.stream()
.collect(Collectors.groupingBy(UniProtKBStatisticsEntry::getStatisticsCategory))
.entrySet()
Expand All @@ -115,6 +96,34 @@ public Collection<StatisticsModuleStatisticsCategory> findAllByVersionAndCategor
.collect(Collectors.toList());
}

private List<UniProtKBStatisticsEntry> getAllEntriesByVersionAndStatisticTypeAndCategoryIn(
String version, String statisticType, Set<String> categories) {
List<UniProtKBStatisticsEntry> entries;
StatisticsModuleStatisticsType statisticsModuleStatisticsType =
getStatisticType(statisticType);
EntryType entryType = statisticsMapper.map(statisticsModuleStatisticsType);
UniProtRelease release = getRelease(version, entryType);
if (categories.isEmpty()) {
entries = statisticsEntryRepository.findAllByUniProtRelease(release);
} else {
entries =
statisticsEntryRepository.findAllByUniProtReleaseAndStatisticsCategoryIn(
release, getCategories(categories));
}
return entries;
}

private UniProtRelease getRelease(String version, EntryType entryType) {
return releaseRepository
.findByNameAndEntryType(version, entryType)
.orElseThrow(
() ->
new IllegalArgumentException(
String.format(
"Invalid Release Version: %s or entry type: %s",
version, entryType)));
}

private Map.Entry<StatisticsCategory, List<UniProtKBStatisticsEntry>> groupEntries(
Map.Entry<StatisticsCategory, List<UniProtKBStatisticsEntry>> entry) {
List<UniProtKBStatisticsEntry> groupedEntries =
Expand Down Expand Up @@ -143,7 +152,7 @@ private UniProtKBStatisticsEntry mapToSingleEntry(
.mapToLong(UniProtKBStatisticsEntry::getEntryCount)
.sum());
uniProtKBStatisticsEntry.setDescription(firstEntry.getDescription());
uniProtKBStatisticsEntry.setReleaseName(firstEntry.getReleaseName());
uniProtKBStatisticsEntry.setUniProtRelease(firstEntry.getUniProtRelease());
return uniProtKBStatisticsEntry;
}

Expand Down Expand Up @@ -187,15 +196,16 @@ private Set<StatisticsCategory> getCategories(Set<String> categories) {

private String getAttributeQuery(UniProtKBStatisticsEntry entry) {
Optional<AttributeQuery> attributeQuery =
attributeQueryRepository.findByStatisticsCategoryAndAttributeNameIgnoreCase(entry.getStatisticsCategory(),entry.getAttributeName());
attributeQueryRepository.findByStatisticsCategoryAndAttributeNameIgnoreCase(
entry.getStatisticsCategory(), entry.getAttributeName());
return attributeQuery.map(query -> prepareQuery(query, entry)).orElse("");
}

private String prepareQuery(AttributeQuery query, UniProtKBStatisticsEntry entry) {
String result = query.getQuery();
if (result.contains(PREVIOUS_RELEASE_DATE)) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
String currentRelease = entry.getReleaseName().getId();
String currentRelease = entry.getUniProtRelease().getName();
Date previousReleaseDate =
releaseRepository
.findPreviousReleaseDate(currentRelease)
Expand Down
13 changes: 7 additions & 6 deletions ...a-rest/src/test/java/org/uniprot/api/support/data/statistics/TestEntityGeneratorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,15 @@ public class TestEntityGeneratorUtil {
};
public static final UniProtRelease[] RELEASES =
new UniProtRelease[] {
createRelease(REL_0, DATES[0]),
createRelease(REL_1, DATES[1]),
createRelease(REL_2, DATES[2])
createRelease(0, REL_0, DATES[0]),
createRelease(1, REL_1, DATES[1]),
createRelease(2, REL_2, DATES[2])
};

private static UniProtRelease createRelease(String name, Date date) {
private static UniProtRelease createRelease(int id, String name, Date date) {
UniProtRelease release = new UniProtRelease();
release.setId(name);
release.setId(id);
release.setName(name);
release.setDate(date);
return release;
}
Expand Down Expand Up @@ -110,7 +111,7 @@ private static UniProtKBStatisticsEntry createStatisticsEntry(int index) {
uniprotkbStatisticsEntry.setValueCount(VALUE_COUNTS[index]);
uniprotkbStatisticsEntry.setEntryCount(ENTRY_COUNTS[index]);
uniprotkbStatisticsEntry.setDescription(DESCRIPTIONS[index]);
uniprotkbStatisticsEntry.setReleaseName(
uniprotkbStatisticsEntry.setUniProtRelease(
Set.of(0, 1, 3, 4, 5).contains(index) ? RELEASES[0] : RELEASES[1]);
uniprotkbStatisticsEntry.setEntryType(ENTRY_TYPES[index]);
return uniprotkbStatisticsEntry;
Expand Down
28 changes: 14 additions & 14 deletions .../test/java/org/uniprot/api/support/data/statistics/controller/StatisticsControllerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ static void postgreSQLProperties(DynamicPropertyRegistry registry) {
@Test
void getByReleaseAndType() throws Exception {
this.mockMvc
.perform(get("/statistics/releases/2021_03/reviewed"))
.perform(get("/statistics/releases/2021_02/reviewed"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(3)))
Expand All @@ -52,7 +52,7 @@ void getByReleaseAndType() throws Exception {
.andExpect(
jsonPath(
"$.results[0].items[0].query",
is("(reviewed:true) AND (previous_release_date:2021-03-25)")))
is("(reviewed:true) AND (previous_release_date:2021-01-25)")))
.andExpect(jsonPath("$.results[0].items[0].count", is(329)))
.andExpect(jsonPath("$.results[0].items[0].entryCount", is(254)))
.andExpect(jsonPath("$.results[1].categoryName", is("TOP_ORGANISM")))
Expand Down Expand Up @@ -91,7 +91,7 @@ void getByReleaseAndType() throws Exception {
@Test
void getByRelease() throws Exception {
this.mockMvc
.perform(get("/statistics/releases/2021_03"))
.perform(get("/statistics/releases/2021_02"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(3)))
Expand All @@ -104,7 +104,7 @@ void getByRelease() throws Exception {
.andExpect(
jsonPath(
"$.results[0].items[0].query",
is("(previous_release_date:2021-03-25)")))
is("(previous_release_date:2021-01-25)")))
.andExpect(jsonPath("$.results[0].items[0].count", is(329)))
.andExpect(jsonPath("$.results[0].items[0].entryCount", is(254)))
.andExpect(jsonPath("$.results[1].categoryName", is("TOP_ORGANISM")))
Expand Down Expand Up @@ -137,7 +137,7 @@ void getByRelease() throws Exception {
@Test
void getByReleaseAndTypeAndSingleCategory() throws Exception {
this.mockMvc
.perform(get("/statistics/releases/2021_03/reviewed?categories=EUKARYOTA"))
.perform(get("/statistics/releases/2021_02/reviewed?categories=EUKARYOTA"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(1)))
Expand Down Expand Up @@ -165,7 +165,7 @@ void getByReleaseAndTypeAndSingleCategory() throws Exception {
@Test
void getByReleaseAndSingleCategory() throws Exception {
this.mockMvc
.perform(get("/statistics/releases/2021_03?categories=EUKARYOTA"))
.perform(get("/statistics/releases/2021_02?categories=EUKARYOTA"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(1)))
Expand All @@ -189,7 +189,7 @@ void getByReleaseAndTypeAndMultipleCategories() throws Exception {
this.mockMvc
.perform(
get(
"/statistics/releases/2021_03/reviewed?categories=EUKARYOTA,TOP_ORGANISM"))
"/statistics/releases/2021_02/reviewed?categories=EUKARYOTA,TOP_ORGANISM"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(2)))
Expand Down Expand Up @@ -229,7 +229,7 @@ void getByReleaseAndTypeAndMultipleCategories() throws Exception {
@Test
void getByReleaseAndMultipleCategories() throws Exception {
this.mockMvc
.perform(get("/statistics/releases/2021_03?categories=EUKARYOTA,TOP_ORGANISM"))
.perform(get("/statistics/releases/2021_02?categories=EUKARYOTA,TOP_ORGANISM"))
.andDo(log())
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(2)))
Expand Down Expand Up @@ -268,8 +268,8 @@ void getAllByAttributeAndStatisticsType() throws Exception {
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(1)))
.andExpect(jsonPath("$.results[0].statisticsType", is("REVIEWED")))
.andExpect(jsonPath("$.results[0].releaseName", is("2021_03")))
.andExpect(jsonPath("$.results[0].releaseDate", is("2021-05-25")))
.andExpect(jsonPath("$.results[0].releaseName", is("2021_02")))
.andExpect(jsonPath("$.results[0].releaseDate", is("2021-03-25")))
.andExpect(jsonPath("$.results[0].valueCount", is(35360)))
.andExpect(jsonPath("$.results[0].entryCount", is(35360)));
}
Expand Down Expand Up @@ -311,13 +311,13 @@ void getAllByAttributeAndAttributeType_withoutAttributeType() throws Exception {
.andExpect(status().isOk())
.andExpect(jsonPath("$.results.size()", is(2)))
.andExpect(jsonPath("$.results[0].statisticsType", is("REVIEWED")))
.andExpect(jsonPath("$.results[0].releaseName", is("2021_03")))
.andExpect(jsonPath("$.results[0].releaseDate", is("2021-05-25")))
.andExpect(jsonPath("$.results[0].releaseName", is("2021_02")))
.andExpect(jsonPath("$.results[0].releaseDate", is("2021-03-25")))
.andExpect(jsonPath("$.results[0].valueCount", is(35360)))
.andExpect(jsonPath("$.results[0].entryCount", is(35360)))
.andExpect(jsonPath("$.results[1].statisticsType", is("UNREVIEWED")))
.andExpect(jsonPath("$.results[1].releaseName", is("2021_03")))
.andExpect(jsonPath("$.results[1].releaseDate", is("2021-05-25")))
.andExpect(jsonPath("$.results[1].releaseName", is("2021_02")))
.andExpect(jsonPath("$.results[1].releaseDate", is("2021-03-25")))
.andExpect(jsonPath("$.results[1].valueCount", is(12793422)))
.andExpect(jsonPath("$.results[1].entryCount", is(12793422)));
}
Expand Down
Loading