diff --git a/pom.xml b/pom.xml index 222100829..a8cdfd3a6 100644 --- a/pom.xml +++ b/pom.xml @@ -302,6 +302,9 @@ false /tmp/atlas/audit/audit.log /tmp/atlas/audit/audit-extra.log + + + false @@ -1256,6 +1259,15 @@ 2.0.1 test + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + @@ -1895,5 +1907,107 @@ + + webapi-shiny + + true + http://localhost/Atlas + src/main/resources/shiny + + default,shiny + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.3.2 + + + + ${shiny.output.directory} + + shiny-cohortCounts.zip + shiny-incidenceRates.zip + shiny-cohortCharacterizations.zip + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + build-cohortCounts-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-cohortCounts.xml + + shiny-cohortCounts + + + + build-incidenceRates-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-incidenceRates.xml + + shiny-incidenceRates + + + + build-cohortCharacterizations-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-cohortCharacterizations.xml + + shiny-cohortCharacterizations + + + + build-cohortPathways-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-cohortPathways.xml + + shiny-cohortPathways + + + + + + + diff --git a/src/main/assembly/shiny-cohortCharacterizations.xml b/src/main/assembly/shiny-cohortCharacterizations.xml new file mode 100644 index 000000000..359932716 --- /dev/null +++ b/src/main/assembly/shiny-cohortCharacterizations.xml @@ -0,0 +1,18 @@ + + + shiny-cohortCharacterizations + + zip + + false + + + ./apps/cohortCharacterization + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/assembly/shiny-cohortCounts.xml b/src/main/assembly/shiny-cohortCounts.xml new file mode 100644 index 000000000..5c43b81be --- /dev/null +++ b/src/main/assembly/shiny-cohortCounts.xml @@ -0,0 +1,18 @@ + + + shiny-cohortCounts + + zip + + false + + + ./apps/cohortCounts + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/assembly/shiny-cohortPathways.xml b/src/main/assembly/shiny-cohortPathways.xml new file mode 100644 index 000000000..3dc1e0fba --- /dev/null +++ b/src/main/assembly/shiny-cohortPathways.xml @@ -0,0 +1,18 @@ + + + shiny-incidenceRates + + zip + + false + + + ./apps/cohortPathways + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/assembly/shiny-incidenceRates.xml b/src/main/assembly/shiny-incidenceRates.xml new file mode 100644 index 000000000..0c03a906b --- /dev/null +++ b/src/main/assembly/shiny-incidenceRates.xml @@ -0,0 +1,18 @@ + + + shiny-incidenceRates + + zip + + false + + + ./apps/IncidenceRate + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/Constants.java b/src/main/java/org/ohdsi/webapi/Constants.java index 2069ed108..7e1f07a4a 100644 --- a/src/main/java/org/ohdsi/webapi/Constants.java +++ b/src/main/java/org/ohdsi/webapi/Constants.java @@ -89,9 +89,13 @@ interface Variables { } interface Headers { + String ACCESS_CONTROL_EXPOSE_HEADERS = "Access-Control-Expose-Headers"; String AUTH_PROVIDER = "x-auth-provider"; String USER_LANGAUGE = "User-Language"; String ACTION_LOCATION = "action-location"; + String BEARER = "Bearer"; + String X_AUTH_ERROR = "x-auth-error"; + String CONTENT_DISPOSITION = "Content-Disposition"; } interface SecurityProviders { diff --git a/src/main/java/org/ohdsi/webapi/JerseyConfig.java b/src/main/java/org/ohdsi/webapi/JerseyConfig.java index eabc3f818..ba14ac91a 100644 --- a/src/main/java/org/ohdsi/webapi/JerseyConfig.java +++ b/src/main/java/org/ohdsi/webapi/JerseyConfig.java @@ -30,6 +30,7 @@ import org.ohdsi.webapi.service.TherapyPathResultsService; import org.ohdsi.webapi.service.UserService; import org.ohdsi.webapi.service.VocabularyService; +import org.ohdsi.webapi.shiny.ShinyController; import org.ohdsi.webapi.source.SourceController; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -47,6 +48,8 @@ public class JerseyConfig extends ResourceConfig implements InitializingBean { @Value("${jersey.resources.root.package}") private String rootPackage; + @Value("${shiny.enabled:false}") + private Boolean shinyEnabled; public JerseyConfig() { RuntimeDelegate.setInstance(new org.glassfish.jersey.internal.RuntimeDelegateImpl()); @@ -94,5 +97,8 @@ protected void configure() { .in(Singleton.class); } }); + if (shinyEnabled) { + register(ShinyController.class); + } } } diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java index fdd1c011d..dcf53b5cc 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java @@ -431,38 +431,7 @@ public String getGenerationDesign( public PathwayPopulationResultsDTO getGenerationResults( @PathParam("generationId") final Long generationId ) { - - PathwayAnalysisResult resultingPathways = pathwayService.getResultingPathways(generationId); - - List eventCodeDtos = resultingPathways.getCodes() - .stream() - .map(entry -> { - PathwayCodeDTO dto = new PathwayCodeDTO(); - dto.setCode(entry.getCode()); - dto.setName(entry.getName()); - dto.setIsCombo(entry.isCombo()); - return dto; - }) - .collect(Collectors.toList()); - - List pathwayDtos = resultingPathways.getCohortPathwaysList() - .stream() - .map(cohortResults -> { - if (cohortResults.getPathwaysCounts() == null) { - return null; - } - - List eventDTOs = cohortResults.getPathwaysCounts() - .entrySet() - .stream() - .map(entry -> new PathwayPopulationEventDTO(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - return new TargetCohortPathwaysDTO(cohortResults.getCohortId(), cohortResults.getTargetCohortCount(), cohortResults.getTotalPathwaysCount(), eventDTOs); - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - return new PathwayPopulationResultsDTO(eventCodeDtos, pathwayDtos); + return pathwayService.getGenerationResults(generationId); } private PathwayAnalysisDTO reloadAndConvert(Integer id) { diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java index dcbd7785a..a3137ab18 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java @@ -4,6 +4,7 @@ import org.ohdsi.webapi.pathway.domain.PathwayAnalysisEntity; import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.PathwayVersionFullDTO; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import org.ohdsi.webapi.shiro.annotations.PathwayAnalysisGenerationId; @@ -69,4 +70,8 @@ public interface PathwayService extends HasTags { PathwayVersion saveVersion(int id); List listByTags(TagNameListRequestDTO requestDTO); + + PathwayAnalysisDTO getByGenerationId(Integer id); + + PathwayPopulationResultsDTO getGenerationResults(Long generationId); } diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java index dd4e62987..65dd22155 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java @@ -24,12 +24,17 @@ import org.ohdsi.webapi.pathway.domain.PathwayEventCohort; import org.ohdsi.webapi.pathway.domain.PathwayTargetCohort; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayCodeDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationEventDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.PathwayVersionFullDTO; +import org.ohdsi.webapi.pathway.dto.TargetCohortPathwaysDTO; import org.ohdsi.webapi.pathway.dto.internal.CohortPathways; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import org.ohdsi.webapi.pathway.dto.internal.PathwayCode; import org.ohdsi.webapi.pathway.repository.PathwayAnalysisEntityRepository; import org.ohdsi.webapi.pathway.repository.PathwayAnalysisGenerationRepository; +import org.ohdsi.webapi.security.PermissionService; import org.ohdsi.webapi.service.AbstractDaoService; import org.ohdsi.webapi.service.CohortDefinitionService; import org.ohdsi.webapi.service.JobService; @@ -63,9 +68,11 @@ import org.springframework.batch.core.job.builder.SimpleJobBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.convert.ConversionService; import org.springframework.core.convert.support.GenericConversionService; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -96,613 +103,653 @@ import static org.ohdsi.webapi.Constants.Params.JOB_NAME; import static org.ohdsi.webapi.Constants.Params.PATHWAY_ANALYSIS_ID; import static org.ohdsi.webapi.Constants.Params.SOURCE_ID; -import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; -import org.ohdsi.webapi.security.PermissionService; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.PageImpl; @Service @Transactional public class PathwayServiceImpl extends AbstractDaoService implements PathwayService, GeneratesNotification { - private final PathwayAnalysisEntityRepository pathwayAnalysisRepository; - private final PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; - private final SourceService sourceService; - private final JobTemplate jobTemplate; - private final EntityManager entityManager; - private final DesignImportService designImportService; - private final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository; - private final UserRepository userRepository; - private final GenerationUtils generationUtils; - private final JobService jobService; - private final GenericConversionService genericConversionService; - private final StepBuilderFactory stepBuilderFactory; - private final CohortDefinitionService cohortDefinitionService; - private final VersionService versionService; - - private PermissionService permissionService; - - @Value("${security.defaultGlobalReadPermissions}") - private boolean defaultGlobalReadPermissions; - - private final List STEP_COLUMNS = Arrays.asList(new String[]{"step_1", "step_2", "step_3", "step_4", "step_5", "step_6", "step_7", "step_8", "step_9", "step_10"}); - - private final EntityGraph defaultEntityGraph = EntityUtils.fromAttributePaths( - "targetCohorts.cohortDefinition", - "eventCohorts.cohortDefinition", - "createdBy", - "modifiedBy" - ); - - @Autowired - public PathwayServiceImpl( - PathwayAnalysisEntityRepository pathwayAnalysisRepository, - PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository, - SourceService sourceService, - ConversionService conversionService, - JobTemplate jobTemplate, - EntityManager entityManager, - Security security, - DesignImportService designImportService, - AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository, - UserRepository userRepository, - GenerationUtils generationUtils, - JobService jobService, - @Qualifier("conversionService") GenericConversionService genericConversionService, - StepBuilderFactory stepBuilderFactory, - CohortDefinitionService cohortDefinitionService, - VersionService versionService, - PermissionService permissionService) { - - this.pathwayAnalysisRepository = pathwayAnalysisRepository; - this.pathwayAnalysisGenerationRepository = pathwayAnalysisGenerationRepository; - this.sourceService = sourceService; - this.jobTemplate = jobTemplate; - this.entityManager = entityManager; - this.jobService = jobService; - this.genericConversionService = genericConversionService; - this.security = security; - this.designImportService = designImportService; - this.analysisGenerationInfoEntityRepository = analysisGenerationInfoEntityRepository; - this.userRepository = userRepository; - this.generationUtils = generationUtils; - this.stepBuilderFactory = stepBuilderFactory; - this.cohortDefinitionService = cohortDefinitionService; - this.versionService = versionService; - this.permissionService = permissionService; - - SerializedPathwayAnalysisToPathwayAnalysisConverter.setConversionService(conversionService); - } - - @Override - public PathwayAnalysisEntity create(PathwayAnalysisEntity toSave) { - - PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); - - copyProps(toSave, newAnalysis); - - toSave.getTargetCohorts().forEach(tc -> { - tc.setId(null); - tc.setPathwayAnalysis(newAnalysis); - newAnalysis.getTargetCohorts().add(tc); - }); - - toSave.getEventCohorts().forEach(ec -> { - ec.setId(null); - ec.setPathwayAnalysis(newAnalysis); - newAnalysis.getEventCohorts().add(ec); - }); - - newAnalysis.setCreatedBy(getCurrentUser()); - newAnalysis.setCreatedDate(new Date()); - // Fields with information about modifications have to be reseted - newAnalysis.setModifiedBy(null); - newAnalysis.setModifiedDate(null); - return save(newAnalysis); - } - - @Override - public PathwayAnalysisEntity importAnalysis(PathwayAnalysisEntity toImport) { - - PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); - - copyProps(toImport, newAnalysis); - - Stream.concat(toImport.getTargetCohorts().stream(), toImport.getEventCohorts().stream()).forEach(pc -> { - CohortDefinition cohortDefinition = designImportService.persistCohortOrGetExisting(pc.getCohortDefinition()); - pc.setId(null); - pc.setName(cohortDefinition.getName()); - pc.setCohortDefinition(cohortDefinition); - pc.setPathwayAnalysis(newAnalysis); - if (pc instanceof PathwayTargetCohort) { - newAnalysis.getTargetCohorts().add((PathwayTargetCohort) pc); - } else { - newAnalysis.getEventCohorts().add((PathwayEventCohort) pc); - } - }); - - newAnalysis.setCreatedBy(getCurrentUser()); - newAnalysis.setCreatedDate(new Date()); - - return save(newAnalysis); - } - - @Override - public Page getPage(final Pageable pageable) { - List pathwayList = pathwayAnalysisRepository.findAll(defaultEntityGraph) - .stream().filter(!defaultGlobalReadPermissions ? entity -> permissionService.hasReadAccess(entity) : entity -> true) - .collect(Collectors.toList()); - return getPageFromResults(pageable, pathwayList); - } - - private Page getPageFromResults(Pageable pageable, List results) { - // Calculate the start and end indices for the current page - int startIndex = pageable.getPageNumber() * pageable.getPageSize(); - int endIndex = Math.min(startIndex + pageable.getPageSize(), results.size()); - - return new PageImpl<>(results.subList(startIndex, endIndex), pageable, results.size()); - } - - @Override - public int getCountPAWithSameName(Integer id, String name) { - - return pathwayAnalysisRepository.getCountPAWithSameName(id, name); - } - - @Override - public PathwayAnalysisEntity getById(Integer id) { - - PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(id, defaultEntityGraph); - if (Objects.nonNull(entity)) { - entity.getTargetCohorts().forEach(tc -> Hibernate.initialize(tc.getCohortDefinition().getDetails())); - entity.getEventCohorts().forEach(ec -> Hibernate.initialize(ec.getCohortDefinition().getDetails())); - } - return entity; - } - - private List getNamesLike(String name) { - - return pathwayAnalysisRepository.findAllByNameStartsWith(name).stream().map(PathwayAnalysisEntity::getName).collect(Collectors.toList()); - } - - @Override - public String getNameForCopy(String dtoName) { - return NameUtils.getNameForCopy(dtoName, this::getNamesLike, pathwayAnalysisRepository.findByName(dtoName)); - } - - @Override - public String getNameWithSuffix(String dtoName) { - return NameUtils.getNameWithSuffix(dtoName, this::getNamesLike); - } - - @Override - public PathwayAnalysisEntity update(PathwayAnalysisEntity forUpdate) { - - PathwayAnalysisEntity existing = getById(forUpdate.getId()); - - copyProps(forUpdate, existing); - updateCohorts(existing, existing.getTargetCohorts(), forUpdate.getTargetCohorts()); - updateCohorts(existing, existing.getEventCohorts(), forUpdate.getEventCohorts()); - - existing.setModifiedBy(getCurrentUser()); - existing.setModifiedDate(new Date()); - - return save(existing); - } - - private void updateCohorts(PathwayAnalysisEntity analysis, Set existing, Set forUpdate) { - - Set removedCohorts = existing - .stream() - .filter(ec -> !forUpdate.contains(ec)) - .collect(Collectors.toSet()); - existing.removeAll(removedCohorts); - forUpdate.forEach(updatedCohort -> existing.stream() - .filter(ec -> ec.equals(updatedCohort)) - .findFirst() - .map(ec -> { - ec.setName(updatedCohort.getName()); - return ec; - }) - .orElseGet(() -> { - updatedCohort.setId(null); - updatedCohort.setPathwayAnalysis(analysis); - existing.add(updatedCohort); - return updatedCohort; - })); - } - - @Override - public void delete(Integer id) { - - pathwayAnalysisRepository.delete(id); - } - - @Override - public Map getEventCohortCodes(PathwayAnalysisEntity pathwayAnalysis) { - - Integer index = 0; - - List sortedEventCohortsCopy = pathwayAnalysis.getEventCohorts() - .stream() - .sorted(Comparator.comparing(PathwayEventCohort::getName)) - .collect(Collectors.toList()); - - Map cohortDefIdToIndexMap = new HashMap<>(); - - for (PathwayEventCohort eventCohort : sortedEventCohortsCopy) { - cohortDefIdToIndexMap.put(eventCohort.getCohortDefinition().getId(), index++); - } - - return cohortDefIdToIndexMap; - } - - @Override - @DataSourceAccess - public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, @SourceId Integer sourceId, String cohortTable, String sessionId) { - - Map eventCohortCodes = getEventCohortCodes(pathwayAnalysis); - Source source = sourceService.findBySourceId(sourceId); - final StringJoiner joiner = new StringJoiner("\n\n"); - - String analysisSql = ResourceHelper.GetResourceAsString("/resources/pathway/runPathwayAnalysis.sql"); - String eventCohortInputSql = ResourceHelper.GetResourceAsString("/resources/pathway/eventCohortInput.sql"); - - String tempTableQualifier = SourceUtils.getTempQualifier(source); - String resultsTableQualifier = SourceUtils.getResultsQualifier(source); - - String eventCohortIdIndexSql = eventCohortCodes.entrySet() - .stream() - .map(ec -> { - String[] params = new String[]{"cohort_definition_id", "event_cohort_index"}; - String[] values = new String[]{ec.getKey().toString(), ec.getValue().toString()}; - return SqlRender.renderSql(eventCohortInputSql, params, values); - }) - .collect(Collectors.joining(" UNION ALL ")); - - pathwayAnalysis.getTargetCohorts().forEach(tc -> { - - String[] params = new String[]{ - GENERATION_ID, - "event_cohort_id_index_map", - "temp_database_schema", - "target_database_schema", - "target_cohort_table", - "pathway_target_cohort_id", - "max_depth", - "combo_window", - "allow_repeats", - "isHive" - }; - String[] values = new String[]{ - generationId.toString(), - eventCohortIdIndexSql, - tempTableQualifier, - resultsTableQualifier, - cohortTable, - tc.getCohortDefinition().getId().toString(), - pathwayAnalysis.getMaxDepth().toString(), - MoreObjects.firstNonNull(pathwayAnalysis.getCombinationWindow(), 1).toString(), - String.valueOf(pathwayAnalysis.isAllowRepeats()), - String.valueOf(Objects.equals(DBMSType.HIVE.getOhdsiDB(), source.getSourceDialect())) - }; - - String renderedSql = SqlRender.renderSql(analysisSql, params, values); - String translatedSql = SqlTranslate.translateSql(renderedSql, source.getSourceDialect(), sessionId, SourceUtils.getTempQualifier(source)); - - joiner.add(translatedSql); - }); - - return joiner.toString(); - } - - @Override - public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, Integer sourceId) { - - return buildAnalysisSql(generationId, pathwayAnalysis, sourceId, "cohort", SessionUtils.sessionId()); - } - - @Override - @DataSourceAccess - public JobExecutionResource generatePathways(final Integer pathwayAnalysisId, final @SourceId Integer sourceId) { - - PathwayService pathwayService = this; - - PathwayAnalysisEntity pathwayAnalysis = getById(pathwayAnalysisId); - Source source = getSourceRepository().findBySourceId(sourceId); - - JobParametersBuilder builder = new JobParametersBuilder(); - builder.addString(JOB_NAME, String.format("Generating Pathway Analysis %d using %s (%s)", pathwayAnalysisId, source.getSourceName(), source.getSourceKey())); - builder.addString(SOURCE_ID, String.valueOf(source.getSourceId())); - builder.addString(PATHWAY_ANALYSIS_ID, pathwayAnalysis.getId().toString()); - builder.addString(JOB_AUTHOR, getCurrentUserLogin()); - - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - - SimpleJobBuilder generateAnalysisJob = generationUtils.buildJobForCohortBasedAnalysisTasklet( - GENERATE_PATHWAY_ANALYSIS, - source, - builder, - jdbcTemplate, - chunkContext -> { - Integer analysisId = Integer.valueOf(chunkContext.getStepContext().getJobParameters().get(PATHWAY_ANALYSIS_ID).toString()); - PathwayAnalysisEntity analysis = pathwayService.getById(analysisId); - return Stream.concat(analysis.getTargetCohorts().stream(), analysis.getEventCohorts().stream()) - .map(PathwayCohort::getCohortDefinition) - .collect(Collectors.toList()); - }, - new GeneratePathwayAnalysisTasklet( - getSourceJdbcTemplate(source), - getTransactionTemplate(), - pathwayService, - analysisGenerationInfoEntityRepository, - userRepository, - sourceService - ) - ); - TransactionalTasklet statisticsTasklet = new PathwayStatisticsTasklet(getSourceJdbcTemplate(source), getTransactionTemplate(), source, this, genericConversionService); - Step generateStatistics = stepBuilderFactory.get(GENERATE_PATHWAY_ANALYSIS + ".generateStatistics") - .tasklet(statisticsTasklet) - .build(); - - generateAnalysisJob.next(generateStatistics); - - final JobParameters jobParameters = builder.toJobParameters(); - - return jobService.runJob(generateAnalysisJob.build(), jobParameters); - } - - @Override - @DataSourceAccess - public void cancelGeneration(Integer pathwayAnalysisId, @SourceId Integer sourceId) { - - PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(pathwayAnalysisId, defaultEntityGraph); - String sourceKey = getSourceRepository().findBySourceId(sourceId).getSourceKey(); - entity.getTargetCohorts().forEach(tc -> cohortDefinitionService.cancelGenerateCohort(tc.getId(), sourceKey)); - entity.getEventCohorts().forEach(ec -> cohortDefinitionService.cancelGenerateCohort(ec.getId(), sourceKey)); - jobService.cancelJobExecution(j -> { - JobParameters jobParameters = j.getJobParameters(); - String jobName = j.getJobInstance().getJobName(); - return Objects.equals(jobParameters.getString(PATHWAY_ANALYSIS_ID), Integer.toString(pathwayAnalysisId)) - && Objects.equals(jobParameters.getString(SOURCE_ID), String.valueOf(sourceId)) - && Objects.equals(GENERATE_PATHWAY_ANALYSIS, jobName); - }); - } - - @Override - public List getPathwayGenerations(final Integer pathwayAnalysisId) { - - return pathwayAnalysisGenerationRepository.findAllByPathwayAnalysisId(pathwayAnalysisId, EntityUtils.fromAttributePaths("source")); - } - - @Override - public PathwayAnalysisGenerationEntity getGeneration(Long generationId) { - - return pathwayAnalysisGenerationRepository.findOne(generationId, EntityUtils.fromAttributePaths("source")); - } - - @Override - @DataSourceAccess - public PathwayAnalysisResult getResultingPathways(final @PathwayAnalysisGenerationId Long generationId) { - - PathwayAnalysisGenerationEntity generation = getGeneration(generationId); - Source source = generation.getSource(); - return queryGenerationResults(source, generationId); - } - - private final RowMapper codeRowMapper = (final ResultSet resultSet, final int arg1) -> { - return new PathwayCode(resultSet.getLong("code"), resultSet.getString("name"), resultSet.getInt("is_combo") != 0); - }; - - private final RowMapper pathwayStatsRowMapper = (final ResultSet rs, final int arg1) -> { - CohortPathways cp = new CohortPathways(); - cp.setCohortId(rs.getInt("target_cohort_id")); - cp.setTargetCohortCount(rs.getInt("target_cohort_count")); - cp.setTotalPathwaysCount(rs.getInt("pathways_count")); - return cp; - }; - - private final ResultSetExtractor>> pathwayExtractor = (final ResultSet rs) -> { - Map> cohortMap = new HashMap<>(); // maps a cohortId to a list of pathways (which is stored as a Map - - while (rs.next()) { - int cohortId = rs.getInt("target_cohort_id"); - if (!cohortMap.containsKey(cohortId)) { - cohortMap.put(cohortId, new HashMap<>()); - } - Map pathList = cohortMap.get(cohortId); - - // build path - List path = new ArrayList<>(); - for (String stepCol : STEP_COLUMNS) { - String step = rs.getString(stepCol); - - if (step == null) break; // cancel for-loop when we encounter a column with a null value - - path.add(step); - } - pathList.put(StringUtils.join(path, "-"), rs.getInt("count_value")); // for a given cohort, a path must be unique, so no need to check - } - return cohortMap; - }; - - @Override - @DataSourceAccess - public String findDesignByGenerationId(@PathwayAnalysisGenerationId final Long id) { - final AnalysisGenerationInfoEntity entity = analysisGenerationInfoEntityRepository.findById(id) - .orElseThrow(() -> new IllegalArgumentException("Analysis with id: " + id + " cannot be found")); - return entity.getDesign(); - } - - @Override - public void assignTag(Integer id, int tagId) { - PathwayAnalysisEntity entity = getById(id); - checkOwnerOrAdminOrGranted(entity); - assignTag(entity, tagId); - } - - @Override - public void unassignTag(Integer id, int tagId) { - PathwayAnalysisEntity entity = getById(id); - checkOwnerOrAdminOrGranted(entity); - unassignTag(entity, tagId); - } - - @Override - public List getVersions(long id) { - List versions = versionService.getVersions(VersionType.PATHWAY, id); - return versions.stream() - .map(v -> genericConversionService.convert(v, VersionDTO.class)) - .collect(Collectors.toList()); - } - - @Override - public PathwayVersionFullDTO getVersion(int id, int version) { - checkVersion(id, version, false); - PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - return genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); - } - - @Override - public VersionDTO updateVersion(int id, int version, VersionUpdateDTO updateDTO) { - checkVersion(id, version); - updateDTO.setAssetId(id); - updateDTO.setVersion(version); - PathwayVersion updated = versionService.update(VersionType.PATHWAY, updateDTO); - - return genericConversionService.convert(updated, VersionDTO.class); - } - - @Override - public void deleteVersion(int id, int version) { - checkVersion(id, version); - versionService.delete(VersionType.PATHWAY, id, version); - } - - @Override - public PathwayAnalysisDTO copyAssetFromVersion(int id, int version) { - checkVersion(id, version, false); - PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - PathwayVersionFullDTO fullDTO = genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); - - PathwayAnalysisDTO dto = fullDTO.getEntityDTO(); - dto.setId(null); - dto.setTags(null); - dto.setName(NameUtils.getNameForCopy(dto.getName(), this::getNamesLike, - pathwayAnalysisRepository.findByName(dto.getName()))); - PathwayAnalysisEntity pathwayAnalysis = genericConversionService.convert(dto, PathwayAnalysisEntity.class); - PathwayAnalysisEntity saved = create(pathwayAnalysis); - return genericConversionService.convert(saved, PathwayAnalysisDTO.class); - } - - @Override - public List listByTags(TagNameListRequestDTO requestDTO) { - List names = requestDTO.getNames().stream() - .map(name -> name.toLowerCase(Locale.ROOT)) - .collect(Collectors.toList()); - List entities = pathwayAnalysisRepository.findByTags(names); - return listByTags(entities, names, PathwayAnalysisDTO.class); - } - - private void checkVersion(int id, int version) { - checkVersion(id, version, true); - } - - private void checkVersion(int id, int version, boolean checkOwnerShip) { - Version pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayVersion, - String.format("There is no pathway analysis version with id = %d.", version)); - - PathwayAnalysisEntity entity = this.pathwayAnalysisRepository.findOne(id); - if (checkOwnerShip) { - checkOwnerOrAdminOrGranted(entity); - } - } - - public PathwayVersion saveVersion(int id) { - PathwayAnalysisEntity def = this.pathwayAnalysisRepository.findOne(id); - PathwayVersion version = genericConversionService.convert(def, PathwayVersion.class); - - UserEntity user = Objects.nonNull(def.getModifiedBy()) ? def.getModifiedBy() : def.getCreatedBy(); - Date versionDate = Objects.nonNull(def.getModifiedDate()) ? def.getModifiedDate() : def.getCreatedDate(); - version.setCreatedBy(user); - version.setCreatedDate(versionDate); - return versionService.create(VersionType.PATHWAY, version); - } - - private PathwayAnalysisResult queryGenerationResults(Source source, Long generationId) { - - // load code lookup - PreparedStatementRenderer pathwayCodesPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayCodeLookup.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - List pathwayCodes = getSourceJdbcTemplate(source).query(pathwayCodesPsr.getSql(), pathwayCodesPsr.getOrderedParams(), codeRowMapper); - - // fetch cohort stats, paths will be populated after - PreparedStatementRenderer pathwayStatsPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayStats.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - List cohortStats = getSourceJdbcTemplate(source).query(pathwayStatsPsr.getSql(), pathwayStatsPsr.getOrderedParams(), pathwayStatsRowMapper); - - // load cohort paths, and assign back to cohortStats - PreparedStatementRenderer pathwayResultsPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayResults.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - Map> pathwayResults = - getSourceJdbcTemplate(source).query(pathwayResultsPsr.getSql(), pathwayResultsPsr.getOrderedParams(), pathwayExtractor); - - cohortStats.stream().forEach((cp) -> { - cp.setPathwaysCounts(pathwayResults.get(cp.getCohortId())); - }); - - PathwayAnalysisResult result = new PathwayAnalysisResult(); - result.setCodes(new HashSet<>(pathwayCodes)); - result.setCohortPathwaysList(new HashSet<>(cohortStats)); - - return result; - } - - private void copyProps(PathwayAnalysisEntity from, PathwayAnalysisEntity to) { - - to.setName(from.getName()); - to.setDescription(from.getDescription()); - to.setMaxDepth(from.getMaxDepth()); - to.setMinCellCount(from.getMinCellCount()); - to.setCombinationWindow(from.getCombinationWindow()); - to.setAllowRepeats(from.isAllowRepeats()); - } - - private int getAnalysisHashCode(PathwayAnalysisEntity pathwayAnalysis) { - - SerializedPathwayAnalysisToPathwayAnalysisConverter designConverter = new SerializedPathwayAnalysisToPathwayAnalysisConverter(); - return designConverter.convertToDatabaseColumn(pathwayAnalysis).hashCode(); - } - - private PathwayAnalysisEntity save(PathwayAnalysisEntity pathwayAnalysis) { - - pathwayAnalysis = pathwayAnalysisRepository.saveAndFlush(pathwayAnalysis); - entityManager.refresh(pathwayAnalysis); - pathwayAnalysis = getById(pathwayAnalysis.getId()); - pathwayAnalysis.setHashCode(getAnalysisHashCode(pathwayAnalysis)); - return pathwayAnalysis; - } - - @Override - public String getJobName() { - return GENERATE_PATHWAY_ANALYSIS; - } - - @Override - public String getExecutionFoldingKey() { - return PATHWAY_ANALYSIS_ID; - } + private final PathwayAnalysisEntityRepository pathwayAnalysisRepository; + private final PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; + private final SourceService sourceService; + private final JobTemplate jobTemplate; + private final EntityManager entityManager; + private final DesignImportService designImportService; + private final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository; + private final UserRepository userRepository; + private final GenerationUtils generationUtils; + private final JobService jobService; + private final GenericConversionService genericConversionService; + private final StepBuilderFactory stepBuilderFactory; + private final CohortDefinitionService cohortDefinitionService; + private final VersionService versionService; + + private PermissionService permissionService; + + @Value("${security.defaultGlobalReadPermissions}") + private boolean defaultGlobalReadPermissions; + + private final List STEP_COLUMNS = Arrays.asList(new String[]{"step_1", "step_2", "step_3", "step_4", "step_5", "step_6", "step_7", "step_8", "step_9", "step_10"}); + + private final EntityGraph defaultEntityGraph = EntityUtils.fromAttributePaths( + "targetCohorts.cohortDefinition", + "eventCohorts.cohortDefinition", + "createdBy", + "modifiedBy" + ); + + @Autowired + public PathwayServiceImpl( + PathwayAnalysisEntityRepository pathwayAnalysisRepository, + PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository, + SourceService sourceService, + ConversionService conversionService, + JobTemplate jobTemplate, + EntityManager entityManager, + Security security, + DesignImportService designImportService, + AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository, + UserRepository userRepository, + GenerationUtils generationUtils, + JobService jobService, + @Qualifier("conversionService") GenericConversionService genericConversionService, + StepBuilderFactory stepBuilderFactory, + CohortDefinitionService cohortDefinitionService, + VersionService versionService, + PermissionService permissionService) { + + this.pathwayAnalysisRepository = pathwayAnalysisRepository; + this.pathwayAnalysisGenerationRepository = pathwayAnalysisGenerationRepository; + this.sourceService = sourceService; + this.jobTemplate = jobTemplate; + this.entityManager = entityManager; + this.jobService = jobService; + this.genericConversionService = genericConversionService; + this.security = security; + this.designImportService = designImportService; + this.analysisGenerationInfoEntityRepository = analysisGenerationInfoEntityRepository; + this.userRepository = userRepository; + this.generationUtils = generationUtils; + this.stepBuilderFactory = stepBuilderFactory; + this.cohortDefinitionService = cohortDefinitionService; + this.versionService = versionService; + this.permissionService = permissionService; + + SerializedPathwayAnalysisToPathwayAnalysisConverter.setConversionService(conversionService); + } + + @Override + public PathwayAnalysisEntity create(PathwayAnalysisEntity toSave) { + + PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); + + copyProps(toSave, newAnalysis); + + toSave.getTargetCohorts().forEach(tc -> { + tc.setId(null); + tc.setPathwayAnalysis(newAnalysis); + newAnalysis.getTargetCohorts().add(tc); + }); + + toSave.getEventCohorts().forEach(ec -> { + ec.setId(null); + ec.setPathwayAnalysis(newAnalysis); + newAnalysis.getEventCohorts().add(ec); + }); + + newAnalysis.setCreatedBy(getCurrentUser()); + newAnalysis.setCreatedDate(new Date()); + // Fields with information about modifications have to be reseted + newAnalysis.setModifiedBy(null); + newAnalysis.setModifiedDate(null); + return save(newAnalysis); + } + + @Override + public PathwayAnalysisEntity importAnalysis(PathwayAnalysisEntity toImport) { + + PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); + + copyProps(toImport, newAnalysis); + + Stream.concat(toImport.getTargetCohorts().stream(), toImport.getEventCohorts().stream()).forEach(pc -> { + CohortDefinition cohortDefinition = designImportService.persistCohortOrGetExisting(pc.getCohortDefinition()); + pc.setId(null); + pc.setName(cohortDefinition.getName()); + pc.setCohortDefinition(cohortDefinition); + pc.setPathwayAnalysis(newAnalysis); + if (pc instanceof PathwayTargetCohort) { + newAnalysis.getTargetCohorts().add((PathwayTargetCohort) pc); + } else { + newAnalysis.getEventCohorts().add((PathwayEventCohort) pc); + } + }); + + newAnalysis.setCreatedBy(getCurrentUser()); + newAnalysis.setCreatedDate(new Date()); + + return save(newAnalysis); + } + + @Override + public Page getPage(final Pageable pageable) { + List pathwayList = pathwayAnalysisRepository.findAll(defaultEntityGraph) + .stream().filter(!defaultGlobalReadPermissions ? entity -> permissionService.hasReadAccess(entity) : entity -> true) + .collect(Collectors.toList()); + return getPageFromResults(pageable, pathwayList); + } + + private Page getPageFromResults(Pageable pageable, List results) { + // Calculate the start and end indices for the current page + int startIndex = pageable.getPageNumber() * pageable.getPageSize(); + int endIndex = Math.min(startIndex + pageable.getPageSize(), results.size()); + + return new PageImpl<>(results.subList(startIndex, endIndex), pageable, results.size()); + } + + @Override + public int getCountPAWithSameName(Integer id, String name) { + + return pathwayAnalysisRepository.getCountPAWithSameName(id, name); + } + + @Override + public PathwayAnalysisEntity getById(Integer id) { + + PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(id, defaultEntityGraph); + if (Objects.nonNull(entity)) { + entity.getTargetCohorts().forEach(tc -> Hibernate.initialize(tc.getCohortDefinition().getDetails())); + entity.getEventCohorts().forEach(ec -> Hibernate.initialize(ec.getCohortDefinition().getDetails())); + } + return entity; + } + + private List getNamesLike(String name) { + + return pathwayAnalysisRepository.findAllByNameStartsWith(name).stream().map(PathwayAnalysisEntity::getName).collect(Collectors.toList()); + } + + @Override + public String getNameForCopy(String dtoName) { + return NameUtils.getNameForCopy(dtoName, this::getNamesLike, pathwayAnalysisRepository.findByName(dtoName)); + } + + @Override + public String getNameWithSuffix(String dtoName) { + return NameUtils.getNameWithSuffix(dtoName, this::getNamesLike); + } + + @Override + public PathwayAnalysisEntity update(PathwayAnalysisEntity forUpdate) { + + PathwayAnalysisEntity existing = getById(forUpdate.getId()); + + copyProps(forUpdate, existing); + updateCohorts(existing, existing.getTargetCohorts(), forUpdate.getTargetCohorts()); + updateCohorts(existing, existing.getEventCohorts(), forUpdate.getEventCohorts()); + + existing.setModifiedBy(getCurrentUser()); + existing.setModifiedDate(new Date()); + + return save(existing); + } + + private void updateCohorts(PathwayAnalysisEntity analysis, Set existing, Set forUpdate) { + + Set removedCohorts = existing + .stream() + .filter(ec -> !forUpdate.contains(ec)) + .collect(Collectors.toSet()); + existing.removeAll(removedCohorts); + forUpdate.forEach(updatedCohort -> existing.stream() + .filter(ec -> ec.equals(updatedCohort)) + .findFirst() + .map(ec -> { + ec.setName(updatedCohort.getName()); + return ec; + }) + .orElseGet(() -> { + updatedCohort.setId(null); + updatedCohort.setPathwayAnalysis(analysis); + existing.add(updatedCohort); + return updatedCohort; + })); + } + + @Override + public void delete(Integer id) { + + pathwayAnalysisRepository.delete(id); + } + + @Override + public Map getEventCohortCodes(PathwayAnalysisEntity pathwayAnalysis) { + + Integer index = 0; + + List sortedEventCohortsCopy = pathwayAnalysis.getEventCohorts() + .stream() + .sorted(Comparator.comparing(PathwayEventCohort::getName)) + .collect(Collectors.toList()); + + Map cohortDefIdToIndexMap = new HashMap<>(); + + for (PathwayEventCohort eventCohort : sortedEventCohortsCopy) { + cohortDefIdToIndexMap.put(eventCohort.getCohortDefinition().getId(), index++); + } + + return cohortDefIdToIndexMap; + } + + @Override + @DataSourceAccess + public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, @SourceId Integer sourceId, String cohortTable, String sessionId) { + + Map eventCohortCodes = getEventCohortCodes(pathwayAnalysis); + Source source = sourceService.findBySourceId(sourceId); + final StringJoiner joiner = new StringJoiner("\n\n"); + + String analysisSql = ResourceHelper.GetResourceAsString("/resources/pathway/runPathwayAnalysis.sql"); + String eventCohortInputSql = ResourceHelper.GetResourceAsString("/resources/pathway/eventCohortInput.sql"); + + String tempTableQualifier = SourceUtils.getTempQualifier(source); + String resultsTableQualifier = SourceUtils.getResultsQualifier(source); + + String eventCohortIdIndexSql = eventCohortCodes.entrySet() + .stream() + .map(ec -> { + String[] params = new String[]{"cohort_definition_id", "event_cohort_index"}; + String[] values = new String[]{ec.getKey().toString(), ec.getValue().toString()}; + return SqlRender.renderSql(eventCohortInputSql, params, values); + }) + .collect(Collectors.joining(" UNION ALL ")); + + pathwayAnalysis.getTargetCohorts().forEach(tc -> { + + String[] params = new String[]{ + GENERATION_ID, + "event_cohort_id_index_map", + "temp_database_schema", + "target_database_schema", + "target_cohort_table", + "pathway_target_cohort_id", + "max_depth", + "combo_window", + "allow_repeats", + "isHive" + }; + String[] values = new String[]{ + generationId.toString(), + eventCohortIdIndexSql, + tempTableQualifier, + resultsTableQualifier, + cohortTable, + tc.getCohortDefinition().getId().toString(), + pathwayAnalysis.getMaxDepth().toString(), + MoreObjects.firstNonNull(pathwayAnalysis.getCombinationWindow(), 1).toString(), + String.valueOf(pathwayAnalysis.isAllowRepeats()), + String.valueOf(Objects.equals(DBMSType.HIVE.getOhdsiDB(), source.getSourceDialect())) + }; + + String renderedSql = SqlRender.renderSql(analysisSql, params, values); + String translatedSql = SqlTranslate.translateSql(renderedSql, source.getSourceDialect(), sessionId, SourceUtils.getTempQualifier(source)); + + joiner.add(translatedSql); + }); + + return joiner.toString(); + } + + @Override + public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, Integer sourceId) { + + return buildAnalysisSql(generationId, pathwayAnalysis, sourceId, "cohort", SessionUtils.sessionId()); + } + + @Override + @DataSourceAccess + public JobExecutionResource generatePathways(final Integer pathwayAnalysisId, final @SourceId Integer sourceId) { + + PathwayService pathwayService = this; + + PathwayAnalysisEntity pathwayAnalysis = getById(pathwayAnalysisId); + Source source = getSourceRepository().findBySourceId(sourceId); + + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Generating Pathway Analysis %d using %s (%s)", pathwayAnalysisId, source.getSourceName(), source.getSourceKey())); + builder.addString(SOURCE_ID, String.valueOf(source.getSourceId())); + builder.addString(PATHWAY_ANALYSIS_ID, pathwayAnalysis.getId().toString()); + builder.addString(JOB_AUTHOR, getCurrentUserLogin()); + + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + + SimpleJobBuilder generateAnalysisJob = generationUtils.buildJobForCohortBasedAnalysisTasklet( + GENERATE_PATHWAY_ANALYSIS, + source, + builder, + jdbcTemplate, + chunkContext -> { + Integer analysisId = Integer.valueOf(chunkContext.getStepContext().getJobParameters().get(PATHWAY_ANALYSIS_ID).toString()); + PathwayAnalysisEntity analysis = pathwayService.getById(analysisId); + return Stream.concat(analysis.getTargetCohorts().stream(), analysis.getEventCohorts().stream()) + .map(PathwayCohort::getCohortDefinition) + .collect(Collectors.toList()); + }, + new GeneratePathwayAnalysisTasklet( + getSourceJdbcTemplate(source), + getTransactionTemplate(), + pathwayService, + analysisGenerationInfoEntityRepository, + userRepository, + sourceService + ) + ); + TransactionalTasklet statisticsTasklet = new PathwayStatisticsTasklet(getSourceJdbcTemplate(source), getTransactionTemplate(), source, this, genericConversionService); + Step generateStatistics = stepBuilderFactory.get(GENERATE_PATHWAY_ANALYSIS + ".generateStatistics") + .tasklet(statisticsTasklet) + .build(); + + generateAnalysisJob.next(generateStatistics); + + final JobParameters jobParameters = builder.toJobParameters(); + + return jobService.runJob(generateAnalysisJob.build(), jobParameters); + } + + @Override + @DataSourceAccess + public void cancelGeneration(Integer pathwayAnalysisId, @SourceId Integer sourceId) { + + PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(pathwayAnalysisId, defaultEntityGraph); + String sourceKey = getSourceRepository().findBySourceId(sourceId).getSourceKey(); + entity.getTargetCohorts().forEach(tc -> cohortDefinitionService.cancelGenerateCohort(tc.getId(), sourceKey)); + entity.getEventCohorts().forEach(ec -> cohortDefinitionService.cancelGenerateCohort(ec.getId(), sourceKey)); + jobService.cancelJobExecution(j -> { + JobParameters jobParameters = j.getJobParameters(); + String jobName = j.getJobInstance().getJobName(); + return Objects.equals(jobParameters.getString(PATHWAY_ANALYSIS_ID), Integer.toString(pathwayAnalysisId)) + && Objects.equals(jobParameters.getString(SOURCE_ID), String.valueOf(sourceId)) + && Objects.equals(GENERATE_PATHWAY_ANALYSIS, jobName); + }); + } + + @Override + public List getPathwayGenerations(final Integer pathwayAnalysisId) { + return pathwayAnalysisGenerationRepository.findAllByPathwayAnalysisId(pathwayAnalysisId, EntityUtils.fromAttributePaths("source")); + } + + @Override + public PathwayAnalysisGenerationEntity getGeneration(Long generationId) { + return pathwayAnalysisGenerationRepository.findOne(generationId, EntityUtils.fromAttributePaths("source")); + } + + @Override + @DataSourceAccess + public PathwayAnalysisResult getResultingPathways(final @PathwayAnalysisGenerationId Long generationId) { + + PathwayAnalysisGenerationEntity generation = getGeneration(generationId); + Source source = generation.getSource(); + return queryGenerationResults(source, generationId); + } + + private final RowMapper codeRowMapper = (final ResultSet resultSet, final int arg1) -> { + return new PathwayCode(resultSet.getLong("code"), resultSet.getString("name"), resultSet.getInt("is_combo") != 0); + }; + + private final RowMapper pathwayStatsRowMapper = (final ResultSet rs, final int arg1) -> { + CohortPathways cp = new CohortPathways(); + cp.setCohortId(rs.getInt("target_cohort_id")); + cp.setTargetCohortCount(rs.getInt("target_cohort_count")); + cp.setTotalPathwaysCount(rs.getInt("pathways_count")); + return cp; + }; + + private final ResultSetExtractor>> pathwayExtractor = (final ResultSet rs) -> { + Map> cohortMap = new HashMap<>(); // maps a cohortId to a list of pathways (which is stored as a Map + + while (rs.next()) { + int cohortId = rs.getInt("target_cohort_id"); + if (!cohortMap.containsKey(cohortId)) { + cohortMap.put(cohortId, new HashMap<>()); + } + Map pathList = cohortMap.get(cohortId); + + // build path + List path = new ArrayList<>(); + for (String stepCol : STEP_COLUMNS) { + String step = rs.getString(stepCol); + + if (step == null) break; // cancel for-loop when we encounter a column with a null value + + path.add(step); + } + pathList.put(StringUtils.join(path, "-"), rs.getInt("count_value")); // for a given cohort, a path must be unique, so no need to check + } + return cohortMap; + }; + + @Override + @DataSourceAccess + public String findDesignByGenerationId(@PathwayAnalysisGenerationId final Long id) { + final AnalysisGenerationInfoEntity entity = analysisGenerationInfoEntityRepository.findById(id) + .orElseThrow(() -> new IllegalArgumentException("Analysis with id: " + id + " cannot be found")); + return entity.getDesign(); + } + + @Override + public void assignTag(Integer id, int tagId) { + PathwayAnalysisEntity entity = getById(id); + checkOwnerOrAdminOrGranted(entity); + assignTag(entity, tagId); + } + + @Override + public void unassignTag(Integer id, int tagId) { + PathwayAnalysisEntity entity = getById(id); + checkOwnerOrAdminOrGranted(entity); + unassignTag(entity, tagId); + } + + @Override + public List getVersions(long id) { + List versions = versionService.getVersions(VersionType.PATHWAY, id); + return versions.stream() + .map(v -> genericConversionService.convert(v, VersionDTO.class)) + .collect(Collectors.toList()); + } + + @Override + public PathwayVersionFullDTO getVersion(int id, int version) { + checkVersion(id, version, false); + PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + return genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); + } + + @Override + public VersionDTO updateVersion(int id, int version, VersionUpdateDTO updateDTO) { + checkVersion(id, version); + updateDTO.setAssetId(id); + updateDTO.setVersion(version); + PathwayVersion updated = versionService.update(VersionType.PATHWAY, updateDTO); + + return genericConversionService.convert(updated, VersionDTO.class); + } + + @Override + public void deleteVersion(int id, int version) { + checkVersion(id, version); + versionService.delete(VersionType.PATHWAY, id, version); + } + + @Override + public PathwayAnalysisDTO copyAssetFromVersion(int id, int version) { + checkVersion(id, version, false); + PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + PathwayVersionFullDTO fullDTO = genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); + + PathwayAnalysisDTO dto = fullDTO.getEntityDTO(); + dto.setId(null); + dto.setTags(null); + dto.setName(NameUtils.getNameForCopy(dto.getName(), this::getNamesLike, + pathwayAnalysisRepository.findByName(dto.getName()))); + PathwayAnalysisEntity pathwayAnalysis = genericConversionService.convert(dto, PathwayAnalysisEntity.class); + PathwayAnalysisEntity saved = create(pathwayAnalysis); + return genericConversionService.convert(saved, PathwayAnalysisDTO.class); + } + + @Override + public List listByTags(TagNameListRequestDTO requestDTO) { + List names = requestDTO.getNames().stream() + .map(name -> name.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); + List entities = pathwayAnalysisRepository.findByTags(names); + return listByTags(entities, names, PathwayAnalysisDTO.class); + } + + private void checkVersion(int id, int version) { + checkVersion(id, version, true); + } + + private void checkVersion(int id, int version, boolean checkOwnerShip) { + Version pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayVersion, + String.format("There is no pathway analysis version with id = %d.", version)); + + PathwayAnalysisEntity entity = this.pathwayAnalysisRepository.findOne(id); + if (checkOwnerShip) { + checkOwnerOrAdminOrGranted(entity); + } + } + + public PathwayVersion saveVersion(int id) { + PathwayAnalysisEntity def = this.pathwayAnalysisRepository.findOne(id); + PathwayVersion version = genericConversionService.convert(def, PathwayVersion.class); + + UserEntity user = Objects.nonNull(def.getModifiedBy()) ? def.getModifiedBy() : def.getCreatedBy(); + Date versionDate = Objects.nonNull(def.getModifiedDate()) ? def.getModifiedDate() : def.getCreatedDate(); + version.setCreatedBy(user); + version.setCreatedDate(versionDate); + return versionService.create(VersionType.PATHWAY, version); + } + + private PathwayAnalysisResult queryGenerationResults(Source source, Long generationId) { + + // load code lookup + PreparedStatementRenderer pathwayCodesPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayCodeLookup.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + List pathwayCodes = getSourceJdbcTemplate(source).query(pathwayCodesPsr.getSql(), pathwayCodesPsr.getOrderedParams(), codeRowMapper); + + // fetch cohort stats, paths will be populated after + PreparedStatementRenderer pathwayStatsPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayStats.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + List cohortStats = getSourceJdbcTemplate(source).query(pathwayStatsPsr.getSql(), pathwayStatsPsr.getOrderedParams(), pathwayStatsRowMapper); + + // load cohort paths, and assign back to cohortStats + PreparedStatementRenderer pathwayResultsPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayResults.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + Map> pathwayResults = + getSourceJdbcTemplate(source).query(pathwayResultsPsr.getSql(), pathwayResultsPsr.getOrderedParams(), pathwayExtractor); + + cohortStats.stream().forEach((cp) -> { + cp.setPathwaysCounts(pathwayResults.get(cp.getCohortId())); + }); + + PathwayAnalysisResult result = new PathwayAnalysisResult(); + result.setCodes(new HashSet<>(pathwayCodes)); + result.setCohortPathwaysList(new HashSet<>(cohortStats)); + + return result; + } + + private void copyProps(PathwayAnalysisEntity from, PathwayAnalysisEntity to) { + + to.setName(from.getName()); + to.setDescription(from.getDescription()); + to.setMaxDepth(from.getMaxDepth()); + to.setMinCellCount(from.getMinCellCount()); + to.setCombinationWindow(from.getCombinationWindow()); + to.setAllowRepeats(from.isAllowRepeats()); + } + + private int getAnalysisHashCode(PathwayAnalysisEntity pathwayAnalysis) { + + SerializedPathwayAnalysisToPathwayAnalysisConverter designConverter = new SerializedPathwayAnalysisToPathwayAnalysisConverter(); + return designConverter.convertToDatabaseColumn(pathwayAnalysis).hashCode(); + } + + private PathwayAnalysisEntity save(PathwayAnalysisEntity pathwayAnalysis) { + + pathwayAnalysis = pathwayAnalysisRepository.saveAndFlush(pathwayAnalysis); + entityManager.refresh(pathwayAnalysis); + pathwayAnalysis = getById(pathwayAnalysis.getId()); + pathwayAnalysis.setHashCode(getAnalysisHashCode(pathwayAnalysis)); + return pathwayAnalysis; + } + + @Override + public String getJobName() { + return GENERATE_PATHWAY_ANALYSIS; + } + + @Override + public String getExecutionFoldingKey() { + return PATHWAY_ANALYSIS_ID; + } + + + @Override + @Transactional + public PathwayAnalysisDTO getByGenerationId(final Integer id) { + PathwayAnalysisGenerationEntity pathwayAnalysisGenerationEntity = getGeneration(id.longValue()); + PathwayAnalysisEntity pathwayAnalysis = pathwayAnalysisGenerationEntity.getPathwayAnalysis(); + Map eventCodes = getEventCohortCodes(pathwayAnalysis); + PathwayAnalysisDTO dto = genericConversionService.convert(pathwayAnalysis, PathwayAnalysisDTO.class); + dto.getEventCohorts().forEach(ec -> ec.setCode(eventCodes.get(ec.getId()))); + return dto; + } + @Override + public PathwayPopulationResultsDTO getGenerationResults(Long generationId) { + PathwayAnalysisResult resultingPathways = getResultingPathways(generationId); + + List eventCodeDtos = resultingPathways.getCodes() + .stream() + .map(entry -> { + PathwayCodeDTO dto = new PathwayCodeDTO(); + dto.setCode(entry.getCode()); + dto.setName(entry.getName()); + dto.setIsCombo(entry.isCombo()); + return dto; + }) + .collect(Collectors.toList()); + + List pathwayDtos = resultingPathways.getCohortPathwaysList() + .stream() + .map(cohortResults -> { + if (cohortResults.getPathwaysCounts() == null) { + return null; + } + + List eventDTOs = cohortResults.getPathwaysCounts() + .entrySet() + .stream() + .map(entry -> new PathwayPopulationEventDTO(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + return new TargetCohortPathwaysDTO(cohortResults.getCohortId(), cohortResults.getTargetCohortCount(), cohortResults.getTotalPathwaysCount(), eventDTOs); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + return new PathwayPopulationResultsDTO(eventCodeDtos, pathwayDtos); + } } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index 867635d64..d51283b49 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -927,7 +927,7 @@ public CheckResult runDiagnosticsWithTags(CohortDTO cohortDTO) { @Path("/printfriendly/cohort") @Consumes(MediaType.APPLICATION_JSON) public Response cohortPrintFriendly(CohortExpression expression, @DefaultValue("html") @QueryParam("format") String format) { - String markdown = markdownPF.renderCohort(expression); + String markdown = convertCohortExpressionToMarkdown(expression); return printFrindly(markdown, format); } @@ -953,16 +953,23 @@ public Response conceptSetListPrintFriendly(List conceptSetList, @De return printFrindly(markdown, format); } + public String convertCohortExpressionToMarkdown(CohortExpression expression){ + return markdownPF.renderCohort(expression); + } + + public String convertMarkdownToHTML(String markdown){ + Parser parser = Parser.builder().extensions(extensions).build(); + Node document = parser.parse(markdown); + HtmlRenderer renderer = HtmlRenderer.builder().extensions(extensions).build(); + return renderer.render(document); + } + private Response printFrindly(String markdown, String format) { ResponseBuilder res; if ("html".equalsIgnoreCase(format)) { - Parser parser = Parser.builder().extensions(extensions).build(); - Node document = parser.parse(markdown); - HtmlRenderer renderer = HtmlRenderer.builder().extensions(extensions).build(); - String html = renderer.render(document); + String html = convertMarkdownToHTML(markdown); res = Response.ok(html, MediaType.TEXT_HTML); - } else if ("markdown".equals(format)) { res = Response.ok(markdown, MediaType.TEXT_PLAIN); } else { diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java new file mode 100644 index 000000000..286cec802 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -0,0 +1,107 @@ +package org.ohdsi.webapi.service; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; + +import org.ohdsi.webapi.shiny.ApplicationBrief; +import org.ohdsi.webapi.shiny.ConflictPositConnectException; +import org.ohdsi.webapi.shiny.PackagingStrategies; +import org.ohdsi.webapi.shiny.PackagingStrategy; +import org.ohdsi.webapi.shiny.PositConnectClient; +import org.ohdsi.webapi.shiny.ShinyPackagingService; +import org.ohdsi.webapi.shiny.ShinyPublishedEntity; +import org.ohdsi.webapi.shiny.ShinyPublishedRepository; +import org.ohdsi.webapi.shiny.TemporaryFile; +import org.ohdsi.webapi.shiro.PermissionManager; +import org.ohdsi.webapi.shiro.Entities.UserRepository; +import org.ohdsi.webapi.shiro.management.Security; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.NotFoundException; + +import java.sql.Date; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +@ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") +public class ShinyService { + private static final Logger log = LoggerFactory.getLogger(ShinyService.class); + private final Map servicesMap; + @Autowired + private ShinyPublishedRepository shinyPublishedRepository; + @Autowired + private PermissionManager permissionManager; + @Autowired + private PositConnectClient connectClient; + @Autowired + protected Security security; + @Autowired + protected UserRepository userRepository; + + @Value("#{!'${security.provider}'.equals('DisabledSecurity')}") + private boolean securityEnabled; + + @Inject + public ShinyService(List services) { + servicesMap = services.stream().collect(Collectors.toMap(ShinyPackagingService::getType, Function.identity())); + } + + public void publishApp(String type, int id, String sourceKey) { + TemporaryFile data = packageShinyApp(type, id, sourceKey, PackagingStrategies.targz()); + ShinyPublishedEntity publication = getPublication(id, sourceKey); + ShinyPackagingService service = findShinyService(CommonAnalysisType.valueOf(type.toUpperCase())); + UUID contentId = Optional.ofNullable(publication.getContentId()) + .orElseGet(() -> createOrFindItem(service.getBrief(id, sourceKey))); + String bundleId = connectClient.uploadBundle(contentId, data); + String taskId = connectClient.deployBundle(contentId, bundleId); + log.debug("Bundle [{}] is deployed to Shiny server, task id: [{}]", id, taskId); + } + + private UUID createOrFindItem(ApplicationBrief brief) { + try { + return connectClient.createContentItem(brief); + } catch (ConflictPositConnectException e) { + log.warn("Content item [{}] already exist, will update", brief.getName()); + return connectClient.listContentItems().stream() + .filter(i -> Objects.equals(i.name, brief.getName())) + .findFirst() + .map(i -> i.guid) + .orElseThrow(NotFoundException::new); + } + } + + private ShinyPublishedEntity getPublication(int id, String sourceKey) { + return shinyPublishedRepository.findByAnalysisIdAndSourceKey(Integer.toUnsignedLong(id), sourceKey).orElseGet(() -> { + ShinyPublishedEntity entity = new ShinyPublishedEntity(); + entity.setAnalysisId(Integer.toUnsignedLong(id)); + entity.setSourceKey(sourceKey); + entity.setCreatedBy(securityEnabled ? permissionManager.getCurrentUser() : userRepository.findByLogin(security.getSubject())); + entity.setCreatedDate(Date.from(Instant.now())); + return entity; + }); + } + + public TemporaryFile packageShinyApp(String type, int id, String sourceKey, PackagingStrategy packaging) { + CommonAnalysisType analysisType = CommonAnalysisType.valueOf(type.toUpperCase()); + ShinyPackagingService service = findShinyService(analysisType); + return service.packageApp(id, sourceKey, packaging); + } + + private ShinyPackagingService findShinyService(CommonAnalysisType type) { + return Optional.ofNullable(servicesMap.get(type)) + .orElseThrow(() -> new NotFoundException(MessageFormat.format("Shiny application download is not supported for [{0}] analyses.", type))); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java b/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java new file mode 100644 index 000000000..7905a8ed5 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java @@ -0,0 +1,31 @@ +package org.ohdsi.webapi.shiny; + +public class ApplicationBrief { + private String name; + private String title; + private String description; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java new file mode 100644 index 000000000..838e036b9 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java @@ -0,0 +1,43 @@ +package org.ohdsi.webapi.shiny; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +@Service +public class CohortCharacterizationAnalysisHeaderToFieldMapper { + + private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationAnalysisHeaderToFieldMapper.class); + private final Map headerFieldMapping; + + public CohortCharacterizationAnalysisHeaderToFieldMapper(@Value("classpath:shiny/cc-header-field-mapping.csv") Resource resource) throws IOException { + this.headerFieldMapping = new HashMap<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + String[] parts = line.split(",", 2); // Split line into two parts + if (parts.length >= 2) { // Ensure that line has header and field + String header = parts[0]; + String field = parts[1]; + headerFieldMapping.put(header, field); + } else { + LOG.warn("ignoring a line due to unexpected count of parameters (!=2): " + line); + } + } + } + } + + public Map getHeaderFieldMapping() { + return headerFieldMapping; + } + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java new file mode 100644 index 000000000..cdc7f8886 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -0,0 +1,247 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.tuple.Pair; +import org.ohdsi.analysis.CohortMetadata; +import org.ohdsi.analysis.WithId; +import org.ohdsi.analysis.cohortcharacterization.design.CohortCharacterization; +import org.ohdsi.webapi.cohortcharacterization.CcService; +import org.ohdsi.webapi.cohortcharacterization.domain.CcGenerationEntity; +import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; +import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest; +import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; +import org.ohdsi.webapi.cohortcharacterization.report.ExportItem; +import org.ohdsi.webapi.cohortcharacterization.report.Report; +import org.ohdsi.webapi.cohortdefinition.CohortDefinition; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortCharacterizationShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); + private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; + private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCharacterizations.zip"; + private static final String APP_TITLE_FORMAT = "Characterization_%s_gv%sx_%s"; + + private final CcService ccService; + + private final CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper; + + @Autowired + public CohortCharacterizationShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, + CcService ccService, + CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); + this.ccService = ccService; + this.cohortCharacterizationAnalysisHeaderToFieldMapper = cohortCharacterizationAnalysisHeaderToFieldMapper; + } + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT_CHARACTERIZATION; + } + + + @Override + public String getAppTemplateFilePath() { + return SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH; + } + + @Override + @Transactional + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); + GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); + ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ANALYSIS_NAME.getValue(), cohortCharacterization.getName()); + + generationResults.getReports() + .stream() + .map(this::convertReportToCSV) + .forEach(csvDataByFilename -> dataConsumers.getTextFiles().accept(csvDataByFilename.getKey(), csvDataByFilename.getValue())); + + CcGenerationEntity generationEntity = ccService.findGenerationById(Long.valueOf(generationId)); + + Long resultsTotalCount = ccService.getCCResultsTotalCount(Long.valueOf(generationId)); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), cohortCharacterization.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), Long.toString(resultsTotalCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), getReferencedCohorts(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), Integer.toString(generationId)); + } + + private String getReferencedCohorts(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity != null) { + return cohortCharacterizationEntity.getCohortDefinitions().stream().map(CohortDefinition::getName).collect(Collectors.joining("; ")); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + + private String getAuthor(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity.getCreatedBy() != null) { + return cohortCharacterizationEntity.getCreatedBy().getLogin(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getGenerationStartTime(CcGenerationEntity generation) { + if (generation != null) { + return dateToString(generation.getStartTime()); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getDescription(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity != null && cohortCharacterizationEntity.getDescription() != null) { + return cohortCharacterizationEntity.getDescription(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + //Pair.left == CSV filename + //Pair.right == CSV contents + private Pair convertReportToCSV(Report report) { + boolean isComparativeAnalysis = report.isComparative; + String analysisName = report.analysisName; + String fileNameFormat = "Export %s(%s).csv"; + String fileName = String.format(fileNameFormat, isComparativeAnalysis ? "comparison " : "", analysisName); + List exportItems = report.items.stream() + .sorted() + .collect(Collectors.toList()); + + String[] header = Iterables.getOnlyElement(report.header); + + String outCsv = prepareCsv(header, exportItems); + return Pair.of(fileName, outCsv); + } + + private String prepareCsv(String[] headers, List exportItems) { + try (StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, + CSVFormat.Builder + .create() + .setQuoteMode(QuoteMode.NON_NUMERIC) + .setHeader(headers) + .build())) { + for (ExportItem item : exportItems) { + List record = new ArrayList<>(); + for (String header : headers) { + String fieldName = cohortCharacterizationAnalysisHeaderToFieldMapper.getHeaderFieldMapping().get(header); // get the corresponding Java field name + Field field; + try { + if (fieldName != null) { + field = findFieldInClassHierarchy(item.getClass(), fieldName); + if (field != null) { + field.setAccessible(true); + record.add(String.valueOf(field.get(item))); + } else { + record.add(null); + } + } + } catch (IllegalAccessException ex) { + LOG.error("Error occurred while accessing field value", ex); + record.add(""); + } + } + csvPrinter.printRecord(record); + } + return stringWriter.toString(); + } catch (IOException e) { + LOG.error("Failed to create a CSV file with Cohort Characterization analysis details", e); + throw new InternalServerErrorException(); + } + } + + private Field findFieldInClassHierarchy(Class clazz, String fieldName) { + if (clazz == null) { + return null; + } + Field field; + try { + field = clazz.getDeclaredField(fieldName); + } catch (NoSuchFieldException ex) { + field = findFieldInClassHierarchy(clazz.getSuperclass(), fieldName); + } + return field; + } + + private GenerationResults fetchGenerationResults(Integer generationId, CohortCharacterization cohortCharacterization) { + ExecutionResultRequest executionResultRequest = new ExecutionResultRequest(); + List cohortIds = cohortCharacterization.getCohorts() + .stream() + .map(CohortMetadata::getId) + .collect(Collectors.toList()); + List analysisIds = cohortCharacterization.getFeatureAnalyses() + .stream() + .map(WithId::getId) + .map(Number::intValue) + .collect(Collectors.toList()); + List domainIds = cohortCharacterization.getFeatureAnalyses() + .stream() + .map(featureAnalysis -> featureAnalysis.getDomain().getName().toUpperCase()) + .distinct() + .collect(Collectors.toList()); + executionResultRequest.setAnalysisIds(analysisIds); + executionResultRequest.setCohortIds(cohortIds); + executionResultRequest.setDomainIds(domainIds); + executionResultRequest.setShowEmptyResults(Boolean.TRUE); + executionResultRequest.setThresholdValuePct(DEFAULT_THRESHOLD_VALUE); + return ccService.findResult(Long.valueOf(generationId), executionResultRequest); + } + + @Override + @Transactional + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT_CHARACTERIZATION.getCode(), generationId, sourceKey)); + applicationBrief.setTitle(prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)); + applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); + return applicationBrief; + } + + private String prepareAppTitle(Long studyAssetId, Integer generationId, String sourceKey) { + return String.format(APP_TITLE_FORMAT, studyAssetId, generationId, sourceKey); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java new file mode 100644 index 000000000..92a41d634 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -0,0 +1,152 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.ohdsi.circe.cohortdefinition.CohortExpression; +import org.ohdsi.webapi.cohortdefinition.CohortDefinition; +import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfo; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoId; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoRepository; +import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.service.CohortDefinitionService; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final String SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCounts.zip"; + private static final String APP_TITLE_FORMAT = "Cohort_%s_gv%sx%s_%s"; + private final CohortDefinitionService cohortDefinitionService; + private final CohortDefinitionRepository cohortDefinitionRepository; + private final CohortGenerationInfoRepository cohortGenerationInfoRepository; + + @Autowired + public CohortCountsShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, + CohortDefinitionService cohortDefinitionService, + CohortDefinitionRepository cohortDefinitionRepository, + SourceRepository sourceRepository, + CohortGenerationInfoRepository cohortGenerationInfoRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter + ) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); + this.cohortDefinitionService = cohortDefinitionService; + this.cohortDefinitionRepository = cohortDefinitionRepository; + this.cohortGenerationInfoRepository = cohortGenerationInfoRepository; + } + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT; + } + + @Override + public String getAppTemplateFilePath() { + return SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH; + } + + @Override + @Transactional + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); + + int sourceId = getSourceRepository().findBySourceKey(sourceKey).getId(); + CohortGenerationInfo cohortGenerationInfo = cohortGenerationInfoRepository.findOne(new CohortGenerationInfoId(cohort.getId(), sourceId)); + + CohortExpression cohortExpression = cohort.getExpression(); + + String cohortSummaryAsMarkdown = cohortDefinitionService.convertCohortExpressionToMarkdown(cohortExpression); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_COHORT_LINK.getValue(), String.format("%s/#/cohortdefinition/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_COHORT_NAME.getValue(), cohort.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(cohort)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), cohort.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), getRecordCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), getPersonCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(cohort)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), cohort.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), getGenerationId(cohortGenerationInfo.getId())); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), getGenerationId(cohortGenerationInfo.getId())); + + dataConsumers.getTextFiles().accept("cohort_summary_markdown.txt", cohortSummaryAsMarkdown); + + InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event + InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person + + dataConsumers.getJsonObjects().accept(sourceKey + "_by_event.json", byEventReport); + dataConsumers.getJsonObjects().accept(sourceKey + "_by_person.json", byPersonReport); + } + + private String getGenerationId(CohortGenerationInfoId id) { + return id == null ? "" : Integer.toString(id.getCohortDefinitionId()).concat("x").concat(Integer.toString(id.getSourceId())); + } + + private String getDescription(CohortDefinition cohort) { + if (cohort != null && cohort.getDescription() != null) { + return cohort.getDescription(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getPersonCount(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getPersonCount() != null) { + return cohortGenerationInfo.getPersonCount().toString(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getRecordCount(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getRecordCount() != null) { + return cohortGenerationInfo.getRecordCount().toString(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getGenerationStartTime(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getStartTime() != null) { + return dateToString(cohortGenerationInfo.getStartTime()); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getAuthor(CohortDefinition cohort) { + if (cohort.getCreatedBy() != null) { + return cohort.getCreatedBy().getLogin(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + @Override + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); + Integer assetId = cohort.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); + + ApplicationBrief brief = new ApplicationBrief(); + brief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT.getCode(), generationId, sourceKey)); + brief.setTitle(prepareAppTitle(generationId, assetId, sourceId, sourceKey)); + brief.setDescription(cohort.getDescription()); + return brief; + } + + private String prepareAppTitle(Integer generationId, Integer assetId, Integer sourceId, String sourceKey) { + return String.format(APP_TITLE_FORMAT, generationId, assetId, sourceId, sourceKey); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java new file mode 100644 index 000000000..50e458cae --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -0,0 +1,135 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayCohortDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; +import org.ohdsi.webapi.pathway.dto.TargetCohortPathwaysDTO; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import java.util.HashSet; +import java.util.Set; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortPathwaysShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final String SHINY_COHORT_PATHWAYS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortPathways.zip"; + private static final String APP_TITLE_FORMAT = "Pathway_%s_gv%sx_%s"; + + private final PathwayService pathwayService; + + @Autowired + public CohortPathwaysShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, PathwayService pathwayService, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); + this.pathwayService = pathwayService; + } + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT_PATHWAY; + } + + @Override + public String getAppTemplateFilePath() { + return SHINY_COHORT_PATHWAYS_APP_TEMPLATE_FILE_PATH; + } + + @Override + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + String designJSON = pathwayService.findDesignByGenerationId(generationId.longValue()); + PathwayPopulationResultsDTO generationResults = pathwayService.getGenerationResults(generationId.longValue()); + ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no pathway analysis generation results with generation id = %d.", generationId)); + ExceptionUtils.throwNotFoundExceptionIfNull(designJSON, String.format("There is no pathway analysis design with generation id = %d.", generationId)); + dataConsumers.getTextFiles().accept("design.json", designJSON); + dataConsumers.getJsonObjects().accept("chartData.json", generationResults); + + PathwayAnalysisDTO pathwayAnalysisDTO = pathwayService.getByGenerationId(generationId); + PathwayAnalysisGenerationEntity generationEntity = pathwayService.getGeneration(generationId.longValue()); + + int totalCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTargetCohortCount).sum(); + int personCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTotalPathwaysCount).sum(); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_NAME.getValue(), pathwayAnalysisDTO.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), pathwayAnalysisDTO.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), Integer.toString(totalCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), Integer.toString(personCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), prepareReferencedCohorts(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), Integer.toString(generationId)); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/pathways/%s", atlasUrl, pathwayAnalysisDTO.getId())); + + } + + private String getAuthor(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO.getCreatedBy() != null) { + return pathwayAnalysisDTO.getCreatedBy().getLogin(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getDescription(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO != null && pathwayAnalysisDTO.getDescription() != null) { + return pathwayAnalysisDTO.getDescription(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + + private String prepareReferencedCohorts(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO == null) { + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + Set referencedCohortNames = new HashSet<>(); + for (PathwayCohortDTO eventCohort : pathwayAnalysisDTO.getEventCohorts()) { + referencedCohortNames.add(eventCohort.getName()); + } + for (PathwayCohortDTO targetCohort : pathwayAnalysisDTO.getTargetCohorts()) { + referencedCohortNames.add(targetCohort.getName()); + } + return String.join("; ", referencedCohortNames); + } + + private String getGenerationStartTime(PathwayAnalysisGenerationEntity generationEntity) { + if (generationEntity != null) { + return dateToString(generationEntity.getStartTime()); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + @Override + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT_PATHWAY.getCode(), generationId, sourceKey)); + applicationBrief.setTitle(prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)); + applicationBrief.setDescription(pathwayAnalysis.getDescription()); + return applicationBrief; + } + + private String prepareAppTitle(Integer studyAssetId, Integer generationId, String sourceKey) { + return String.format(APP_TITLE_FORMAT, studyAssetId, generationId, sourceKey); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java new file mode 100644 index 000000000..7bf0b04dc --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java @@ -0,0 +1,203 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.ohdsi.webapi.report.CDMDashboard; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummary; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public abstract class CommonShinyPackagingService { + private static final Logger LOG = LoggerFactory.getLogger(CommonShinyPackagingService.class); + protected final String atlasUrl; + protected String repoLink; + protected final FileWriter fileWriter; + protected final ManifestUtils manifestUtils; + protected final ObjectMapper objectMapper; + protected final SourceRepository sourceRepository; + protected final CDMResultsService cdmResultsService; + protected final DataSourceSummaryConverter dataSourceSummaryConverter; + + public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper, SourceRepository sourceRepository, CDMResultsService cdmResultsService, DataSourceSummaryConverter dataSourceSummaryConverter) { + this.atlasUrl = atlasUrl; + this.repoLink = repoLink; + this.fileWriter = fileWriter; + this.manifestUtils = manifestUtils; + this.objectMapper = objectMapper; + this.sourceRepository = sourceRepository; + this.cdmResultsService = cdmResultsService; + this.dataSourceSummaryConverter = dataSourceSummaryConverter; + } + + public abstract CommonAnalysisType getType(); + + + public abstract ApplicationBrief getBrief(Integer generationId, String sourceKey); + + public abstract String getAppTemplateFilePath(); + + public abstract void populateAppData( + Integer generationId, + String sourceKey, + ShinyAppDataConsumers shinyAppDataConsumers + ); + + public String getAtlasUrl() { + return atlasUrl; + } + + public String getRepoLink() { + return repoLink; + } + + public void setRepoLink(String repoLink) { + this.repoLink = repoLink; + } + + public FileWriter getFileWriter() { + return fileWriter; + } + + public ManifestUtils getManifestUtils() { + return manifestUtils; + } + + public ObjectMapper getObjectMapper() { + return objectMapper; + } + + public SourceRepository getSourceRepository() { + return sourceRepository; + } + + public CDMResultsService getCdmResultsService() { + return cdmResultsService; + } + + public DataSourceSummaryConverter getDataSourceSummaryConverter() { + return dataSourceSummaryConverter; + } + + + class ShinyAppDataConsumers { + private final Map applicationProperties = new HashMap<>(); + private final Map jsonObjectsToSave = new HashMap<>(); + private final Map textFilesToSave = new HashMap<>(); + private final BiConsumer appPropertiesConsumer = applicationProperties::put; + private final BiConsumer textFilesConsumer = textFilesToSave::put; + private final BiConsumer jsonObjectsConsumer = jsonObjectsToSave::put; + + public BiConsumer getAppProperties() { + return appPropertiesConsumer; + } + + public BiConsumer getTextFiles() { + return textFilesConsumer; + } + + public BiConsumer getJsonObjects() { + return jsonObjectsConsumer; + } + } + + public final TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { + return TempFileUtils.doInDirectory(path -> { + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(getAppTemplateFilePath(), "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = getManifestUtils().parseManifest(manifestPath); + + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + + Source source = getSourceRepository().findBySourceKey(sourceKey); + + ShinyAppDataConsumers shinyAppDataConsumers = new ShinyAppDataConsumers(); + + //Default properties common for each shiny app + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_REPO_LINK.getValue(), getRepoLink()); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_ATLAS_URL.getValue(), getAtlasUrl()); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_DATASOURCE_KEY.getValue(), sourceKey); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_DATASOURCE_NAME.getValue(), source.getSourceName()); + + populateCDMDataSourceSummaryIfPresent(source, shinyAppDataConsumers); + + populateAppData(generationId, sourceKey, shinyAppDataConsumers); + + Stream textFilesPaths = shinyAppDataConsumers.textFilesToSave.entrySet() + .stream() + .map(entry -> getFileWriter().writeTextFile(dataDir.resolve(entry.getKey()), pw -> pw.print(entry.getValue()))); + + Stream jsonFilesPaths = shinyAppDataConsumers.jsonObjectsToSave.entrySet() + .stream() + .map(entry -> getFileWriter().writeObjectAsJsonFile(dataDir, entry.getValue(), entry.getKey())); + + Stream appPropertiesFilePath = Stream.of( + getFileWriter().writeTextFile(dataDir.resolve("app.properties"), pw -> pw.print(convertAppPropertiesToString(shinyAppDataConsumers.applicationProperties))) + ); + + Stream.of(textFilesPaths, jsonFilesPaths, appPropertiesFilePath) + .flatMap(Function.identity()) + .forEach(getManifestUtils().addDataToManifest(manifest, path)); + + getFileWriter().writeJsonNodeToFile(manifest, manifestPath); + Path appArchive = packaging.apply(path); + ApplicationBrief applicationBrief = getBrief(generationId, sourceKey); + return new TemporaryFile(String.format("%s.zip", applicationBrief.getTitle()), appArchive); + } catch (IOException e) { + LOG.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + private void populateCDMDataSourceSummaryIfPresent(Source source, ShinyAppDataConsumers shinyAppDataConsumers) { + DataSourceSummary dataSourceSummary; + try { + CDMDashboard cdmDashboard = getCdmResultsService().getDashboard(source.getSourceKey()); + dataSourceSummary = getDataSourceSummaryConverter().convert(cdmDashboard); + } catch (Exception e) { + LOG.warn("Could not populate datasource summary", e); + dataSourceSummary = getDataSourceSummaryConverter().emptySummary(source.getSourceName()); + } + shinyAppDataConsumers.jsonObjectsToSave.put("datasource_summary.json", dataSourceSummary); + } + + private String convertAppPropertiesToString(Map appProperties) { + return appProperties.entrySet().stream() + .map(entry -> String.format("%s=%s\n", entry.getKey(), entry.getValue())) + .collect(Collectors.joining()); + } + + protected String dateToString(Date date) { + if (date == null) return null; + DateFormat df = new SimpleDateFormat(ShinyConstants.DATE_TIME_FORMAT.getValue()); + return df.format(date); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java b/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java new file mode 100644 index 000000000..3f51cd85c --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +public class ConflictPositConnectException extends PositConnectClientException { + public ConflictPositConnectException(String message) { + super(message); + } + + public ConflictPositConnectException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java new file mode 100644 index 000000000..2300f50b4 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java @@ -0,0 +1,56 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Consumer; + +@Component +public class FileWriter { + + private static final Logger LOG = LoggerFactory.getLogger(FileWriter.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + + public Path writeTextFile(Path path, Consumer writer) { + try (OutputStream out = Files.newOutputStream(path); PrintWriter printWriter = new PrintWriter(out)) { + writer.accept(printWriter); + return path; + } catch (IOException e) { + LOG.error("Failed to write file", e); + throw new InternalServerErrorException(); + } + } + + public Path writeObjectAsJsonFile(Path parentDir, Object object, String filename) { + try { + Path file = Files.createFile(parentDir.resolve(filename)); + try (OutputStream out = Files.newOutputStream(file)) { + objectMapper.writeValue(out, object); + } + return file; + } catch (IOException e) { + LOG.error("Failed to package Shiny application", e); + throw new InternalServerErrorException(); + } + } + + public void writeJsonNodeToFile(JsonNode object, Path path) { + try { + objectMapper.writeValue(path.toFile(), object); + } catch (IOException e) { + LOG.error("Failed to write json file", e); + throw new InternalServerErrorException(); + } + } + + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java new file mode 100644 index 000000000..ada600e18 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -0,0 +1,234 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; +import org.ohdsi.webapi.ircalc.AnalysisReport; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.service.IRAnalysisResource; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.service.dto.AnalysisInfoDTO; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.StringWriter; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +@Service +@ConditionalOnBean(ShinyService.class) +public class IncidenceRatesShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final Logger LOG = LoggerFactory.getLogger(IncidenceRatesShinyPackagingService.class); + private static final String SHINY_INCIDENCE_RATES_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-incidenceRates.zip"; + private static final String COHORT_TYPE_TARGET = "target"; + private static final String COHORT_TYPE_OUTCOME = "outcome"; + private static final String APP_NAME_FORMAT = "Incidence_%s_gv%sx%s_%s"; + private final IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; + private final IRAnalysisResource irAnalysisResource; + + @Autowired + public IncidenceRatesShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, + IncidenceRateAnalysisRepository incidenceRateAnalysisRepository, + IRAnalysisResource irAnalysisResource, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); + this.incidenceRateAnalysisRepository = incidenceRateAnalysisRepository; + this.irAnalysisResource = irAnalysisResource; + } + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.INCIDENCE; + } + + @Override + public String getAppTemplateFilePath() { + return SHINY_INCIDENCE_RATES_APP_TEMPLATE_FILE_PATH; + } + + @Override + @Transactional + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); + try { + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/iranalysis/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ANALYSIS_NAME.getValue(), analysis.getName()); + + IncidenceRateAnalysisExportExpression expression = objectMapper.readValue(analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); + AnalysisInfoDTO analysisInfoDTO = irAnalysisResource.getAnalysisInfo(analysis.getId(), sourceKey); + + Integer assetId = analysis.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), analysis.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), getRecordCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), getPersonCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), prepareReferencedCohorts(expression)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), getGenerationId(assetId, sourceId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), getGenerationId(assetId, sourceId)); + + String csvWithCohortDetails = prepareCsvWithCohorts(expression); + + dataConsumers.getTextFiles().accept("cohorts.csv", csvWithCohortDetails); + + streamAnalysisReportsForAllCohortCombinations(expression, generationId, sourceKey) + .forEach(analysisReport -> + dataConsumers.getJsonObjects().accept( + String.format("%s_targetId%s_outcomeId%s.json", sourceKey, analysisReport.summary.targetId, analysisReport.summary.outcomeId), + analysisReport + ) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getAuthor(IncidenceRateAnalysis analysis) { + if (analysis.getCreatedBy() != null) { + return analysis.getCreatedBy().getLogin(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getGenerationStartTime(IncidenceRateAnalysis analysis) { + if (analysis != null) { + if (CollectionUtils.isNotEmpty(analysis.getExecutionInfoList())) { + return dateToString(Iterables.getLast(analysis.getExecutionInfoList()).getStartTime()); + } + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getDescription(IncidenceRateAnalysis analysis) { + if (analysis != null && analysis.getDescription() != null) { + return analysis.getDescription(); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getPersonCount(AnalysisInfoDTO analysisInfo) { + if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { + return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).cases); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getRecordCount(AnalysisInfoDTO analysisInfo) { + if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { + return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).totalPersons); + } + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); + } + + private String getGenerationId(Integer assetId, Integer sourceId) { + return assetId == null || sourceId == null ? "" : Integer.toString(assetId).concat("x").concat(Integer.toString(sourceId)); + } + + private String prepareReferencedCohorts(IncidenceRateAnalysisExportExpression expression) { + if (expression == null) { + return ""; + } + Set referencedCohortNames = new HashSet<>(); + for (CohortDTO targetCohort : expression.targetCohorts) { + referencedCohortNames.add(targetCohort.getName()); + } + for (CohortDTO outcomeCohort : expression.outcomeCohorts) { + referencedCohortNames.add(outcomeCohort.getName()); + } + return String.join("; ", referencedCohortNames); + } + + private Stream streamAnalysisReportsForAllCohortCombinations(IncidenceRateAnalysisExportExpression expression, Integer analysisId, String sourceKey) { + List targetCohorts = expression.targetCohorts; + List outcomeCohorts = expression.outcomeCohorts; + return targetCohorts.stream() + .map(CohortDTO::getId) + .flatMap(targetCohortId -> streamAnalysisReportsForOneCohortCombination(targetCohortId, outcomeCohorts, analysisId, sourceKey)); + } + + private Stream streamAnalysisReportsForOneCohortCombination(Integer targetCohortId, List outcomeCohorts, Integer analysisId, String sourceKey) { + return outcomeCohorts.stream() + .map(outcomeCohort -> { + AnalysisReport analysisReport = irAnalysisResource.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId()); + if (analysisReport.summary == null) { + analysisReport.summary = new AnalysisReport.Summary(); + analysisReport.summary.targetId = targetCohortId; + analysisReport.summary.outcomeId = outcomeCohort.getId(); + } + return analysisReport; + }); + } + + @Override + @Transactional + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); + Integer assetId = analysis.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.INCIDENCE.getCode(), generationId, sourceKey)); + applicationBrief.setTitle(prepareAppTitle(generationId, assetId, sourceId, sourceKey)); + applicationBrief.setDescription(analysis.getDescription()); + return applicationBrief; + } + + private String prepareCsvWithCohorts(IncidenceRateAnalysisExportExpression expression) { + final String[] HEADER = {"cohort_id", "cohort_name", "type"}; + List targetCohorts = expression.targetCohorts; + List outcomeCohorts = expression.outcomeCohorts; + try (StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, + CSVFormat.Builder.create() + .setQuoteMode(QuoteMode.NON_NUMERIC) + .setHeader(HEADER) + .build())) { + + for (CohortDTO targetCohort : targetCohorts) { + csvPrinter.printRecord(targetCohort.getId(), targetCohort.getName(), COHORT_TYPE_TARGET); + } + for (CohortDTO outcomeCohort : outcomeCohorts) { + csvPrinter.printRecord(outcomeCohort.getId(), outcomeCohort.getName(), COHORT_TYPE_OUTCOME); + } + return stringWriter.toString(); + } catch (IOException e) { + LOG.error("Failed to create a CSV file with Cohort details", e); + throw new InternalServerErrorException(); + } + } + + private String prepareAppTitle(Integer generationId, Integer assetId, Integer sourceId, String sourceKey) { + return String.format(APP_NAME_FORMAT, generationId, assetId, sourceId, sourceKey); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java b/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java new file mode 100644 index 000000000..b39dd368c --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java @@ -0,0 +1,57 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Consumer; + +@Component +public class ManifestUtils { + private final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger LOG = LoggerFactory.getLogger(ManifestUtils.class); + + public JsonNode parseManifest(Path path) { + try (InputStream in = Files.newInputStream(path)) { + return objectMapper.readTree(in); + } catch (IOException e) { + LOG.error("Failed to parse manifest", e); + throw new InternalServerErrorException(); + } + } + + public Consumer addDataToManifest(JsonNode manifest, Path root) { + return file -> { + JsonNode node = manifest.get("files"); + if (node.isObject()) { + ObjectNode filesNode = (ObjectNode) node; + Path relative = root.relativize(file); + ObjectNode item = filesNode.putObject(relative.toString().replace("\\", "/")); + item.put("checksum", checksum(file)); + } else { + LOG.error("Wrong manifest.json, there is no files section"); + throw new InternalServerErrorException(); + } + }; + } + + private String checksum(Path path) { + try (InputStream in = Files.newInputStream(path)) { + return DigestUtils.md5Hex(in); + } catch (IOException e) { + LOG.error("Failed to calculate checksum", e); + throw new InternalServerErrorException(); + } + } + + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java new file mode 100644 index 000000000..b09fbe134 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java @@ -0,0 +1,75 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.utils.ZipUtils; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class PackagingStrategies { + public static PackagingStrategy zip() { + return path -> { + try { + Path appArchive = Files.createTempFile("shinyapp_", ".zip"); + ZipUtils.zipDirectory(appArchive, path); + return appArchive; + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + + public static PackagingStrategy targz() { + return path -> { + try { + Path archive = Files.createTempFile("shinyapp_", ".tar.gz"); + try (OutputStream out = Files.newOutputStream(archive); OutputStream gzout = new GzipCompressorOutputStream(out); ArchiveOutputStream arch = new TarArchiveOutputStream(gzout)) { + packDirectoryFiles(path, arch); + } + return archive; + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + + private static void packDirectoryFiles(Path path, ArchiveOutputStream arch) throws IOException { + packDirectoryFiles(path, null, arch); + } + + private static void packDirectoryFiles(Path path, String parentDir, ArchiveOutputStream arch) throws IOException { + try (Stream files = Files.list(path)) { + files.forEach(p -> { + try { + File file = p.toFile(); + String filePath = Stream.of(parentDir, p.getFileName().toString()).filter(Objects::nonNull).collect(Collectors.joining("/")); + ArchiveEntry entry = arch.createArchiveEntry(file, filePath); + arch.putArchiveEntry(entry); + if (file.isFile()) { + try (InputStream in = Files.newInputStream(p)) { + IOUtils.copy(in, arch); + } + } + arch.closeArchiveEntry(); + if (file.isDirectory()) { + packDirectoryFiles(p, filePath, arch); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java new file mode 100644 index 000000000..a2ecfb0c6 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java @@ -0,0 +1,7 @@ +package org.ohdsi.webapi.shiny; + +import java.nio.file.Path; +import java.util.function.Function; + +public interface PackagingStrategy extends Function { +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java new file mode 100644 index 000000000..f20d3c822 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -0,0 +1,215 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.ohdsi.webapi.service.ShinyService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Scope; +import org.springframework.context.annotation.ScopedProxyMode; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Service +@ConditionalOnBean(ShinyService.class) +@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS) +public class PositConnectClient implements InitializingBean { + + private static final Logger log = LoggerFactory.getLogger(PositConnectClient.class); + private static final MediaType JSON_TYPE = MediaType.parse(org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE); + private static final MediaType OCTET_STREAM_TYPE = MediaType.parse(org.springframework.http.MediaType.APPLICATION_OCTET_STREAM_VALUE); + private static final String HEADER_AUTH = "Authorization"; + private static final String AUTH_PREFIX = "Key"; + + private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); + + @Autowired(required = false) + private PositConnectProperties properties; + + public UUID createContentItem(ApplicationBrief brief) { + ContentItem contentItem = new ContentItem(); + contentItem.accessType = "acl"; + contentItem.name = brief.getName(); + contentItem.title = brief.getTitle(); + contentItem.description = brief.getDescription(); + RequestBody body = RequestBody.create(toJson(contentItem), JSON_TYPE); + String url = connect("/v1/content"); + ContentItemResponse response = doPost(ContentItemResponse.class, url, body); + return response.guid; + } + + public List listContentItems() { + String url = connect("/v1/content"); + Request.Builder request = new Request.Builder() + .url(url); + return doCall(new TypeReference>() {}, request, url); + } + + public String uploadBundle(UUID contentId, TemporaryFile bundle) { + String url = connect(MessageFormat.format("/v1/content/{0}/bundles", contentId)); + BundleResponse response = doPost(BundleResponse.class, url, RequestBody.create(bundle.getFile().toFile(), OCTET_STREAM_TYPE)); + return response.id; + } + + public String deployBundle(UUID contentId, String bundleId) { + String url = connect(MessageFormat.format("/v1/content/{0}/deploy", contentId)); + BundleRequest request = new BundleRequest(); + request.bundleId = bundleId; + RequestBody requestBody = RequestBody.create(toJson(request), JSON_TYPE); + BundleDeploymentResponse response = doPost(BundleDeploymentResponse.class, url, requestBody); + return response.taskId; + } + + private T doPost(Class responseClass, String url, RequestBody requestBody) { + Request.Builder request = new Request.Builder() + .url(url) + .post(requestBody); + return doCall(responseClass, request, url); + } + + private T doCall(Class responseClass, Request.Builder request, String url) { + return doCall(new TypeReference() { + @Override + public Type getType() { + return responseClass; + } + }, request, url); + } + + private T doCall(TypeReference responseClass, Request.Builder request, String url) { + Call call = call(request, properties.getApiKey()); + try(Response response = call.execute()) { + if (!response.isSuccessful()) { + log.error("Request [{}] returned code: [{}], message: [{}]", url, response.code(), response.message()); + String message = MessageFormat.format("Request [{0}] returned code: [{1}], message: [{2}]", url, response.code(), response.message()); + if (response.code() == 409) { + throw new ConflictPositConnectException(message); + } + throw new PositConnectClientException(message); + } + if (response.body() == null) { + log.error("Failed to create a content, an empty result returned [{}]", url); + throw new PositConnectClientException("Failed to create a content, an empty result returned"); + } + return objectMapper.readValue(response.body().charStream(), responseClass); + } catch (IOException e) { + log.error("Failed to execute call [{}]", url, e); + throw new PositConnectClientException(MessageFormat.format("Failed to execute call [{0}]: {1}", url, e.getMessage())); + } + } + + private String toJson(T value) { + try { + return objectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + log.error("Failed to execute Connect request", e); + throw new PositConnectClientException("Failed to execute Connect request", e); + } + } + + private Call call(Request.Builder request, String token) { + OkHttpClient client = new OkHttpClient.Builder() + .retryOnConnectionFailure(false) + .connectTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) + .readTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) + .writeTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) + .build(); + return client.newCall(request.header(HEADER_AUTH, AUTH_PREFIX + " " + token).build()); + } + + @Override + public void afterPropertiesSet() throws Exception { + if (properties != null) { + if (StringUtils.isBlank(properties.getApiKey())) { + log.error("Set Posit Connect API Key to property \"shiny.connect.api.key\""); + throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api.key\""); + } + if (StringUtils.isBlank(properties.getUrl())) { + log.error("Set Posit Connect URL to property \"shiny.connect.url\""); + throw new BeanInitializationException("Set Posit Connect URL to property \"shiny.connect.url\""); + } + if (Objects.isNull(properties.getTimeoutSeconds())) { + log.error("Set Posit Connect HTTP Connect/Read/Write Timeout to property \"shiny.connect.okhttp.timeout.seconds\""); + throw new BeanInitializationException("Set Posit Connect HTTP Connect/Read/Write Timeout to property \"shiny.connect.okhttp.timeout.seconds\""); + } + } + } + + private String connect(String path) { + return StringUtils.removeEnd(properties.getUrl(), "/") + "/__api__/" + StringUtils.removeStart(path, "/"); + } + + public static class ContentItem { + public String name; + public String title; + public String description; + @JsonProperty("access_type") + public String accessType; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ContentItemResponse extends ContentItem { + public UUID guid; + @JsonProperty("owner_guid") + public UUID ownerGuid; + public String id; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class BundleResponse { + public String id; + @JsonProperty("content_guid") + public String contentGuid; + @JsonProperty("created_time") + public Instant createdTime; + @JsonProperty("cluster_name") + public String clusterName; + @JsonProperty("image_name") + public String imageName; + @JsonProperty("r_version") + public String rVersion; + @JsonProperty("r_environment_management") + public Boolean rEnvironmentManagement; + @JsonProperty("py_version") + public String pyVersion; + @JsonProperty("py_environment_management") + public Boolean pyEnvironmentManagement; + @JsonProperty("quarto_version") + public String quartoVersion; + public Boolean active; + public Integer size; + } + + static class BundleRequest { + @JsonProperty("bundle_id") + public String bundleId; + } + + static class BundleDeploymentResponse { + @JsonProperty("task_id") + public String taskId; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java new file mode 100644 index 000000000..37a98fc18 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +public class PositConnectClientException extends RuntimeException { + public PositConnectClientException(String message) { + super(message); + } + + public PositConnectClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java new file mode 100644 index 000000000..a603e35df --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java @@ -0,0 +1,37 @@ +package org.ohdsi.webapi.shiny; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "shiny.connect") +public class PositConnectProperties { + @Value("${shiny.connect.api.key}") + private String apiKey; + private String url; + @Value("${shiny.connect.okhttp.timeout.seconds}") + private Integer timeoutSeconds; + + public String getApiKey() { + return apiKey; + } + + public void setApiKey(String apiKey) { + this.apiKey = apiKey; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Integer getTimeoutSeconds() { + return timeoutSeconds; + } + + public void setTimeoutSeconds(Integer timeoutSeconds) { + this.timeoutSeconds = timeoutSeconds; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java new file mode 100644 index 000000000..f7f2b413f --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java @@ -0,0 +1,12 @@ +package org.ohdsi.webapi.shiny; + +import org.ohdsi.webapi.service.ShinyService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnBean(ShinyService.class) +@EnableConfigurationProperties(PositConnectProperties.class) +public class ShinyConfiguration { +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java new file mode 100644 index 000000000..0439ff205 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java @@ -0,0 +1,34 @@ +package org.ohdsi.webapi.shiny; + +public enum ShinyConstants { + VALUE_NOT_AVAILABLE("N/A"), + DATE_TIME_FORMAT("yyyy-MM-dd HH:mm:ss"), + PROPERTY_NAME_REPO_LINK("repo_link"), + PROPERTY_NAME_COHORT_LINK("cohort_link"), + PROPERTY_NAME_COHORT_NAME("cohort_name"), + PROPERTY_NAME_ATLAS_URL("atlas_url"), + PROPERTY_NAME_ATLAS_LINK("atlas_link"), + PROPERTY_NAME_DATASOURCE_KEY("datasource"), + PROPERTY_NAME_DATASOURCE_NAME("datasource_name"), + PROPERTY_NAME_ASSET_ID("asset_id"), + PROPERTY_NAME_ASSET_NAME("asset_name"), + PROPERTY_NAME_ANALYSIS_NAME("analysis_name"), + PROPERTY_NAME_AUTHOR("author"), + PROPERTY_NAME_AUTHOR_NOTES("author_notes"), + PROPERTY_NAME_GENERATED_DATE("generated_date"), + PROPERTY_NAME_RECORD_COUNT("record_count"), + PROPERTY_NAME_REFERENCED_COHORTS("referenced_cohorts"), + PROPERTY_NAME_VERSION_ID("version_id"), + PROPERTY_NAME_GENERATION_ID("generation_id"), + PROPERTY_NAME_PERSON_COUNT("person_count"); + + private final String value; + + ShinyConstants(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java new file mode 100644 index 000000000..f7ba193e1 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java @@ -0,0 +1,66 @@ +package org.ohdsi.webapi.shiny; + +import org.glassfish.jersey.media.multipart.ContentDisposition; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; +import org.ohdsi.webapi.shiro.annotations.SourceKey; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.nio.file.Files; + +@Component +@ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") +@Path("/shiny") +public class ShinyController { + + @Autowired + private ShinyService service; + + @GET + @Path("/download/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + public Response downloadShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) throws IOException { + TemporaryFile data = service.packageShinyApp(type, id, sourceKey, PackagingStrategies.zip()); + ContentDisposition contentDisposition = ContentDisposition.type("attachment") + .fileName(data.getFilename()) + .build(); + return Response + .ok(Files.newInputStream(data.getFile())) + .header(HttpHeaders.CONTENT_TYPE, "application/zip") + .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition) + .build(); + } + + @GET + @Path("/publish/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + @Transactional + public Response publishShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) { + service.publishApp(type, id, sourceKey); + return Response.ok().build(); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java new file mode 100644 index 000000000..6e1c85238 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; + +public interface ShinyPackagingService { + CommonAnalysisType getType(); + + TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging); + + ApplicationBrief getBrief(Integer generationId, String sourceKey); +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java new file mode 100644 index 000000000..92e207a82 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java @@ -0,0 +1,87 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; +import org.ohdsi.webapi.model.CommonEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.UUID; + +@Entity +@Table(name = "shiny_published") +public class ShinyPublishedEntity extends CommonEntity { + + @Id + @GenericGenerator( + name = "shiny_published_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "shiny_published_sequence"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "shiny_published_generator") + private Long id; + private CommonAnalysisType type; + @Column(name = "analysis_id") + private Long analysisId; + @Column(name = "source_key") + private String sourceKey; + @Column(name = "execution_id") + private Long executionId; + @Column(name = "content_id") + private UUID contentId; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public CommonAnalysisType getType() { + return type; + } + + public void setType(CommonAnalysisType type) { + this.type = type; + } + + public Long getAnalysisId() { + return analysisId; + } + + public void setAnalysisId(Long analysisId) { + this.analysisId = analysisId; + } + + public Long getExecutionId() { + return executionId; + } + + public void setExecutionId(Long executionId) { + this.executionId = executionId; + } + + public UUID getContentId() { + return contentId; + } + + public void setContentId(UUID contentId) { + this.contentId = contentId; + } + + public String getSourceKey() { + return sourceKey; + } + + public void setSourceKey(String sourceKey) { + this.sourceKey = sourceKey; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java new file mode 100644 index 000000000..e76130b10 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface ShinyPublishedRepository extends JpaRepository { + Optional findByAnalysisIdAndSourceKey(Long id, String sourceKey); +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java b/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java new file mode 100644 index 000000000..406f83ebe --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java @@ -0,0 +1,22 @@ +package org.ohdsi.webapi.shiny; + + +import java.nio.file.Path; + +public class TemporaryFile { + private final String filename; + private final Path file; + + public TemporaryFile(String filename, Path file) { + this.filename = filename; + this.file = file; + } + + public String getFilename() { + return filename; + } + + public Path getFile() { + return file; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java new file mode 100644 index 000000000..aa8e8b00e --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java @@ -0,0 +1,67 @@ +package org.ohdsi.webapi.shiny.summary; + +public class DataSourceSummary { + private String sourceName; + private String numberOfPersons; + private String female; + private String male; + private String ageAtFirstObservation; + private String cumulativeObservation; + private String continuousObservationCoverage; + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + + public void setNumberOfPersons(String numberOfPersons) { + this.numberOfPersons = numberOfPersons; + } + + public void setFemale(String female) { + this.female = female; + } + + public void setMale(String male) { + this.male = male; + } + + public void setAgeAtFirstObservation(String ageAtFirstObservation) { + this.ageAtFirstObservation = ageAtFirstObservation; + } + + public void setCumulativeObservation(String cumulativeObservation) { + this.cumulativeObservation = cumulativeObservation; + } + + public void setContinuousObservationCoverage(String continuousObservationCoverage) { + this.continuousObservationCoverage = continuousObservationCoverage; + } + + public String getSourceName() { + return sourceName; + } + + public String getNumberOfPersons() { + return numberOfPersons; + } + + public String getFemale() { + return female; + } + + public String getMale() { + return male; + } + + public String getAgeAtFirstObservation() { + return ageAtFirstObservation; + } + + public String getCumulativeObservation() { + return cumulativeObservation; + } + + public String getContinuousObservationCoverage() { + return continuousObservationCoverage; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java new file mode 100644 index 000000000..6d4490fd8 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java @@ -0,0 +1,127 @@ +package org.ohdsi.webapi.shiny.summary; + +import org.ohdsi.webapi.report.CDMAttribute; +import org.ohdsi.webapi.report.CDMDashboard; +import org.ohdsi.webapi.report.ConceptCountRecord; +import org.ohdsi.webapi.report.ConceptDistributionRecord; +import org.ohdsi.webapi.report.CumulativeObservationRecord; +import org.ohdsi.webapi.report.MonthObservationRecord; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.ShinyConstants; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import java.text.DecimalFormat; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@ConditionalOnBean(ShinyService.class) +public class DataSourceSummaryConverter { + + private double calculateVariance(List values, double mean) { + double variance = 0; + for (double value : values) { + variance += Math.pow(value - mean, 2); + } + return variance / values.size(); + } + + public DataSourceSummary convert(CDMDashboard cdmDashboard) { + DataSourceSummary dataSourceSummary = new DataSourceSummary(); + + if (cdmDashboard.getSummary() != null) { + for (CDMAttribute attribute : cdmDashboard.getSummary()) { + switch (attribute.getAttributeName()) { + case "Source name": + dataSourceSummary.setSourceName(attribute.getAttributeValue()); + break; + case "Number of persons": + double number = Double.parseDouble(attribute.getAttributeValue()); + String formattedNumber = new DecimalFormat("#,###.###M").format(number / 1_000_000); + dataSourceSummary.setNumberOfPersons(formattedNumber); + break; + } + } + } + + if (cdmDashboard.getGender() != null) { + long maleCount = 0; + long femaleCount = 0; + for (ConceptCountRecord record : cdmDashboard.getGender()) { + if (record.getConceptName().equalsIgnoreCase("MALE")) { + maleCount = record.getCountValue(); + } else if (record.getConceptName().equalsIgnoreCase("FEMALE")) { + femaleCount = record.getCountValue(); + } + } + long totalGenderCount = maleCount + femaleCount; + String malePercentage = String.format("%,.1f %%", 100 * (double) maleCount / totalGenderCount); + String femalePercentage = String.format("%,.1f %%", 100 * (double) femaleCount / totalGenderCount); + dataSourceSummary.setMale(String.format("%,d (%s)", maleCount, malePercentage)); + dataSourceSummary.setFemale(String.format("%,d (%s)", femaleCount, femalePercentage)); + } + + if (cdmDashboard.getAgeAtFirstObservation() != null) { + List ages = cdmDashboard.getAgeAtFirstObservation().stream() + .map(ConceptDistributionRecord::getIntervalIndex) + .collect(Collectors.toList()); + double sum = ages.stream() + .mapToInt(Integer::intValue) + .sum(); + double mean = sum / ages.size(); + double variance = calculateVariance(ages, mean); + + int minYear = cdmDashboard.getAgeAtFirstObservation().stream() + .min(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) + .orElse(new ConceptDistributionRecord()).getIntervalIndex(); + int maxYear = cdmDashboard.getAgeAtFirstObservation().stream() + .max(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) + .orElse(new ConceptDistributionRecord()).getIntervalIndex(); + dataSourceSummary.setAgeAtFirstObservation(String.format("[%d - %d] (M = %.1f; SD = %.1f)", + minYear, maxYear, mean, Math.sqrt(variance))); + } + + if (cdmDashboard.getCumulativeObservation() != null) { + List observationLengths = cdmDashboard.getCumulativeObservation().stream() + .map(CumulativeObservationRecord::getxLengthOfObservation) + .collect(Collectors.toList()); + double sum = observationLengths.stream().mapToInt(Integer::intValue).sum(); + double mean = sum / observationLengths.size(); + double variance = calculateVariance(observationLengths, mean); + + int minObs = cdmDashboard.getCumulativeObservation().stream() + .min(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) + .orElse(new CumulativeObservationRecord()).getxLengthOfObservation(); + int maxObs = cdmDashboard.getCumulativeObservation().stream() + .max(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) + .orElse(new CumulativeObservationRecord()).getxLengthOfObservation(); + dataSourceSummary.setCumulativeObservation(String.format("[%d - %d] (M = %.1f; SD = %.1f)", + minObs, maxObs, mean, Math.sqrt(variance))); + } + + if (cdmDashboard.getObservedByMonth() != null && !cdmDashboard.getObservedByMonth().isEmpty()) { + MonthObservationRecord startRecord = cdmDashboard.getObservedByMonth().get(0); + MonthObservationRecord endRecord = cdmDashboard.getObservedByMonth() + .get(cdmDashboard.getObservedByMonth().size() - 1); + dataSourceSummary.setContinuousObservationCoverage(String.format("Start: %02d/%02d, End: %02d/%02d", + startRecord.getMonthYear() % 100, startRecord.getMonthYear() / 100, + endRecord.getMonthYear() % 100, endRecord.getMonthYear() / 100)); + } + + return dataSourceSummary; + } + + public DataSourceSummary emptySummary(String dataSourceName) { + DataSourceSummary dataSourceSummary = new DataSourceSummary(); + dataSourceSummary.setSourceName(dataSourceName); + dataSourceSummary.setFemale(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setMale(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setCumulativeObservation(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setAgeAtFirstObservation(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setContinuousObservationCoverage(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setNumberOfPersons(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + return dataSourceSummary; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java b/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java index 320d64c25..4c62e2347 100644 --- a/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java +++ b/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java @@ -12,6 +12,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.StringJoiner; @@ -64,8 +66,16 @@ protected boolean preHandle(ServletRequest request, ServletResponse response) th // continue processing request // - httpResponse.setHeader("Access-Control-Expose-Headers", "Bearer,x-auth-error," + - Joiner.on(",").join(Constants.Headers.AUTH_PROVIDER, Constants.Headers.USER_LANGAUGE)); + + List exposedHeaders = Arrays.asList( + Constants.Headers.BEARER, + Constants.Headers.X_AUTH_ERROR, + Constants.Headers.AUTH_PROVIDER, + Constants.Headers.USER_LANGAUGE, + Constants.Headers.CONTENT_DISPOSITION + ); + httpResponse.setHeader(Constants.Headers.ACCESS_CONTROL_EXPOSE_HEADERS, Joiner.on(",").join(exposedHeaders)); + return true; } } diff --git a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java index c2f555e32..091211cd6 100644 --- a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java +++ b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java @@ -1,8 +1,16 @@ package org.ohdsi.webapi.util; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Function; public class TempFileUtils { @@ -10,10 +18,26 @@ public static File copyResourceToTempFile(String resource, String prefix, String File tempFile = File.createTempFile(prefix, suffix); try(InputStream in = TempFileUtils.class.getResourceAsStream(resource)) { - try(OutputStream out = new FileOutputStream(tempFile)) { + try(OutputStream out = Files.newOutputStream(tempFile.toPath())) { + if(in == null) { + throw new IOException("File not found: " + resource); + } IOUtils.copy(in, out); } } return tempFile; } + + public static F doInDirectory(Function action) { + try { + Path tempDir = Files.createTempDirectory("webapi-"); + try { + return action.apply(tempDir); + } finally { + FileUtils.deleteQuietly(tempDir.toFile()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to create temp directory, " + e.getMessage()); + } + } } diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties new file mode 100644 index 000000000..079030668 --- /dev/null +++ b/src/main/resources/application-shiny.properties @@ -0,0 +1,7 @@ +flyway.locations=${flyway.locations},classpath:shiny/migration + +shiny.atlas.url=${shiny.atlas.url} +shiny.repo.link=${shiny.repo.link} +shiny.connect.api.key=${shiny.connect.api.key} +shiny.connect.url=${shiny.connect.url} +shiny.connect.okhttp.timeout.seconds=${shiny.connect.okhttp.timeout.seconds} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index beaafd1e0..b1f89c915 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -282,3 +282,6 @@ versioning.maxAttempt=${versioning.maxAttempt} audit.trail.enabled=${audit.trail.enabled} audit.trail.log.file=${audit.trail.log.file} audit.trail.log.extraFile=${audit.trail.log.extraFile} + +#Shiny +shiny.enabled=${shiny.enabled} \ No newline at end of file diff --git a/src/main/resources/i18n/messages_en.json b/src/main/resources/i18n/messages_en.json index e4ea6edf6..4b01ee2a4 100644 --- a/src/main/resources/i18n/messages_en.json +++ b/src/main/resources/i18n/messages_en.json @@ -1229,6 +1229,15 @@ "tag": "Tag:" } } + }, + "shiny": { + "button": { + "title": "Shiny App", + "menu": { + "download": "Download", + "publish": "Publish" + } + } } }, "facets": { diff --git a/src/main/resources/shiny/cc-header-field-mapping.csv b/src/main/resources/shiny/cc-header-field-mapping.csv new file mode 100644 index 000000000..a91f645f9 --- /dev/null +++ b/src/main/resources/shiny/cc-header-field-mapping.csv @@ -0,0 +1,31 @@ +Analysis ID,analysisId +Analysis name,analysisName +Strata ID,strataId +Strata name,strataName +Cohort ID,cohortId +Cohort name,cohortName +Covariate ID,covariateId +Covariate name,covariateName +Covariate short name,covariateShortName +Count,count +Percent,pct +Value field, +Missing Means Zero,missingMeansZero +Avg,avg +StdDev,stdDev +Min,min +P10,p10 +P25,p25 +Median,median +P75,p75 +P90,p90 +Max,max +Target cohort ID,targetCohortId +Target cohort name,targetCohortName +Comparator cohort ID,comparatorCohortId +Comparator cohort name,comparatorCohortName +Target count,targetCount +Target percent,targetPct +Comparator count,comparatorCount +Comparator percent,comparatorPct +Std. Diff Of Mean,diff diff --git a/src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql b/src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql new file mode 100644 index 000000000..977bcbab3 --- /dev/null +++ b/src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql @@ -0,0 +1,35 @@ +CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; + +CREATE TABLE ${ohdsiSchema}.shiny_published( + id BIGINT PRIMARY KEY default nextval('${ohdsiSchema}.shiny_published_sequence'), + type VARCHAR NOT NULL, + analysis_id BIGINT NOT NULL, + execution_id BIGINT, + source_key VARCHAR, + content_id UUID, + created_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + modified_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), + modified_date TIMESTAMP WITH TIME ZONE +); + +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:download:*:*:*:get', + 'Download Shiny Application presenting analysis results'; + +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:publish:*:*:*:get', + 'Publish Shiny Application presenting analysis results to external resource'; + +INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission sp, + ${ohdsiSchema}.sec_role sr +WHERE sp."value" in + ( + 'shiny:download:*:*:*:get', + 'shiny:publish:*:*:*:get' + ) + AND sr.name IN ('Atlas users'); diff --git a/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java b/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java new file mode 100644 index 000000000..f2d4a0de7 --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java @@ -0,0 +1,46 @@ +package org.ohdsi.webapi.pathway; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisEntity; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.repository.PathwayAnalysisGenerationRepository; +import org.springframework.core.convert.support.GenericConversionService; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PathwayServiceTest { + + @Mock + private PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; + @Mock + private GenericConversionService genericConversionService; + @Mock + private PathwayAnalysisGenerationEntity pathwayAnalysisGenerationEntity; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private PathwayAnalysisDTO pathwayAnalysisDTO; + @Mock + private PathwayAnalysisEntity pathwayAnalysisEntity; + @InjectMocks + private PathwayServiceImpl sut; + + @Test + public void shouldGetByGenerationId() { + when(pathwayAnalysisGenerationRepository.findOne(anyLong(), any())).thenReturn(pathwayAnalysisGenerationEntity); + when(pathwayAnalysisGenerationEntity.getPathwayAnalysis()).thenReturn(pathwayAnalysisEntity); + when(genericConversionService.convert(eq(pathwayAnalysisEntity), eq(PathwayAnalysisDTO.class))).thenReturn(pathwayAnalysisDTO); + PathwayAnalysisDTO result = sut.getByGenerationId(1); + assertEquals(result, pathwayAnalysisDTO); + } + +} diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java new file mode 100644 index 000000000..f01e05bb3 --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -0,0 +1,93 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; +import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CohortPathwaysShinyPackagingServiceTest { + + private static final int GENERATION_ID = 1; + private static final String SOURCE_KEY = "SynPuf110k"; + + @Mock + private PathwayService pathwayService; + @Spy + private ManifestUtils manifestUtils; + @Spy + private FileWriter fileWriter; + + @InjectMocks + private CohortPathwaysShinyPackagingService sut; + + @Test + public void shouldGetBrief() { + when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(createPathwayAnalysisDTO()); + when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); + + ApplicationBrief brief = sut.getBrief(GENERATION_ID, SOURCE_KEY); + assertEquals(brief.getName(), "txp_" + GENERATION_ID + "_" + SOURCE_KEY); + assertEquals(brief.getTitle(), "Pathway_8_gv1x_SynPuf110k"); + assertEquals(brief.getDescription(), "desc"); + } + + @Test + public void shouldPopulateAppData() { + when(pathwayService.findDesignByGenerationId(eq((long) GENERATION_ID))).thenReturn("design json"); + when(pathwayService.getGenerationResults(eq((long) GENERATION_ID))).thenReturn(createPathwayGenerationResults()); + + PathwayAnalysisDTO pathwayAnalysisDTO = Mockito.mock(PathwayAnalysisDTO.class); + PathwayAnalysisGenerationEntity generationEntity = Mockito.mock(PathwayAnalysisGenerationEntity.class); + when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(pathwayAnalysisDTO); + when(pathwayService.getGeneration(eq((long) GENERATION_ID))).thenReturn(generationEntity); + + CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = Mockito.mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); + sut.populateAppData(GENERATION_ID, SOURCE_KEY, dataConsumers); + + verify(dataConsumers.getTextFiles(), times(1)).accept(eq("design.json"), anyString()); + verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("chartData.json"), any(PathwayPopulationResultsDTO.class)); + } + + private PathwayPopulationResultsDTO createPathwayGenerationResults() { + return new PathwayPopulationResultsDTO(Collections.emptyList(), Collections.emptyList()); + } + + @Test + public void shouldReturnIncidenceType() { + assertEquals(sut.getType(), CommonAnalysisType.COHORT_PATHWAY); + } + + + private PathwayAnalysisResult createPathwayAnalysisResult() { + return new PathwayAnalysisResult(); + } + + private PathwayAnalysisDTO createPathwayAnalysisDTO() { + PathwayAnalysisDTO pathwayAnalysisDTO = new PathwayAnalysisDTO(); + pathwayAnalysisDTO.setId(8); + pathwayAnalysisDTO.setName("pathwayAnalysis"); + pathwayAnalysisDTO.setDescription("desc"); + return pathwayAnalysisDTO; + } +} diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java new file mode 100644 index 000000000..8ca6193ce --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -0,0 +1,134 @@ +package org.ohdsi.webapi.shiny; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; +import org.ohdsi.webapi.ircalc.AnalysisReport; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisDetails; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; +import org.ohdsi.webapi.service.IRAnalysisResource; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class IncidenceRatesShinyPackagingServiceTest { + + @Mock + private IncidenceRateAnalysisRepository repository; + @Spy + private ManifestUtils manifestUtils; + @Spy + private FileWriter fileWriter; + @Mock + private IRAnalysisResource irAnalysisResource; + @Mock + private SourceRepository sourceRepository; + @Spy + private ObjectMapper objectMapper; + + @InjectMocks + private IncidenceRatesShinyPackagingService sut; + + private final Integer analysisId = 1; + private final String sourceKey = "sourceKey"; + + @Test + public void shouldGetBrief() { + IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); + + when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); + Source source = new Source(); + source.setSourceId(3); + when(sourceRepository.findBySourceKey("sourceKey")).thenReturn(source); + ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); + assertEquals(brief.getName(), "ir_" + analysisId + "_" + sourceKey); + assertEquals(brief.getTitle(), "Incidence_1_gv1x3_sourceKey"); + assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); + } + + @Test + public void shouldPopulateAppDataWithValidData() throws JsonProcessingException { + Integer generationId = 1; + String sourceKey = "source"; + + Source source = new Source(); + source.setSourceId(3); + when(sourceRepository.findBySourceKey("source")).thenReturn(source); + + IncidenceRateAnalysis analysis = Mockito.mock(IncidenceRateAnalysis.class, Answers.RETURNS_DEEP_STUBS.get()); + when(analysis.getDetails().getExpression()).thenReturn("{}"); + when(repository.findOne(generationId)).thenReturn(analysis); + + CohortDTO targetCohort = new CohortDTO(); + targetCohort.setId(101); + targetCohort.setName("Target Cohort"); + + CohortDTO outcomeCohort = new CohortDTO(); + outcomeCohort.setId(201); + outcomeCohort.setName("Outcome Cohort"); + + + IncidenceRateAnalysisExportExpression expression = new IncidenceRateAnalysisExportExpression(); + expression.outcomeCohorts.add(outcomeCohort); + expression.targetCohorts.add(targetCohort); + + when(objectMapper.readValue("{}", IncidenceRateAnalysisExportExpression.class)).thenReturn(expression); + AnalysisReport analysisReport = new AnalysisReport(); + analysisReport.summary = new AnalysisReport.Summary(); + when(irAnalysisResource.getAnalysisReport(1, "source", 101, 201)).thenReturn(analysisReport); + + CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); + + sut.populateAppData(generationId, sourceKey, dataConsumers); + + verify(dataConsumers.getAppProperties(), times(11)).accept(anyString(), anyString()); + verify(dataConsumers.getTextFiles(), times(1)).accept(anyString(), anyString()); + verify(dataConsumers.getJsonObjects(), times(1)).accept(anyString(), any()); + } + + @Test + public void shouldReturnIncidenceType() { + assertEquals(sut.getType(), CommonAnalysisType.INCIDENCE); + } + + private IncidenceRateAnalysis createIncidenceRateAnalysis() { + IncidenceRateAnalysis incidenceRateAnalysis = new IncidenceRateAnalysis(); + + IncidenceRateAnalysisDetails incidenceRateAnalysisDetails = new IncidenceRateAnalysisDetails(incidenceRateAnalysis); + incidenceRateAnalysisDetails.setExpression("{\"ConceptSets\":[],\"targetIds\":[11,7],\"outcomeIds\":[12,6],\"timeAtRisk\":{\"start\":{\"DateField\":\"StartDate\",\"Offset\":0},\"end\":{\"DateField\":\"EndDate\",\"Offset\":0}},\"studyWindow\":null,\"strata\":[{\"name\":\"Male\",\"description\":null,\"expression\":{\"Type\":\"ALL\",\"Count\":null,\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":null,\"Gender\":[{\"CONCEPT_ID\":8507,\"CONCEPT_NAME\":\"MALE\",\"STANDARD_CONCEPT\":null,\"STANDARD_CONCEPT_CAPTION\":\"Unknown\",\"INVALID_REASON\":null,\"INVALID_REASON_CAPTION\":\"Unknown\",\"CONCEPT_CODE\":\"M\",\"DOMAIN_ID\":\"Gender\",\"VOCABULARY_ID\":\"Gender\",\"CONCEPT_CLASS_ID\":null}],\"Race\":null,\"Ethnicity\":null,\"OccurrenceStartDate\":null,\"OccurrenceEndDate\":null}],\"Groups\":[]}},{\"name\":\"Female\",\"description\":null,\"expression\":{\"Type\":\"ALL\",\"Count\":null,\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":null,\"Gender\":[{\"CONCEPT_ID\":8532,\"CONCEPT_NAME\":\"FEMALE\",\"STANDARD_CONCEPT\":null,\"STANDARD_CONCEPT_CAPTION\":\"Unknown\",\"INVALID_REASON\":null,\"INVALID_REASON_CAPTION\":\"Unknown\",\"CONCEPT_CODE\":\"F\",\"DOMAIN_ID\":\"Gender\",\"VOCABULARY_ID\":\"Gender\",\"CONCEPT_CLASS_ID\":null}],\"Race\":null,\"Ethnicity\":null,\"OccurrenceStartDate\":null,\"OccurrenceEndDate\":null}],\"Groups\":[]}}],\"targetCohorts\":[{\"id\":11,\"name\":\"All population-IR\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"ConditionOccurrence\":{\"ConditionTypeExclude\":false}},{\"DrugExposure\":{\"DrugTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":0,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}},{\"id\":7,\"name\":\"Test Cohort 4\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expressionType\":\"SIMPLE_EXPRESSION\",\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"DrugExposure\":{\"CodesetId\":0,\"DrugTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":30,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"celecoxib\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":1118084,\"CONCEPT_NAME\":\"celecoxib\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"140587\",\"DOMAIN_ID\":\"Drug\",\"VOCABULARY_ID\":\"RxNorm\",\"CONCEPT_CLASS_ID\":\"Ingredient\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":true}]}},{\"id\":1,\"name\":\"Major gastrointestinal (GI) bleeding\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":4280942,\"CONCEPT_NAME\":\"Acute gastrojejunal ulcer with perforation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"66636001\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":28779,\"CONCEPT_NAME\":\"Bleeding esophageal varices\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"17709002\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":198798,\"CONCEPT_NAME\":\"Dieulafoy's vascular malformation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"109558001\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":4112183,\"CONCEPT_NAME\":\"Esophageal varices with bleeding, associated with another disorder\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"195475003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":194382,\"CONCEPT_NAME\":\"External hemorrhoids\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"23913003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":false,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":192671,\"CONCEPT_NAME\":\"Gastrointestinal hemorrhage\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"74474003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":196436,\"CONCEPT_NAME\":\"Internal hemorrhoids\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"90458007\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":false,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":4338225,\"CONCEPT_NAME\":\"Peptic ulcer with perforation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"88169003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":194158,\"CONCEPT_NAME\":\"Perinatal gastrointestinal hemorrhage\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"48729005\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"All\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[{\"name\":\"No prior GI\",\"expression\":{\"Type\":\"ALL\",\"CriteriaList\":[{\"Criteria\":{\"ConditionOccurrence\":{\"CodesetId\":1}},\"StartWindow\":{\"Start\":{\"Coeff\":-1},\"End\":{\"Days\":0,\"Coeff\":1},\"UseIndexEnd\":false,\"UseEventEnd\":false},\"RestrictVisit\":false,\"IgnoreObservationPeriod\":false,\"Occurrence\":{\"Type\":1,\"Count\":0,\"IsDistinct\":false}}],\"DemographicCriteriaList\":[],\"Groups\":[]}}],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{\"StartDate\":\"2010-04-01\",\"EndDate\":\"2010-12-01\"}}}],\"outcomeCohorts\":[{\"id\":12,\"name\":\"Diabetes-IR\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"ConditionOccurrence\":{\"CodesetId\":0,\"First\":true,\"ConditionTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":365,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"Diabetes-IR\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":201826,\"CONCEPT_NAME\":\"Type 2 diabetes mellitus\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"44054006\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[{\"name\":\"Age over 18\",\"expression\":{\"Type\":\"ALL\",\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":{\"Value\":18,\"Op\":\"gte\"}}],\"Groups\":[]}}],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}},{\"id\":6,\"name\":\"TEST COHORT 2\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expressionType\":\"SIMPLE_EXPRESSION\",\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"DrugEra\":{\"CodesetId\":0}}],\"ObservationWindow\":{\"PriorDays\":0,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"All\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"Simvastatin1\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":1539403,\"CONCEPT_NAME\":\"Simvastatin\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"36567\",\"DOMAIN_ID\":\"Drug\",\"VOCABULARY_ID\":\"RxNorm\",\"CONCEPT_CLASS_ID\":\"Ingredient\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"All\"},\"InclusionRules\":[],\"EndStrategy\":{\"DateOffset\":{\"DateField\":\"EndDate\",\"Offset\":0}},\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}}]}"); + + incidenceRateAnalysis.setId(analysisId); + incidenceRateAnalysis.setName("Analysis Name"); + incidenceRateAnalysis.setDescription("Analysis Description"); + incidenceRateAnalysis.setDetails(incidenceRateAnalysisDetails); + return incidenceRateAnalysis; + } + + private AnalysisReport createAnalysisReport(int targetId, int outcomeId) { + AnalysisReport analysisReport = new AnalysisReport(); + analysisReport.summary = new AnalysisReport.Summary(); + analysisReport.summary.targetId = targetId; + analysisReport.summary.outcomeId = outcomeId; + return analysisReport; + } +}