Calendar Improvements

.github/workflows/IOnIntegration_Production.yml

@@ -2,18 +2,47 @@ name: I-On Integration Production
 
 on:
   push:
+    branches: ['master']
     tags:
       - v*
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   push_production:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && contains(github.ref, 'tags')
+    permissions:
+      contents: read
+      packages: write
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Build image
-        id: production_build_image
-        run: ./gradlew buildDockerImage
+      - name: Login to Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+              type=semver,pattern={{version}}
+              type=semver,pattern={{major}}.{{minor}}
+              type=semver,pattern={{major}}
+              type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/IOnIntegration_Staging.yml

@@ -51,7 +51,7 @@ jobs:
           heroku_app_name: ${{secrets.HEROKU_APP_NAME}}
           heroku_email: ${{secrets.HEROKU_EMAIL}}
           usedocker: true
-          docker_heroku_process_type: worker
+          docker_heroku_process_type: web
       - run: |
           echo "::add-mask::$HD_SQL_HOST"
           echo "::add-mask::$HD_SQL_USER"
@@ -72,4 +72,4 @@ jobs:
         if: ${{ env.ACT }}
         run: |
           echo GIT_SERVER_ADDRESS=$GIT_SERVER_ADDRESS
-          echo GIT_BRANCH=$GIT_BRANCH
+          echo GIT_BRANCH=$GIT_BRANCH

Dockerfile

@@ -27,6 +27,5 @@ ARG EXTRACT_DEPENDENCY_PATH=/src/build/dependency
 COPY --from=build-env ${EXTRACT_DEPENDENCY_PATH}/BOOT-INF/classes /app
 COPY --from=build-env ${EXTRACT_DEPENDENCY_PATH}/BOOT-INF/lib /app/lib
 
-EXPOSE ${SERVER_PORT}
 
-ENTRYPOINT [ "java", "-cp", "app:app/lib/*", "org.ionproject.integration.IOnIntegrationApplicationKt", "-XX:+UseContainerSupport" ]
+ENTRYPOINT [ "java", "-cp", "app:app/lib/*", "org.ionproject.integration.IOnIntegrationApplicationKt", "-XX:+UseContainerSupport", "echo --server.port=$PORT" ]

README.md

@@ -22,7 +22,7 @@ Academic information such as class schedules or academic calendars is often scat
 
 ![I-On Integration Architecture](img/ion_integration_architecture.png)
 
-**I-On Integration** uses *batch processing* techniques to acquire and process all unstructured data and write it to the common **File Repository** (a Git repo whose sole purpose is to host this data).
+**I-On Integration** uses *batch processing* techniques to acquire and process all unstructured data and write it to the common **File Repository** (a Git repo whose sole purpose is to host this data). The data is available for anyone that wishes to use this data on their own projects. It is available at [GitHub i-on Integration Data repository](https://github.com/i-on-project/integration-data).
 
 The **Scheduler** component, as the name suggests, is responsible for periodically triggering job executions through Integration's Web API (not yet available).
 
@@ -62,10 +62,6 @@ By **default** the app containers will be accessible via the following ports:
 
 If you wish to change these values please refer to the [Customizing Containers](#customizing-containers) section.
 
-On Windows 10 you may see a **warning** about poor performance due to filesharing to a WSL container. This is due to the fact that we make use of [Docker Bind Mounts](https://docs.docker.com/storage/bind-mounts/) to share the necessary server files.
-
-![Warning](img/bindmountwarning.jpg)
-
 #### Using the containerized database
 While the Integration app will be able to connect to the database out of the box without any human intervention you might also be interested in connecting directly to run your own queries.
 
@@ -89,6 +85,8 @@ The Git server currently deployed for local development is based on the [GitBuck
 
 It's Web front-end can be accessed by pointing your browser to `localhost:8080` (default port, see the [Customizing Containers section](#customizing-containers) if you need to use a different port value).
 
+On startup a repository named **"integration-data"** is created and initialized to mirror the existing production repository.
+
 ##### Git Credentials
 The server is deployed with a single **account** that can be accessed by entering `root` for both username and password fields.
 

docker-compose.yml

@@ -15,16 +15,16 @@ services:
       - GIT_USER=${GIT_USER}
       - GIT_PASSWORD=${GIT_PASSWORD}
       - GIT_BRANCH=${GIT_BRANCH}
-      - SERVER_PORT=${SERVER_PORT}
+      - PORT=${SERVER_PORT}
     volumes:
       - db-data:/var/lib/postgresql/data
     ports:
       - "${SERVER_PORT}:${SERVER_PORT}"
     depends_on:
       ion-db:
         condition: service_started
-      git-server:
-        condition: service_healthy
+      git-setup:
+        condition: service_started
   ion-db:
     image: postgres:13.2-alpine
     ports:
@@ -38,11 +38,19 @@ services:
     ports:
       - "${GIT_PORT}:8080"
     volumes:
-      - "./git-server:/gitbucket"
+      - git-data:/gitbucket
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:${GIT_PORT}" ]
       interval: 30s
       timeout: 10s
       retries: 5
+  git-setup:
+    image: gitbucket/gitbucket:4.35.3
+    depends_on:
+      git-server:
+        condition: service_healthy
+    restart: "no"
+    entrypoint: [ "curl", "-X", "POST", "-u", "root:root", "git-server:${GIT_PORT}/api/v3/user/repos", "-d", "{\"name\":\"integration-data\", \"auto_init\": true}" ]
 volumes:
-  db-data:
+  db-data:
+  git-data:

docs/infrastructure/ArgumentException.md

@@ -0,0 +1,3 @@
+This **error** will occur in Integration's Web API associated with the status code [400: Bad Request](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400) and signals that either an expected argument is missing or incorrect.
+
+The response body will be in the `application/json+problem` mediatype and its `detail` field will include more context-specific information about the error cause.

docs/infrastructure/JobNotFoundException.md

@@ -0,0 +1,3 @@
+This **error** will occur in Integration's Web API associated with the status code [404: Not Found](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404) and indicates that the request Job ID does not exist.
+
+The response body will be in the `application/json+problem` mediatype and its `detail` field will include the given Job ID. The `instance` field will contain the requested path.

src/main/kotlin/org/ionproject/integration/application/JobEngine.kt

@@ -3,13 +3,14 @@ package org.ionproject.integration.application
 import org.ionproject.integration.application.config.AppProperties
 import org.ionproject.integration.application.config.LAUNCHER_NAME
 import org.ionproject.integration.application.job.CALENDAR_JOB_NAME
-import org.ionproject.integration.infrastructure.file.OutputFormat
-import org.ionproject.integration.domain.common.InstitutionModel
-import org.ionproject.integration.domain.common.ProgrammeModel
-import org.ionproject.integration.infrastructure.repository.IIntegrationJobRepository
+import org.ionproject.integration.application.job.EVALUATIONS_JOB_NAME
 import org.ionproject.integration.application.job.JobType
 import org.ionproject.integration.application.job.TIMETABLE_JOB_NAME
+import org.ionproject.integration.domain.common.InstitutionModel
+import org.ionproject.integration.domain.common.ProgrammeModel
 import org.ionproject.integration.infrastructure.exception.JobNotFoundException
+import org.ionproject.integration.infrastructure.file.OutputFormat
+import org.ionproject.integration.infrastructure.repository.IIntegrationJobRepository
 import org.slf4j.LoggerFactory
 import org.springframework.batch.core.Job
 import org.springframework.batch.core.JobParameters
@@ -49,6 +50,7 @@ class JobEngine(
     fun runJob(request: AbstractJobRequest): JobStatus {
         return when (request) {
             is TimetableJobRequest -> runTimetableJob(request)
+            is EvaluationsJobRequest -> runEvaluationsJob(request)
             is CalendarJobRequest -> runCalendarJob(request)
         }
     }
@@ -62,6 +64,11 @@ class JobEngine(
         return runJob(TIMETABLE_JOB_NAME, jobParams)
     }
 
+    private fun runEvaluationsJob(request: EvaluationsJobRequest): JobStatus {
+        val jobParams = getJobParameters(request, EVALUATIONS_JOB_NAME)
+        return runJob(EVALUATIONS_JOB_NAME, jobParams)
+    }
+
     private fun runCalendarJob(request: CalendarJobRequest): JobStatus {
         val jobParams = getJobParameters(request, CALENDAR_JOB_NAME)
         return runJob(CALENDAR_JOB_NAME, jobParams)
@@ -72,7 +79,11 @@ class JobEngine(
 
         val uri = when (request) {
             is TimetableJobRequest ->
-                request.programme.timetableUri.also {
+                request.programme.resources.timetableUri.also {
+                    parametersBuilder.addString(PROGRAMME_PARAMETER, request.programme.acronym)
+                }
+            is EvaluationsJobRequest ->
+                request.programme.resources.evaluationsUri.also {
                     parametersBuilder.addString(PROGRAMME_PARAMETER, request.programme.acronym)
                 }
             is CalendarJobRequest -> request.institution.academicCalendarUri
@@ -154,6 +165,30 @@ class JobEngine(
         }
     }
 
+    class EvaluationsJobRequest(
+        format: OutputFormat,
+        institution: InstitutionModel,
+        val programme: ProgrammeModel
+    ) : AbstractJobRequest(format, institution, JobType.EXAM_SCHEDULE) {
+        override fun equals(other: Any?): Boolean {
+            if (this === other) return true
+            if (javaClass != other?.javaClass) return false
+            if (!super.equals(other)) return false
+
+            other as EvaluationsJobRequest
+
+            if (programme != other.programme) return false
+
+            return true
+        }
+
+        override fun hashCode(): Int {
+            var result = super.hashCode()
+            result = 31 * result + programme.hashCode()
+            return result
+        }
+    }
+
     data class IntegrationJob(
         val type: JobType,
         val status: JobStatus,
@@ -162,7 +197,8 @@ class JobEngine(
 
     data class IntegrationJobParameters(
         val creationDate: LocalDateTime,
-        val startDate: LocalDateTime,
+        val startDate: LocalDateTime? = null,
+        val endDate: LocalDateTime? = null,
         val format: OutputFormat,
         val institution: InstitutionModel,
         val programme: ProgrammeModel? = null,

src/main/kotlin/org/ionproject/integration/application/dto/ParsedData.kt

@@ -1,8 +1,10 @@
 package org.ionproject.integration.application.dto
 
+import org.ionproject.integration.domain.calendar.AcademicCalendarDto
+import org.ionproject.integration.domain.common.Term
+import org.ionproject.integration.domain.evaluations.EvaluationsDto
 import org.ionproject.integration.domain.timetable.dto.TimetableDto
 import org.ionproject.integration.infrastructure.file.Filepath
-import org.ionproject.integration.domain.calendar.AcademicCalendarDto
 
 /**
  * ParsedData will be used to "transport" the final data along with the required metadata.
@@ -74,4 +76,54 @@ data class AcademicCalendarData(
 
         return staging + segments
     }
+
+    data class EvaluationsData(
+        val programme: ProgrammeMetadata,
+        val term: CalendarTerm,
+        private val dto: EvaluationsDto
+    ) : ParsedData(dto) {
+        companion object Factory {
+            private const val ACADEMIC_YEAR_LENGTH = 9
+            private const val ACADEMIC_YEARS_FOLDER_NAME = "academic_years"
+
+            fun from(evaluationsDto: EvaluationsDto, identifier: String): EvaluationsData =
+                EvaluationsData(
+                    ProgrammeMetadata(
+                        InstitutionMetadata(
+                            evaluationsDto.school.name,
+                            evaluationsDto.school.acr,
+                            identifier
+                        ),
+                        evaluationsDto.programme.name,
+                        evaluationsDto.programme.acr
+                    ),
+                    CalendarTerm(
+                        evaluationsDto.calendarTerm.take(4).toInt(),
+                        when (evaluationsDto.calendarTerm.takeLast(1).toInt()) {
+                            1 -> Term.FALL
+                            2 -> Term.SPRING
+                            else -> throw IllegalArgumentException("Invalid Term ${evaluationsDto.calendarTerm}")
+                        }
+                    ),
+                    evaluationsDto
+                )
+        }
+
+        private val PROGRAMMES = "programmes"
+
+        override val identifier: String
+            get() = "${javaClass.simpleName}:${programme.acronym}:$term"
+
+        override fun getDirectory(repositoryName: String, staging: Filepath): Filepath {
+            val segments = listOf(
+                repositoryName,
+                programme.institution.domain,
+                PROGRAMMES,
+                programme.acronym,
+                term.toString()
+            )
+
+            return staging + segments
+        }
+    }
 }