From 01a1ddd38834a4a6a2e1dea023c7249e829f631d Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 6 Nov 2024 14:53:13 -0800 Subject: [PATCH 01/12] fix(github-actions): added default read-only permissions for codeql fixes #723 --- .github/workflows/codeql.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d763ccb6..976f257b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,14 +1,3 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL Advanced" on: @@ -17,7 +6,10 @@ on: pull_request: branches: [ "main" ] schedule: - - cron: '34 9 * * 1' + - cron: '0 0 * * *' + +# Declare default permissions as read only. +permissions: read-all jobs: analyze: From 51a03ca89f105a3afa841180fbbc232021f2b689 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 6 Nov 2024 14:53:58 -0800 Subject: [PATCH 02/12] fix(github-actions): changed openssf scorecard to run daily at midnight --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 39d337aa..c48ae086 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -10,7 +10,7 @@ on: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - - cron: '37 23 * * 3' + - cron: '0 0 * * *' push: branches: [ "main" ] From a20e7fc1d7595d0d6e77e658b42b38cf803646cc Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 6 Nov 2024 15:21:10 -0800 Subject: [PATCH 03/12] feat(gradle): added spotless linting and formatting checks for java, yaml and gradle files --- skaha/build.gradle | 24 +++++++++++++++++++++++- skaha/opencadc.gradle | 28 ---------------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/skaha/build.gradle b/skaha/build.gradle index 619ec064..b7e695ed 100644 --- a/skaha/build.gradle +++ b/skaha/build.gradle @@ -1,7 +1,7 @@ plugins { id 'war' id 'maven-publish' - id 'checkstyle' + id("com.diffplug.spotless") version "6.25.0" } repositories { @@ -53,3 +53,25 @@ dependencies { testImplementation 'org.json:json:20231013' testImplementation 'org.mockito:mockito-core:5.12.0' } + +spotless { + java { + // Use the default importOrder configuration + importOrder() + // Remove unused imports + removeUnusedImports() + // Google Java Format, Android Open Source Project style which uses 4 spaces for indentation + palantirJavaFormat() + // Format annotations on a single line + formatAnnotations() + } + yaml { + target '**/*.yaml, **/*.yml, ../.github/workflows/*.yml' + } + format 'misc', { + target '**/*.gradle' + trimTrailingWhitespace() + indentWithSpaces(4) + endWithNewline() + } +} \ No newline at end of file diff --git a/skaha/opencadc.gradle b/skaha/opencadc.gradle index 73b037b6..6cc16522 100644 --- a/skaha/opencadc.gradle +++ b/skaha/opencadc.gradle @@ -1,21 +1,7 @@ configurations { - checkstyleDep - intTestImplementation.extendsFrom testImplementation intTestRuntime.extendsFrom testRuntime } -dependencies { - testImplementation 'com.puppycrawl.tools:checkstyle:8.2' - checkstyleDep 'org.opencadc:cadc-quality:[1.0,)' -} - -checkstyle { - ignoreFailures = false - config = resources.text.fromArchiveEntry(configurations.checkstyleDep, 'cadc_checkstyle.xml') - toolVersion = '8.2' - sourceSets = [] -} - sourceSets { test { resources.srcDirs += 'src/test/resources' @@ -30,20 +16,6 @@ sourceSets { } } -// Temporary work around for issue https://github.com/gradle/gradle/issues/881 - -// gradle not displaying fail build status when warnings reported --> - -tasks.withType(Checkstyle).each { checkstyleTask -> - checkstyleTask.doLast { - reports.all { report -> - def outputFile = report.destination - if (outputFile.exists() && outputFile.text.contains(" Date: Wed, 6 Nov 2024 15:21:52 -0800 Subject: [PATCH 04/12] feat(github-actions): added ci for linting code --- .github/workflows/ci.linting.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/ci.linting.yml diff --git a/.github/workflows/ci.linting.yml b/.github/workflows/ci.linting.yml new file mode 100644 index 00000000..193fa6b0 --- /dev/null +++ b/.github/workflows/ci.linting.yml @@ -0,0 +1,30 @@ +name: "CI: Linting" + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: '0 0 * * *' + +jobs: + spotless: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Setup Java + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '11' + cache: 'gradle' + + - name: Run Spotless + run: | + cd skaha + ./gradlew clean spotlessCheck + From 4eafb629761624ce4d9fc59133fdfb6759398941 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 6 Nov 2024 15:31:23 -0800 Subject: [PATCH 05/12] style(spotless): applied misc changes --- skaha/build.gradle | 2 +- skaha/opencadc.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/skaha/build.gradle b/skaha/build.gradle index b7e695ed..075ecef3 100644 --- a/skaha/build.gradle +++ b/skaha/build.gradle @@ -74,4 +74,4 @@ spotless { indentWithSpaces(4) endWithNewline() } -} \ No newline at end of file +} diff --git a/skaha/opencadc.gradle b/skaha/opencadc.gradle index 6cc16522..57772675 100644 --- a/skaha/opencadc.gradle +++ b/skaha/opencadc.gradle @@ -20,7 +20,7 @@ tasks.withType(Test) { // reset the report destinations so that intTests go to their own page //reports.html.destination = file("${reporting.baseDir}/${name}") reports.html.destination = file(reporting.baseDir.getAbsolutePath() + '/' + name) - + // Assign all Java system properties from // the command line to the tests systemProperties System.properties @@ -30,7 +30,7 @@ task intTest(type: Test) { // set the configuration context testClassesDirs = sourceSets.intTest.output.classesDirs classpath = sourceSets.intTest.runtimeClasspath - + // run the tests always outputs.upToDateWhen { false } } @@ -38,7 +38,7 @@ task intTest(type: Test) { test { testLogging { events "PASSED", "FAILED", "SKIPPED" - // "STARTED", + // "STARTED", } } From 4e207a511bc89e3c34f4b12f1e434800b0a3b4b5 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Wed, 6 Nov 2024 16:00:22 -0800 Subject: [PATCH 06/12] feat(github-actions): added testing and code coverage in ci also fixed build.gradle issues from pr commits --- .github/workflows/ci.testing.yml | 56 ++++++++++++++++++++++++++++++++ skaha/build.gradle | 15 +++++++-- skaha/opencadc.gradle | 1 - 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.testing.yml diff --git a/.github/workflows/ci.testing.yml b/.github/workflows/ci.testing.yml new file mode 100644 index 00000000..65ee3cdb --- /dev/null +++ b/.github/workflows/ci.testing.yml @@ -0,0 +1,56 @@ +name: "CI: Testing" + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Setup Java + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '11' + cache: 'gradle' + + - name: Run Tests + run: | + cd skaha + ./gradlew clean check + - + name: Upload coverage artifacts + uses: actions/upload-artifact@v4.4.3 + with: + name: skaha-unittests-coverage + path: build/reports/jacoco/test/jacocoTestReport.xml + if-no-files-found: error + retention-days: 1 + overwrite: true + + codecov: + runs-on: ubuntu-latest + needs: tests + steps: + - + name: Download coverage artifacts + uses: actions/download-artifact@v4.1.8 + with: + name: skaha-unittests-coverage + path: jacocoTestReport.xml + - + name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: jacocoTestReport.xml + flags: skaha-unittests-coverage + name: skaha-unittests-coverage + fail_ci_if_error: true \ No newline at end of file diff --git a/skaha/build.gradle b/skaha/build.gradle index 075ecef3..917ab422 100644 --- a/skaha/build.gradle +++ b/skaha/build.gradle @@ -1,7 +1,9 @@ plugins { id 'war' id 'maven-publish' - id("com.diffplug.spotless") version "6.25.0" + id 'com.diffplug.spotless' version '6.25.0' + id 'java' + id 'jacoco' } repositories { @@ -49,7 +51,7 @@ dependencies { runtimeOnly 'org.opencadc:cadc-registry:[1.3.5,)' - testImplementation 'junit:junit:[4.0,)' + testImplementation 'junit:junit:[4.13,)' testImplementation 'org.json:json:20231013' testImplementation 'org.mockito:mockito-core:5.12.0' } @@ -75,3 +77,12 @@ spotless { endWithNewline() } } + +jacocoTestReport { + reports { + xml.enabled true + html.enabled true + } +} + +check.dependsOn jacocoTestReport diff --git a/skaha/opencadc.gradle b/skaha/opencadc.gradle index 57772675..e70af818 100644 --- a/skaha/opencadc.gradle +++ b/skaha/opencadc.gradle @@ -30,7 +30,6 @@ task intTest(type: Test) { // set the configuration context testClassesDirs = sourceSets.intTest.output.classesDirs classpath = sourceSets.intTest.runtimeClasspath - // run the tests always outputs.upToDateWhen { false } } From 9e119ac5f264e6c72ee72077fdb6fbad078f8277 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 10:50:35 -0800 Subject: [PATCH 07/12] fix(github-actions): updated setup-java from v2 to v4.5.0 --- .github/workflows/ci.linting.yml | 2 +- .github/workflows/ci.testing.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.linting.yml b/.github/workflows/ci.linting.yml index 193fa6b0..4ec839bc 100644 --- a/.github/workflows/ci.linting.yml +++ b/.github/workflows/ci.linting.yml @@ -17,7 +17,7 @@ jobs: with: fetch-depth: 0 - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v4.5.0 with: distribution: 'temurin' java-version: '11' diff --git a/.github/workflows/ci.testing.yml b/.github/workflows/ci.testing.yml index 65ee3cdb..e089db63 100644 --- a/.github/workflows/ci.testing.yml +++ b/.github/workflows/ci.testing.yml @@ -15,7 +15,7 @@ jobs: with: fetch-depth: 0 - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v4.5.0 with: distribution: 'temurin' java-version: '11' From a1bad03b2877cc160f2302972613a2919b9910a5 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 10:51:23 -0800 Subject: [PATCH 08/12] style(linting): reformatted code according to spotless config --- .../skaha/DesktopAppLifecycleTest.java | 34 +-- .../opencadc/skaha/ExpiryTimeRenewalTest.java | 53 +++-- .../java/org/opencadc/skaha/ImagesTest.java | 6 +- .../opencadc/skaha/SessionLifecycleTest.java | 36 ++- .../java/org/opencadc/skaha/SessionUtil.java | 95 ++++---- .../main/java/org/opencadc/skaha/K8SUtil.java | 143 ++++++------ .../java/org/opencadc/skaha/SkahaAction.java | 66 +++--- .../org/opencadc/skaha/SkahaAvailability.java | 158 ++++++------- .../org/opencadc/skaha/context/GetAction.java | 7 +- .../skaha/context/ResourceContexts.java | 15 +- .../org/opencadc/skaha/image/GetAction.java | 8 +- .../java/org/opencadc/skaha/image/Image.java | 15 +- .../skaha/registry/ImageRegistryAuth.java | 1 - .../opencadc/skaha/session/DeleteAction.java | 171 +++++++------- .../org/opencadc/skaha/session/GetAction.java | 210 ++++++++++-------- .../opencadc/skaha/session/PostAction.java | 194 +++++++++------- .../opencadc/skaha/session/PublicSession.java | 9 +- .../opencadc/skaha/session/ResourceStats.java | 29 ++- .../org/opencadc/skaha/session/Session.java | 50 +++-- .../opencadc/skaha/session/SessionAction.java | 42 ++-- .../opencadc/skaha/session/SessionDAO.java | 73 +++--- .../skaha/session/SessionJobBuilder.java | 50 +++-- .../skaha/utils/CommandExecutioner.java | 30 ++- .../org/opencadc/skaha/utils/CommonUtils.java | 1 - .../org/opencadc/skaha/utils/PosixCache.java | 30 +-- .../org/opencadc/skaha/utils/RedisCache.java | 8 +- .../java/org/opencadc/skaha/K8SUtilTest.java | 9 +- .../opencadc/skaha/image/GetActionTest.java | 34 +-- .../skaha/registry/ImageRegistryAuthTest.java | 3 +- .../skaha/session/GetActionTests.java | 18 +- .../skaha/session/GetSessionsTests.java | 42 ++-- .../skaha/session/PostActionTest.java | 16 +- .../skaha/session/SessionDAOTest.java | 28 ++- .../skaha/session/SessionJobBuilderTest.java | 44 ++-- .../skaha/session/TestPosixMapperClient.java | 3 +- .../skaha/utils/CommandExecutionerTest.java | 10 +- .../opencadc/skaha/utils/RedisCacheTest.java | 34 ++- 37 files changed, 942 insertions(+), 833 deletions(-) diff --git a/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java b/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java index 03d82b5f..a21d9df7 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java @@ -99,7 +99,8 @@ public class DesktopAppLifecycleTest { public DesktopAppLifecycleTest() { try { RegistryClient regClient = new RegistryClient(); - final URL sessionServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL sessionServiceURL = regClient.getServiceURL( + SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); sessionURL = new URL(sessionServiceURL.toString() + "/session"); log.info("sessions URL: " + sessionURL); @@ -118,21 +119,27 @@ public void testCreateDeleteDesktopApp() throws Exception { SessionUtil.initializeCleanup(this.sessionURL); // create desktop session - final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), - SessionAction.SESSION_TYPE_DESKTOP); - - final Session desktopSession = SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); - SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); + final String desktopSessionID = SessionUtil.createSession( + this.sessionURL, + "inttest" + SessionAction.SESSION_TYPE_DESKTOP, + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP) + .getId(), + SessionAction.SESSION_TYPE_DESKTOP); + + final Session desktopSession = + SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); + SessionUtil.verifySession( + desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); final URL desktopAppURL = new URL(this.sessionURL.toString() + "/" + desktopSession.getId() + "/app"); log.info("desktop-app URL: " + desktopAppURL); // create a terminal desktop-app - String desktopAppID = SessionUtil.createDesktopAppSession(SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), - desktopAppURL); + String desktopAppID = SessionUtil.createDesktopAppSession( + SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), desktopAppURL); - Session appSession = SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopAppID, Session.STATUS_RUNNING); + Session appSession = + SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopAppID, Session.STATUS_RUNNING); Assert.assertNotNull("no desktop app", desktopAppID); Assert.assertEquals("Wrong app session ID", appSession.getAppId(), desktopAppID); @@ -150,9 +157,10 @@ public void testCreateDeleteDesktopApp() throws Exception { // create desktop-app specifying resources int cores = 1; int ram = 4; - desktopAppID = SessionUtil.createDesktopAppSession(SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), - desktopAppURL, cores, ram); - appSession = SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopAppID, Session.STATUS_RUNNING); + desktopAppID = SessionUtil.createDesktopAppSession( + SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), desktopAppURL, cores, ram); + appSession = + SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopAppID, Session.STATUS_RUNNING); Assert.assertEquals("wrong number of cores", cores, Integer.parseInt(appSession.getRequestedCPUCores())); Assert.assertEquals("wrong amount of ram", ram + "G", appSession.getRequestedRAM()); diff --git a/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java b/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java index e3bf05dd..09715562 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java @@ -105,7 +105,8 @@ public class ExpiryTimeRenewalTest { public ExpiryTimeRenewalTest() throws Exception { RegistryClient regClient = new RegistryClient(); - final URL sessionServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL sessionServiceURL = + regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); this.sessionURL = new URL(sessionServiceURL.toString() + "/session"); log.info("sessions URL: " + sessionURL); @@ -120,9 +121,12 @@ public void testRenewCARTA() throws Exception { SessionUtil.initializeCleanup(this.sessionURL); // create carta session - final String cartaSessionID = SessionUtil.createSession(this.sessionURL, "inttest-" + SessionAction.SESSION_TYPE_CARTA, - SessionUtil.getImageByName(ExpiryTimeRenewalTest.CARTA_IMAGE_SUFFIX).getId(), - SessionAction.SESSION_TYPE_CARTA); + final String cartaSessionID = SessionUtil.createSession( + this.sessionURL, + "inttest-" + SessionAction.SESSION_TYPE_CARTA, + SessionUtil.getImageByName(ExpiryTimeRenewalTest.CARTA_IMAGE_SUFFIX) + .getId(), + SessionAction.SESSION_TYPE_CARTA); Session cartaSession = SessionUtil.waitForSession(this.sessionURL, cartaSessionID, Session.STATUS_RUNNING); // Sleep to force time to pass before renewal @@ -145,7 +149,8 @@ public void testRenewCARTA() throws Exception { long changedTime = timeToLiveAfterRenewal - timeToLive; if (changedTime <= SLEEP_TIME_SECONDS) { // renew failed - Assert.fail("activeDeadlineSeconds and/or skaha.sessionexpiry for a CARTA session has been changed, please update the test."); + Assert.fail( + "activeDeadlineSeconds and/or skaha.sessionexpiry for a CARTA session has been changed, please update the test."); } SessionUtil.deleteSession(this.sessionURL, cartaSessionID); @@ -163,11 +168,12 @@ public void testRenewHeadless() throws Exception { // create headless session final String headlessSessionID = SessionUtil.createHeadlessSession( - SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), this.sessionURL); - Session headlessSession = SessionUtil.waitForSession(this.sessionURL, headlessSessionID, Session.STATUS_RUNNING); + SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), this.sessionURL); + Session headlessSession = + SessionUtil.waitForSession(this.sessionURL, headlessSessionID, Session.STATUS_RUNNING); final Instant headlessExpiryTime = Instant.parse(headlessSession.getExpiryTime()); - //renew session + // renew session renewSession(sessionURL, headlessSessionID); headlessSession = SessionUtil.waitForSession(this.sessionURL, headlessSessionID, Session.STATUS_RUNNING); @@ -191,32 +197,39 @@ public void testRenewDesktop() throws Exception { SessionUtil.initializeCleanup(this.sessionURL); // create desktop session - final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest-" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), - SessionAction.SESSION_TYPE_DESKTOP); + final String desktopSessionID = SessionUtil.createSession( + this.sessionURL, + "inttest-" + SessionAction.SESSION_TYPE_DESKTOP, + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP) + .getId(), + SessionAction.SESSION_TYPE_DESKTOP); SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); // create desktop app final URL desktopAppURL = new URL(sessionURL.toString() + "/" + desktopSessionID + "/app"); - final String desktopApplicationSessionID = - SessionUtil.createDesktopAppSession(SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), desktopAppURL); - Session desktopApplicationSession = SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopApplicationSessionID, - Session.STATUS_RUNNING); + final String desktopApplicationSessionID = SessionUtil.createDesktopAppSession( + SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), desktopAppURL); + Session desktopApplicationSession = SessionUtil.waitForDesktopApplicationSession( + desktopAppURL, desktopApplicationSessionID, Session.STATUS_RUNNING); Instant desktopAppTimeToLiveStartTime = Instant.parse(desktopApplicationSession.getStartTime()); Instant desktopAppTimeToLiveExpiryTime = Instant.parse(desktopApplicationSession.getExpiryTime()); - final long desktopAppTimeToLive = desktopAppTimeToLiveStartTime.until(desktopAppTimeToLiveExpiryTime, ChronoUnit.SECONDS); + final long desktopAppTimeToLive = + desktopAppTimeToLiveStartTime.until(desktopAppTimeToLiveExpiryTime, ChronoUnit.SECONDS); Assert.assertNotEquals("failed to calculate desktop app time-to-live", 0L, desktopAppTimeToLive); - //renew desktop session, the associated desktop-app should also be renewed + // renew desktop session, the associated desktop-app should also be renewed renewSession(this.sessionURL, desktopSessionID); - desktopApplicationSession = SessionUtil.waitForDesktopApplicationSession(desktopAppURL, desktopApplicationSessionID, Session.STATUS_RUNNING); + desktopApplicationSession = SessionUtil.waitForDesktopApplicationSession( + desktopAppURL, desktopApplicationSessionID, Session.STATUS_RUNNING); Instant appStartTimeAfterRenewal = Instant.parse(desktopApplicationSession.getStartTime()); Instant appExpiryTimeAfterRenewal = Instant.parse(desktopApplicationSession.getExpiryTime()); - final long desktopAppTimeToLiveAfterRenewal = appStartTimeAfterRenewal.until(appExpiryTimeAfterRenewal, ChronoUnit.SECONDS); + final long desktopAppTimeToLiveAfterRenewal = + appStartTimeAfterRenewal.until(appExpiryTimeAfterRenewal, ChronoUnit.SECONDS); - Assert.assertNotEquals("failed to calculate the renewed desktop app time-to-live", 0L, desktopAppTimeToLiveAfterRenewal); + Assert.assertNotEquals( + "failed to calculate the renewed desktop app time-to-live", 0L, desktopAppTimeToLiveAfterRenewal); // delete desktop session, no need to delete the desktop-app SessionUtil.deleteSession(this.sessionURL, desktopSessionID); diff --git a/skaha/src/intTest/java/org/opencadc/skaha/ImagesTest.java b/skaha/src/intTest/java/org/opencadc/skaha/ImagesTest.java index 95f6589e..e690ea17 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/ImagesTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/ImagesTest.java @@ -103,7 +103,8 @@ public class ImagesTest { public ImagesTest() { try { RegistryClient regClient = new RegistryClient(); - final URL imageServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL imageServiceURL = regClient.getServiceURL( + SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); imageURL = new URL(imageServiceURL.toExternalForm() + "/image"); log.info("sessions URL: " + imageURL); @@ -122,8 +123,7 @@ protected static List getImages(final URL imageURL) { Assert.assertEquals("response code", 200, get.getResponseCode()); Assert.assertEquals("content-type", "application/json", get.getContentType()); String json = out.toString(); - Type listType = new TypeToken>() { - }.getType(); + Type listType = new TypeToken>() {}.getType(); Gson gson = new Gson(); return gson.fromJson(json, listType); } diff --git a/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java b/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java index e1c22e40..d3720412 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java @@ -97,7 +97,8 @@ public class SessionLifecycleTest { public SessionLifecycleTest() throws Exception { RegistryClient regClient = new RegistryClient(); - final URL sessionServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL sessionServiceURL = + regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); sessionURL = new URL(sessionServiceURL.toString() + "/session"); log.info("sessions URL: " + sessionURL); @@ -113,22 +114,33 @@ public void testCreateDeleteSessions() throws Exception { SessionUtil.initializeCleanup(this.sessionURL); // create desktop session - final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), - SessionAction.SESSION_TYPE_DESKTOP); - - final Session desktopSession = SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); - SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); + final String desktopSessionID = SessionUtil.createSession( + this.sessionURL, + "inttest" + SessionAction.SESSION_TYPE_DESKTOP, + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP) + .getId(), + SessionAction.SESSION_TYPE_DESKTOP); + + final Session desktopSession = + SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); + SessionUtil.verifySession( + desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); // create carta session - final String cartaSessionID = SessionUtil.createSession(sessionURL, "inttest" + SessionAction.SESSION_TYPE_CARTA, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_CARTA).getId(), - SessionAction.SESSION_TYPE_CARTA); + final String cartaSessionID = SessionUtil.createSession( + sessionURL, + "inttest" + SessionAction.SESSION_TYPE_CARTA, + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_CARTA).getId(), + SessionAction.SESSION_TYPE_CARTA); Session cartaSession = SessionUtil.waitForSession(sessionURL, cartaSessionID, Session.STATUS_RUNNING); - SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_CARTA, "inttest" + SessionAction.SESSION_TYPE_CARTA); + SessionUtil.verifySession( + desktopSession, SessionAction.SESSION_TYPE_CARTA, "inttest" + SessionAction.SESSION_TYPE_CARTA); Assert.assertNotNull("CARTA session not running.", cartaSession); - Assert.assertEquals("CARTA session name is wrong", "inttest" + SessionAction.SESSION_TYPE_CARTA, cartaSession.getName()); + Assert.assertEquals( + "CARTA session name is wrong", + "inttest" + SessionAction.SESSION_TYPE_CARTA, + cartaSession.getName()); Assert.assertNotNull("CARTA session id is null", cartaSession.getId()); Assert.assertNotNull("CARTA connect URL is null", cartaSession.getConnectURL()); Assert.assertNotNull("CARTA up since is null", cartaSession.getStartTime()); diff --git a/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java b/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java index e1b249be..e2d2d26a 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java @@ -175,10 +175,12 @@ static Subject getCurrentUser(final URL sessionURL, final boolean allowAnonymous } final RegistryClient registryClient = new RegistryClient(); - URL newLoginURL = registryClient.getServiceURL(URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_10, AuthMethod.ANON); + URL newLoginURL = registryClient.getServiceURL( + URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_10, AuthMethod.ANON); final URL loginURL = newLoginURL == null - ? registryClient.getServiceURL(URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_01, AuthMethod.ANON) - : newLoginURL; + ? registryClient.getServiceURL( + URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_01, AuthMethod.ANON) + : newLoginURL; final NetrcFile netrcFile = new NetrcFile(); final PasswordAuthentication passwordAuthentication = netrcFile.getCredentials(loginURL.getHost(), true); final Map loginPayload = new HashMap<>(); @@ -206,7 +208,8 @@ static Subject getCurrentUser(final URL sessionURL, final boolean allowAnonymous private static AuthorizationToken getBearerToken(final URL sessionURL) throws Exception { final File bearerTokenFile = FileUtil.getFileFromResource("skaha-test.token", SessionUtil.class); final String bearerToken = new String(Files.readAllBytes(bearerTokenFile.toPath())); - return new AuthorizationToken("Bearer", bearerToken.replaceAll("\n", ""), List.of(NetUtil.getDomainName(sessionURL))); + return new AuthorizationToken( + "Bearer", bearerToken.replaceAll("\n", ""), List.of(NetUtil.getDomainName(sessionURL))); } private static X509CertificateChain getProxyCertificate() throws Exception { @@ -265,7 +268,8 @@ static String createDesktopAppSession(final String image, final URL desktopSessi return SessionUtil.createDesktopAppSession(image, desktopSessionURL, 1, 1); } - static String createDesktopAppSession(final String image, final URL desktopSessionURL, final int cores, final int ram) { + static String createDesktopAppSession( + final String image, final URL desktopSessionURL, final int cores, final int ram) { final Map params = new HashMap<>(); final String name = new RandomStringGenerator(16).getID(); @@ -289,9 +293,10 @@ static String createDesktopAppSession(final String image, final URL desktopSessi return outputStream.toString().trim(); } - static Session waitForDesktopApplicationSession(final URL desktopApplicationURL, final String desktopAppID, final String expectedState) - throws Exception { - Session requestedSession = SessionUtil.getDesktopApplicationSessionWithoutWait(desktopApplicationURL, desktopAppID, expectedState); + static Session waitForDesktopApplicationSession( + final URL desktopApplicationURL, final String desktopAppID, final String expectedState) throws Exception { + Session requestedSession = + SessionUtil.getDesktopApplicationSessionWithoutWait(desktopApplicationURL, desktopAppID, expectedState); long currentWaitTime = 0L; while (requestedSession == null) { LOGGER.info("Waiting for Desktop Application Session " + desktopAppID + " to reach " + expectedState); @@ -299,11 +304,12 @@ static Session waitForDesktopApplicationSession(final URL desktopApplicationURL, currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS) { - throw new TimeoutException("Timed out waiting for Desktop Application session " + desktopAppID + " and status " + expectedState + " after " - + currentWaitTime + "ms"); + throw new TimeoutException("Timed out waiting for Desktop Application session " + desktopAppID + + " and status " + expectedState + " after " + currentWaitTime + "ms"); } - requestedSession = SessionUtil.getDesktopApplicationSessionWithoutWait(desktopApplicationURL, desktopAppID, expectedState); + requestedSession = SessionUtil.getDesktopApplicationSessionWithoutWait( + desktopApplicationURL, desktopAppID, expectedState); } LOGGER.info("Desktop Application Session " + desktopAppID + " reached " + expectedState); @@ -311,9 +317,11 @@ static Session waitForDesktopApplicationSession(final URL desktopApplicationURL, return requestedSession; } - static void deleteDesktopApplicationSession(URL desktopApplicationSessionURL, String desktopApplicationSessionID) throws Exception { + static void deleteDesktopApplicationSession(URL desktopApplicationSessionURL, String desktopApplicationSessionID) + throws Exception { LOGGER.info("Deleting desktop application session " + desktopApplicationSessionID); - HttpDelete delete = new HttpDelete(new URL(desktopApplicationSessionURL.toString() + "/" + desktopApplicationSessionID), true); + HttpDelete delete = new HttpDelete( + new URL(desktopApplicationSessionURL.toString() + "/" + desktopApplicationSessionID), true); delete.run(); SessionUtil.waitForSessionToTerminate(desktopApplicationSessionURL, desktopApplicationSessionID); @@ -341,12 +349,13 @@ static List getSessions(final URL sessionURL, String... omitStatuses) t return active; } - private static Session getDesktopApplicationSessionWithoutWait(final URL desktopApplicationURL, final String desktopApplicationSessionID, - final String expectedState) { + private static Session getDesktopApplicationSessionWithoutWait( + final URL desktopApplicationURL, final String desktopApplicationSessionID, final String expectedState) { return SessionUtil.getAllDesktopApplicationSessions(desktopApplicationURL).stream() - .filter(session -> session.getAppId().equals(desktopApplicationSessionID) && session.getStatus().equals(expectedState)) - .findFirst() - .orElse(null); + .filter(session -> session.getAppId().equals(desktopApplicationSessionID) + && session.getStatus().equals(expectedState)) + .findFirst() + .orElse(null); } static List getAllDesktopApplicationSessions(final URL desktopAppURL) { @@ -356,8 +365,7 @@ static List getAllDesktopApplicationSessions(final URL desktopAppURL) { Assert.assertNull("get desktop app error", get.getThrowable()); Assert.assertEquals("content-type", "application/json", get.getContentType()); String json = out.toString(); - Type listType = new TypeToken>() { - }.getType(); + Type listType = new TypeToken>() {}.getType(); Gson gson = new Gson(); List sessions = gson.fromJson(json, listType); List active = new ArrayList<>(); @@ -370,11 +378,13 @@ static List getAllDesktopApplicationSessions(final URL desktopAppURL) { return active; } - private static Session getSessionWithoutWait(final URL sessionURL, final String sessionID, final String expectedState) throws Exception { + private static Session getSessionWithoutWait( + final URL sessionURL, final String sessionID, final String expectedState) throws Exception { return SessionUtil.getAllSessions(sessionURL).stream() - .filter(session -> session.getId().equals(sessionID) && session.getStatus().equals(expectedState)) - .findFirst() - .orElse(null); + .filter(session -> + session.getId().equals(sessionID) && session.getStatus().equals(expectedState)) + .findFirst() + .orElse(null); } static void waitForSessionToTerminate(final URL sessionURL, final String sessionID) throws Exception { @@ -386,8 +396,8 @@ static void waitForSessionToTerminate(final URL sessionURL, final String session currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_TERMINATE_MS) { - throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + Session.STATUS_TERMINATING + " after " - + currentWaitTime + "ms"); + throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + + Session.STATUS_TERMINATING + " after " + currentWaitTime + "ms"); } requestedSession = SessionUtil.getSessionWithoutWait(sessionURL, sessionID, Session.STATUS_TERMINATING); @@ -396,7 +406,8 @@ static void waitForSessionToTerminate(final URL sessionURL, final String session LOGGER.info("Session " + sessionID + " terminated."); } - static Session waitForSession(final URL sessionURL, final String sessionID, final String expectedState) throws Exception { + static Session waitForSession(final URL sessionURL, final String sessionID, final String expectedState) + throws Exception { Session requestedSession = SessionUtil.getSessionWithoutWait(sessionURL, sessionID, expectedState); long currentWaitTime = 0L; while (requestedSession == null) { @@ -405,8 +416,8 @@ static Session waitForSession(final URL sessionURL, final String sessionID, fina currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS) { - throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + expectedState + " after " - + currentWaitTime + "ms"); + throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + expectedState + + " after " + currentWaitTime + "ms"); } requestedSession = SessionUtil.getSessionWithoutWait(sessionURL, sessionID, expectedState); @@ -434,7 +445,7 @@ private static List getAllSessions(final URL sessionURL) throws Excepti Assert.assertEquals("content-type", "application/json", get.getContentType()); final String json; try (final Writer writer = new StringWriter(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(get.getInputStream()))) { + final BufferedReader reader = new BufferedReader(new InputStreamReader(get.getInputStream()))) { String line; while ((line = reader.readLine()) != null) { writer.write(line); @@ -443,19 +454,22 @@ private static List getAllSessions(final URL sessionURL) throws Excepti json = writer.toString(); } - final Type listType = new TypeToken>() { - }.getType(); + final Type listType = new TypeToken>() {}.getType(); final Gson gson = new Gson(); return gson.fromJson(json, listType); } static Image getImageByName(final String imagePath) throws Exception { final RegistryClient registryClient = new RegistryClient(); - final URL imageServiceURL = registryClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL imageServiceURL = registryClient.getServiceURL( + SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); final URL imageURL = new URL(imageServiceURL.toExternalForm() + "/image"); final List allImagesList = ImagesTest.getImages(imageURL); - return allImagesList.stream().filter(image -> image.getId().endsWith(imagePath)).findFirst().orElseThrow(); + return allImagesList.stream() + .filter(image -> image.getId().endsWith(imagePath)) + .findFirst() + .orElseThrow(); } protected static Image getImageOfType(final String type) throws Exception { @@ -464,17 +478,20 @@ protected static Image getImageOfType(final String type) throws Exception { protected static List getImagesOfType(final String type) throws Exception { final RegistryClient registryClient = new RegistryClient(); - final URL imageServiceURL = registryClient.getServiceURL(SessionUtil.SKAHA_SERVICE_ID, - Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL imageServiceURL = registryClient.getServiceURL( + SessionUtil.SKAHA_SERVICE_ID, Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); final URL imageURL = new URL(imageServiceURL.toExternalForm() + "/image"); final List allImagesList = ImagesTest.getImages(imageURL); - return allImagesList.stream().filter(image -> image.getTypes().contains(type)).collect(Collectors.toList()); + return allImagesList.stream() + .filter(image -> image.getTypes().contains(type)) + .collect(Collectors.toList()); } protected static Image getDesktopAppImageOfType(final String fuzzySearch) throws Exception { return SessionUtil.getImagesOfType(SessionAction.TYPE_DESKTOP_APP).stream() - .filter(image -> image.getId().contains(fuzzySearch)) - .findFirst().orElseThrow(); + .filter(image -> image.getId().contains(fuzzySearch)) + .findFirst() + .orElseThrow(); } } diff --git a/skaha/src/main/java/org/opencadc/skaha/K8SUtil.java b/skaha/src/main/java/org/opencadc/skaha/K8SUtil.java index f86eb714..85debe24 100644 --- a/skaha/src/main/java/org/opencadc/skaha/K8SUtil.java +++ b/skaha/src/main/java/org/opencadc/skaha/K8SUtil.java @@ -1,73 +1,72 @@ /* -************************************************************************ -******************* CANADIAN ASTRONOMY DATA CENTRE ******************* -************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -* -* (c) 2019. (c) 2019. -* Government of Canada Gouvernement du Canada -* National Research Council Conseil national de recherches -* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -* All rights reserved Tous droits réservés -* -* NRC disclaims any warranties, Le CNRC dénie toute garantie -* expressed, implied, or énoncée, implicite ou légale, -* statutory, of any kind with de quelque nature que ce -* respect to the software, soit, concernant le logiciel, -* including without limitation y compris sans restriction -* any warranty of merchantability toute garantie de valeur -* or fitness for a particular marchande ou de pertinence -* purpose. NRC shall not be pour un usage particulier. -* liable in any event for any Le CNRC ne pourra en aucun cas -* damages, whether direct or être tenu responsable de tout -* indirect, special or general, dommage, direct ou indirect, -* consequential or incidental, particulier ou général, -* arising from the use of the accessoire ou fortuit, résultant -* software. Neither the name de l'utilisation du logiciel. Ni -* of the National Research le nom du Conseil National de -* Council of Canada nor the Recherches du Canada ni les noms -* names of its contributors may de ses participants ne peuvent -* be used to endorse or promote être utilisés pour approuver ou -* products derived from this promouvoir les produits dérivés -* software without specific prior de ce logiciel sans autorisation -* written permission. préalable et particulière -* par écrit. -* -* This file is part of the Ce fichier fait partie du projet -* OpenCADC project. OpenCADC. -* -* OpenCADC is free software: OpenCADC est un logiciel libre ; -* you can redistribute it and/or vous pouvez le redistribuer ou le -* modify it under the terms of modifier suivant les termes de -* the GNU Affero General Public la “GNU Affero General Public -* License as published by the License” telle que publiée -* Free Software Foundation, par la Free Software Foundation -* either version 3 of the : soit la version 3 de cette -* License, or (at your option) licence, soit (à votre gré) -* any later version. toute version ultérieure. -* -* OpenCADC is distributed in the OpenCADC est distribué -* hope that it will be useful, dans l’espoir qu’il vous -* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -* without even the implied GARANTIE : sans même la garantie -* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence -* General Public License for Générale Publique GNU Affero -* more details. pour plus de détails. -* -* You should have received Vous devriez avoir reçu une -* a copy of the GNU Affero copie de la Licence Générale -* General Public License along Publique GNU Affero avec -* with OpenCADC. If not, see OpenCADC ; si ce n’est -* . pas le cas, consultez : -* . -* -************************************************************************ -*/ + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2019. (c) 2019. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ package org.opencadc.skaha; - public class K8SUtil { static final String ARC_USER_QUOTA_IN_GB_NAME = "skaha.defaultquotagb"; @@ -101,24 +100,24 @@ public static String getJobName(String sessionID, String type, String userID) { final String userJobID = userID.replaceAll("[^0-9a-zA-Z-]", "-"); return ("skaha-" + type + "-" + userJobID + "-" + sessionID).toLowerCase(); } - - //skaha-notebook-svc-rdcc0219 + + // skaha-notebook-svc-rdcc0219 public static String getServiceName(String sessionID, String type) { return "skaha-" + type + "-svc-" + sessionID; } - + public static String getIngressName(String sessionID, String type) { return "skaha-" + type + "-ingress-" + sessionID; } - + public static String getMiddlewareName(String sessionID, String type) { return "skaha-" + type + "-middleware-" + sessionID; } - + public static String getHomeDir() { return System.getenv("skaha.homedir"); } - + public static String getScratchDir() { return System.getenv("skaha.scratchdir"); } diff --git a/skaha/src/main/java/org/opencadc/skaha/SkahaAction.java b/skaha/src/main/java/org/opencadc/skaha/SkahaAction.java index d61598cc..17659e62 100644 --- a/skaha/src/main/java/org/opencadc/skaha/SkahaAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/SkahaAction.java @@ -112,7 +112,6 @@ import org.opencadc.skaha.utils.CommonUtils; import org.opencadc.skaha.utils.RedisCache; - public abstract class SkahaAction extends RestAction { public static final String SESSION_TYPE_CARTA = "carta"; @@ -126,8 +125,12 @@ public abstract class SkahaAction extends RestAction { private static final Logger log = Logger.getLogger(SkahaAction.class); private static final String POSIX_MAPPER_RESOURCE_ID_KEY = "skaha.posixmapper.resourceid"; public static List SESSION_TYPES = Arrays.asList( - SESSION_TYPE_CARTA, SESSION_TYPE_NOTEBOOK, SESSION_TYPE_DESKTOP, - SESSION_TYPE_CONTRIB, SESSION_TYPE_HEADLESS, TYPE_DESKTOP_APP); + SESSION_TYPE_CARTA, + SESSION_TYPE_NOTEBOOK, + SESSION_TYPE_DESKTOP, + SESSION_TYPE_CONTRIB, + SESSION_TYPE_HEADLESS, + TYPE_DESKTOP_APP); protected final PosixMapperConfiguration posixMapperConfiguration; private final String redisHost; private final String redisPort; @@ -152,7 +155,6 @@ public abstract class SkahaAction extends RestAction { protected boolean skahaCallbackFlow = false; protected String callbackSupplementalGroups = null; - public SkahaAction() { server = System.getenv("skaha.hostname"); homedir = System.getenv("skaha.homedir"); @@ -219,8 +221,8 @@ protected static TokenTool getTokenTool() throws Exception { private static EncodedKeyPair getPreAuthorizedTokenSecret() throws Exception { // Check the current secret - final JSONObject secretData = CommandExecutioner.getSecretData(K8SUtil.getPreAuthorizedTokenSecretName(), - K8SUtil.getWorkloadNamespace()); + final JSONObject secretData = CommandExecutioner.getSecretData( + K8SUtil.getPreAuthorizedTokenSecretName(), K8SUtil.getWorkloadNamespace()); final String publicKeyPropertyName = "public"; final String privateKeyPropertyName = "private"; @@ -231,10 +233,17 @@ private static EncodedKeyPair getPreAuthorizedTokenSecret() throws Exception { // create new secret final String[] createCmd = new String[] { - "kubectl", "--namespace", K8SUtil.getWorkloadNamespace(), "create", "secret", "generic", + "kubectl", + "--namespace", + K8SUtil.getWorkloadNamespace(), + "create", + "secret", + "generic", K8SUtil.getPreAuthorizedTokenSecretName(), - String.format("--from-literal=%s=%s", publicKeyPropertyName, CommonUtils.encodeBase64(encodedPublicKey)), - String.format("--from-literal=%s=%s", privateKeyPropertyName, CommonUtils.encodeBase64(encodedPrivateKey)) + String.format( + "--from-literal=%s=%s", publicKeyPropertyName, CommonUtils.encodeBase64(encodedPublicKey)), + String.format( + "--from-literal=%s=%s", privateKeyPropertyName, CommonUtils.encodeBase64(encodedPrivateKey)) }; final String createResult = CommandExecutioner.execute(createCmd); @@ -244,10 +253,9 @@ private static EncodedKeyPair getPreAuthorizedTokenSecret() throws Exception { } else { final Base64.Decoder base64Decoder = Base64.getDecoder(); // Decode twice since Kubernetes does a separate Base64 encoding. - return new EncodedKeyPair(base64Decoder.decode(base64Decoder.decode( - secretData.getString(publicKeyPropertyName))), - base64Decoder.decode(base64Decoder.decode( - secretData.getString(privateKeyPropertyName)))); + return new EncodedKeyPair( + base64Decoder.decode(base64Decoder.decode(secretData.getString(publicKeyPropertyName))), + base64Decoder.decode(base64Decoder.decode(secretData.getString(privateKeyPropertyName)))); } } @@ -276,7 +284,7 @@ protected ImageRegistryAuth getRegistryAuth(final String registryHost) { final String registryAuthValue = this.syncInput.getHeader(SkahaAction.X_REGISTRY_AUTH_HEADER); if (!StringUtil.hasText(registryAuthValue)) { throw new IllegalArgumentException("No authentication provided for unknown or private image. Use " - + SkahaAction.X_REGISTRY_AUTH_HEADER + " request header with base64Encode(username:secret)."); + + SkahaAction.X_REGISTRY_AUTH_HEADER + " request header with base64Encode(username:secret)."); } return ImageRegistryAuth.fromEncoded(registryAuthValue, registryHost); } @@ -293,15 +301,16 @@ private void initiateSkahaCallbackFlow(Subject currentSubject, URI skahaUsersUri final String xAuthTokenSkaha = syncInput.getHeader(X_AUTH_TOKEN_SKAHA); log.debug("x-auth-token-skaha header is " + xAuthTokenSkaha); try { - final String callbackSessionId = SkahaAction.getTokenTool().validateToken(xAuthTokenSkaha, skahaUsersUri, WriteGrant.class); + final String callbackSessionId = + SkahaAction.getTokenTool().validateToken(xAuthTokenSkaha, skahaUsersUri, WriteGrant.class); final Session session = SessionDAO.getSession(null, callbackSessionId, skahaTld); this.posixPrincipal = session.getPosixPrincipal(); currentSubject.getPrincipals().add(posixPrincipal); this.callbackSupplementalGroups = Arrays.stream(session.getSupplementalGroups()) - .map(i -> Integer.toString(i)) - .collect(Collectors.joining(",")); + .map(i -> Integer.toString(i)) + .collect(Collectors.joining(",")); } catch (Exception ex) { log.error("Unable to retrieve information for for callback flow", ex); if (ex instanceof IllegalStateException) { @@ -313,7 +322,7 @@ private void initiateSkahaCallbackFlow(Subject currentSubject, URI skahaUsersUri } private void initiateGeneralFlow(Subject currentSubject, URI skahaUsersUri) - throws IOException, InterruptedException, ResourceNotFoundException { + throws IOException, InterruptedException, ResourceNotFoundException { if (currentSubject == null || currentSubject.getPrincipals().isEmpty()) { throw new NotAuthenticatedException("Unauthorized"); } @@ -343,17 +352,16 @@ private void initiateGeneralFlow(Subject currentSubject, URI skahaUsersUri) IvoaGroupClient ivoaGroupClient = new IvoaGroupClient(); Set skahaUsersGroupUriSet = ivoaGroupClient.getMemberships(gmsSearchURI); - final GroupURI skahaHeadlessGroupURI = StringUtil.hasText(this.skahaHeadlessGroup) - ? new GroupURI(URI.create(this.skahaHeadlessGroup)) - : null; + final GroupURI skahaHeadlessGroupURI = + StringUtil.hasText(this.skahaHeadlessGroup) ? new GroupURI(URI.create(this.skahaHeadlessGroup)) : null; if (skahaHeadlessGroupURI != null && skahaUsersGroupUriSet.contains(skahaHeadlessGroupURI)) { headlessUser = true; } final GroupURI skahaPriorityHeadlessGroupURI = StringUtil.hasText(this.skahaPriorityHeadlessGroup) - ? new GroupURI(URI.create(this.skahaPriorityHeadlessGroup)) - : null; + ? new GroupURI(URI.create(this.skahaPriorityHeadlessGroup)) + : null; if (skahaPriorityHeadlessGroupURI != null && skahaUsersGroupUriSet.contains(skahaPriorityHeadlessGroupURI)) { priorityHeadlessUser = true; @@ -380,8 +388,8 @@ private void initiateGeneralFlow(Subject currentSubject, URI skahaUsersUri) } List groups = isNotEmpty(skahaUsersGroupUriSet) - ? skahaUsersGroupUriSet.stream().map(Group::new).collect(toList()) - : Collections.emptyList(); + ? skahaUsersGroupUriSet.stream().map(Group::new).collect(toList()) + : Collections.emptyList(); // adding all groups to the Subject currentSubject.getPublicCredentials().add(groups); @@ -407,9 +415,9 @@ public Image getPublicImage(String imageID) { return null; } return images.parallelStream() - .filter(image -> image.getId().equals(imageID)) - .findFirst() - .orElse(null); + .filter(image -> image.getId().equals(imageID)) + .findFirst() + .orElse(null); } /** @@ -429,7 +437,7 @@ protected PosixMapperConfiguration(final URI configuredPosixMapperID) throws IOE baseURL = configuredPosixMapperID.toURL(); } else { throw new IllegalStateException("Incorrect configuration for specified posix mapper service (" - + configuredPosixMapperID + ")."); + + configuredPosixMapperID + ")."); } } diff --git a/skaha/src/main/java/org/opencadc/skaha/SkahaAvailability.java b/skaha/src/main/java/org/opencadc/skaha/SkahaAvailability.java index 51774d08..fc7966a4 100644 --- a/skaha/src/main/java/org/opencadc/skaha/SkahaAvailability.java +++ b/skaha/src/main/java/org/opencadc/skaha/SkahaAvailability.java @@ -1,92 +1,83 @@ /* -************************************************************************ -******************* CANADIAN ASTRONOMY DATA CENTRE ******************* -************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -* -* (c) 2019. (c) 2019. -* Government of Canada Gouvernement du Canada -* National Research Council Conseil national de recherches -* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -* All rights reserved Tous droits réservés -* -* NRC disclaims any warranties, Le CNRC dénie toute garantie -* expressed, implied, or énoncée, implicite ou légale, -* statutory, of any kind with de quelque nature que ce -* respect to the software, soit, concernant le logiciel, -* including without limitation y compris sans restriction -* any warranty of merchantability toute garantie de valeur -* or fitness for a particular marchande ou de pertinence -* purpose. NRC shall not be pour un usage particulier. -* liable in any event for any Le CNRC ne pourra en aucun cas -* damages, whether direct or être tenu responsable de tout -* indirect, special or general, dommage, direct ou indirect, -* consequential or incidental, particulier ou général, -* arising from the use of the accessoire ou fortuit, résultant -* software. Neither the name de l'utilisation du logiciel. Ni -* of the National Research le nom du Conseil National de -* Council of Canada nor the Recherches du Canada ni les noms -* names of its contributors may de ses participants ne peuvent -* be used to endorse or promote être utilisés pour approuver ou -* products derived from this promouvoir les produits dérivés -* software without specific prior de ce logiciel sans autorisation -* written permission. préalable et particulière -* par écrit. -* -* This file is part of the Ce fichier fait partie du projet -* OpenCADC project. OpenCADC. -* -* OpenCADC is free software: OpenCADC est un logiciel libre ; -* you can redistribute it and/or vous pouvez le redistribuer ou le -* modify it under the terms of modifier suivant les termes de -* the GNU Affero General Public la “GNU Affero General Public -* License as published by the License” telle que publiée -* Free Software Foundation, par la Free Software Foundation -* either version 3 of the : soit la version 3 de cette -* License, or (at your option) licence, soit (à votre gré) -* any later version. toute version ultérieure. -* -* OpenCADC is distributed in the OpenCADC est distribué -* hope that it will be useful, dans l’espoir qu’il vous -* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -* without even the implied GARANTIE : sans même la garantie -* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence -* General Public License for Générale Publique GNU Affero -* more details. pour plus de détails. -* -* You should have received Vous devriez avoir reçu une -* a copy of the GNU Affero copie de la Licence Générale -* General Public License along Publique GNU Affero avec -* with OpenCADC. If not, see OpenCADC ; si ce n’est -* . pas le cas, consultez : -* . -* -************************************************************************ -*/ + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2019. (c) 2019. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ package org.opencadc.skaha; -import org.apache.log4j.Logger; -import org.opencadc.skaha.session.SessionAction; - import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; +import org.apache.log4j.Logger; import org.opencadc.skaha.utils.CommandExecutioner; -import java.io.IOException; - - -public class SkahaAvailability implements AvailabilityPlugin -{ +public class SkahaAvailability implements AvailabilityPlugin { private static final Logger LOG = Logger.getLogger(SkahaAvailability.class); - private static final Availability STATUS_UP = - new Availability(true, "skaha service is available."); + private static final Availability STATUS_UP = new Availability(true, "skaha service is available."); - public SkahaAvailability() - { - } + public SkahaAvailability() {} @Override public void setAppName(String appName) { @@ -99,13 +90,11 @@ public boolean heartbeat() { } @Override - public Availability getStatus() - { + public Availability getStatus() { // ensure we can run kubectl try { String k8sNamespace = K8SUtil.getWorkloadNamespace(); - String[] getPods = new String[] { - "kubectl", "get", "--namespace", k8sNamespace, "pods"}; + String[] getPods = new String[] {"kubectl", "get", "--namespace", k8sNamespace, "pods"}; CommandExecutioner.execute(getPods); return STATUS_UP; } catch (Exception e) { @@ -114,8 +103,7 @@ public Availability getStatus() } @Override - public void setState(String string) - { - //no-op + public void setState(String string) { + // no-op } } diff --git a/skaha/src/main/java/org/opencadc/skaha/context/GetAction.java b/skaha/src/main/java/org/opencadc/skaha/context/GetAction.java index 2315b61f..123f2f6b 100644 --- a/skaha/src/main/java/org/opencadc/skaha/context/GetAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/context/GetAction.java @@ -67,19 +67,18 @@ package org.opencadc.skaha.context; -import org.opencadc.skaha.SkahaAction; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import org.opencadc.skaha.SkahaAction; /** * Output the resource context information. - * + * * @author majorb */ public class GetAction extends SkahaAction { - + public GetAction() { super(); } diff --git a/skaha/src/main/java/org/opencadc/skaha/context/ResourceContexts.java b/skaha/src/main/java/org/opencadc/skaha/context/ResourceContexts.java index 4bab0057..59fe5c28 100644 --- a/skaha/src/main/java/org/opencadc/skaha/context/ResourceContexts.java +++ b/skaha/src/main/java/org/opencadc/skaha/context/ResourceContexts.java @@ -67,13 +67,11 @@ package org.opencadc.skaha.context; import ca.nrc.cadc.util.PropertiesReader; - +import com.google.gson.*; import java.io.File; import java.io.FileReader; import java.util.ArrayList; import java.util.List; - -import com.google.gson.*; import org.apache.log4j.Logger; import org.opencadc.skaha.SkahaAction; @@ -82,22 +80,22 @@ * */ public class ResourceContexts { - + private static final Logger log = Logger.getLogger(ResourceContexts.class); - + private final Integer defaultRequestCores; private final Integer defaultLimitCores; private final Integer defaultCores; private final Integer defaultCoresHeadless; private final List availableCores = new ArrayList<>(); - + // units in GB private final Integer defaultRequestRAM; private final Integer defaultLimitRAM; private final Integer defaultRAM; private final Integer defaultRAMHeadless; private final List availableRAM = new ArrayList<>(); - + private final List availableGPUs = new ArrayList<>(); public ResourceContexts() { @@ -177,9 +175,8 @@ public Integer getDefaultRAM(String sessionType) { public List getAvailableRAM() { return availableRAM; } - + public List getAvailableGPUs() { return availableGPUs; } - } diff --git a/skaha/src/main/java/org/opencadc/skaha/image/GetAction.java b/skaha/src/main/java/org/opencadc/skaha/image/GetAction.java index 9226ecbe..b37252bd 100644 --- a/skaha/src/main/java/org/opencadc/skaha/image/GetAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/image/GetAction.java @@ -66,14 +66,12 @@ */ package org.opencadc.skaha.image; - import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.log4j.Logger; -import org.opencadc.skaha.SkahaAction; - import java.util.List; import java.util.stream.Collectors; +import org.apache.log4j.Logger; +import org.opencadc.skaha.SkahaAction; /** * @author majorb @@ -97,7 +95,7 @@ public void doAction() throws Exception { List images = getImages(type); Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); String json = gson.toJson(images); - + syncOutput.setHeader("Content-Type", "application/json"); syncOutput.getOutputStream().write(json.getBytes()); } diff --git a/skaha/src/main/java/org/opencadc/skaha/image/Image.java b/skaha/src/main/java/org/opencadc/skaha/image/Image.java index 2cafaaf9..0b3e3c79 100644 --- a/skaha/src/main/java/org/opencadc/skaha/image/Image.java +++ b/skaha/src/main/java/org/opencadc/skaha/image/Image.java @@ -74,7 +74,7 @@ * */ public class Image { - + private String id; private Set types; private String digest; @@ -82,8 +82,7 @@ public class Image { /** * The following constructor is used by the ObjectMapper class for creating instances using reflection */ - public Image() { - } + public Image() {} public Image(String id, Set types, String digest) { if (id == null) { @@ -111,7 +110,7 @@ public Set getTypes() { public String getDigest() { return digest; } - + @Override public boolean equals(Object o) { if (o instanceof Image) { @@ -130,16 +129,16 @@ private boolean hasSameTypes(Set inputTypes) { boolean found = false; for (String type : this.types) { if (type.equals(iType)) { - found = true; - break; + found = true; + break; } } - + if (!found) { return false; } } - + return true; } } diff --git a/skaha/src/main/java/org/opencadc/skaha/registry/ImageRegistryAuth.java b/skaha/src/main/java/org/opencadc/skaha/registry/ImageRegistryAuth.java index 50bfa4fd..fba295c2 100644 --- a/skaha/src/main/java/org/opencadc/skaha/registry/ImageRegistryAuth.java +++ b/skaha/src/main/java/org/opencadc/skaha/registry/ImageRegistryAuth.java @@ -4,7 +4,6 @@ import ca.nrc.cadc.util.StringUtil; import java.nio.charset.StandardCharsets; - /** * Represents credentials to an image registry. Will be used from the request input as a base64 encoded value. */ diff --git a/skaha/src/main/java/org/opencadc/skaha/session/DeleteAction.java b/skaha/src/main/java/org/opencadc/skaha/session/DeleteAction.java index df7ad072..46a1e95d 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/DeleteAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/DeleteAction.java @@ -1,78 +1,76 @@ /* -************************************************************************ -******************* CANADIAN ASTRONOMY DATA CENTRE ******************* -************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -* -* (c) 2018. (c) 2018. -* Government of Canada Gouvernement du Canada -* National Research Council Conseil national de recherches -* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -* All rights reserved Tous droits réservés -* -* NRC disclaims any warranties, Le CNRC dénie toute garantie -* expressed, implied, or énoncée, implicite ou légale, -* statutory, of any kind with de quelque nature que ce -* respect to the software, soit, concernant le logiciel, -* including without limitation y compris sans restriction -* any warranty of merchantability toute garantie de valeur -* or fitness for a particular marchande ou de pertinence -* purpose. NRC shall not be pour un usage particulier. -* liable in any event for any Le CNRC ne pourra en aucun cas -* damages, whether direct or être tenu responsable de tout -* indirect, special or general, dommage, direct ou indirect, -* consequential or incidental, particulier ou général, -* arising from the use of the accessoire ou fortuit, résultant -* software. Neither the name de l'utilisation du logiciel. Ni -* of the National Research le nom du Conseil National de -* Council of Canada nor the Recherches du Canada ni les noms -* names of its contributors may de ses participants ne peuvent -* be used to endorse or promote être utilisés pour approuver ou -* products derived from this promouvoir les produits dérivés -* software without specific prior de ce logiciel sans autorisation -* written permission. préalable et particulière -* par écrit. -* -* This file is part of the Ce fichier fait partie du projet -* OpenCADC project. OpenCADC. -* -* OpenCADC is free software: OpenCADC est un logiciel libre ; -* you can redistribute it and/or vous pouvez le redistribuer ou le -* modify it under the terms of modifier suivant les termes de -* the GNU Affero General Public la “GNU Affero General Public -* License as published by the License” telle que publiée -* Free Software Foundation, par la Free Software Foundation -* either version 3 of the : soit la version 3 de cette -* License, or (at your option) licence, soit (à votre gré) -* any later version. toute version ultérieure. -* -* OpenCADC is distributed in the OpenCADC est distribué -* hope that it will be useful, dans l’espoir qu’il vous -* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -* without even the implied GARANTIE : sans même la garantie -* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence -* General Public License for Générale Publique GNU Affero -* more details. pour plus de détails. -* -* You should have received Vous devriez avoir reçu une -* a copy of the GNU Affero copie de la Licence Générale -* General Public License along Publique GNU Affero avec -* with OpenCADC. If not, see OpenCADC ; si ce n’est -* . pas le cas, consultez : -* . -* -************************************************************************ -*/ + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2018. (c) 2018. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ package org.opencadc.skaha.session; import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.util.StringUtil; - import java.io.IOException; import java.security.AccessControlException; - import org.apache.log4j.Logger; import org.opencadc.skaha.K8SUtil; import org.opencadc.skaha.utils.CommandExecutioner; @@ -82,7 +80,7 @@ * @author majorb */ public class DeleteAction extends SessionAction { - + private static final Logger log = Logger.getLogger(DeleteAction.class); public DeleteAction() { @@ -96,16 +94,21 @@ public void doAction() throws Exception { if (sessionID == null) { throw new UnsupportedOperationException("Cannot kill all sessions."); } else { - + String k8sNamespace = K8SUtil.getWorkloadNamespace(); String[] getSessionCMD = new String[] { - "kubectl", "get", "--namespace", k8sNamespace, "pod", + "kubectl", + "get", + "--namespace", + k8sNamespace, + "pod", "--selector=canfar-net-sessionID=" + sessionID, "--no-headers=true", - "-o", "custom-columns=" + - "TYPE:.metadata.labels.canfar-net-sessionType," + - "USERID:.metadata.labels.canfar-net-userid"}; - + "-o", + "custom-columns=" + "TYPE:.metadata.labels.canfar-net-sessionType," + + "USERID:.metadata.labels.canfar-net-userid" + }; + String session = CommandExecutioner.execute(getSessionCMD); if (StringUtil.hasText(session)) { final String[] lines = session.split("\n"); @@ -126,7 +129,7 @@ public void doAction() throws Exception { } } } - + // no session to delete throw new ResourceNotFoundException(sessionID); } @@ -136,12 +139,12 @@ public void doAction() throws Exception { deleteSession(posixPrincipal.username, TYPE_DESKTOP_APP, sessionID); } } - + public void deleteSession(String userID, String type, String sessionID) throws Exception { // kill the session specified by sessionID log.debug("Stopping " + type + " session: " + sessionID); String k8sNamespace = K8SUtil.getWorkloadNamespace(); - + if (TYPE_DESKTOP_APP.equalsIgnoreCase(type)) { // deleting a desktop-app if (StringUtil.hasText(appID)) { @@ -150,7 +153,8 @@ public void deleteSession(String userID, String type, String sessionID) throws E if (StringUtil.hasText(jobName)) { delete(k8sNamespace, "job", jobName); } else { - log.warn("no job deleted, desktop-app job name not found for userID " + userID + ", sessionID " + sessionID + ", appID " + appID); + log.warn("no job deleted, desktop-app job name not found for userID " + userID + ", sessionID " + + sessionID + ", appID " + appID); } } else { throw new IllegalArgumentException("Missing app ID"); @@ -159,24 +163,23 @@ public void deleteSession(String userID, String type, String sessionID) throws E // deleting a session String jobName = K8SUtil.getJobName(sessionID, type, userID); delete(k8sNamespace, "job", jobName); - + if (!SESSION_TYPE_HEADLESS.equals(type)) { String ingressName = K8SUtil.getIngressName(sessionID, type); delete(k8sNamespace, "ingressroute", ingressName); - + String serviceName = K8SUtil.getServiceName(sessionID, type); delete(k8sNamespace, "service", serviceName); - + String middlewareName = K8SUtil.getMiddlewareName(sessionID, type); delete(k8sNamespace, "middleware", middlewareName); } } } - + private void delete(String k8sNamespace, String type, String name) throws InterruptedException, IOException { try { - String[] cmd = new String[] { - "kubectl", "delete", "--namespace", k8sNamespace, type, name}; + String[] cmd = new String[] {"kubectl", "delete", "--namespace", k8sNamespace, type, name}; CommandExecutioner.execute(cmd); } catch (Exception ex) { // fail to delete the object, just log a warning and continue diff --git a/skaha/src/main/java/org/opencadc/skaha/session/GetAction.java b/skaha/src/main/java/org/opencadc/skaha/session/GetAction.java index ff41cb02..ffc83e53 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/GetAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/GetAction.java @@ -1,77 +1,75 @@ /* -************************************************************************ -******************* CANADIAN ASTRONOMY DATA CENTRE ******************* -************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -* -* (c) 2020. (c) 2020. -* Government of Canada Gouvernement du Canada -* National Research Council Conseil national de recherches -* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -* All rights reserved Tous droits réservés -* -* NRC disclaims any warranties, Le CNRC dénie toute garantie -* expressed, implied, or énoncée, implicite ou légale, -* statutory, of any kind with de quelque nature que ce -* respect to the software, soit, concernant le logiciel, -* including without limitation y compris sans restriction -* any warranty of merchantability toute garantie de valeur -* or fitness for a particular marchande ou de pertinence -* purpose. NRC shall not be pour un usage particulier. -* liable in any event for any Le CNRC ne pourra en aucun cas -* damages, whether direct or être tenu responsable de tout -* indirect, special or general, dommage, direct ou indirect, -* consequential or incidental, particulier ou général, -* arising from the use of the accessoire ou fortuit, résultant -* software. Neither the name de l'utilisation du logiciel. Ni -* of the National Research le nom du Conseil National de -* Council of Canada nor the Recherches du Canada ni les noms -* names of its contributors may de ses participants ne peuvent -* be used to endorse or promote être utilisés pour approuver ou -* products derived from this promouvoir les produits dérivés -* software without specific prior de ce logiciel sans autorisation -* written permission. préalable et particulière -* par écrit. -* -* This file is part of the Ce fichier fait partie du projet -* OpenCADC project. OpenCADC. -* -* OpenCADC is free software: OpenCADC est un logiciel libre ; -* you can redistribute it and/or vous pouvez le redistribuer ou le -* modify it under the terms of modifier suivant les termes de -* the GNU Affero General Public la “GNU Affero General Public -* License as published by the License” telle que publiée -* Free Software Foundation, par la Free Software Foundation -* either version 3 of the : soit la version 3 de cette -* License, or (at your option) licence, soit (à votre gré) -* any later version. toute version ultérieure. -* -* OpenCADC is distributed in the OpenCADC est distribué -* hope that it will be useful, dans l’espoir qu’il vous -* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -* without even the implied GARANTIE : sans même la garantie -* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence -* General Public License for Générale Publique GNU Affero -* more details. pour plus de détails. -* -* You should have received Vous devriez avoir reçu une -* a copy of the GNU Affero copie de la Licence Générale -* General Public License along Publique GNU Affero avec -* with OpenCADC. If not, see OpenCADC ; si ce n’est -* . pas le cas, consultez : -* . -* -************************************************************************ -*/ + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2020. (c) 2020. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ package org.opencadc.skaha.session; import ca.nrc.cadc.util.StringUtil; - import com.google.gson.Gson; import com.google.gson.GsonBuilder; - import java.io.OutputStream; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; @@ -83,7 +81,6 @@ import java.util.Locale; import java.util.Map; import java.util.Set; - import org.apache.log4j.Logger; import org.opencadc.skaha.K8SUtil; import org.opencadc.skaha.utils.CommandExecutioner; @@ -115,7 +112,10 @@ public void doAction() throws Exception { if (sessionID == null) { if (SESSION_VIEW_STATS.equals(view)) { ResourceStats resourceStats = getResourceStats(); - Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); + Gson gson = new GsonBuilder() + .disableHtmlEscaping() + .setPrettyPrinting() + .create(); String json = gson.toJson(resourceStats); syncOutput.setHeader("Content-Type", "application/json"); syncOutput.getOutputStream().write(json.getBytes()); @@ -158,7 +158,7 @@ public void doAction() throws Exception { String json = listSessions(SessionAction.TYPE_DESKTOP_APP, statusFilter, allUsers); syncOutput.setHeader("Content-Type", "application/json"); syncOutput.getOutputStream().write(json.getBytes()); - } else if (sessionID == null){ + } else if (sessionID == null) { throw new IllegalArgumentException("Missing session ID for desktop-app ID " + appID); } else { String json = getSingleDesktopApp(sessionID, appID); @@ -219,13 +219,24 @@ private ResourceStats getResourceStats() throws Exception { String withRAMStr = String.valueOf(formatter.format(Double.valueOf(normalizeToLong(withRAM))/(1024 * 1024 * 1024))) + "G"; String maxRAMStr = String.valueOf(formatter.format(Double.valueOf(maxRAM)/(1024 * 1024 * 1024))) + "G"; */ - String withRAMStr = formatter.format(Double.valueOf( - (double) (normalizeToLong(withRAM)) / (1024 * 1024 * 1024))) + "G"; + String withRAMStr = + formatter.format(Double.valueOf((double) (normalizeToLong(withRAM)) / (1024 * 1024 * 1024))) + "G"; String maxRAMStr = formatter.format(Double.valueOf((double) (maxRAM) / (1024 * 1024 * 1024))) + "G"; String requestedRAMStr = formatter.format(requestedRAM) + "G"; - String ramAvailableStr = formatter.format(Double.valueOf((double) (ramAvailable) / (1024 * 1024 * 1024))) + "G"; - return new ResourceStats(desktopCount, headlessCount, totalCount, requestedCPUCores, requestedRAMStr, - coresAvailable, ramAvailableStr, maxCores, withRAMStr, maxRAMStr, withCores); + String ramAvailableStr = + formatter.format(Double.valueOf((double) (ramAvailable) / (1024 * 1024 * 1024))) + "G"; + return new ResourceStats( + desktopCount, + headlessCount, + totalCount, + requestedCPUCores, + requestedRAMStr, + coresAvailable, + ramAvailableStr, + maxCores, + withRAMStr, + maxRAMStr, + withCores); } catch (Exception e) { log.error(e); throw new IllegalStateException("failed to gather resource statistics", e); @@ -257,11 +268,11 @@ protected long normalizeToLong(String ramString) { } private Map> getNodeResources(String k8sNamespace) throws Exception { - String getCPUCoresCmd = "kubectl -n " + k8sNamespace + " get pods --no-headers=true -o custom-columns=" + - "NODENAME:.spec.nodeName,PODNAME:.metadata.name," + - "REQCPUCORES:.spec.containers[].resources.requests.cpu," + - "REQRAM:.spec.containers[].resources.requests.memory " + - "--field-selector status.phase=Running --sort-by=.spec.nodeName"; + String getCPUCoresCmd = "kubectl -n " + k8sNamespace + " get pods --no-headers=true -o custom-columns=" + + "NODENAME:.spec.nodeName,PODNAME:.metadata.name," + + "REQCPUCORES:.spec.containers[].resources.requests.cpu," + + "REQRAM:.spec.containers[].resources.requests.memory " + + "--field-selector status.phase=Running --sort-by=.spec.nodeName"; String cpuCores = CommandExecutioner.execute(getCPUCoresCmd.split(" ")); Map> nodeToResourcesMap = new HashMap<>(); if (StringUtil.hasLength(cpuCores)) { @@ -302,21 +313,27 @@ private Map initResourcesMap() { rMap.put(REQ_RAM_KEY, Double.valueOf(0L)); return rMap; } - + private Map getResourcesMap(Map resourcesMap, String[] resources) { if (!NONE.equalsIgnoreCase(resources[2])) { - resourcesMap.put(REQ_CPU_CORES_KEY, resourcesMap.get(REQ_CPU_CORES_KEY) + Double.parseDouble(toCoreUnit(resources[2]))); - log.debug("Node: " + resources[0] + " " + REQ_CPU_CORES_KEY + ": "+ resourcesMap.get(REQ_CPU_CORES_KEY)); + resourcesMap.put( + REQ_CPU_CORES_KEY, + resourcesMap.get(REQ_CPU_CORES_KEY) + Double.parseDouble(toCoreUnit(resources[2]))); + log.debug("Node: " + resources[0] + " " + REQ_CPU_CORES_KEY + ": " + resourcesMap.get(REQ_CPU_CORES_KEY)); } if (!NONE.equalsIgnoreCase(resources[3])) { - resourcesMap.put(REQ_RAM_KEY, resourcesMap.get(REQ_RAM_KEY) + Double.valueOf((double) (normalizeToLong(toCommonUnit(resources[3]))) / (1024 * 1024 * 1024))); - log.debug("Node: " + resources[0] + " " + REQ_RAM_KEY + ": "+ resourcesMap.get(REQ_RAM_KEY)); + resourcesMap.put( + REQ_RAM_KEY, + resourcesMap.get(REQ_RAM_KEY) + + Double.valueOf( + (double) (normalizeToLong(toCommonUnit(resources[3]))) / (1024 * 1024 * 1024))); + log.debug("Node: " + resources[0] + " " + REQ_RAM_KEY + ": " + resourcesMap.get(REQ_RAM_KEY)); } - + return resourcesMap; } - + private Map getAvailableResources(String k8sNamespace) throws Exception { String getAvailableResourcesCmd = "kubectl -n " + k8sNamespace + " describe nodes "; String rawResources = CommandExecutioner.execute(getAvailableResourcesCmd.split(" ")); @@ -324,7 +341,8 @@ private Map getAvailableResources(String k8sNamespace) throws if (StringUtil.hasLength(rawResources)) { String[] lines = rawResources.split("\n"); if (lines.length > 0) { - List keywords = Arrays.asList("Name:", "Capacity:", "cpu:", "memory:", "nvidia.com/gpu:", "Allocatable:"); + List keywords = + Arrays.asList("Name:", "Capacity:", "cpu:", "memory:", "nvidia.com/gpu:", "Allocatable:"); String nodeName = ""; boolean hasName = false; boolean hasCapacity = false; @@ -383,9 +401,8 @@ private Map getAvailableResources(String k8sNamespace) throws nodeToResourcesMap.put(nodeName, resources); hasAllocatable = true; // TODO: add RAM unit to debug message - log.debug("node: " + nodeName + ", cores=" + resources[0] - + ", RAM=" + resources[1]); - + log.debug( + "node: " + nodeName + ", cores=" + resources[0] + ", RAM=" + resources[1]); } } } @@ -396,13 +413,13 @@ private Map getAvailableResources(String k8sNamespace) throws return nodeToResourcesMap; } - + public String getSingleDesktopApp(String sessionID, String appID) throws Exception { Session session = this.getDesktopApp(sessionID, appID); Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); return gson.toJson(session); } - + public String getSingleSession(String sessionID) throws Exception { Session session = this.getSession(getUsername(), sessionID); Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); @@ -421,7 +438,8 @@ public String listSessions(String typeFilter, String statusFilter, boolean allUs // if for all users, only show public information String json; - final Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); + final Gson gson = + new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create(); if (allUsers) { final List publicSessions = new ArrayList<>(filteredSessions.size()); for (final Session s : filteredSessions) { @@ -438,8 +456,8 @@ public String listSessions(String typeFilter, String statusFilter, boolean allUs public List filter(List sessions, String typeFilter, String statusFilter) { List ret = new ArrayList<>(); for (Session session : sessions) { - if ((typeFilter == null || session.getType().equalsIgnoreCase(typeFilter)) && - (statusFilter == null || session.getStatus().equalsIgnoreCase(statusFilter))) { + if ((typeFilter == null || session.getType().equalsIgnoreCase(typeFilter)) + && (statusFilter == null || session.getStatus().equalsIgnoreCase(statusFilter))) { ret.add(session); } } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java b/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java index ee33228d..38453e73 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java @@ -146,7 +146,6 @@ public class PostAction extends SessionAction { private static final Logger log = Logger.getLogger(PostAction.class); - public PostAction() { super(); } @@ -164,7 +163,8 @@ private static List getRenewJobNamesCmd(String forUserID, String session getRenewJobNamesCmd.add("--no-headers=true"); getRenewJobNamesCmd.add("-o"); - String customColumns = "custom-columns=NAME:.metadata.name,UID:.metadata.uid,STATUS:.status.active,START:.status.startTime"; + String customColumns = + "custom-columns=NAME:.metadata.name,UID:.metadata.uid,STATUS:.status.active,START:.status.startTime"; getRenewJobNamesCmd.add(customColumns); return getRenewJobNamesCmd; @@ -185,7 +185,8 @@ public void doAction() throws Exception { ResourceContexts rc = new ResourceContexts(); String image = syncInput.getParameter("image"); if (image == null) { - if (requestType.equals(REQUEST_TYPE_APP) || (requestType.equals(REQUEST_TYPE_SESSION) && sessionID == null)) { + if (requestType.equals(REQUEST_TYPE_APP) + || (requestType.equals(REQUEST_TYPE_SESSION) && sessionID == null)) { throw new IllegalArgumentException("Missing parameter 'image'"); } } @@ -195,7 +196,8 @@ public void doAction() throws Exception { final String requestedType = syncInput.getParameter("type"); // Absence of type is assumed to be headless - final String type = StringUtil.hasText(requestedType) ? requestedType : PostAction.SESSION_TYPE_HEADLESS; + final String type = + StringUtil.hasText(requestedType) ? requestedType : PostAction.SESSION_TYPE_HEADLESS; validatedType = validateImage(image, type); Integer cores = getCoresParam(); @@ -249,15 +251,15 @@ public void doAction() throws Exception { String action = syncInput.getParameter("action"); if (StringUtil.hasLength(action)) { if (action.equalsIgnoreCase("renew")) { - Map> jobNameToAttributesMap = getJobsToRenew(posixPrincipal.username, - sessionID); + Map> jobNameToAttributesMap = + getJobsToRenew(posixPrincipal.username, sessionID); if (!jobNameToAttributesMap.isEmpty()) { for (Map.Entry> entry : jobNameToAttributesMap.entrySet()) { renew(entry); } } else { throw new IllegalArgumentException( - "No active job for user " + posixPrincipal + " with session " + sessionID); + "No active job for user " + posixPrincipal + " with session " + sessionID); } } else { throw new UnsupportedOperationException("unrecognized action"); @@ -306,13 +308,16 @@ void allocateUser() throws Exception { log.debug("PostAction.makeUserBase()"); final Path userHomePath = getUserHomeDirectory(); final String[] allocateUserCommand = new String[] { - PostAction.CREATE_USER_BASE_COMMAND, getUsername(), Integer.toString(getUID()), - getDefaultQuota(), userHomePath.toAbsolutePath().toString() + PostAction.CREATE_USER_BASE_COMMAND, + getUsername(), + Integer.toString(getUID()), + getDefaultQuota(), + userHomePath.toAbsolutePath().toString() }; log.debug("Executing " + Arrays.toString(allocateUserCommand)); try (final ByteArrayOutputStream standardOutput = new ByteArrayOutputStream(); - final ByteArrayOutputStream standardError = new ByteArrayOutputStream()) { + final ByteArrayOutputStream standardError = new ByteArrayOutputStream()) { executeCommand(allocateUserCommand, standardOutput, standardError); final String errorOutput = standardError.toString(); @@ -320,8 +325,8 @@ PostAction.CREATE_USER_BASE_COMMAND, getUsername(), Integer.toString(getUID()), if (StringUtil.hasText(errorOutput)) { throw new IOException("Unable to create user home." - + "\nError message from server: " + errorOutput - + "\nOutput from command: " + commandOutput); + + "\nError message from server: " + errorOutput + + "\nOutput from command: " + commandOutput); } else { log.debug("PostAction.makeUserBase() success creating: " + commandOutput); } @@ -331,7 +336,7 @@ PostAction.CREATE_USER_BASE_COMMAND, getUsername(), Integer.toString(getUID()), } void executeCommand(final String[] command, final OutputStream standardOut, final OutputStream standardErr) - throws IOException, InterruptedException { + throws IOException, InterruptedException { execute(command, standardOut, standardErr); } @@ -393,7 +398,8 @@ private void renew(Map.Entry> entry) throws Exception { renewExpiryTimeCmd.add(entry.getKey()); renewExpiryTimeCmd.add("--type=json"); renewExpiryTimeCmd.add("-p"); - renewExpiryTimeCmd.add("[{\"op\":\"add\",\"path\":\"/spec/activeDeadlineSeconds\", \"value\":" + newExpiryTime + "}]"); + renewExpiryTimeCmd.add( + "[{\"op\":\"add\",\"path\":\"/spec/activeDeadlineSeconds\", \"value\":" + newExpiryTime + "}]"); execute(renewExpiryTimeCmd.toArray(new String[0])); } } @@ -528,23 +534,33 @@ public void checkExistingSessions(String userid, String type) throws Exception { int count = 0; for (Session session : sessions) { log.debug("checking session: " + session); - if (!SESSION_TYPE_HEADLESS.equalsIgnoreCase(session.getType()) && !TYPE_DESKTOP_APP.equals(session.getType())) { + if (!SESSION_TYPE_HEADLESS.equalsIgnoreCase(session.getType()) + && !TYPE_DESKTOP_APP.equals(session.getType())) { final String status = session.getStatus(); if (!(status.equalsIgnoreCase(Session.STATUS_TERMINATING) - || status.equalsIgnoreCase(Session.STATUS_SUCCEEDED))) { + || status.equalsIgnoreCase(Session.STATUS_SUCCEEDED))) { count++; } } } log.debug("active interactive sessions: " + count); if (count >= maxUserSessions) { - throw new IllegalArgumentException("User " + posixPrincipal.username + " has reached the maximum of " + maxUserSessions + " active sessions."); + throw new IllegalArgumentException("User " + posixPrincipal.username + " has reached the maximum of " + + maxUserSessions + " active sessions."); } } - public void createSession(String type, String image, String name, Integer cores, Integer ram, Integer gpus, - String cmd, String args, List envs) - throws Exception { + public void createSession( + String type, + String image, + String name, + Integer cores, + Integer ram, + Integer gpus, + String cmd, + String args, + List envs) + throws Exception { String supplementalGroups = getSupplementalGroupsList(); log.debug("supplementalGroups are " + supplementalGroups); @@ -587,7 +603,8 @@ public void createSession(String type, String image, String name, Integer cores, final String headlessImageBundle = getHeadlessImageBundle(image, cmd, args, envs); final String imageRegistrySecretName; - // In the absence of the existence of a public image, assume Private. The validateImage() step above will have caught a non-existent Image already. + // In the absence of the existence of a public image, assume Private. The validateImage() step above will have + // caught a non-existent Image already. if (getPublicImage(image) == null) { final ImageRegistryAuth userRegistryAuth = getRegistryAuth(getRegistryHost(image)); imageRegistrySecretName = createRegistryImageSecret(userRegistryAuth); @@ -598,29 +615,29 @@ public void createSession(String type, String image, String name, Integer cores, String jobName = K8SUtil.getJobName(sessionID, type, posixPrincipal.username); SessionJobBuilder sessionJobBuilder = SessionJobBuilder.fromPath(Paths.get(jobLaunchPath)) - .withGPUEnabled(this.gpuEnabled) - .withGPUCount(gpus) - .withImageSecret(imageRegistrySecretName) - .withParameter(PostAction.SKAHA_SESSIONID, this.sessionID) - .withParameter(PostAction.SKAHA_SESSIONNAME, name.toLowerCase()) - .withParameter(PostAction.SKAHA_SESSIONEXPIRY, K8SUtil.getSessionExpiry()) - .withParameter(PostAction.SKAHA_JOBNAME, jobName) - .withParameter(PostAction.SKAHA_HOSTNAME, K8SUtil.getHostName()) - .withParameter(PostAction.SKAHA_USERID, getUsername()) - .withParameter(PostAction.SKAHA_POSIXID, Integer.toString(this.posixPrincipal.getUidNumber())) - .withParameter(PostAction.SKAHA_SESSIONTYPE, type) - .withParameter(PostAction.SOFTWARE_IMAGEID, image) - .withParameter(PostAction.SOFTWARE_HOSTNAME, name.toLowerCase()) - .withParameter(PostAction.HEADLESS_IMAGE_BUNDLE, headlessImageBundle) - .withParameter(PostAction.HEADLESS_PRIORITY, headlessPriority) - .withParameter(PostAction.SOFTWARE_REQUESTS_CORES, cores.toString()) - .withParameter(PostAction.SOFTWARE_REQUESTS_RAM, ram.toString() + "Gi") - .withParameter(PostAction.SOFTWARE_LIMITS_CORES, cores.toString()) - .withParameter(PostAction.SOFTWARE_LIMITS_RAM, ram + "Gi") - .withParameter(PostAction.SKAHA_TLD, this.skahaTld); - - sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) - ? supplementalGroups : ""); + .withGPUEnabled(this.gpuEnabled) + .withGPUCount(gpus) + .withImageSecret(imageRegistrySecretName) + .withParameter(PostAction.SKAHA_SESSIONID, this.sessionID) + .withParameter(PostAction.SKAHA_SESSIONNAME, name.toLowerCase()) + .withParameter(PostAction.SKAHA_SESSIONEXPIRY, K8SUtil.getSessionExpiry()) + .withParameter(PostAction.SKAHA_JOBNAME, jobName) + .withParameter(PostAction.SKAHA_HOSTNAME, K8SUtil.getHostName()) + .withParameter(PostAction.SKAHA_USERID, getUsername()) + .withParameter(PostAction.SKAHA_POSIXID, Integer.toString(this.posixPrincipal.getUidNumber())) + .withParameter(PostAction.SKAHA_SESSIONTYPE, type) + .withParameter(PostAction.SOFTWARE_IMAGEID, image) + .withParameter(PostAction.SOFTWARE_HOSTNAME, name.toLowerCase()) + .withParameter(PostAction.HEADLESS_IMAGE_BUNDLE, headlessImageBundle) + .withParameter(PostAction.HEADLESS_PRIORITY, headlessPriority) + .withParameter(PostAction.SOFTWARE_REQUESTS_CORES, cores.toString()) + .withParameter(PostAction.SOFTWARE_REQUESTS_RAM, ram.toString() + "Gi") + .withParameter(PostAction.SOFTWARE_LIMITS_CORES, cores.toString()) + .withParameter(PostAction.SOFTWARE_LIMITS_RAM, ram + "Gi") + .withParameter(PostAction.SKAHA_TLD, this.skahaTld); + + sessionJobBuilder = sessionJobBuilder.withParameter( + PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) ? supplementalGroups : ""); if (type.equals(SessionAction.SESSION_TYPE_DESKTOP)) { sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.DESKTOP_SESSION_APP_TOKEN, generateToken()); @@ -629,7 +646,8 @@ public void createSession(String type, String image, String name, Integer cores, String jobLaunchString = sessionJobBuilder.build(); String jsonLaunchFile = super.stageFile(jobLaunchString); - // insert the user's proxy cert in the home dir. Do this first, so they're available to initContainer configurations. + // insert the user's proxy cert in the home dir. Do this first, so they're available to initContainer + // configurations. injectCredentials(); // inject the entries from the POSIX Mapper @@ -663,12 +681,14 @@ public void createSession(String type, String image, String name, Integer cores, } private void injectPOSIXDetails() throws Exception { - final PosixCache posixCache = new PosixCache(this.skahaPosixCacheURL, this.homedir, this.posixMapperConfiguration.getPosixMapperClient()); + final PosixCache posixCache = new PosixCache( + this.skahaPosixCacheURL, this.homedir, this.posixMapperConfiguration.getPosixMapperClient()); posixCache.writePOSIXEntries(); } private String generateToken() throws Exception { - return SkahaAction.getTokenTool().generateToken(URI.create(this.skahaUsersGroup), WriteGrant.class, this.sessionID); + return SkahaAction.getTokenTool() + .generateToken(URI.create(this.skahaUsersGroup), WriteGrant.class, this.sessionID); } /** @@ -687,9 +707,13 @@ private String createRegistryImageSecret(final ImageRegistryAuth registryAuth) t } private String getRegistryHost(final String imageID) { - final String registryHost = this.harborHosts.stream().filter(imageID::startsWith).findFirst().orElse(null); + final String registryHost = this.harborHosts.stream() + .filter(imageID::startsWith) + .findFirst() + .orElse(null); if (registryHost == null) { - throw new IllegalArgumentException("session image '" + imageID + "' must come from one of " + Arrays.toString(this.harborHosts.toArray())); + throw new IllegalArgumentException("session image '" + imageID + "' must come from one of " + + Arrays.toString(this.harborHosts.toArray())); } return registryHost; @@ -707,12 +731,19 @@ private String getRegistryHost(final String imageID) { * @param limitRAM Max amount of RAM in Gi. * @throws Exception For any unexpected errors. */ - public void attachDesktopApp(String image, Integer requestCores, Integer limitCores, Integer requestRAM, Integer limitRAM) throws Exception { + public void attachDesktopApp( + String image, Integer requestCores, Integer limitCores, Integer requestRAM, Integer limitRAM) + throws Exception { String k8sNamespace = K8SUtil.getWorkloadNamespace(); // Get the IP address based on the session String[] getIPCommand = new String[] { - "kubectl", "-n", k8sNamespace, "get", "pod", "--selector=canfar-net-sessionID=" + sessionID, + "kubectl", + "-n", + k8sNamespace, + "get", + "pod", + "--selector=canfar-net-sessionID=" + sessionID, "--no-headers=true", "-o", "custom-columns=IPADDR:.status.podIP,DT:.metadata.deletionTimestamp,TYPE:.metadata.labels.canfar-net-sessionType,NAME:.metadata.name" @@ -772,34 +803,32 @@ public void attachDesktopApp(String image, Integer requestCores, Integer limitCo final String launchSoftwarePath = System.getProperty("user.home") + "/config/launch-desktop-app.yaml"; SessionJobBuilder sessionJobBuilder = SessionJobBuilder.fromPath(Paths.get(launchSoftwarePath)) - .withGPUEnabled(this.gpuEnabled) - .withImageSecret(PostAction.DEFAULT_SOFTWARE_IMAGESECRET_VALUE) - .withParameter(PostAction.SKAHA_SESSIONID, this.sessionID) - .withParameter(PostAction.SKAHA_SESSIONEXPIRY, K8SUtil.getSessionExpiry()) - .withParameter(PostAction.SKAHA_SESSIONTYPE, SessionAction.TYPE_DESKTOP_APP) - .withParameter(PostAction.SKAHA_HOSTNAME, K8SUtil.getHostName()) - .withParameter(PostAction.SKAHA_USERID, getUsername()) - .withParameter(PostAction.SKAHA_POSIXID, Integer.toString(this.posixPrincipal.getUidNumber())) - .withParameter(PostAction.SOFTWARE_IMAGEID, image) - .withParameter(PostAction.SOFTWARE_APPID, this.appID) - .withParameter(PostAction.SOFTWARE_JOBNAME, jobName) - .withParameter(PostAction.SOFTWARE_HOSTNAME, name.toLowerCase()) - .withParameter(PostAction.SOFTWARE_REQUESTS_CORES, requestCores.toString()) - .withParameter(PostAction.SOFTWARE_LIMITS_CORES, limitCores.toString()) - .withParameter(PostAction.SOFTWARE_REQUESTS_RAM, requestRAM + "Gi") - .withParameter(PostAction.SOFTWARE_LIMITS_RAM, limitRAM + "Gi") - .withParameter(PostAction.SOFTWARE_TARGETIP, targetIP + ":1") - .withParameter(PostAction.SOFTWARE_CONTAINERNAME, containerName) - .withParameter(PostAction.SOFTWARE_CONTAINERPARAM, param) - .withParameter(PostAction.SKAHA_TLD, this.skahaTld); + .withGPUEnabled(this.gpuEnabled) + .withImageSecret(PostAction.DEFAULT_SOFTWARE_IMAGESECRET_VALUE) + .withParameter(PostAction.SKAHA_SESSIONID, this.sessionID) + .withParameter(PostAction.SKAHA_SESSIONEXPIRY, K8SUtil.getSessionExpiry()) + .withParameter(PostAction.SKAHA_SESSIONTYPE, SessionAction.TYPE_DESKTOP_APP) + .withParameter(PostAction.SKAHA_HOSTNAME, K8SUtil.getHostName()) + .withParameter(PostAction.SKAHA_USERID, getUsername()) + .withParameter(PostAction.SKAHA_POSIXID, Integer.toString(this.posixPrincipal.getUidNumber())) + .withParameter(PostAction.SOFTWARE_IMAGEID, image) + .withParameter(PostAction.SOFTWARE_APPID, this.appID) + .withParameter(PostAction.SOFTWARE_JOBNAME, jobName) + .withParameter(PostAction.SOFTWARE_HOSTNAME, name.toLowerCase()) + .withParameter(PostAction.SOFTWARE_REQUESTS_CORES, requestCores.toString()) + .withParameter(PostAction.SOFTWARE_LIMITS_CORES, limitCores.toString()) + .withParameter(PostAction.SOFTWARE_REQUESTS_RAM, requestRAM + "Gi") + .withParameter(PostAction.SOFTWARE_LIMITS_RAM, limitRAM + "Gi") + .withParameter(PostAction.SOFTWARE_TARGETIP, targetIP + ":1") + .withParameter(PostAction.SOFTWARE_CONTAINERNAME, containerName) + .withParameter(PostAction.SOFTWARE_CONTAINERPARAM, param) + .withParameter(PostAction.SKAHA_TLD, this.skahaTld); final String supplementalGroups = getSupplementalGroupsList(); - sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) - ? supplementalGroups : ""); + sessionJobBuilder = sessionJobBuilder.withParameter( + PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) ? supplementalGroups : ""); String launchFile = super.stageFile(sessionJobBuilder.build()); - String[] launchCmd = new String[] { - "kubectl", "create", "--namespace", k8sNamespace, "-f", launchFile - }; + String[] launchCmd = new String[] {"kubectl", "create", "--namespace", k8sNamespace, "-f", launchFile}; String createResult = execute(launchCmd); log.debug("Create result: " + createResult); @@ -822,20 +851,17 @@ private String getSupplementalGroupsList() throws Exception { } Set> groupCredentials = getCachedGroupsFromSubject(); if (groupCredentials.size() == 1) { - return buildGroupUriList(groupCredentials) - .stream() - .map(posixGroup -> Integer.toString(posixGroup.getGID())) - .collect(Collectors.joining(",")); + return buildGroupUriList(groupCredentials).stream() + .map(posixGroup -> Integer.toString(posixGroup.getGID())) + .collect(Collectors.joining(",")); } else { return ""; } } private List buildGroupUriList(Set> groupCredentials) throws Exception { - return toGIDs(groupCredentials.iterator().next().stream() - .map(Group::getID) - .collect(Collectors.toList()) - ); + return toGIDs( + groupCredentials.iterator().next().stream().map(Group::getID).collect(Collectors.toList())); } List toGIDs(final List groupURIS) throws Exception { diff --git a/skaha/src/main/java/org/opencadc/skaha/session/PublicSession.java b/skaha/src/main/java/org/opencadc/skaha/session/PublicSession.java index 09b441bf..c559a433 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/PublicSession.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/PublicSession.java @@ -70,14 +70,14 @@ /** * @author majorb - * + * * Public session information. * */ public class PublicSession { - + private static final Logger log = Logger.getLogger(PublicSession.class); - + private String userid; private String type; private String status; @@ -89,7 +89,7 @@ public PublicSession(String userid, String type, String status, String startTime this.status = status; this.startTime = startTime; } - + public String getUserid() { return userid; } @@ -105,5 +105,4 @@ public String getStatus() { public String getStartTime() { return startTime; } - } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/ResourceStats.java b/skaha/src/main/java/org/opencadc/skaha/session/ResourceStats.java index 1464f64d..881c4846 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/ResourceStats.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/ResourceStats.java @@ -73,16 +73,25 @@ * */ public class ResourceStats { - + private static final Logger log = Logger.getLogger(ResourceStats.class); - + private JobInstances instances; private Core cores; private Ram ram; - public ResourceStats(int desktopCount, int headlessCount, int totalCount, - Double requestedCPUCores, String requestedRAM, Double coresAvailable, String ramAvailable, - Double mCores, String withRAM, String mRAM, Double withCores) { + public ResourceStats( + int desktopCount, + int headlessCount, + int totalCount, + Double requestedCPUCores, + String requestedRAM, + Double coresAvailable, + String ramAvailable, + Double mCores, + String withRAM, + String mRAM, + Double withCores) { instances = new JobInstances(desktopCount, headlessCount, totalCount); MaxCoreResource maxCores = new MaxCoreResource(); @@ -101,13 +110,13 @@ public ResourceStats(int desktopCount, int headlessCount, int totalCount, ram.ramAvailable = ramAvailable; ram.requestedRAM = requestedRAM; } - + class JobInstances { private int session; private int desktopApp; private int headless; private int total; - + public JobInstances(int desktopCount, int headlessCount, int totalCount) { desktopApp = desktopCount; headless = headlessCount; @@ -115,13 +124,13 @@ public JobInstances(int desktopCount, int headlessCount, int totalCount) { session = totalCount - desktopCount - headlessCount; } } - + class Core { Double requestedCPUCores = 0.0; Double cpuCoresAvailable = 0.0; MaxCoreResource maxCPUCores; } - + class Ram { String requestedRAM = "0G"; String ramAvailable = "0G"; @@ -135,7 +144,7 @@ class MaxCoreResource { class MaxRamResource { public String ram = "0G"; - + public Double withCPUCores = 0.0; } } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/Session.java b/skaha/src/main/java/org/opencadc/skaha/session/Session.java index 487ff5ee..9fe46e0e 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/Session.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/Session.java @@ -67,21 +67,19 @@ package org.opencadc.skaha.session; import ca.nrc.cadc.auth.PosixPrincipal; -import org.apache.log4j.Logger; - -import java.lang.reflect.Array; import java.util.Arrays; +import org.apache.log4j.Logger; /** * @author majorb - * + * * Represents a session running in skaha. * */ public class Session { - + private static final Logger log = Logger.getLogger(Session.class); - + public static final String STATUS_TERMINATING = "Terminating"; public static final String STATUS_SUCCEEDED = "Succeeded"; public static final String STATUS_RUNNING = "Running"; @@ -97,7 +95,7 @@ public class Session { private final String status; private final String name; private final String startTime; - private String expiryTime; // in seconds + private String expiryTime; // in seconds private final String connectURL; private String requestedRAM; private String requestedCPUCores; @@ -107,9 +105,19 @@ public class Session { private String cpuCoresInUse; private String gpuUtilization; - public Session(String id, String userid, String runAsUID, String runAsGID, Integer[] supplementalGroups, - String image, String type, String status, String name, String startTime, String connectURL, - String appID) { + public Session( + String id, + String userid, + String runAsUID, + String runAsGID, + Integer[] supplementalGroups, + String image, + String type, + String status, + String name, + String startTime, + String connectURL, + String appID) { if (id == null) { throw new IllegalArgumentException("id is required"); } @@ -131,15 +139,15 @@ public Session(String id, String userid, String runAsUID, String runAsGID, Integ this.supplementalGroups = new Integer[0]; } } - + public String getId() { return id; } - + public String getUserid() { return userid; } - + public String getImage() { return image; } @@ -163,11 +171,11 @@ public String getStartTime() { public String getConnectURL() { return connectURL; } - + public String getRequestedRAM() { return requestedRAM; } - + public void setRequestedRAM(String ram) { this.requestedRAM = ram; } @@ -227,7 +235,7 @@ public String getExpiryTime() { public void setExpiryTime(String timeInSeconds) { this.expiryTime = timeInSeconds; } - + public String getAppId() { return appid; } @@ -241,7 +249,6 @@ public Integer[] getSupplementalGroups() { return Arrays.copyOf(this.supplementalGroups, this.supplementalGroups.length); } - public PosixPrincipal getPosixPrincipal() { final PosixPrincipal posixPrincipal = new PosixPrincipal(Integer.parseInt(this.runAsUID)); posixPrincipal.username = this.userid; @@ -249,7 +256,7 @@ public PosixPrincipal getPosixPrincipal() { return posixPrincipal; } - + @Override public boolean equals(Object o) { if (o instanceof Session) { @@ -258,12 +265,11 @@ public boolean equals(Object o) { } return false; } - + @Override public String toString() { return String.format( - "Session[id=%s,userid=%s,image=%s,type=%s,status=%s,name=%s,startTime=%s,connectURL=%s]", - id, userid, image, type, status, name, startTime, connectURL); + "Session[id=%s,userid=%s,image=%s,type=%s,status=%s,name=%s,startTime=%s,connectURL=%s]", + id, userid, image, type, status, name, startTime, connectURL); } - } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/SessionAction.java b/skaha/src/main/java/org/opencadc/skaha/session/SessionAction.java index b357e8d5..6d7e4fca 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/SessionAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/SessionAction.java @@ -100,7 +100,6 @@ import org.opencadc.skaha.utils.CommandExecutioner; import org.opencadc.skaha.utils.CommonUtils; - public abstract class SessionAction extends SkahaAction { protected static final String REQUEST_TYPE_SESSION = "session"; @@ -125,7 +124,7 @@ public SessionAction() { public static String getVNCURL(String host, String sessionID) { // vnc.html does not... return "https://" + host + "/session/desktop/" + sessionID + "/?password=" + sessionID - + "&path=session/desktop/" + sessionID + "/"; + + "&path=session/desktop/" + sessionID + "/"; } public static String getCartaURL(String host, String sessionID, boolean altSocketUrl) { @@ -137,8 +136,9 @@ public static String getCartaURL(String host, String sessionID, boolean altSocke } public static String getNotebookURL(String host, String sessionID, String userid, String skahaTLD) { - return String.format("https://%s/session/notebook/%s/lab/tree/%s/home/%s?token=%s", host, sessionID, - skahaTLD.replaceAll("/", ""), userid, sessionID); + return String.format( + "https://%s/session/notebook/%s/lab/tree/%s/home/%s?token=%s", + host, sessionID, skahaTLD.replaceAll("/", ""), userid, sessionID); } public static String getContributedURL(String host, String sessionID) { @@ -189,19 +189,24 @@ private void injectProxyCertificate() { // Should throw a NoSuchElementException if it's missing, but check here anyway. if (credServiceID != null) { final RegistryClient registryClient = new RegistryClient(); - final URL credServiceURL = registryClient.getServiceURL(credServiceID, Standards.CRED_PROXY_10, AuthMethod.CERT); + final URL credServiceURL = + registryClient.getServiceURL(credServiceID, Standards.CRED_PROXY_10, AuthMethod.CERT); if (credServiceURL != null) { final CredClient credClient = new CredClient(credServiceID); final Subject currentSubject = AuthenticationUtil.getCurrentSubject(); - final X509CertificateChain proxyCert = Subject.doAs(CredUtil.createOpsSubject(), (PrivilegedExceptionAction) () - -> credClient.getProxyCertificate(currentSubject, SessionAction.ONE_WEEK_DAYS)); + final X509CertificateChain proxyCert = + Subject.doAs(CredUtil.createOpsSubject(), (PrivilegedExceptionAction) + () -> credClient.getProxyCertificate(currentSubject, SessionAction.ONE_WEEK_DAYS)); log.debug("Proxy cert: " + proxyCert); // inject the proxy cert log.debug("Running docker exec to insert cert"); - writeClientCertificate(proxyCert, Path.of(homedir, this.posixPrincipal.username, ".ssl", "cadcproxy.pem").toString()); + writeClientCertificate( + proxyCert, + Path.of(homedir, this.posixPrincipal.username, ".ssl", "cadcproxy.pem") + .toString()); log.debug("injectProxyCertificate(): OK"); } } @@ -214,7 +219,8 @@ private void injectProxyCertificate() { } } - private void writeClientCertificate(X509CertificateChain clientCertificateChain, String path) throws IOException, InterruptedException { + private void writeClientCertificate(X509CertificateChain clientCertificateChain, String path) + throws IOException, InterruptedException { final int uid = posixPrincipal.getUidNumber(); // stage file @@ -249,7 +255,6 @@ protected String getImageName(String image) { log.warn("failed to determine name for image: " + image); return "unknown"; } - } protected String stageFile(String data) throws IOException { @@ -299,15 +304,16 @@ public String getEvents(String forUserID, String sessionID) throws Exception { getEventsCmd.add("event"); getEventsCmd.add("--field-selector"); getEventsCmd.add("involvedObject.name=" + podID); - //getEventsCmd.add("--no-headers=true"); + // getEventsCmd.add("--no-headers=true"); getEventsCmd.add("-o"); - String customColumns = "TYPE:.type,REASON:.reason,MESSAGE:.message,FIRST-TIME:.firstTimestamp,LAST-TIME:.lastTimestamp"; + String customColumns = + "TYPE:.type,REASON:.reason,MESSAGE:.message,FIRST-TIME:.firstTimestamp,LAST-TIME:.lastTimestamp"; getEventsCmd.add("custom-columns=" + customColumns); String events = CommandExecutioner.execute(getEventsCmd.toArray(new String[0])); log.debug("events: " + events); if (events != null) { String[] lines = events.split("\n"); - if (lines.length > 1) { // header row returned + if (lines.length > 1) { // header row returned return events; } } @@ -334,14 +340,15 @@ public Session getDesktopApp(String sessionID, String appID) throws Exception { for (Session session : sessions) { // only include 'desktop-app' if (SkahaAction.TYPE_DESKTOP_APP.equalsIgnoreCase(session.getType()) - && (sessionID.equals(session.getId())) && (appID.equals(session.getAppId()))) { + && (sessionID.equals(session.getId())) + && (appID.equals(session.getAppId()))) { return session; } } } throw new ResourceNotFoundException( - "desktop app with session " + sessionID + " and app ID " + appID + " was not found"); + "desktop app with session " + sessionID + " and app ID " + appID + " was not found"); } public Session getSession(String forUserID, String sessionID) throws Exception { @@ -424,8 +431,8 @@ private List getJobExpiryTimeCMD(String k8sNamespace, String forUserID) return getSessionJobCMD; } - protected String getAppJobName(String sessionID, String userID, String appID) throws - IOException, InterruptedException { + protected String getAppJobName(String sessionID, String userID, String appID) + throws IOException, InterruptedException { String k8sNamespace = K8SUtil.getWorkloadNamespace(); List getAppJobNameCMD = getAppJobNameCMD(k8sNamespace, userID, sessionID, appID); return CommandExecutioner.execute(getAppJobNameCMD.toArray(new String[0])); @@ -457,5 +464,4 @@ private List getAppJobNameCMD(String k8sNamespace, String userID, String getAppJobNameCMD.add(customColumns); return getAppJobNameCMD; } - } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/SessionDAO.java b/skaha/src/main/java/org/opencadc/skaha/session/SessionDAO.java index 206bdc60..529aa6f6 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/SessionDAO.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/SessionDAO.java @@ -1,11 +1,9 @@ package org.opencadc.skaha.session; +import static org.opencadc.skaha.utils.CommandExecutioner.execute; + import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.util.StringUtil; -import org.apache.log4j.Logger; -import org.opencadc.skaha.K8SUtil; -import org.opencadc.skaha.SkahaAction; - import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -15,8 +13,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - -import static org.opencadc.skaha.utils.CommandExecutioner.execute; +import org.apache.log4j.Logger; +import org.opencadc.skaha.K8SUtil; +import org.opencadc.skaha.SkahaAction; public class SessionDAO { public static final Logger LOGGER = Logger.getLogger(SessionDAO.class); @@ -25,8 +24,7 @@ public class SessionDAO { // Ordered dictionary of columns requested from Kubernetes - static List getSessionsCMD(final String k8sNamespace, final String forUserID, - final String sessionID) { + static List getSessionsCMD(final String k8sNamespace, final String forUserID, final String sessionID) { final List sessionsCMD = new ArrayList<>(); sessionsCMD.add("kubectl"); sessionsCMD.add("get"); @@ -45,11 +43,10 @@ static List getSessionsCMD(final String k8sNamespace, final String forUs sessionsCMD.add("-o"); final String customColumns = "custom-columns=" - + Arrays.stream(CustomColumns.values()) - .filter(customColumn -> !customColumn.forUserOnly || forUserID != null) - .map(customColumn -> String.format("%s:%s", customColumn.name(), - customColumn.columnDefinition)) - .collect(Collectors.joining(",")); + + Arrays.stream(CustomColumns.values()) + .filter(customColumn -> !customColumn.forUserOnly || forUserID != null) + .map(customColumn -> String.format("%s:%s", customColumn.name(), customColumn.columnDefinition)) + .collect(Collectors.joining(",")); sessionsCMD.add(customColumns); return sessionsCMD; @@ -140,9 +137,9 @@ protected static List getSessions(String forUserID, String sessionID, f } // if this session usages GPU, get the GPU usage - if (StringUtil.hasText(session.getRequestedGPUCores()) && - !NONE.equals(session.getRequestedGPUCores()) && - Double.parseDouble(session.getRequestedGPUCores()) > 0.0) { + if (StringUtil.hasText(session.getRequestedGPUCores()) + && !NONE.equals(session.getRequestedGPUCores()) + && Double.parseDouble(session.getRequestedGPUCores()) > 0.0) { List sessionGPUUsageCMD = getSessionGPUUsageCMD(k8sNamespace, fullName); String sessionGPUUsage = execute(sessionGPUUsageCMD.toArray(new String[0])); List gpuUsage = getGPUUsage(sessionGPUUsage); @@ -324,9 +321,7 @@ private static List getJobExpiryTimeCMD(String k8sNamespace, String forU getSessionJobCMD.add("--no-headers=true"); getSessionJobCMD.add("-o"); - String customColumns = "custom-columns=" - + "UID:.metadata.uid," - + "EXPIRY:.spec.activeDeadlineSeconds"; + String customColumns = "custom-columns=" + "UID:.metadata.uid," + "EXPIRY:.spec.activeDeadlineSeconds"; getSessionJobCMD.add(customColumns); return getSessionJobCMD; @@ -362,8 +357,8 @@ static Session constructSession(String k8sOutput, final String topLevelDirectory String type = parts[allColumns.indexOf(CustomColumns.TYPE)]; String deletionTimestamp = parts[allColumns.indexOf(CustomColumns.DELETION)]; final String status = (deletionTimestamp != null && !NONE.equals(deletionTimestamp)) - ? Session.STATUS_TERMINATING - : parts[allColumns.indexOf(CustomColumns.STATUS)]; + ? Session.STATUS_TERMINATING + : parts[allColumns.indexOf(CustomColumns.STATUS)]; final String host = K8SUtil.getHostName(); final String connectURL; @@ -384,19 +379,19 @@ static Session constructSession(String k8sOutput, final String topLevelDirectory connectURL = "not-applicable"; } - final Session session = new Session(id, - userid, - parts[allColumns.indexOf(CustomColumns.RUN_AS_UID)], - parts[allColumns.indexOf(CustomColumns.RUN_AS_GID)], - SessionDAO.fromStringArray(parts[allColumns.indexOf( - CustomColumns.SUPPLEMENTAL_GROUPS)]), - image, - type, - status, - parts[allColumns.indexOf(CustomColumns.NAME)], - parts[allColumns.indexOf(CustomColumns.STARTED)], - connectURL, - parts[allColumns.indexOf(CustomColumns.APP_ID)]); + final Session session = new Session( + id, + userid, + parts[allColumns.indexOf(CustomColumns.RUN_AS_UID)], + parts[allColumns.indexOf(CustomColumns.RUN_AS_GID)], + SessionDAO.fromStringArray(parts[allColumns.indexOf(CustomColumns.SUPPLEMENTAL_GROUPS)]), + image, + type, + status, + parts[allColumns.indexOf(CustomColumns.NAME)], + parts[allColumns.indexOf(CustomColumns.STARTED)], + connectURL, + parts[allColumns.indexOf(CustomColumns.APP_ID)]); // Check if all columns were requested (set by forUserId) final int requestedRamIndex = allColumns.indexOf(CustomColumns.REQUESTED_RAM); @@ -427,11 +422,11 @@ private static Integer[] fromStringArray(final String inputArray) { if (inputArray.equals(SessionDAO.NONE)) { return new Integer[0]; } else { - final Object[] parsedArray = - Arrays.stream(inputArray.replace("[", "").replace("]", "") - .trim().split(" ")) - .filter(StringUtil::hasText) - .map(Integer::parseInt).toArray(); + final Object[] parsedArray = Arrays.stream( + inputArray.replace("[", "").replace("]", "").trim().split(" ")) + .filter(StringUtil::hasText) + .map(Integer::parseInt) + .toArray(); return Arrays.copyOf(parsedArray, parsedArray.length, Integer[].class); } } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/SessionJobBuilder.java b/skaha/src/main/java/org/opencadc/skaha/session/SessionJobBuilder.java index 1a739377..86e31963 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/SessionJobBuilder.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/SessionJobBuilder.java @@ -21,7 +21,6 @@ import java.util.Map; import org.apache.log4j.Logger; - /** * Class to interface with Kubernetes. */ @@ -34,10 +33,7 @@ public class SessionJobBuilder { private boolean gpuEnabled; private Integer gpuCount; - - private SessionJobBuilder() { - - } + private SessionJobBuilder() {} /** * Create a new builder from the provided path. @@ -52,19 +48,21 @@ static SessionJobBuilder fromPath(final Path jobFilePath) { return sessionJobBuilder; } - private static V1NodeSelectorRequirement getV1NodeSelectorRequirement(List gpuRequiredNodeSelectorTerms) { + private static V1NodeSelectorRequirement getV1NodeSelectorRequirement( + List gpuRequiredNodeSelectorTerms) { if (gpuRequiredNodeSelectorTerms.size() != 1) { throw new IllegalStateException("GPU Node Selector cannot exceed one selector."); } final V1NodeSelectorTerm gpuNodeSelectorTerm = gpuRequiredNodeSelectorTerms.get(0); - final List gpuNodeSelectorMatchExpressions = gpuNodeSelectorTerm.getMatchExpressions(); + final List gpuNodeSelectorMatchExpressions = + gpuNodeSelectorTerm.getMatchExpressions(); if (gpuNodeSelectorMatchExpressions == null) { throw new IllegalStateException("Preset GPU Node Selector match expressions are missing."); } else if (gpuNodeSelectorMatchExpressions.size() != 1) { throw new IllegalStateException("Preset GPU Node Selector match expressions must be exactly one (found " - + gpuNodeSelectorMatchExpressions.size() + ")"); + + gpuNodeSelectorMatchExpressions.size() + ")"); } return gpuNodeSelectorMatchExpressions.get(0); @@ -170,14 +168,15 @@ private String mergeAffinity(final String jobFileString) throws IOException { affinity.setNodeAffinity(gpuAffinity.getNodeAffinity()); } else { final List existingPreferredSchedulingTerms = - nodeAffinity.getPreferredDuringSchedulingIgnoredDuringExecution(); + nodeAffinity.getPreferredDuringSchedulingIgnoredDuringExecution(); final V1NodeAffinity gpuNodeAffinity = gpuAffinity.getNodeAffinity(); if (gpuNodeAffinity != null) { final List gpuAffinityPreferredSchedulingTerms = - gpuNodeAffinity.getPreferredDuringSchedulingIgnoredDuringExecution(); + gpuNodeAffinity.getPreferredDuringSchedulingIgnoredDuringExecution(); - final List mergedPreferredSchedulingTerms = new ArrayList<>(); + final List mergedPreferredSchedulingTerms = + new ArrayList<>(); if (existingPreferredSchedulingTerms != null) { mergedPreferredSchedulingTerms.addAll(existingPreferredSchedulingTerms); } @@ -187,32 +186,41 @@ private String mergeAffinity(final String jobFileString) throws IOException { } if (!mergedPreferredSchedulingTerms.isEmpty()) { - nodeAffinity.setPreferredDuringSchedulingIgnoredDuringExecution(mergedPreferredSchedulingTerms); + nodeAffinity.setPreferredDuringSchedulingIgnoredDuringExecution( + mergedPreferredSchedulingTerms); } // spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - final V1NodeSelector requiredNodeSelector = nodeAffinity.getRequiredDuringSchedulingIgnoredDuringExecution(); - final V1NodeSelector gpuRequiredNodeSelector = gpuNodeAffinity.getRequiredDuringSchedulingIgnoredDuringExecution(); + final V1NodeSelector requiredNodeSelector = + nodeAffinity.getRequiredDuringSchedulingIgnoredDuringExecution(); + final V1NodeSelector gpuRequiredNodeSelector = + gpuNodeAffinity.getRequiredDuringSchedulingIgnoredDuringExecution(); // No preset one from the configuration, so assume the GPU setting is the only one. if (requiredNodeSelector == null) { - nodeAffinity.setRequiredDuringSchedulingIgnoredDuringExecution(gpuRequiredNodeSelector); + nodeAffinity.setRequiredDuringSchedulingIgnoredDuringExecution( + gpuRequiredNodeSelector); } else if (gpuRequiredNodeSelector != null) { - final List requiredNodeSelectorTerms = requiredNodeSelector.getNodeSelectorTerms(); - final List gpuRequiredNodeSelectorTerms = gpuRequiredNodeSelector.getNodeSelectorTerms(); + final List requiredNodeSelectorTerms = + requiredNodeSelector.getNodeSelectorTerms(); + final List gpuRequiredNodeSelectorTerms = + gpuRequiredNodeSelector.getNodeSelectorTerms(); if (requiredNodeSelectorTerms.isEmpty()) { requiredNodeSelector.setNodeSelectorTerms(gpuRequiredNodeSelectorTerms); } else { final V1NodeSelectorRequirement gpuNodeSelectorMatchExpression = - SessionJobBuilder.getV1NodeSelectorRequirement(gpuRequiredNodeSelectorTerms); + SessionJobBuilder.getV1NodeSelectorRequirement( + gpuRequiredNodeSelectorTerms); requiredNodeSelectorTerms.forEach(requiredNodeSelectorTerm -> { final List requiredNodeSelectorMatchExpressions = - requiredNodeSelectorTerm.getMatchExpressions(); + requiredNodeSelectorTerm.getMatchExpressions(); if (requiredNodeSelectorMatchExpressions == null) { - requiredNodeSelectorTerm.setMatchExpressions(Collections.singletonList(gpuNodeSelectorMatchExpression)); + requiredNodeSelectorTerm.setMatchExpressions( + Collections.singletonList(gpuNodeSelectorMatchExpression)); } else { - requiredNodeSelectorTerm.addMatchExpressionsItem(gpuNodeSelectorMatchExpression); + requiredNodeSelectorTerm.addMatchExpressionsItem( + gpuNodeSelectorMatchExpression); } }); } diff --git a/skaha/src/main/java/org/opencadc/skaha/utils/CommandExecutioner.java b/skaha/src/main/java/org/opencadc/skaha/utils/CommandExecutioner.java index 43a6a6d3..4a093e2a 100644 --- a/skaha/src/main/java/org/opencadc/skaha/utils/CommandExecutioner.java +++ b/skaha/src/main/java/org/opencadc/skaha/utils/CommandExecutioner.java @@ -64,11 +64,11 @@ public static void execute(String[] command, OutputStream out) throws IOExceptio } public static void execute(final String[] command, final OutputStream standardOut, final OutputStream standardErr) - throws IOException, InterruptedException { + throws IOException, InterruptedException { final Process p = Runtime.getRuntime().exec(command); final int code = p.waitFor(); try (final InputStream stdOut = new BufferedInputStream(p.getInputStream()); - final InputStream stdErr = new BufferedInputStream(p.getErrorStream())) { + final InputStream stdErr = new BufferedInputStream(p.getErrorStream())) { final String commandOutput = readStream(stdOut); if (code != 0) { final String errorOutput = readStream(stdErr); @@ -93,7 +93,7 @@ public static void execute(final String[] command, final OutputStream standardOu * @throws Exception If there is an error creating the secret. */ public static void ensureRegistrySecret(final ImageRegistryAuth registryAuth, final String secretName) - throws Exception { + throws Exception { // delete any old secret by this name final String[] deleteCmd = CommandExecutioner.getDeleteSecretCommand(secretName); log.debug("delete secret command: " + Arrays.toString(deleteCmd)); @@ -143,7 +143,12 @@ static String[] getRegistryCreateSecretCommand(final ImageRegistryAuth registryA } return new String[] { - "kubectl", "--namespace", K8SUtil.getWorkloadNamespace(), "create", "secret", "docker-registry", + "kubectl", + "--namespace", + K8SUtil.getWorkloadNamespace(), + "create", + "secret", + "docker-registry", secretName, "--docker-server=" + registryAuth.getHost(), "--docker-username=" + registryAuth.getUsername(), @@ -154,16 +159,23 @@ static String[] getRegistryCreateSecretCommand(final ImageRegistryAuth registryA public static JSONObject getSecretData(final String secretName, final String secretNamespace) throws Exception { // Check the current secret final String[] getSecretCommand = new String[] { - "kubectl", "--namespace", secretNamespace, "get", "--ignore-not-found", "secret", - secretName, "-o", "jsonpath=\"{.data}\"" + "kubectl", + "--namespace", + secretNamespace, + "get", + "--ignore-not-found", + "secret", + secretName, + "-o", + "jsonpath=\"{.data}\"" }; final String data = CommandExecutioner.execute(getSecretCommand); // The data from the output begins with a double-quote and ends with one, so strip them. - return StringUtil.hasText(data) ? new JSONObject(data.replaceFirst("\"", "") - .substring(0, data.lastIndexOf("\""))) - : new JSONObject(); + return StringUtil.hasText(data) + ? new JSONObject(data.replaceFirst("\"", "").substring(0, data.lastIndexOf("\""))) + : new JSONObject(); } protected static String readStream(InputStream in) throws IOException { diff --git a/skaha/src/main/java/org/opencadc/skaha/utils/CommonUtils.java b/skaha/src/main/java/org/opencadc/skaha/utils/CommonUtils.java index ea296e04..55e71894 100644 --- a/skaha/src/main/java/org/opencadc/skaha/utils/CommonUtils.java +++ b/skaha/src/main/java/org/opencadc/skaha/utils/CommonUtils.java @@ -6,7 +6,6 @@ import java.util.Collection; import java.util.Set; - public class CommonUtils { public static boolean isNotEmpty(Collection collection) { return null != collection && !collection.isEmpty(); diff --git a/skaha/src/main/java/org/opencadc/skaha/utils/PosixCache.java b/skaha/src/main/java/org/opencadc/skaha/utils/PosixCache.java index 3ae3fbab..dd24478d 100644 --- a/skaha/src/main/java/org/opencadc/skaha/utils/PosixCache.java +++ b/skaha/src/main/java/org/opencadc/skaha/utils/PosixCache.java @@ -10,7 +10,6 @@ import redis.clients.jedis.AbstractTransaction; import redis.clients.jedis.JedisPooled; - /** * A simple Redis Cache for POSIX information. This will update the underlying Redis Set in a transaction to ensure single access. * BEWARE - Changes to the items in the Set (i.e. the POSIX entries) will require a purge of the cache to properly reset it. @@ -50,18 +49,17 @@ public PosixCache(final String cacheURL, final String rootHomeFolder, final Posi * @return String POSIX entry, never null; */ private static String uidMapping(final PosixPrincipalEntry posixPrincipalEntry) { - return String.format("%s:x:%d:%d::%s:%s", - posixPrincipalEntry.posixPrincipal.username, - posixPrincipalEntry.posixPrincipal.getUidNumber(), - posixPrincipalEntry.posixPrincipal.getUidNumber(), - posixPrincipalEntry.homeFolder, - PosixCache.NO_LOGIN_SHELL); + return String.format( + "%s:x:%d:%d::%s:%s", + posixPrincipalEntry.posixPrincipal.username, + posixPrincipalEntry.posixPrincipal.getUidNumber(), + posixPrincipalEntry.posixPrincipal.getUidNumber(), + posixPrincipalEntry.homeFolder, + PosixCache.NO_LOGIN_SHELL); } private static String gidMapping(final PosixGroup posixGroup) { - return String.format("%s:x:%d:", - posixGroup.getGroupURI().getURI().getQuery(), - posixGroup.getGID()); + return String.format("%s:x:%d:", posixGroup.getGroupURI().getURI().getQuery(), posixGroup.getGID()); } public void writePOSIXEntries() throws Exception { @@ -72,11 +70,15 @@ public void writePOSIXEntries() throws Exception { private void writeUserEntries() throws Exception { LOGGER.debug("writeUserEntries()"); try (final ResourceIterator posixPrincipalIterator = this.posixMapperClient.getUserMap(); - final AbstractTransaction transaction = this.jedisPool.transaction(true)) { + final AbstractTransaction transaction = this.jedisPool.transaction(true)) { while (posixPrincipalIterator.hasNext()) { final PosixPrincipal posixPrincipal = posixPrincipalIterator.next(); - transaction.sadd(PosixCache.UID_MAP_KEY, PosixCache.uidMapping( - new PosixPrincipalEntry(posixPrincipal, Path.of(this.rootHomeFolder, posixPrincipal.username).toString()))); + transaction.sadd( + PosixCache.UID_MAP_KEY, + PosixCache.uidMapping(new PosixPrincipalEntry( + posixPrincipal, + Path.of(this.rootHomeFolder, posixPrincipal.username) + .toString()))); } final List setEntries = transaction.exec(); @@ -87,7 +89,7 @@ private void writeUserEntries() throws Exception { private void writeGroupEntries() throws Exception { LOGGER.debug("writeGroupEntries()"); try (final ResourceIterator posixGroupIterator = this.posixMapperClient.getGroupMap(); - final AbstractTransaction transaction = this.jedisPool.transaction(true)) { + final AbstractTransaction transaction = this.jedisPool.transaction(true)) { while (posixGroupIterator.hasNext()) { final PosixGroup posixGroup = posixGroupIterator.next(); transaction.sadd(PosixCache.GID_MAP_FIELD, PosixCache.gidMapping(posixGroup)); diff --git a/skaha/src/main/java/org/opencadc/skaha/utils/RedisCache.java b/skaha/src/main/java/org/opencadc/skaha/utils/RedisCache.java index 5cd7789b..9f82943e 100644 --- a/skaha/src/main/java/org/opencadc/skaha/utils/RedisCache.java +++ b/skaha/src/main/java/org/opencadc/skaha/utils/RedisCache.java @@ -3,12 +3,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; - import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.log4j.Logger; +import redis.clients.jedis.Jedis; public class RedisCache { private static final Logger log = Logger.getLogger(RedisCache.class); @@ -38,8 +37,7 @@ public List getAll(String key) { public List getAll(String key, Class className) { List list = getAll(key); if (list.isEmpty()) return List.of(); - return list - .parallelStream() + return list.parallelStream() .map(item -> deserialize(className, item)) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/skaha/src/test/java/org/opencadc/skaha/K8SUtilTest.java b/skaha/src/test/java/org/opencadc/skaha/K8SUtilTest.java index daad4c88..fbbf918a 100644 --- a/skaha/src/test/java/org/opencadc/skaha/K8SUtilTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/K8SUtilTest.java @@ -74,13 +74,10 @@ public class K8SUtilTest { @Test public void sanitizeJobName() { - Assert.assertEquals("Wrong name", "skaha-type-userid-sess", - K8SUtil.getJobName("SESS", "TYPE", "USERID")); + Assert.assertEquals("Wrong name", "skaha-type-userid-sess", K8SUtil.getJobName("SESS", "TYPE", "USERID")); - Assert.assertEquals("Wrong name", "skaha-type-my-user-sess", - K8SUtil.getJobName("SESS", "TYPE", "my_user")); + Assert.assertEquals("Wrong name", "skaha-type-my-user-sess", K8SUtil.getJobName("SESS", "TYPE", "my_user")); - Assert.assertEquals("Wrong name", "skaha-type-my-us-e-r-sess", - K8SUtil.getJobName("SESS", "TYPE", "my|us+e&r")); + Assert.assertEquals("Wrong name", "skaha-type-my-us-e-r-sess", K8SUtil.getJobName("SESS", "TYPE", "my|us+e&r")); } } diff --git a/skaha/src/test/java/org/opencadc/skaha/image/GetActionTest.java b/skaha/src/test/java/org/opencadc/skaha/image/GetActionTest.java index 500bccf2..967c6233 100644 --- a/skaha/src/test/java/org/opencadc/skaha/image/GetActionTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/image/GetActionTest.java @@ -66,6 +66,14 @@ */ package org.opencadc.skaha.image; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.when; +import static org.opencadc.skaha.utils.TestUtils.set; +import static org.opencadc.skaha.utils.TestUtils.setEnv; + +import java.util.List; +import java.util.Set; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -75,16 +83,6 @@ import org.opencadc.skaha.SkahaAction; import org.opencadc.skaha.utils.RedisCache; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.mockito.Mockito.when; -import static org.opencadc.skaha.utils.TestUtils.set; -import static org.opencadc.skaha.utils.TestUtils.setEnv; - - /** * @author majorb * @@ -113,11 +111,9 @@ public void setUp() { public void testGetImagesWithNullType() throws Exception { List expectedImages = List.of( new Image("image1", Set.of("type1", "type2"), "digest1"), - new Image("image2", Set.of("type2", "type3"), "digest2") - ); + new Image("image2", Set.of("type2", "type3"), "digest2")); - when(redis.getAll("public", Image.class)) - .thenReturn(expectedImages); + when(redis.getAll("public", Image.class)).thenReturn(expectedImages); List result = getAction.getImages(null); @@ -138,18 +134,12 @@ public void testGetImagesWithImageType() throws Exception { // Arrange String notebook = "notebook"; Image notebookImage = new Image("image2", Set.of(notebook), "digest2"); - List expectedImages = List.of( - new Image("image1", Set.of("type1", "type2"), "digest1"), - notebookImage - ); + List expectedImages = List.of(new Image("image1", Set.of("type1", "type2"), "digest1"), notebookImage); - when(redis.getAll("public", Image.class)) - .thenReturn(expectedImages); + when(redis.getAll("public", Image.class)).thenReturn(expectedImages); List result = getAction.getImages(notebook); assertEquals(List.of(notebookImage), result); } - - } diff --git a/skaha/src/test/java/org/opencadc/skaha/registry/ImageRegistryAuthTest.java b/skaha/src/test/java/org/opencadc/skaha/registry/ImageRegistryAuthTest.java index 94ef3cda..4ae6e08a 100644 --- a/skaha/src/test/java/org/opencadc/skaha/registry/ImageRegistryAuthTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/registry/ImageRegistryAuthTest.java @@ -32,7 +32,8 @@ public void testFromEncodedBadInputs() { @Test public void testFromEncoded() { - final String encodedValue = new String(Base64.getEncoder().encode("username:supersecret".getBytes(StandardCharsets.UTF_8))); + final String encodedValue = + new String(Base64.getEncoder().encode("username:supersecret".getBytes(StandardCharsets.UTF_8))); final ImageRegistryAuth auth = ImageRegistryAuth.fromEncoded(encodedValue, "host.example.com"); Assert.assertEquals("Wrong username", "username", auth.getUsername()); diff --git a/skaha/src/test/java/org/opencadc/skaha/session/GetActionTests.java b/skaha/src/test/java/org/opencadc/skaha/session/GetActionTests.java index 9470bcc9..366aaf8c 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/GetActionTests.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/GetActionTests.java @@ -67,7 +67,6 @@ package org.opencadc.skaha.session; import ca.nrc.cadc.util.Log4jInit; - import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; @@ -78,18 +77,18 @@ * */ public class GetActionTests { - + private static final Logger log = Logger.getLogger(GetActionTests.class); - + static { Log4jInit.setLevel("org.opencadc.skaha", Level.DEBUG); } - + private static final long K_UNIT = 1024; private static final long M_UNIT = K_UNIT * K_UNIT; private static final long G_UNIT = K_UNIT * M_UNIT; private static final long T_UNIT = K_UNIT * G_UNIT; - + private static final long NO_UNIT_VALUE = 100; private static final long K_VALUE = 2 * K_UNIT; private static final long M_VALUE = 3 * M_UNIT; @@ -103,10 +102,9 @@ public class GetActionTests { private static final String G_VALUE_STR = String.valueOf(4) + "G"; private static final String T_VALUE_STR = String.valueOf(5) + "T"; private static final String INVALID_VALUE_STR = String.valueOf(5) + "A"; - - public GetActionTests() { - } - + + public GetActionTests() {} + @Test public void testNormalizeToLong() { try { @@ -126,7 +124,7 @@ public void testNormalizeToLong() { Assert.fail("Unexpected: " + t.getMessage()); } } - + static class TestGetAction extends GetAction { @Override protected String getUsername() { diff --git a/skaha/src/test/java/org/opencadc/skaha/session/GetSessionsTests.java b/skaha/src/test/java/org/opencadc/skaha/session/GetSessionsTests.java index b6cd59cb..611ef6e0 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/GetSessionsTests.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/GetSessionsTests.java @@ -67,46 +67,40 @@ package org.opencadc.skaha.session; import ca.nrc.cadc.util.Log4jInit; - import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - import java.lang.reflect.Type; import java.time.Instant; import java.util.ArrayList; import java.util.List; - import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Test; - /** * @author majorb * */ public class GetSessionsTests { - + private static final Logger log = Logger.getLogger(GetSessionsTests.class); - + static { Log4jInit.setLevel("org.opencadc.skaha", Level.DEBUG); } - private static final String K8S_LIST = - "pud05npw majorb 1001 1001 [23 24 25] imageID carta Running brian 2021-02-02T17:49:55Z \n" + - "e37lmx4m majorb 1001 1001 [23 24 25] imageID desktop Terminating brian 2021-01-28T21:52:51Z \n" + - "gspc0n8m majorb 1001 1001 [23 24 25] imageID notebook Running brian 2021-01-29T22:56:21Z \n" + - "abcd0n8m majorb 1001 1001 [23 25] imageID notebook Terminating brian 2021-01-29T22:56:21Z \n" + - "defg0n8m majorb 1001 1001 [1992] imageID notebook Running brian 2021-01-29T22:56:21Z \n" + - "shd89sfg majorb 1001 1001 [] imageID notebook Running brian 2021-02-09T22:56:21Z \n" + - "bbn3829s majorb 1001 1001 imageID notebook Running brian 2021-02-27T22:56:21Z \n"; - - public GetSessionsTests() { - } - + "pud05npw majorb 1001 1001 [23 24 25] imageID carta Running brian 2021-02-02T17:49:55Z \n" + + "e37lmx4m majorb 1001 1001 [23 24 25] imageID desktop Terminating brian 2021-01-28T21:52:51Z \n" + + "gspc0n8m majorb 1001 1001 [23 24 25] imageID notebook Running brian 2021-01-29T22:56:21Z \n" + + "abcd0n8m majorb 1001 1001 [23 25] imageID notebook Terminating brian 2021-01-29T22:56:21Z \n" + + "defg0n8m majorb 1001 1001 [1992] imageID notebook Running brian 2021-01-29T22:56:21Z \n" + + "shd89sfg majorb 1001 1001 [] imageID notebook Running brian 2021-02-09T22:56:21Z \n" + + "bbn3829s majorb 1001 1001 imageID notebook Running brian 2021-02-27T22:56:21Z \n"; + + public GetSessionsTests() {} + @Test public void testListSessions() { try { @@ -115,7 +109,7 @@ public void testListSessions() { log.info("json: \n" + json); List sessions1 = get.getAllSessions(null); Gson gson = new Gson(); - Type listType = new TypeToken>(){}.getType(); + Type listType = new TypeToken>() {}.getType(); List sessions2 = gson.fromJson(json, listType); Assert.assertEquals(sessions1.size(), K8S_LIST.split("\n").length); Assert.assertEquals("session count", sessions1.size(), sessions2.size()); @@ -125,13 +119,13 @@ public void testListSessions() { // All start times should be parsable. Instant.parse(s.getStartTime()); } - + } catch (Throwable t) { log.error("Unexpected", t); Assert.fail("Unexpected: " + t.getMessage()); } } - + @Test public void testFilterType() { try { @@ -146,7 +140,7 @@ public void testFilterType() { Assert.fail("Unexpected: " + t.getMessage()); } } - + @Test public void testFilterStatus() throws Exception { GetAction get = new TestGetAction(); @@ -156,7 +150,7 @@ public void testFilterStatus() throws Exception { Assert.assertEquals(s.getId(), "Running", s.getStatus()); } } - + @Test public void testFilterTypeStatus() { try { @@ -172,7 +166,7 @@ public void testFilterTypeStatus() { Assert.fail("Unexpected: " + t.getMessage()); } } - + static class TestGetAction extends GetAction { @Override diff --git a/skaha/src/test/java/org/opencadc/skaha/session/PostActionTest.java b/skaha/src/test/java/org/opencadc/skaha/session/PostActionTest.java index 3610d023..28ac83ed 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/PostActionTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/PostActionTest.java @@ -69,14 +69,12 @@ package org.opencadc.skaha.session; import ca.nrc.cadc.util.Log4jInit; +import java.io.IOException; +import java.io.OutputStream; import org.apache.log4j.Level; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; -import java.io.OutputStream; - - public class PostActionTest { static { Log4jInit.setLevel("org.opencadc.skaha", Level.DEBUG); @@ -113,9 +111,12 @@ void executeCommand(String[] command, OutputStream standardOut, OutputStream sta Assert.fail("Should throw IOException"); } catch (IOException exception) { // Good. - Assert.assertEquals("Wrong message.", "Unable to create user home." - + "\nError message from server: Forbidden to write." - + "\nOutput from command: ", exception.getMessage()); + Assert.assertEquals( + "Wrong message.", + "Unable to create user home." + + "\nError message from server: Forbidden to write." + + "\nOutput from command: ", + exception.getMessage()); } } @@ -137,7 +138,6 @@ String getDefaultQuota() { return "14"; } - @Override void executeCommand(String[] command, OutputStream standardOut, OutputStream standardErr) throws IOException { diff --git a/skaha/src/test/java/org/opencadc/skaha/session/SessionDAOTest.java b/skaha/src/test/java/org/opencadc/skaha/session/SessionDAOTest.java index 013ff3b3..7bab57ab 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/SessionDAOTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/SessionDAOTest.java @@ -69,27 +69,25 @@ package org.opencadc.skaha.session; import java.util.List; - import org.junit.Assert; import org.junit.Test; - public class SessionDAOTest { @Test public void testCommand() { - final List commandValues = - SessionDAO.getSessionsCMD("ns", null, "session-id-1"); + final List commandValues = SessionDAO.getSessionsCMD("ns", null, "session-id-1"); final String command = String.join(" ", commandValues.toArray(new String[0])); - Assert.assertEquals("Wrong command.", - "kubectl get --namespace ns pod -l canfar-net-sessionID=session-id-1 " - + "--no-headers=true -o custom-columns=SESSION_ID:.metadata.labels.canfar-net-sessionID," - + "USERID:.metadata.labels.canfar-net-userid,RUN_AS_UID:.spec.securityContext.runAsUser," - + "RUN_AS_GID:.spec.securityContext.runAsGroup," - + "SUPPLEMENTAL_GROUPS:.spec.securityContext.supplementalGroups," - + "IMAGE:.spec.containers[0].image," - + "TYPE:.metadata.labels.canfar-net-sessionType,STATUS:.status.phase," - + "NAME:.metadata.labels.canfar-net-sessionName,STARTED:.status.startTime," - + "DELETION:.metadata.deletionTimestamp,APP_ID:.metadata.labels.canfar-net-appID", - command); + Assert.assertEquals( + "Wrong command.", + "kubectl get --namespace ns pod -l canfar-net-sessionID=session-id-1 " + + "--no-headers=true -o custom-columns=SESSION_ID:.metadata.labels.canfar-net-sessionID," + + "USERID:.metadata.labels.canfar-net-userid,RUN_AS_UID:.spec.securityContext.runAsUser," + + "RUN_AS_GID:.spec.securityContext.runAsGroup," + + "SUPPLEMENTAL_GROUPS:.spec.securityContext.supplementalGroups," + + "IMAGE:.spec.containers[0].image," + + "TYPE:.metadata.labels.canfar-net-sessionType,STATUS:.status.phase," + + "NAME:.metadata.labels.canfar-net-sessionName,STARTED:.status.startTime," + + "DELETION:.metadata.deletionTimestamp,APP_ID:.metadata.labels.canfar-net-appID", + command); } } diff --git a/skaha/src/test/java/org/opencadc/skaha/session/SessionJobBuilderTest.java b/skaha/src/test/java/org/opencadc/skaha/session/SessionJobBuilderTest.java index 0b415cc0..a774ae39 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/SessionJobBuilderTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/SessionJobBuilderTest.java @@ -20,12 +20,17 @@ public class SessionJobBuilderTest { @Test public void testParsing() throws Exception { - final Path testBaseValuesPath = FileUtil.getFileFromResource("test-base-values.yaml", SessionJobBuilderTest.class).toPath(); + final Path testBaseValuesPath = FileUtil.getFileFromResource( + "test-base-values.yaml", SessionJobBuilderTest.class) + .toPath(); final String fileContent = Files.readString(testBaseValuesPath); final Map parametersToReplaceValues = new HashMap<>(); final String[] parametersToReplace = new String[] { - PostAction.SKAHA_SESSIONID, PostAction.SKAHA_HOSTNAME, PostAction.SKAHA_SESSIONTYPE, PostAction.SKAHA_POSIXID + PostAction.SKAHA_SESSIONID, + PostAction.SKAHA_HOSTNAME, + PostAction.SKAHA_SESSIONTYPE, + PostAction.SKAHA_POSIXID }; for (final String param : parametersToReplace) { @@ -33,7 +38,8 @@ public void testParsing() throws Exception { parametersToReplaceValues.put(param, RandomStringUtils.randomAlphanumeric(12)); } - SessionJobBuilder testSubject = SessionJobBuilder.fromPath(testBaseValuesPath).withParameters(parametersToReplaceValues); + SessionJobBuilder testSubject = + SessionJobBuilder.fromPath(testBaseValuesPath).withParameters(parametersToReplaceValues); final String output = testSubject.build(); for (final Map.Entry entry : parametersToReplaceValues.entrySet()) { @@ -44,13 +50,13 @@ public void testParsing() throws Exception { @Test public void testWithAffinityMerging() throws Exception { - final Path testBaseValuesPath = FileUtil.getFileFromResource("test-base-values-affinity.yaml", SessionJobBuilderTest.class).toPath(); + final Path testBaseValuesPath = FileUtil.getFileFromResource( + "test-base-values-affinity.yaml", SessionJobBuilderTest.class) + .toPath(); final String fileContent = Files.readString(testBaseValuesPath); final Map parametersToReplaceValues = new HashMap<>(); - final String[] parametersToReplace = new String[] { - PostAction.SKAHA_SESSIONID - }; + final String[] parametersToReplace = new String[] {PostAction.SKAHA_SESSIONID}; for (final String param : parametersToReplace) { Assert.assertTrue("Test file is missing required field.", fileContent.contains(param)); @@ -58,10 +64,10 @@ public void testWithAffinityMerging() throws Exception { } final SessionJobBuilder testSubject = SessionJobBuilder.fromPath(testBaseValuesPath) - .withGPUEnabled(true) - .withParameters(parametersToReplaceValues) - .withImageSecret("my-secret") - .withGPUCount(2); + .withGPUEnabled(true) + .withParameters(parametersToReplaceValues) + .withImageSecret("my-secret") + .withGPUCount(2); final String output = testSubject.build(); for (final Map.Entry entry : parametersToReplaceValues.entrySet()) { @@ -74,14 +80,20 @@ public void testWithAffinityMerging() throws Exception { final V1NodeAffinity nodeAffinity = podSpec.getAffinity().getNodeAffinity(); final List testMatchExpressions = new ArrayList<>(); - final List matchExpressions = - nodeAffinity.getRequiredDuringSchedulingIgnoredDuringExecution().getNodeSelectorTerms().get(0).getMatchExpressions(); + final List matchExpressions = nodeAffinity + .getRequiredDuringSchedulingIgnoredDuringExecution() + .getNodeSelectorTerms() + .get(0) + .getMatchExpressions(); if (matchExpressions != null) { testMatchExpressions.addAll(matchExpressions); } - Assert.assertEquals("Wrong pull secret.", "my-secret", podSpec.getImagePullSecrets().get(0).getName()); + Assert.assertEquals( + "Wrong pull secret.", + "my-secret", + podSpec.getImagePullSecrets().get(0).getName()); final V1NodeSelectorRequirement gpuRequirement = new V1NodeSelectorRequirement(); gpuRequirement.setKey("nvidia.com/gpu.count"); @@ -93,6 +105,8 @@ public void testWithAffinityMerging() throws Exception { providedRequirement.setOperator("Exists"); Assert.assertTrue("Missing GPU required match expression.", testMatchExpressions.contains(gpuRequirement)); - Assert.assertTrue("Missing provided (custom) required match expression.", testMatchExpressions.contains(providedRequirement)); + Assert.assertTrue( + "Missing provided (custom) required match expression.", + testMatchExpressions.contains(providedRequirement)); } } diff --git a/skaha/src/test/java/org/opencadc/skaha/session/TestPosixMapperClient.java b/skaha/src/test/java/org/opencadc/skaha/session/TestPosixMapperClient.java index 282cbc56..59d24643 100644 --- a/skaha/src/test/java/org/opencadc/skaha/session/TestPosixMapperClient.java +++ b/skaha/src/test/java/org/opencadc/skaha/session/TestPosixMapperClient.java @@ -69,10 +69,9 @@ package org.opencadc.skaha.session; import ca.nrc.cadc.net.ResourceNotFoundException; -import org.opencadc.auth.PosixMapperClient; - import java.io.IOException; import java.net.URI; +import org.opencadc.auth.PosixMapperClient; public class TestPosixMapperClient extends PosixMapperClient { public TestPosixMapperClient() throws IOException, ResourceNotFoundException { diff --git a/skaha/src/test/java/org/opencadc/skaha/utils/CommandExecutionerTest.java b/skaha/src/test/java/org/opencadc/skaha/utils/CommandExecutionerTest.java index a614a36d..7447e0f3 100644 --- a/skaha/src/test/java/org/opencadc/skaha/utils/CommandExecutionerTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/utils/CommandExecutionerTest.java @@ -17,8 +17,10 @@ public void testGetDeleteSecretCommand() { } final String[] deleteCommand = CommandExecutioner.getDeleteSecretCommand("mysecret"); - Assert.assertArrayEquals("Wrong delete command.", new String[] { "kubectl", "--namespace", K8SUtil.getWorkloadNamespace(), "delete", "secret", - "mysecret" }, deleteCommand); + Assert.assertArrayEquals( + "Wrong delete command.", + new String[] {"kubectl", "--namespace", K8SUtil.getWorkloadNamespace(), "delete", "secret", "mysecret"}, + deleteCommand); } @Test @@ -29,8 +31,8 @@ public void testGetRegistryCreateSecretCommand() { Assert.assertEquals("registryAuth is required.", e.getMessage()); } - final ImageRegistryAuth imageRegistryAuth = ImageRegistryAuth.fromEncoded(new String(Base64.getEncoder().encode("username:password".getBytes())), - "host"); + final ImageRegistryAuth imageRegistryAuth = ImageRegistryAuth.fromEncoded( + new String(Base64.getEncoder().encode("username:password".getBytes())), "host"); try { CommandExecutioner.getRegistryCreateSecretCommand(imageRegistryAuth, ""); diff --git a/skaha/src/test/java/org/opencadc/skaha/utils/RedisCacheTest.java b/skaha/src/test/java/org/opencadc/skaha/utils/RedisCacheTest.java index 18be6542..1595aa7d 100644 --- a/skaha/src/test/java/org/opencadc/skaha/utils/RedisCacheTest.java +++ b/skaha/src/test/java/org/opencadc/skaha/utils/RedisCacheTest.java @@ -1,6 +1,14 @@ package org.opencadc.skaha.utils; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; +import static org.opencadc.skaha.utils.TestUtils.set; + import com.google.gson.Gson; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -9,16 +17,6 @@ import org.opencadc.skaha.image.Image; import redis.clients.jedis.Jedis; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.mockito.Mockito.when; -import static org.opencadc.skaha.utils.TestUtils.set; - @RunWith(MockitoJUnitRunner.class) public class RedisCacheTest { private final RedisCache redisCache = new RedisCache(); @@ -27,7 +25,6 @@ public class RedisCacheTest { Gson gson = new Gson(); - private static final String OK = "OK"; @Before @@ -36,7 +33,6 @@ public void setUp() throws Exception { set(redisCache, "jedis", jedis); } - @Test public void testGetAllWithRange() { // Arrange @@ -91,11 +87,13 @@ public void testGetAllWithWrongTypeClass() { long start = 0; long stop = -1; - Map image = new HashMap<>() {{ - put("id-", "id"); - put("types-", Set.of("type1", "type2")); - put("digest-", "digest"); - }}; + Map image = new HashMap<>() { + { + put("id-", "id"); + put("types-", Set.of("type1", "type2")); + put("digest-", "digest"); + } + }; List expectedList = List.of(); when(jedis.lrange(key, start, stop)).thenReturn(List.of(gson.toJson(image))); @@ -103,4 +101,4 @@ public void testGetAllWithWrongTypeClass() { List result = redisCache.getAll(key, Image.class); assertEquals(expectedList, result); } -} \ No newline at end of file +} From 514670e3135d07785937b09a08b2b8c3a825281c Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 11:01:36 -0800 Subject: [PATCH 09/12] fix(github-actions): fix for CI:Testing upload artifacts --- .github/workflows/ci.testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.testing.yml b/.github/workflows/ci.testing.yml index e089db63..bec8b8d6 100644 --- a/.github/workflows/ci.testing.yml +++ b/.github/workflows/ci.testing.yml @@ -30,7 +30,7 @@ jobs: uses: actions/upload-artifact@v4.4.3 with: name: skaha-unittests-coverage - path: build/reports/jacoco/test/jacocoTestReport.xml + path: skaha/build/reports/jacoco/test/jacocoTestReport.xml if-no-files-found: error retention-days: 1 overwrite: true From e5dc3994233bd714bff3eed2fefff61e48fc83e4 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 11:20:05 -0800 Subject: [PATCH 10/12] fix(github-actions): fix for download artifact section --- .github/workflows/ci.testing.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.testing.yml b/.github/workflows/ci.testing.yml index bec8b8d6..8b3fc27c 100644 --- a/.github/workflows/ci.testing.yml +++ b/.github/workflows/ci.testing.yml @@ -44,7 +44,11 @@ jobs: uses: actions/download-artifact@v4.1.8 with: name: skaha-unittests-coverage - path: jacocoTestReport.xml + - + name: List Downloaded Artifacts + run: | + echo "Downloaded artifacts:" + ls -lah $GITHUB_WORKSPACE - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 54ed5ff370361f2b83a28655b976d94c4df7b9bc Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 11:47:23 -0800 Subject: [PATCH 11/12] fix(codecov): updated codecov action and added verbose logging --- .github/workflows/ci.testing.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.testing.yml b/.github/workflows/ci.testing.yml index 8b3fc27c..0d224a16 100644 --- a/.github/workflows/ci.testing.yml +++ b/.github/workflows/ci.testing.yml @@ -38,6 +38,8 @@ jobs: codecov: runs-on: ubuntu-latest needs: tests + permissions: + id-token: write steps: - name: Download coverage artifacts @@ -51,10 +53,11 @@ jobs: ls -lah $GITHUB_WORKSPACE - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4.6.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: jacocoTestReport.xml flags: skaha-unittests-coverage name: skaha-unittests-coverage - fail_ci_if_error: true \ No newline at end of file + fail_ci_if_error: true + verbose: true \ No newline at end of file From 3488c58b67efa3b501f5c93f2e57ca001c994f73 Mon Sep 17 00:00:00 2001 From: Shiny Brar Date: Fri, 8 Nov 2024 12:34:01 -0800 Subject: [PATCH 12/12] fix(opencadc.gradle): fixed configurations needed for integration tests --- skaha/opencadc.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skaha/opencadc.gradle b/skaha/opencadc.gradle index e70af818..64a46442 100644 --- a/skaha/opencadc.gradle +++ b/skaha/opencadc.gradle @@ -1,5 +1,7 @@ configurations { + // Configuration for integration tests intTestRuntime.extendsFrom testRuntime + intTestImplementation.extendsFrom testImplementation } sourceSets {