diff --git a/deployment/helm/skaha/desktop-template/start-software-sh.template b/deployment/helm/skaha/desktop-template/start-software-sh.template index 259735cf..ff357dea 100644 --- a/deployment/helm/skaha/desktop-template/start-software-sh.template +++ b/deployment/helm/skaha/desktop-template/start-software-sh.template @@ -3,7 +3,6 @@ HOST=(HOST) # Callback token TOKEN="${DESKTOP_SESSION_APP_TOKEN}" -CURL_APP_SESSION_HEADERS="--header \"x-auth-token-skaha: ${TOKEN}\"" handle_error() { echo "$1" @@ -19,10 +18,10 @@ get_resource_options() { else resources=`curl -s -L -k --header "x-auth-token-skaha: ${TOKEN}" https://(HOST)/skaha/(SKAHA_API_VERSION)/context` fi - core_default=`echo $resources | jq .defaultCores` - core_options=`echo $resources | jq .availableCores[] | tr '\n' ' '` - ram_default=`echo $resources | jq .defaultRAM` - ram_options=`echo $resources | jq .availableRAM[] | tr '\n' ' '` + core_default=`echo $resources | jq .cores.default` + core_options=`echo $resources | jq .cores.options[] | tr '\n' ' '` + ram_default=`echo $resources | jq .memoryGB.default` + ram_options=`echo $resources | jq .memoryGB.options[] | tr '\n' ' '` } get_cores() { @@ -90,7 +89,7 @@ prompt_user() { if [ -z "${TOKEN}" ]; then handle_error "[skaha] No credentials to call back to Skaha with." else - app_id=`curl -s -L -k "${CURL_APP_SESSION_HEADERS}" -d "image=(IMAGE_ID)" --data-urlencode "param=(NAME)" https://(HOST)/skaha/(SKAHA_API_VERSION)/session/${VNC_PW}/app` + app_id=`curl -s -L -k "x-auth-token-skaha: ${TOKEN}" -d "image=(IMAGE_ID)" --data-urlencode "param=(NAME)" https://(HOST)/skaha/(SKAHA_API_VERSION)/session/${VNC_PW}/app` fi break elif [[ ${yn} == "y" || ${yn} == "Y" ]]; then @@ -102,7 +101,7 @@ prompt_user() { if [ -z "${TOKEN}" ]; then handle_error "[skaha] No credentials to call back to Skaha with." else - app_id=`curl -s -L -k "${CURL_APP_SESSION_HEADERS}" -d "cores=${cores}" -d "ram=$ram" -d "image=(IMAGE_ID)" --data-urlencode "param=(NAME)" https://(HOST)/skaha/(SKAHA_API_VERSION)/session/${VNC_PW}/app` + app_id=`curl -s -L -k "x-auth-token-skaha: ${TOKEN}" -d "cores=${cores}" -d "ram=$ram" -d "image=(IMAGE_ID)" --data-urlencode "param=(NAME)" https://(HOST)/skaha/(SKAHA_API_VERSION)/session/${VNC_PW}/app` fi break else diff --git a/deployment/helm/skaha/launch-scripts/build-menu.sh b/deployment/helm/skaha/launch-scripts/build-menu.sh index d3ec129f..d86169ee 100644 --- a/deployment/helm/skaha/launch-scripts/build-menu.sh +++ b/deployment/helm/skaha/launch-scripts/build-menu.sh @@ -228,7 +228,7 @@ build_menu_item () { echo "[skaha] Start building menu." init create_merged_applications_menu -curl_out=$(curl -s -k --header "x-auth-token-skaha:${TOKEN}" "https://${HOST}/skaha/${SKAHA_API_VERSION}/image?type=desktop-app") +curl_out=$(curl -s -k --header "x-auth-token-skaha: ${TOKEN}" "https://${HOST}/skaha/${SKAHA_API_VERSION}/image?type=desktop-app") if [[ $(echo ${curl_out} | jq '[.[] | .id | length] | add') == 0 ]]; then echo "[skaha] no desktop-app" echo "${curl_out}" diff --git a/deployment/helm/skaha/skaha-config/k8s-resources.properties b/deployment/helm/skaha/skaha-config/k8s-resources.properties deleted file mode 100644 index a7a75c92..00000000 --- a/deployment/helm/skaha/skaha-config/k8s-resources.properties +++ /dev/null @@ -1,36 +0,0 @@ -# Defines the resources available to science containers -### - -# Default number of requested cores when not specified -cores-default-request = 1 - -# Default maximum number of cores when not specified -cores-default-limit = 16 - -# Default number of cores (request=limit) when not specified -cores-default = 2 - -# Default cores for headless jobs -cores-default-headless = 1 - -# Default requested memory (RAM) in GB when not specified -mem-gb-default-request = 4 - -# Default maximum memory (RAM) in GB when not specified -mem-gb-default-limit = 192 - -# Default memory (RAM) in GB (request=limit) when not specified -mem-gb-default = 8 - -# Default RAM for headless jobs -mem-gb-default-headless = 4 - -# Other options for cores -# TODO: Make this a range? -cores-options = 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 - -# Other options for memory (RAM) in GB -mem-gb-options = 1 2 4 6 8 10 12 14 16 20 24 26 28 30 32 36 40 44 48 56 64 80 92 112 128 140 170 192 - - # GPU options -gpus-options = 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 diff --git a/deployment/helm/skaha/templates/skaha-config-configmap.yaml b/deployment/helm/skaha/templates/skaha-config-configmap.yaml index 6087e7ab..66a7f803 100644 --- a/deployment/helm/skaha/templates/skaha-config-configmap.yaml +++ b/deployment/helm/skaha/templates/skaha-config-configmap.yaml @@ -10,6 +10,6 @@ data: {{ base $path }}: | {{- tpl ($.Files.Get $path) $currContext | nindent 4 }} {{ end }} -{{ ($.Files.Glob "skaha-config/*.properties").AsConfig | indent 2 }} +{{ ($.Files.Glob "skaha-config/*.json").AsConfig | indent 2 }} {{- include "utils.extraConfig" (dict "extraConfigData" .Values.deployment.skaha.extraConfigData) -}} {{- (.Files.Glob "image-cache/*").AsConfig | nindent 2 }} diff --git a/image-cache/README.md b/image-cache/README.md new file mode 100644 index 00000000..95a85b85 --- /dev/null +++ b/image-cache/README.md @@ -0,0 +1,7 @@ +# Redis cache client for Image Listings + +This builds a simple image with some formatting tools to support the image caching feature in Skaha. This image acts as a client +to a running Redis cache. See the [cache-images.sh script](https://github.com/opencadc/science-platform/blob/main/deployment/helm/skaha/image-cache/cache-images.sh) in +Skaha, which is run from _within_ this Image. + +See also the [`CronJob` and initialization `Job`](https://github.com/opencadc/science-platform/blob/main/deployment/helm/skaha/templates/image-caching-cronjob.yaml) on how this image is used from a Skaha deployment. \ No newline at end of file diff --git a/image-cache/VERSION b/image-cache/VERSION new file mode 100644 index 00000000..954e1460 --- /dev/null +++ b/image-cache/VERSION @@ -0,0 +1,6 @@ +## deployable containers have a semantic and build tag +# version tag: major.minor.patch +# build version tag: timestamp +VER=0.1.0 +TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" +unset VER diff --git a/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java b/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java index dd1b95b8..03d82b5f 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/DesktopAppLifecycleTest.java @@ -68,19 +68,12 @@ package org.opencadc.skaha; import ca.nrc.cadc.auth.AuthMethod; -import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.Log4jInit; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Type; import java.net.URL; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; import javax.security.auth.Subject; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -122,11 +115,12 @@ public DesktopAppLifecycleTest() { public void testCreateDeleteDesktopApp() throws Exception { Subject.doAs(userSubject, (PrivilegedExceptionAction) () -> { // ensure that there is no active session - initialize(); + SessionUtil.initializeCleanup(this.sessionURL); // create desktop session final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId()); + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), + SessionAction.SESSION_TYPE_DESKTOP); final Session desktopSession = SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); @@ -175,21 +169,4 @@ public void testCreateDeleteDesktopApp() throws Exception { return null; }); } - - private void initialize() throws Exception { - List sessions = SessionUtil.getSessions(this.sessionURL); - for (Session session : sessions) { - if (session.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { - // delete desktop-app - String sessionID = session.getId(); - final URL desktopAppURL = new URL(sessionURL.toString() + "/" + sessionID + "/app"); - SessionUtil.deleteDesktopApplicationSession(desktopAppURL, session.getAppId()); - } else { - // delete session - SessionUtil.deleteSession(sessionURL, session.getId()); - } - } - sessions = SessionUtil.getSessions(this.sessionURL); - Assert.assertEquals("zero sessions #1", 0, sessions.size()); - } } diff --git a/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java b/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java index 58dc5c76..3b9f883a 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/ExpiryTimeRenewalTest.java @@ -73,7 +73,6 @@ import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.Log4jInit; - import java.net.MalformedURLException; import java.net.URL; import java.security.PrivilegedExceptionAction; @@ -82,10 +81,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import java.util.concurrent.TimeUnit; import javax.security.auth.Subject; - import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; @@ -98,12 +95,9 @@ */ public class ExpiryTimeRenewalTest { - private static final Logger log = Logger.getLogger(ExpiryTimeRenewalTest.class); - private static final String HOST_PROPERTY = RegistryClient.class.getName() + ".host"; public static final String CARTA_IMAGE_SUFFIX = "/skaha/carta:3.0"; - public static final String PROD_IMAGE_HOST = "images.canfar.net"; - public static final String DEV_IMAGE_HOST = "images-rc.canfar.net"; public static final int SLEEP_TIME_SECONDS = 5; + private static final Logger log = Logger.getLogger(ExpiryTimeRenewalTest.class); static { Log4jInit.setLevel("org.opencadc.skaha", Level.INFO); @@ -111,22 +105,8 @@ public class ExpiryTimeRenewalTest { protected final URL sessionURL; protected final Subject userSubject; - protected final String imageHost; public ExpiryTimeRenewalTest() throws Exception { - // determine image host - String hostP = System.getProperty(HOST_PROPERTY); - if (hostP == null || hostP.trim().isEmpty()) { - throw new IllegalArgumentException("missing server host, check " + HOST_PROPERTY); - } else { - hostP = hostP.trim(); - if (hostP.startsWith("rc-")) { - imageHost = DEV_IMAGE_HOST; - } else { - imageHost = PROD_IMAGE_HOST; - } - } - RegistryClient regClient = new RegistryClient(); final URL sessionServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); this.sessionURL = new URL(sessionServiceURL.toString() + "/session"); @@ -140,11 +120,12 @@ public ExpiryTimeRenewalTest() throws Exception { public void testRenewCARTA() throws Exception { Subject.doAs(userSubject, (PrivilegedExceptionAction) () -> { // ensure that there is no active session - initialize(); + SessionUtil.initializeCleanup(this.sessionURL); // create carta session final String cartaSessionID = SessionUtil.createSession(this.sessionURL, "inttest-" + SessionAction.SESSION_TYPE_CARTA, - imageHost + CARTA_IMAGE_SUFFIX); + SessionUtil.getImageByName(ExpiryTimeRenewalTest.CARTA_IMAGE_SUFFIX).getId(), + SessionAction.SESSION_TYPE_CARTA); Session cartaSession = SessionUtil.waitForSession(this.sessionURL, cartaSessionID, Session.STATUS_RUNNING); // Sleep to force time to pass before renewal @@ -181,11 +162,11 @@ public void testRenewHeadless() throws Exception { Subject.doAs(userSubject, (PrivilegedExceptionAction) () -> { // ensure that there is no active session - initialize(); + SessionUtil.initializeCleanup(this.sessionURL); // create headless session final String headlessSessionID = SessionUtil.createHeadlessSession( - SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), this.sessionURL); + SessionUtil.getDesktopAppImageOfType("/skaha/terminal").getId(), this.sessionURL); Session headlessSession = SessionUtil.waitForSession(this.sessionURL, headlessSessionID, Session.STATUS_RUNNING); final Instant headlessExpiryTime = Instant.parse(headlessSession.getExpiryTime()); @@ -210,11 +191,12 @@ public void testRenewHeadless() throws Exception { public void testRenewDesktop() throws Exception { Subject.doAs(userSubject, (PrivilegedExceptionAction) () -> { // ensure that there is no active session - initialize(); + SessionUtil.initializeCleanup(this.sessionURL); // create desktop session final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest-" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId()); + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), + SessionAction.SESSION_TYPE_DESKTOP); SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); // create desktop app @@ -246,39 +228,10 @@ public void testRenewDesktop() throws Exception { }); } - private void initialize() throws Exception { - List sessions = getSessions(); - for (Session session : sessions) { - // skip dekstop-app, deletion of desktop-app is not supported - if (!session.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { - deleteSession(sessionURL, session.getId()); - } - } - - int count = 0; - sessions = getSessions(); - for (Session s : sessions) { - if (!s.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { - count++; - } - } - Assert.assertEquals("zero sessions #1", 0, count); - } - - private void deleteSession(URL sessionURL, String sessionID) throws MalformedURLException { - HttpDelete delete = new HttpDelete(new URL(sessionURL.toString() + "/" + sessionID), true); - delete.run(); - Assert.assertNull("delete session error", delete.getThrowable()); - } - private void renewSession(URL sessionURL, String sessionID) throws Exception { Map params = new HashMap<>(); params.put("action", "renew"); HttpPost post = new HttpPost(new URL(sessionURL.toString() + "/" + sessionID), params, false); post.prepare(); } - - private List getSessions() throws Exception { - return SessionUtil.getSessions(this.sessionURL, Session.STATUS_TERMINATING); - } } diff --git a/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java b/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java index cb2b02f9..e1c22e40 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/SessionLifecycleTest.java @@ -73,7 +73,6 @@ import ca.nrc.cadc.util.Log4jInit; import java.net.URL; import java.security.PrivilegedExceptionAction; -import java.util.List; import javax.security.auth.Subject; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -87,10 +86,7 @@ */ public class SessionLifecycleTest { - public static final String PROD_IMAGE_HOST = "images.canfar.net"; - public static final String DEV_IMAGE_HOST = "images-rc.canfar.net"; private static final Logger log = Logger.getLogger(SessionLifecycleTest.class); - private static final String HOST_PROPERTY = RegistryClient.class.getName() + ".host"; static { Log4jInit.setLevel("org.opencadc.skaha", Level.INFO); @@ -98,22 +94,8 @@ public class SessionLifecycleTest { protected final URL sessionURL; protected final Subject userSubject; - protected final String imageHost; public SessionLifecycleTest() throws Exception { - // determine image host - String hostP = System.getProperty(HOST_PROPERTY); - if (hostP == null || hostP.trim().isEmpty()) { - throw new IllegalArgumentException("missing server host, check " + HOST_PROPERTY); - } else { - hostP = hostP.trim(); - if (hostP.startsWith("rc-")) { - imageHost = DEV_IMAGE_HOST; - } else { - imageHost = PROD_IMAGE_HOST; - } - } - RegistryClient regClient = new RegistryClient(); final URL sessionServiceURL = regClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); sessionURL = new URL(sessionServiceURL.toString() + "/session"); @@ -128,18 +110,20 @@ public void testCreateDeleteSessions() throws Exception { Subject.doAs(userSubject, (PrivilegedExceptionAction) () -> { // ensure that there is no active session - initialize(); + SessionUtil.initializeCleanup(this.sessionURL); // create desktop session final String desktopSessionID = SessionUtil.createSession(this.sessionURL, "inttest" + SessionAction.SESSION_TYPE_DESKTOP, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId()); + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_DESKTOP).getId(), + SessionAction.SESSION_TYPE_DESKTOP); final Session desktopSession = SessionUtil.waitForSession(this.sessionURL, desktopSessionID, Session.STATUS_RUNNING); SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_DESKTOP, "inttest" + SessionAction.SESSION_TYPE_DESKTOP); // create carta session final String cartaSessionID = SessionUtil.createSession(sessionURL, "inttest" + SessionAction.SESSION_TYPE_CARTA, - SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_CARTA).getId()); + SessionUtil.getImageOfType(SessionAction.SESSION_TYPE_CARTA).getId(), + SessionAction.SESSION_TYPE_CARTA); Session cartaSession = SessionUtil.waitForSession(sessionURL, cartaSessionID, Session.STATUS_RUNNING); SessionUtil.verifySession(desktopSession, SessionAction.SESSION_TYPE_CARTA, "inttest" + SessionAction.SESSION_TYPE_CARTA); @@ -166,27 +150,4 @@ public void testCreateDeleteSessions() throws Exception { return null; }); } - - private void initialize() throws Exception { - List sessions = getSessions(); - for (Session session : sessions) { - // skip dekstop-app, deletion of desktop-app is not supported - if (!session.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { - SessionUtil.deleteSession(sessionURL, session.getId()); - } - } - - int count = 0; - sessions = getSessions(); - for (Session s : sessions) { - if (!s.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { - count++; - } - } - Assert.assertEquals("zero sessions #1", 0, count); - } - - private List getSessions() throws Exception { - return SessionUtil.getSessions(sessionURL, Session.STATUS_TERMINATING, Session.STATUS_SUCCEEDED); - } } diff --git a/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java b/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java index 31e369b0..0a4d8e81 100644 --- a/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java +++ b/skaha/src/intTest/java/org/opencadc/skaha/SessionUtil.java @@ -87,49 +87,71 @@ import ca.nrc.cadc.uws.server.RandomStringGenerator; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; -import java.net.PasswordAuthentication; -import java.nio.file.Files; -import java.util.MissingResourceException; -import java.util.concurrent.TimeoutException; -import javax.security.auth.Subject; -import org.apache.log4j.Logger; -import org.junit.Assert; -import org.opencadc.skaha.image.Image; -import org.opencadc.skaha.session.Session; -import org.opencadc.skaha.session.SessionAction; - -import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.StringWriter; import java.io.Writer; import java.lang.reflect.Type; +import java.net.PasswordAuthentication; import java.net.URI; import java.net.URL; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.MissingResourceException; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import javax.security.auth.Subject; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.opencadc.skaha.image.Image; +import org.opencadc.skaha.session.Session; +import org.opencadc.skaha.session.SessionAction; public class SessionUtil { public static final URI SKAHA_SERVICE_ID = URI.create("ivo://cadc.nrc.ca/skaha"); private static final Logger LOGGER = Logger.getLogger(SessionUtil.class); private static final long ONE_SECOND = 1000L; - private static final long TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS = 45L * SessionUtil.ONE_SECOND; - private static final long TIMEOUT_WAIT_FOR_SESSION_TERMINATE_MS = 40L * SessionUtil.ONE_SECOND; + private static final long TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS = 120L * SessionUtil.ONE_SECOND; + private static final long TIMEOUT_WAIT_FOR_SESSION_TERMINATE_MS = 120L * SessionUtil.ONE_SECOND; static URI getSkahaServiceID() { final String configuredServiceID = System.getenv("SKAHA_SERVICE_ID"); return StringUtil.hasText(configuredServiceID) ? URI.create(configuredServiceID) : SessionUtil.SKAHA_SERVICE_ID; } + static void initializeCleanup(final URL sessionURL) throws Exception { + for (Session session : SessionUtil.getSessions(sessionURL, Session.STATUS_TERMINATING, Session.STATUS_SUCCEEDED)) { + if (session.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { + // delete desktop-app + String sessionID = session.getId(); + final URL desktopAppURL = new URL(sessionURL.toString() + "/" + sessionID + "/app"); + SessionUtil.deleteDesktopApplicationSession(desktopAppURL, session.getAppId()); + } else { + // delete session + SessionUtil.deleteSession(sessionURL, session.getId()); + } + } + + int count = 0; + for (Session s : SessionUtil.getSessions(sessionURL, Session.STATUS_TERMINATING, Session.STATUS_SUCCEEDED)) { + if (!s.getType().equals(SessionAction.TYPE_DESKTOP_APP)) { + count++; + } + } + Assert.assertEquals("zero sessions #1", 0, count); + } + /** * Read in the current user's credentials from the local path. - * @param sessionURL The current URL to use to deduce a domain. - * @return Subject instance, never null. + * + * @param sessionURL The current URL to use to deduce a domain. + * @return Subject instance, never null. */ static Subject getCurrentUser(final URL sessionURL, final boolean allowAnonymous) throws Exception { final Subject subject = new Subject(); @@ -155,8 +177,8 @@ static Subject getCurrentUser(final URL sessionURL, final boolean allowAnonymous final RegistryClient registryClient = new RegistryClient(); URL newLoginURL = registryClient.getServiceURL(URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_10, AuthMethod.ANON); final URL loginURL = newLoginURL == null - ? registryClient.getServiceURL(URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_01, AuthMethod.ANON) - : newLoginURL; + ? registryClient.getServiceURL(URI.create("ivo://cadc.nrc.ca/gms"), Standards.UMS_LOGIN_01, AuthMethod.ANON) + : newLoginURL; final NetrcFile netrcFile = new NetrcFile(); final PasswordAuthentication passwordAuthentication = netrcFile.getCredentials(loginURL.getHost(), true); final Map loginPayload = new HashMap<>(); @@ -194,17 +216,19 @@ private static X509CertificateChain getProxyCertificate() throws Exception { /** * Create a Session and return the Session ID. Call waitForSession() after to obtain the Session object. - * @param sessionURL The Session URL to use. - * @param name The name of the Session. - * @param image The image URI to use. - * @return String session ID, never null. + * + * @param sessionURL The Session URL to use. + * @param name The name of the Session. + * @param image The image URI to use. + * @return String session ID, never null. */ - static String createSession(final URL sessionURL, final String name, String image) { + static String createSession(final URL sessionURL, final String name, String image, String type) { final Map params = new HashMap<>(); params.put("name", name); params.put("image", image); params.put("cores", 1); params.put("ram", 1); + params.put("type", type); final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); final HttpPost post = new HttpPost(sessionURL, params, outputStream); @@ -216,39 +240,6 @@ static String createSession(final URL sessionURL, final String name, String imag return outputStream.toString().trim(); } - /** - * Start a session and return the ID. - * - * @param image The Image to spin up. - * @param sessionURL The base URL for sessions - * @return String sessionID. - * @throws Exception For any badness. - */ - protected static Session createSession(final String image, final URL sessionURL) throws Exception { - final Map params = new HashMap<>(); - final String name = new RandomStringGenerator(16).getID(); - params.put("name", name); - params.put("image", image); - params.put("cores", 1); - params.put("ram", 1); - - final HttpPost post = new HttpPost(sessionURL, params, false); - post.prepare(); - - final String sessionID; - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(post.getInputStream()))) { - sessionID = reader.readLine(); - } - - final HttpGet httpGet = new HttpGet(new URL(sessionURL.toExternalForm() + "/" + sessionID), true); - httpGet.prepare(); - - final Gson gson = new Gson(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(httpGet.getInputStream()))) { - return gson.fromJson(reader, Session.class); - } - } - static String createHeadlessSession(final String image, final URL sessionURL) { final Map params = new HashMap<>(); final String name = new RandomStringGenerator(16).getID(); @@ -274,7 +265,7 @@ static String createDesktopAppSession(final String image, final URL desktopSessi return SessionUtil.createDesktopAppSession(image, desktopSessionURL, 1, 1); } - static String createDesktopAppSession(final String image, final URL desktopSessionURL, final int cores, final int ram) throws Exception { + static String createDesktopAppSession(final String image, final URL desktopSessionURL, final int cores, final int ram) { final Map params = new HashMap<>(); final String name = new RandomStringGenerator(16).getID(); @@ -282,9 +273,12 @@ static String createDesktopAppSession(final String image, final URL desktopSessi params.put("image", image); params.put("cores", cores); params.put("ram", ram); + params.put("type", SessionAction.TYPE_DESKTOP_APP); params.put("cmd", "sleep"); params.put("args", "260"); + LOGGER.info("Creating desktop app session with image " + image + " and name " + name); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); final HttpPost post = new HttpPost(desktopSessionURL, params, outputStream); post.setFollowRedirects(false); @@ -305,7 +299,8 @@ static Session waitForDesktopApplicationSession(final URL desktopApplicationURL, currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS) { - throw new TimeoutException("Timed out waiting for Desktop Application session " + desktopAppID + " and status " + expectedState); + throw new TimeoutException("Timed out waiting for Desktop Application session " + desktopAppID + " and status " + expectedState + " after " + + currentWaitTime + "ms"); } requestedSession = SessionUtil.getDesktopApplicationSessionWithoutWait(desktopApplicationURL, desktopAppID, expectedState); @@ -377,9 +372,9 @@ static List getAllDesktopApplicationSessions(final URL desktopAppURL) { private static Session getSessionWithoutWait(final URL sessionURL, final String sessionID, final String expectedState) throws Exception { return SessionUtil.getAllSessions(sessionURL).stream() - .filter(session -> session.getId().equals(sessionID) && session.getStatus().equals(expectedState)) - .findFirst() - .orElse(null); + .filter(session -> session.getId().equals(sessionID) && session.getStatus().equals(expectedState)) + .findFirst() + .orElse(null); } static void waitForSessionToTerminate(final URL sessionURL, final String sessionID) throws Exception { @@ -391,7 +386,8 @@ static void waitForSessionToTerminate(final URL sessionURL, final String session currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_TERMINATE_MS) { - throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + Session.STATUS_TERMINATING); + throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + Session.STATUS_TERMINATING + " after " + + currentWaitTime + "ms"); } requestedSession = SessionUtil.getSessionWithoutWait(sessionURL, sessionID, Session.STATUS_TERMINATING); @@ -409,7 +405,8 @@ static Session waitForSession(final URL sessionURL, final String sessionID, fina currentWaitTime += SessionUtil.ONE_SECOND; if (currentWaitTime > SessionUtil.TIMEOUT_WAIT_FOR_SESSION_STARTUP_MS) { - throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + expectedState); + throw new TimeoutException("Timed out waiting for session " + sessionID + " and status " + expectedState + " after " + + currentWaitTime + "ms"); } requestedSession = SessionUtil.getSessionWithoutWait(sessionURL, sessionID, expectedState); @@ -452,6 +449,15 @@ private static List getAllSessions(final URL sessionURL) throws Excepti return gson.fromJson(json, listType); } + static Image getImageByName(final String imagePath) throws Exception { + final RegistryClient registryClient = new RegistryClient(); + final URL imageServiceURL = registryClient.getServiceURL(SessionUtil.getSkahaServiceID(), Standards.PROC_SESSIONS_10, AuthMethod.TOKEN); + final URL imageURL = new URL(imageServiceURL.toExternalForm() + "/image"); + + final List allImagesList = ImagesTest.getImages(imageURL); + return allImagesList.stream().filter(image -> image.getId().endsWith(imagePath)).findFirst().orElseThrow(); + } + protected static Image getImageOfType(final String type) throws Exception { return SessionUtil.getImagesOfType(type).stream().findFirst().orElseThrow(); } diff --git a/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java b/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java index 8bda5da2..ee33228d 100644 --- a/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java +++ b/skaha/src/main/java/org/opencadc/skaha/session/PostAction.java @@ -619,9 +619,8 @@ public void createSession(String type, String image, String name, Integer cores, .withParameter(PostAction.SOFTWARE_LIMITS_RAM, ram + "Gi") .withParameter(PostAction.SKAHA_TLD, this.skahaTld); - if (StringUtil.hasText(supplementalGroups)) { - sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, supplementalGroups); - } + sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) + ? supplementalGroups : ""); if (type.equals(SessionAction.SESSION_TYPE_DESKTOP)) { sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.DESKTOP_SESSION_APP_TOKEN, generateToken()); @@ -794,9 +793,8 @@ public void attachDesktopApp(String image, Integer requestCores, Integer limitCo .withParameter(PostAction.SOFTWARE_CONTAINERPARAM, param) .withParameter(PostAction.SKAHA_TLD, this.skahaTld); final String supplementalGroups = getSupplementalGroupsList(); - if (StringUtil.hasText(supplementalGroups)) { - sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, supplementalGroups); - } + sessionJobBuilder = sessionJobBuilder.withParameter(PostAction.SKAHA_SUPPLEMENTALGROUPS, StringUtil.hasText(supplementalGroups) + ? supplementalGroups : ""); String launchFile = super.stageFile(sessionJobBuilder.build()); String[] launchCmd = new String[] { diff --git a/skaha/src/main/webapp/service.yaml b/skaha/src/main/webapp/service.yaml index c3b8882d..5cbb57c4 100644 --- a/skaha/src/main/webapp/service.yaml +++ b/skaha/src/main/webapp/service.yaml @@ -1,6 +1,6 @@ swagger: '2.0' info: - version: 0.11.0 + version: 0.23.0 title: skaha description: | skaha API Documentation. This API allows authorized users to create and interact with desktop (NoVNC), CARTA Visualization, and Jupyter Notebook sessions in the skaha processing environment. Desktop apps can also be launched and attached to skaha desktop sessions through this API.

Clients may authenticate to this service by:
1. Providing a bearer token in the Authorization header.
2. Using a client certificate.
3. Using a browser cookie from CADC Login.

The main skaha github page with documentation and source code is here: https://github.com/opencadc/science-platform

Documentation on using Skaha can be found here: https://www.opencadc.org/science-containers/ @@ -100,6 +100,14 @@ paths: description: | Only applies to session type 'headless'. Add additional environment to the container. Format is key=value. Multiple env parameters supported. 'key=value' must be URL encoded, so, for example, PATH=/usr/local/bin should be supplied as PATH%3D%2Fusr%2Flocal%2Fbin. required: false + - name: x-skaha-registry-auth + in: header + type: string + description: | + If the image is not in a supported harbor registry, the x-registry-auth header must be provided. This requires additional privileges. The value is a base64 encoded string from "username:secret": + ``` + $ echo -n "username:my-registry-secret" | base64 -i - + ``` responses: '200': description: Successful response