Added forced close of scheduler when connection seems lost

xenon-middleware · Mar 26, 2020 · 5a38f2e · 5a38f2e
1 parent b2b727b
commit 5a38f2e
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/src/main/java/nl/esciencecenter/computeservice/service/XenonMonitor.java b/src/main/java/nl/esciencecenter/computeservice/service/XenonMonitor.java
@@ -142,6 +142,10 @@ private void updateRunningJobs(Scheduler scheduler) {
                         jobLogger.error("Exception while retrieving status of job (scheduler connection lost?)", status.getException());
                         logger.error("Exception while retrieving status of job " + job.getName() + "(" + job.getId() + ") -- scheduler connection lost?",
                                 status.getException());
+
+                        // Force the scheduler to close when we seem to lose the connection
+                        logger.error("Forcing the scheduler to close");
+                        xenonService.closeScheduler();
                     } else if (status.getState().equals("UNKNOWN")) {
                         // We seem to have lost the job completely? -- try to recover the output
                         jobLogger.error("Could not find job ", status.getException());
@@ -196,6 +200,9 @@ private void updateWaitingJobs(Scheduler scheduler) {
                         jobLogger.error("Exception while retrieving status of job (scheduler connection lost?)", status.getException());
                         logger.error("Exception while retrieving status of job " + job.getName() + "(" + job.getId() + ") -- scheduler connection lost?",
                                 status.getException());
+
+                        logger.error("Forcing the scheduler to close");
+                        xenonService.closeScheduler();
                     } else if (status.getState().equals("UNKNOWN")) {
                         // We seem to have lost the job completely? -- try to recover the output
                         jobLogger.error("Could not find job ", status.getException());
@@ -249,6 +256,7 @@ private void cancelRunningJobs(Scheduler scheduler) {
                         status = scheduler.cancelJob(job.getXenonId());
                         jobService.setXenonState(job.getId(), status.getState());
                     } else {
+                        // FIXME: we may leak jobs here if the connection to the machine is lost!
                         if (status.hasException() && !(status.getException() instanceof JobCanceledException)) {
                             jobService.setXenonExitcode(job.getId(), status.getExitCode());
                             jobService.setErrorAndState(job.getId(), status.getException(), JobState.RUNNING_CR, JobState.PERMANENT_FAILURE);
@@ -300,6 +308,7 @@ private void cancelWaitingJobs(Scheduler scheduler) {
                         logger.debug("Cancelled job: " + job.getId() + " new status: " + status);
                     }
 
+                    // FIXME: we may leak jobs here if the connection to the machine is lost?
                     jobService.setJobState(job.getId(), JobState.WAITING_CR, JobState.CANCELLED);
                 }
             } catch (NoSuchJobException e) {

diff --git a/src/main/java/nl/esciencecenter/computeservice/service/XenonService.java b/src/main/java/nl/esciencecenter/computeservice/service/XenonService.java
@@ -149,10 +149,17 @@ public Scheduler getScheduler() throws XenonException {
         return scheduler;
     }
 
-    public Scheduler forceNewScheduler() throws XenonException {
+    public void closeScheduler() throws XenonException {
         if (scheduler != null) {
+            if (scheduler.isOpen()) {
+                scheduler.close();
+            }
             scheduler = null;
         }
+    }
+
+    public Scheduler forceNewScheduler() throws XenonException {
+        closeScheduler();
         return getScheduler();
     }