Fix bug when scheduler disappears

If the scheduler disappeared, the program would get stuck in a loop trying to process messages. Instead, detect the disconnection and periodically try to re-discover the scheduler until it returns.
JPEWdev · Jun 8, 2018 · 8111abe · 8111abe
1 parent 6f11d79
commit 8111abe
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 9 deletions.
diff --git a/src/draw.cpp b/src/draw.cpp
@@ -47,12 +47,16 @@ class NCursesInterface: public UserInterface {
         return STDIN_FILENO;
     }
 
+    virtual void suspend() override;
+    virtual void resume() override;
+
     void print_job_graph(Job::Map const &jobs, int max_jobs) const;
 
 private:
     static gboolean on_idle_draw(gpointer user_data);
     static gboolean on_redraw_timer(gpointer user_data);
 
+    void init();
     void doRender();
     void doRedraw();
     int assign_color(int fg, int bg);
@@ -375,6 +379,19 @@ int NCursesInterface::processInput()
     return consumed ? 0 : c;
 }
 
+void NCursesInterface::suspend()
+{
+    clear();
+    refresh();
+    endwin();
+    redraw_source.clear();
+}
+
+void NCursesInterface::resume()
+{
+    init();
+}
+
 void NCursesInterface::print_job_graph(Job::Map const &jobs, int max_jobs) const
 {
     addch('[');
@@ -667,8 +684,7 @@ void NCursesInterface::triggerRedraw()
         idle_source.set(g_idle_add(reinterpret_cast<GSourceFunc>(on_idle_draw), this));
 }
 
-NCursesInterface::NCursesInterface() :
-    UserInterface()
+void NCursesInterface::init()
 {
     initscr();
 
@@ -681,6 +697,7 @@ NCursesInterface::NCursesInterface() :
     nodelay(stdscr, TRUE);
     keypad(stdscr, TRUE);
 
+    Host::clearColors();
     Host::addColor(assign_color(COLOR_RED, -1));
     Host::addColor(assign_color(COLOR_GREEN, -1));
     Host::addColor(assign_color(COLOR_YELLOW, -1));
@@ -693,6 +710,16 @@ NCursesInterface::NCursesInterface() :
     expand_color = assign_color(COLOR_GREEN, -1);
     highlight_color = assign_color(COLOR_BLACK, COLOR_CYAN);
 
+    redraw_source.set(g_timeout_add(1000, on_redraw_timer, this));
+
+    triggerRedraw();
+}
+
+NCursesInterface::NCursesInterface() :
+    UserInterface()
+{
+    init();
+
     columns.emplace_back(std::make_unique<IDColumn>());
     columns.emplace_back(std::make_unique<NameColumn>());
     columns.emplace_back(std::make_unique<InJobsColumn>());
@@ -704,10 +731,6 @@ NCursesInterface::NCursesInterface() :
     columns.emplace_back(std::make_unique<ActiveJobsColumn>());
     columns.emplace_back(std::make_unique<PendingJobsColumn>());
     columns.emplace_back(std::make_unique<SpeedColumn>());
-
-    redraw_source.set(g_timeout_add(1000, on_redraw_timer, this));
-
-    triggerRedraw();
 }
 
 NCursesInterface::~NCursesInterface()

diff --git a/src/main.hpp b/src/main.hpp
@@ -123,6 +123,11 @@ struct Host {
         host_color_ids.push_back(ident);
     }
 
+    static void clearColors()
+    {
+        host_color_ids.clear();
+    }
+
     static std::shared_ptr<Host> create(uint32_t id);
     static std::shared_ptr<Host> find(uint32_t id);
     static void remove(uint32_t id);
@@ -185,6 +190,8 @@ class UserInterface {
     virtual void triggerRedraw() = 0;
     virtual int processInput() = 0;
     virtual int getInputFd() = 0;
+    virtual void suspend() = 0;
+    virtual void resume() = 0;
 };
 
 class GlibSource {

diff --git a/src/scheduler.cpp b/src/scheduler.cpp
@@ -42,6 +42,7 @@ class IcecreamScheduler: public Scheduler {
 
 private:
     static gboolean scheduler_process(gint fd, GIOCondition condition, gpointer);
+    static gboolean on_reconnect_timer(gpointer);
 
     bool process_message(MsgChannel *sched);
     void discover_scheduler(std::string const &netname, std::string const &schedname);
@@ -51,6 +52,7 @@ class IcecreamScheduler: public Scheduler {
     GlibSource scheduler_source;
     std::string current_net_name;
     std::string current_scheduler_name;
+    GlibSource reconnect_source;
 };
 
 gboolean IcecreamScheduler::scheduler_process(gint, GIOCondition, gpointer user_data)
@@ -62,6 +64,9 @@ gboolean IcecreamScheduler::scheduler_process(gint, GIOCondition, gpointer user_
             break;
     }
 
+    if (self->scheduler->at_eof())
+        self->reconnect(self->current_net_name, self->current_scheduler_name);
+
     return TRUE;
 }
 
@@ -89,7 +94,7 @@ void IcecreamScheduler::discover_scheduler(std::string const &netname, std::stri
             pfd.events = POLLIN;
 
             std::cout << "Waiting " << pfd.fd << std::endl;
-            poll(&pfd, 1, -1);
+            poll(&pfd, 1, 10000);
 
             scheduler.reset(discover->try_get_scheduler());
         }
@@ -180,18 +185,36 @@ bool IcecreamScheduler::process_message(MsgChannel *sched)
     return true;
 }
 
+gboolean IcecreamScheduler::on_reconnect_timer(gpointer user_data)
+{
+    auto *self = static_cast<IcecreamScheduler*>(user_data);
+
+    self->reconnect(self->current_net_name, self->current_scheduler_name);
+
+    return TRUE;
+}
+
 void IcecreamScheduler::reconnect(std::string const &netname, std::string const &schedname)
 {
     scheduler.reset();
     scheduler_source.remove();
 
+    if (interface)
+        interface->suspend();
+
     discover_scheduler(netname, schedname);
 
-    if (scheduler)
+    if (scheduler) {
         scheduler_source.set(g_unix_fd_add(scheduler->fd, G_IO_IN, scheduler_process, this));
+        reconnect_source.remove();
+    } else {
+        reconnect_source.set(g_timeout_add(5000, on_reconnect_timer, this));
+    }
 
-    if (interface)
+    if (interface) {
+        interface->resume();
         interface->triggerRedraw();
+    }
 }