Merge pull request #206 from veg/develop

1.4.1-rc
veg · Dec 9, 2019 · 3de67ab · 3de67ab
2 parents 5ff9a4b + d351ecd
commit 3de67ab
Show file tree

Hide file tree

Showing 40 changed files with 705 additions and 208 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,7 @@ lib-cov
 *.out
 *.pid
 *.gz
-tags
+tags*
 
 pids
 logs

diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ datamonkey-js
 INSTALL
 ===========================
 ## System Dependencies
-* node
+* node >= 12
 * mongodb-server
 * redis
 

diff --git a/app/models/analysis.js b/app/models/analysis.js
@@ -9,7 +9,10 @@ var mongoose = require("mongoose"),
   extend = require("mongoose-schema-extend");
 
 var redis = require("redis"),
-  client = redis.createClient();
+  client = redis.createClient({
+    host: setup.redisHost,
+    port: setup.redisPort
+  });
 
 var Schema = mongoose.Schema,
   ObjectId = Schema.ObjectId;

diff --git a/app/models/fubar.js b/app/models/fubar.js
@@ -10,10 +10,6 @@ var FUBAR = AnalysisSchema.extend({
   last_status_msg: String,
   results: Object,
   number_of_grid_points: Number,
-  number_of_mcmc_chains: Number,
-  length_of_each_chain: Number,
-  number_of_burn_in_samples: Number,
-  number_of_samples: Number,
   concentration_of_dirichlet_prior: Number
 });
 

diff --git a/app/models/msa.js b/app/models/msa.js
@@ -167,30 +167,26 @@ Msa.statics.deliverFasta = function(filepath) {
   return new Promise(function(resolve, reject) {
     fs.readFile(filepath, function(err, data) {
       if (err) reject(err);
-      if (data.slice(0, 6) != "#NEXUS") resolve(data.toString());
-      const split_data = data.toString().split("\n"),
-        start_index = split_data.indexOf("MATRIX") + 1,
-        end_index = split_data.indexOf("END;", start_index),
-        tax_labels_index = split_data.indexOf("\tTAXLABELS") + 1,
-        tax_labels = split_data[tax_labels_index]
-          .slice(0, -1)
-          .trim()
-          .split(" ")
-          .map(name => name.slice(1, -1));
-      if ((start_index == 0) | (tax_labels == 0)) {
-        resolve(null);
-      } else {
-        resolve(
-          split_data
-            .slice(start_index, end_index)
-            .map((line, index) => {
-              const header = ">" + tax_labels[index],
-                sequence = line.trim().replace(";", "");
-              return [header, sequence].join("\n");
-            })
-            .join("\n") + "\n"
-        );
-      }
+      if (data.slice(0, 1) == ">") resolve(data.toString());
+      const converted_filepath = filepath.split(".")[0] + "-converted.fasta";
+      const hyphy = spawn(globals.hyphy, [
+        __dirname + "/../../.hyphy/res/TemplateBatchFiles/ConvertDataFile.bf"
+      ]);
+
+      hyphy.on("exit", function(code) {
+        fs.readFile(converted_filepath, function(err, data) {
+          if (err) reject(err);
+          const fasta = data.toString();
+          fs.unlink(converted_filepath, function(err) {
+            resolve(fasta);
+          });
+        });
+      });
+
+      hyphy.stdin.write("1\n");
+      hyphy.stdin.write(filepath + "\n");
+      hyphy.stdin.write("10\n");
+      hyphy.stdin.write(converted_filepath + "\n");
     });
   });
 };

diff --git a/app/routes/hivtrace.js b/app/routes/hivtrace.js
@@ -16,7 +16,10 @@ var mongoose = require("mongoose"),
   HivTrace = mongoose.model("HivTrace"),
   Msa = mongoose.model("Msa");
 
-var publisher = redis.createClient();
+var publisher = redis.createClient({
+  host: setup.redisHost,
+  port: setup.redisPort
+});
 
 /**
  * Request a new job ID
@@ -88,7 +91,10 @@ exports.uploadFile = function(req, res) {
           hivtrace.mail = postdata.mail;
         }
 
-        var publisher = redis.createClient(),
+        var publisher = redis.createClient({
+            host: setup.redisHost,
+            port: setup.redisPort
+          }),
           channel_id = "fasta_parsing_progress_" + hivtrace._id;
 
         var file_path = req.files.files.file;

diff --git a/app/templates/busted/upload_msa.ejs b/app/templates/busted/upload_msa.ejs
@@ -88,7 +88,7 @@
         <input name="mail" type="text" class="form-control" placeholder="Email Address">
       </div><!-- /form-group -->
 
-      <button type="submit" class="dm-continue-btn btn float-right">
+      <button id="submit-button" type="submit" class="dm-continue-btn btn float-right">
         Run Analysis
         <span class="fa fa-play"></span></button>
       </button>

diff --git a/app/templates/fel/msa_form.ejs b/app/templates/fel/msa_form.ejs
@@ -92,7 +92,7 @@
         </div>
       </div><!-- /form-group -->
 
-        <button type="submit" class="dm-continue-btn btn float-right">
+        <button type="submit" id="submit-button" class="dm-continue-btn btn float-right">
           Run Analysis
         <span class="fa fa-play"></span></button>
 

diff --git a/app/templates/index.ejs b/app/templates/index.ejs
@@ -1,5 +1,5 @@
 <%if (warning_message) { %>
-  <div class="alert alert-warning" role="alert"> <h1><%= warning_message %></h1> </div>
+  <div class="alert alert-warning sticky-top" role="alert" style="margin-bottom: 0px;"> <h2><%= warning_message %></h2> </div>
 <% } %>
 <%- include includes/header.ejs %>
   <div id="main">

diff --git a/app/templates/partials/msa/form.ejs b/app/templates/partials/msa/form.ejs
@@ -78,7 +78,7 @@
       <input name="mail" type="text" class="form-control" placeholder="Email Address">
     </div><!-- /form-group -->
 
-    <button type="submit" class="dm-continue-btn btn float-right">
+    <button id="submit-button" type="submit" class="dm-continue-btn btn float-right">
       Run Analysis 
       <span class="fa fa-play"></span></button>
     </button>

diff --git a/app/templates/relax/upload_msa.ejs b/app/templates/relax/upload_msa.ejs
@@ -91,7 +91,7 @@
           <label id="datatype-content">Notify When Completed?</label>
           <input name="mail" type="text" class="form-control" placeholder="Email Address">
         </div><!-- /form-group -->
-        <button type="submit" class="dm-continue-btn btn float-right">
+        <button type="submit" id="submit-button" class="dm-continue-btn btn float-right">
           Run Analysis <span class="fa fa-play"></span></button>
         </button>
       </form>

diff --git a/config/setup.js.tpl b/config/setup.js.tpl
@@ -17,6 +17,8 @@ var database_name = "datamonkey-dev";
 
 define('host','datamonkey-dev');
 define('port', 4002);
+define('redisHost', 'localhost');
+define('redisPort', '6379');
 define('database', database_path + database_name);
 define('database_name', database_name);
 define('log_level', 'warn');
@@ -26,6 +28,4 @@ define('fasta_validator', '/home/sweaver/TN93/validate_fasta');
 define('hyphy', __dirname + '/node_modules/hyphy/HYPHYMP');
 define('cluster_ip_urls_array', cluster_ip_urls_array);
 define('default_url', 'http://silverback.temple.edu:7013');
-define('redisPort', 6379);
-define('redisHost', "localhost");
 //define('warning_message', "Hi");
diff --git a/lib/hpcsocket.js b/lib/hpcsocket.js
@@ -1,13 +1,16 @@
 var path = require("path"),
   redis = require("redis"),
   winston = require("winston"),
-  publisher = redis.createClient();
+  setup = require(path.join(__dirname, "../config/setup.js")),
+  publisher = redis.createClient({
+    host: setup.redisHost,
+    port: setup.redisPort
+  });
 
-var setup = require(path.join(__dirname, "../config/setup.js")),
-  fs = require("fs"),
-  ioclient = require("socket.io-client"),
-  mailer = require("./mailer.js"),
-  ss = require("socket.io-stream");
+(fs = require("fs")),
+  (ioclient = require("socket.io-client")),
+  (mailer = require("./mailer.js")),
+  (ss = require("socket.io-stream"));
 
 var logger = require(path.join(__dirname, "/logger"));
 
@@ -30,7 +33,10 @@ function ClientSocket(socket, job_id) {
 
   self.channel_id = "job_" + job_id;
   self.socket = socket;
-  self.subscriber = redis.createClient();
+  self.subscriber = redis.createClient({
+    host: setup.redisHost,
+    port: setup.redisPort
+  });
   self.subscriber.subscribe(self.channel_id);
   self.initializeServer();
 }
@@ -128,14 +134,11 @@ HPCSocket.prototype.connectionError = function(err) {
 HPCSocket.prototype.initializeServer = function(url_input, callback) {
   var self = this;
 
-  self.server_socket = ioclient.connect(
-    url_input,
-    {
-      "connect timeout": 1000,
-      "force new connection": true,
-      reconnection: false
-    }
-  );
+  self.server_socket = ioclient.connect(url_input, {
+    "connect timeout": 1000,
+    "force new connection": true,
+    reconnection: false
+  });
 
   self.server_socket.on("connect_error", function() {
     self.connectionError();
@@ -402,14 +405,11 @@ function ClusterStatus(url, callback) {
 ClusterStatus.prototype.initializeServer = function(url, callback) {
   var self = this;
 
-  var server_socket = ioclient.connect(
-    url,
-    {
-      "connect timeout": 1000,
-      "force new connection": true,
-      reconnection: false
-    }
-  );
+  var server_socket = ioclient.connect(url, {
+    "connect timeout": 1000,
+    "force new connection": true,
+    reconnection: false
+  });
 
   var connectionError = function(err) {
     callback({ successful_connection: false, msg: err });
@@ -441,14 +441,11 @@ function HPCSocketRecheck(job, callback) {
 
 HPCSocketRecheck.prototype.initializeServer = function(job, callback) {
   var self = this;
-  var server_socket = ioclient.connect(
-    default_setup_url,
-    {
-      "connect timeout": 1000,
-      "force new connection": true,
-      reconnection: false
-    }
-  );
+  var server_socket = ioclient.connect(default_setup_url, {
+    "connect timeout": 1000,
+    "force new connection": true,
+    reconnection: false
+  });
 
   var connectionError = function(err) {
     callback({ successful_connection: false, msg: err });
@@ -538,14 +535,11 @@ function JobQueue(callback) {
 JobQueue.prototype.initializeServer = function(callback) {
   var self = this;
 
-  var server_socket = ioclient.connect(
-    default_setup_url,
-    {
-      "connect timeout": 1000,
-      "force new connection": true,
-      reconnection: false
-    }
-  );
+  var server_socket = ioclient.connect(default_setup_url, {
+    "connect timeout": 1000,
+    "force new connection": true,
+    reconnection: false
+  });
 
   var connectionError = function(err) {
     winston.warn(err);

diff --git a/lib/loadBal.js b/lib/loadBal.js
@@ -1,10 +1,13 @@
 var redis = require("redis"),
   _ = require("underscore"),
-  setup = require("./../config/setup.js"),
+  setup = require("../config/setup.js"),
   globals = require("./../config/globals.js");
 
 // Use redis as our key-value store
-var client = redis.createClient({ host: "localhost", port: 6379 });
+const client = redis.createClient({
+  host: setup.redisHost,
+  port: setup.redisPort
+});
 
 //Reference for iteration variable used by function.
 var redis_iteration = globals.redis_iteration;

diff --git a/lib/queue.js b/lib/queue.js
@@ -1,10 +1,13 @@
 var path = require("path"),
   redis = require("redis"),
-  client = redis.createClient({ host: "localhost", port: 6379 }),
   hpcsocket = require(path.join(__dirname, "./hpcsocket.js")),
   winston = require("winston"),
-  setup = require("./../config/setup");
+  setup = require("../config/setup");
 
+const client = redis.createClient({
+  host: setup.redisHost,
+  port: setup.redisPort
+});
 const jobQueueRedisKey = "jobQueue_" + setup.port;
 
 function queueSet() {

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "datamonkey.js",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "A free public server for comparative analysis of sequence alignments using state-of-the-art statistical models.",
   "private": true,
   "author": {
@@ -9,6 +9,7 @@
   },
   "scripts": {
     "test": "mocha",
+    "ui-tests": "mocha-parallel-tests -t 86400000 --max-parallel 10 test/ui/",
     "lint": "./node_modules/.bin/eslint",
     "format": "prettier 'src/**/*.{js,jsx}' 'app/**/*.{js,jsx}' 'lib/**/*.{js,jsx}' 'config/**/*.{js,jsx}' 'test/**/*.{js,jsx}' --write",
     "precommit": "pretty-quick --staged"
@@ -61,7 +62,7 @@
     "underscore": "^1.9.1",
     "validator": "1.2.x",
     "winston": "^0.8.3",
-    "yarn": "^0.27.5"
+    "yarn": "^1.19.1"
   },
   "devDependencies": {
     "@fortawesome/fontawesome-free": "^5.3.1",
@@ -97,10 +98,12 @@
     "less": "^2.7.2",
     "less-loader": "^4.0.4",
     "lodash.clonedeep": "^4.5.0",
-    "mocha": "^2.0.0",
+    "mocha": "^6.2.2",
+    "mocha-parallel-tests": "^2.2.2",
     "node-inspect": "^1.11.2",
     "node-sass": "^4.9.3",
     "prettier": "^1.14.2",
+    "puppeteer": "^2.0.0",
     "reactify": "^1.1.1",
     "sass-loader": "6.0.7",
     "scss-loader": "^0.0.1",

diff --git a/server.js b/server.js
@@ -20,10 +20,7 @@ var express = require("express"),
   bb = require("express-busboy");
 
 // Connect to database
-mongoose.connect(
-  setup.database,
-  { useMongoClient: true }
-);
+mongoose.connect(setup.database, { useMongoClient: true });
 
 //Ensure that upload paths exists
 mkdirErrorLogger = error.errorLogger(["EEXIST"]);
@@ -112,7 +109,10 @@ io.sockets.on("connection", function(socket) {
   });
 
   socket.on("fasta_parsing_progress_start", function(data) {
-    var fasta_listener = redis.createClient();
+    var fasta_listener = redis.createClient({
+      host: setup.redisHost,
+      port: setup.redisPort
+    });
     fasta_listener.subscribe("fasta_parsing_progress_" + data.id);
     fasta_listener.on("message", function(channel, message) {
       socket.emit("fasta_parsing_update", message);
@@ -123,7 +123,10 @@ io.sockets.on("connection", function(socket) {
   });
 
   socket.on("attribute_parsing_progress_start", function(data) {
-    var attr_listener = redis.createClient();
+    var attr_listener = redis.createClient({
+      host: setup.redisHost,
+      port: setup.redisPort
+    });
     attr_listener.subscribe("attribute_parsing_progress_" + data.id);
     attr_listener.on("message", function(channel, message) {
       socket.emit("attribute_parsing_progress", message);