Merge branch 'add-db-file-lock'

Makefile

@@ -2,7 +2,7 @@ SHELL ?= /bin/bash -e
 #BINARY ?= cmd/server/mendel-web-ui
 BINARY ?= mendel-web-ui
 # Set these 2 vars before building the pkg, and set Requires in pkg/rpm/mendel-web-ui.spec if necessary
-export VERSION ?= 1.1.7
+export VERSION ?= 1.1.8
 # Release is only needed for the rpm, and only needs to be incremented if you have to rebuild/reinstall this version multiple times due to packaging fixes
 export RELEASE ?= 1
 # rpmbuild does not give us a good way to set topdir, so use the default location
@@ -13,11 +13,11 @@ MAC_PKG_INSTALL_DIR ?= /Users/Shared/mendel-web-ui
 
 default: runserver
 
-cmd/server/$(BINARY): cmd/server/*.go Makefile
+cmd/server/$(BINARY): cmd/server/*.go cmd/server/*/*.go Makefile
 	echo 'package main; const MENDEL_UI_VERSION = "$(VERSION)-$(RELEASE)"' > cmd/server/version.go
 	scripts/build_go
 
-tools/mendel-chg-pw: tools/mendel-chg-pw.go
+tools/mendel-chg-pw: tools/mendel-chg-pw.go cmd/server/*/*.go
 	glide --quiet install
 	go build -o $@ $<
 

README.md

@@ -31,6 +31,7 @@ First, set the `VERSION` and `RELEASE` variables in `Makefile`.
 
 Note: so far the RPM has only been tested on Amazon Linux.
 
+- Increment the `VERSION` value in `Makefile`
 - Build (you build the linux rpm on mac if you 1st: `brew install rpm`):
 ```
 make rpmbuild
@@ -40,16 +41,13 @@ make rpmbuild
 ```
 yum install mendel-web-ui-*.x86_64.rpm
 ```
-- The server is started automatically by `initctl` running `/etc/init/mendel-web-ui.conf`.
-- If you just installed the rpm for the 1st time, get the admin password:
-```
-cat /var/log/mendel-web-ui/stdout.log
-```
+- The server is started automatically by `systemctl` running `/etc/init/mendel-web-ui.conf`.
 - Browse http://hostname-or-ip:8581/
-- Change the admin pw, and then create additional users as necessary.
+- If you just installed the rpm for the 1st time, the `admin` password is `changeme!` . Once logged in, change the `admin` password by clicking on `USERS` and then the `admin` user, and set the password to something you will remember. Create additional users as necessary.
 
 ### For the Mac install package:
 
+- Increment the `VERSION` value in `Makefile`
 - Build:
 ```
 make macpkg
@@ -64,12 +62,8 @@ make macinstall
 ```
 start-mendel-ui.sh
 ```
-- If you just installed the rpm for the 1st time, get the admin password:
-```
-cat /var/log/mendel-web-ui/stdout.log
-```
 - Browse http://hostname-or-ip:8581/
-- Change the admin pw, and then create additional users as necessary.
+- If you just installed the package for the 1st time, the `admin` password is `changeme!` . Once logged in, change the `admin` password by clicking on `USERS` and then the `admin` user, and set the password to something you will remember. Create additional users as necessary.
 
 To stop the web UI server:
 ```

cmd/server/api_create_edit_user_handler.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/db"
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/mutils"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -22,7 +24,7 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if !isValidPostJson(r) {
+	if !mutils.IsValidPostJson(r) {
 		http.Error(w, "400 Bad Request (method or content-type)", http.StatusBadRequest)
 		return
 	}
@@ -34,21 +36,20 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "400 Bad Request (parsing body)", http.StatusBadRequest)
 		return
 	}
+	mutils.Verbose("/api/create-edit-user/ Username=%s, Id=%s", postUser.Username, postUser.Id)
 
 	usernameExists := false
-	globalDbLock.RLock()
-	for _, u := range globalDb.Users {
+	db.Db.RLock()
+	for _, u := range db.Db.Data.Users {
 		if u.Username == postUser.Username && u.Id != postUser.Id {
 			usernameExists = true
 			break
 		}
 	}
-	globalDbLock.RUnlock()
+	db.Db.RUnlock()
 
 	if usernameExists {
-		writeJsonResponse(w, map[string]string{
-			"status": "username_exists",
-		})
+		mutils.WriteJsonResponse(w, map[string]string{"status": "username_exists",})
 		return
 	}
 
@@ -61,7 +62,7 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	var newUser DatabaseUser
+	var newUser db.DatabaseUser
 	if postUser.Id == "" {
 		// Create user
 
@@ -70,13 +71,13 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		userId, err := generateUuid()
+		userId, err := mutils.GenerateUuid()
 		if err != nil {
 			http.Error(w, "500 Internal Server Error (could not generate userId)", http.StatusInternalServerError)
 			return
 		}
 
-		newUser = DatabaseUser{
+		newUser = db.DatabaseUser{
 			Id:       userId,
 			Username: postUser.Username,
 			Password: hashedPassword,
@@ -90,10 +91,10 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		globalDbLock.RLock()
+		db.Db.RLock()
 		var ok bool
-		newUser, ok = globalDb.Users[postUser.Id]
-		globalDbLock.RUnlock()
+		newUser, ok = db.Db.Data.Users[postUser.Id]
+		db.Db.RUnlock()
 
 		if !ok {
 			http.Error(w, "400 Bad Request (user does not exist)", http.StatusBadRequest)
@@ -107,16 +108,16 @@ func apiCreateEditUserHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	globalDbLock.Lock()
-	globalDb.Users[newUser.Id] = newUser
-	err = persistDatabase()
-	globalDbLock.Unlock()
+	db.Db.Lock()
+	db.Db.Data.Users[newUser.Id] = newUser
+	err = db.Db.Persist()
+	db.Db.Unlock()
 	if err != nil {
 		http.Error(w, "500 Internal Server Error (could not persist database)", http.StatusInternalServerError)
 		return
 	}
 
-	writeJsonResponse(w, map[string]string{
+	mutils.WriteJsonResponse(w, map[string]string{
 		"status": "success",
 	})
 }

cmd/server/api_create_job_handler.go

@@ -13,6 +13,8 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/db"
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/mutils"
 )
 
 // Called for /api/create-job/ route
@@ -24,7 +26,7 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if !isValidPostJson(r) {
+	if !mutils.IsValidPostJson(r) {
 		http.Error(w, "400 Bad Request (method or content-type)", http.StatusBadRequest)
 		return
 	}
@@ -39,6 +41,7 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "400 Bad Request (parsing body)", http.StatusBadRequest)
 		return
 	}
+	//mutils.Verbose("/api/create-job/ job config:%s", data.Config)
 
 	var config map[string]map[string]interface{}
 	_, err = toml.Decode(data.Config, &config)
@@ -47,7 +50,8 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	jobId, err := generateJobId()
+	// Start building the job directory content
+	jobId, err := mutils.GenerateJobId()
 	if err != nil {
 		http.Error(w, "500 Internal Server Error (could not generate jobId)", http.StatusInternalServerError)
 		return
@@ -70,44 +74,50 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Write out the config file to the job dir
 	err = toml.NewEncoder(configFile).Encode(config)
+	// close the file before handling the potential encode err, because we want to close it regardless
+	if errClose := configFile.Close(); errClose != nil {
+		http.Error(w, "500 Internal Server Error (could not close job config)", http.StatusInternalServerError)
+		return
+	}
 	if err != nil {
-		configFile.Close()
 		http.Error(w, "500 Internal Server Error (could not encode job config)", http.StatusInternalServerError)
 		return
 	}
-	configFile.Close()
 
+	// This will collect the output of the job we are able to run
 	outputBuilder := &strings.Builder{}
-
 	globalRunningJobsLock.Lock()
 	globalRunningJobsOutput[jobId] = outputBuilder
 	globalRunningJobsLock.Unlock()
 
-	job := DatabaseJob{
+	// Put info about the job in the db
+	job := db.DatabaseJob{
 		Id:          jobId,
 		Description: config["basic"]["description"].(string),
 		Time:        time.Now().UTC(),
 		OwnerId:     user.Id,
 		Status:      "running",
 	}
+	db.Db.Lock()
+	db.Db.Data.Jobs[jobId] = job	// we will persist this to the db file when the job completes
+	db.Db.Unlock()
 
-	globalDbLock.Lock()
-	globalDb.Jobs[jobId] = job
-	globalDbLock.Unlock()
-
+	// Run the job in its own thread
 	go func() {
 		cmd := exec.Command(globalMendelGoBinaryPath, "-f", configFilePath)
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
-			log.Println(err)
+			log.Printf("Error getting stdout pipe for job %s: %v", jobId, err)
 		}
 
 		err = cmd.Start()
 		if err != nil {
-			log.Println(err)
+			log.Printf("Error starting job %s: %v", jobId, err)
 		}
 
+		// Repeatedly get the latest bytes of output from the running job and add them to our in-memory copy so /api/job-output/ can get them and return them to the frontend
 		scanner := bufio.NewScanner(stdout)
 		for scanner.Scan() {
 			globalRunningJobsLock.Lock()
@@ -116,26 +126,29 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 			globalRunningJobsLock.Unlock()
 		}
 
+		// Wait for the job to complete and update the completion status of the job entry in the db
 		err = cmd.Wait()
 		if err != nil {
-			log.Println(err)
+			log.Printf("Job %s failed: %v", jobId, err)
 			job.Status = "failed"
-			globalDbLock.Lock()
-			globalDb.Jobs[jobId] = job
-			err = persistDatabase()
-			globalDbLock.Unlock()
+			db.Db.Lock()
+			db.Db.Data.Jobs[jobId] = job
+			err = db.Db.Persist()
+			db.Db.Unlock()
 			if err != nil {
 				http.Error(w, "500 Internal Server Error (could not persist database)", http.StatusInternalServerError)
 				return
 			}
 		}
 
+		mutils.Verbose("Job %s completed successfully. Setting job status in db, and removing the in-memory copy of job output", jobId)
 		job.Status = "succeeded"
-		globalDbLock.Lock()
-		globalDb.Jobs[jobId] = job
-		err = persistDatabase()
-		globalDbLock.Unlock()
+		db.Db.Lock()
+		db.Db.Data.Jobs[jobId] = job
+		err = db.Db.Persist()
+		db.Db.Unlock()
 		if err != nil {
+			mutils.Verbose("Error persisting job %s in the db: %v", jobId, err)
 			http.Error(w, "500 Internal Server Error (could not persist database)", http.StatusInternalServerError)
 			return
 		}
@@ -146,12 +159,13 @@ func apiCreateJobHandler(w http.ResponseWriter, r *http.Request) {
 
 		err = ioutil.WriteFile(filepath.Join(jobDir, "mendel_go.out"), []byte(outputBuilder.String()), 0644)
 		if err != nil {
+			mutils.Verbose("Error writing job %s output to %s: %v", jobId, filepath.Join(jobDir, "mendel_go.out"), err)
 			http.Error(w, "500 Internal Server Error (could not write to mendel_go.out file)", http.StatusInternalServerError)
 			return
 		}
 	}()
 
-	writeJsonResponse(w, map[string]string{
+	mutils.WriteJsonResponse(w, map[string]string{
 		"job_id": jobId,
 	})
 }

cmd/server/api_delete_job_handler.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"encoding/json"
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/db"
+	"github.com/genetic-algorithms/mendel-web-ui/cmd/server/mutils"
 	"log"
 	"net/http"
 	"os"
@@ -21,7 +23,7 @@ func apiDeleteJobHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if !isValidPostJson(r) {
+	if !mutils.IsValidPostJson(r) {
 		http.Error(w, "400 Bad Request (method or content-type)", http.StatusBadRequest)
 		return
 	}
@@ -33,8 +35,9 @@ func apiDeleteJobHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "400 Bad Request (parsing body)", http.StatusBadRequest)
 		return
 	}
+	mutils.Verbose("/api/delete-job/ Is=%s", postJob.Id)
 
-	globalDbLock.Lock()
+	db.Db.Lock()
 	// Remove the job data files first
 	jobDir := filepath.Join(globalJobsDir, postJob.Id)
 	//log.Printf("In /api/delete-job/: deleting %s ...", jobDir)
@@ -43,15 +46,15 @@ func apiDeleteJobHandler(w http.ResponseWriter, r *http.Request) {
 	if err == nil {
 		// Now remove the job from the db
 		//log.Printf("In /api/delete-job/: now deleting %s from db ...", postJob.Id)
-		delete(globalDb.Jobs, postJob.Id)
-		err = persistDatabase()
+		delete(db.Db.Data.Jobs, postJob.Id)
+		err = db.Db.Persist()
 	}
-	globalDbLock.Unlock()
+	db.Db.Unlock()
 	if err != nil {
 		log.Printf("Error in /api/delete-job/: %v", err)
 		http.Error(w, "500 Internal Server Error (could not persist database)", http.StatusInternalServerError)
 		return
 	}
 
-	writeJsonResponse(w, map[string]string{})
+	mutils.WriteJsonResponse(w, map[string]string{})
 }