Merge pull request #7 from oolong-sh/patrick-dev

feat: Graph serialization, parallel state handling, and front-facing API endpoints
oolong-sh · Nov 24, 2024 · 56b9378 · 56b9378
2 parents 17f5f1e + 3c5e591
commit 56b9378
Show file tree

Hide file tree

Showing 23 changed files with 1,214 additions and 196 deletions.
diff --git a/LICENSE b/LICENSE
diff --git a/examples/data/note.md b/examples/data/note.md
@@ -0,0 +1,23 @@
+# Oolong Development Notes
+Oct 2, 2024
+
+## Tokenizing -> NGram Flow
+
+- Frequency Score (tf-idf)
+    - requires number of occurrences in document
+    - number of occurrences across documents
+    - requires number of documents
+    - TF should be calculated at the document level
+    - IDF should be calculated after all documents are fully processed
+
+- Token/NGram info:
+    - Need to store all tokens in a document (and Ngrams)
+    - Store filepaths for each document
+    - Need occurences of ngrams in each document and in all documents
+
+## API
+
+- Build API for frontend-backend communication
+    - CRUD endpoints for files
+    - Weight recalculation trigger
+    - Graph data getter
diff --git a/index.html b/index.html
@@ -0,0 +1,59 @@
+<head>
+  <style>
+    body {
+      margin: 0;
+    }
+  </style>
+
+  <script src="//unpkg.com/3d-force-graph"></script>
+</head>
+
+<body>
+  <div id="3d-graph"></div>
+
+  <script type="importmap">{ "imports": { "three": "//unpkg.com/three/build/three.module.js" }}</script>
+  <script type="module">
+    import SpriteText from "//unpkg.com/three-spritetext/dist/three-spritetext.mjs";
+
+    const normalizeValue = (val, min, max) => (val - min) / (max - min);
+
+    const minWeight = 0.00
+
+    // CHANGE: move handling of color to backend where calculations are faster
+    function mapValueToColor(value) {
+      const clampedValue = Math.max(minWeight, Math.min(value, 1));
+      // interpolate color between dark gray (#333333 -- 51) and white (#ffffff 255)
+      // const grayScale = Math.round(51 + (204 * clampedValue));
+      const grayScale = Math.round(1 + (204 * clampedValue)) * 1.5;
+      return `rgb(${grayScale}, ${grayScale}, ${grayScale})`;
+    }
+
+    fetch("http://localhost:11975/graph").then(res => res.json())
+      // fetch("./graph.json").then(res => res.json())
+      .then(data => {
+        const Graph = ForceGraph3D()
+          (document.getElementById('3d-graph'))
+          // .jsonUrl('./graph.json')
+          .graphData(data)
+          .nodeAutoColorBy('group')
+          .nodeThreeObject(node => {
+            const splitname = node.id.split("/") // CHANGE: add new field on daemon side for split name
+            const sprite = new SpriteText(splitname[splitname.length - 1]);
+            sprite.material.depthWrite = false;
+            sprite.color = node.color;
+            sprite.textHeight = 8;
+            return sprite;
+          })
+          .linkOpacity(.8) // NOTE: baseline opacity can be adjusted, but keep high
+          .linkColor(link => {
+            const value = link.strength !== undefined ? link.strength : minWeight;
+            return mapValueToColor(value);
+          });
+
+        // Spread nodes a little wider
+        Graph.d3Force('charge').strength(-120);
+      })
+
+    // TODO: line weight can be set here, use documentWeight to do this
+  </script>
+</body>
diff --git a/internal/api/api_utils.go b/internal/api/api_utils.go
@@ -0,0 +1,18 @@
+package api
+
+import "os"
+
+// exists returns whether the given file or directory exists
+func exists(path string) (bool, error) {
+	_, err := os.Stat(path)
+
+	if err == nil {
+		return true, nil
+	}
+
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+
+	return false, err
+}
diff --git a/internal/api/endpoints.go b/internal/api/endpoints.go
@@ -0,0 +1,31 @@
+package api
+
+import (
+	"log"
+	"net/http"
+)
+
+// DOC:
+func SpawnServer() {
+	mux := http.NewServeMux()
+
+	// graph endpoints
+	mux.HandleFunc("GET /graph", handleGetGraph)
+
+	// note endpoints
+	mux.HandleFunc("GET /notes", handleGetNotes)
+	mux.HandleFunc("GET /note", handleGetNote)
+	mux.HandleFunc("POST /note", handleCreateNote)
+	mux.HandleFunc("PUT /note", handleUpdateNote)
+	mux.HandleFunc("DELETE /note", handleDeleteNote)
+
+	// keyword endpoints?
+
+	// plugin endpoints? (probably not these outside of official ones)
+
+	// start server
+	log.Println("Starting server on :11975...")
+	if err := http.ListenAndServe(":11975", mux); err != nil {
+		log.Fatalf("Server failed to start: %v", err)
+	}
+}
diff --git a/internal/api/graph_handlers.go b/internal/api/graph_handlers.go
@@ -0,0 +1,53 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"slices"
+
+	"github.com/oolong-sh/oolong/internal/graph"
+	"github.com/oolong-sh/oolong/internal/keywords"
+	"github.com/oolong-sh/oolong/internal/notes"
+	"github.com/oolong-sh/oolong/internal/state"
+)
+
+var allowedOrigins = []string{
+	"http://localhost:8000",
+}
+
+func handleGetGraph(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+	w.Header().Set("Content-Type", "application/json")
+	origin := r.Header.Get("Origin")
+
+	// check if the origin is whitelisted
+	if !slices.Contains(allowedOrigins, origin) {
+		log.Println("Requesting client not in allow list. Origin:", origin)
+		http.Error(w, "Request origin not in allow list", http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Access-Control-Allow-Origin", origin)
+
+	// get snapshot of current state
+	s := state.State()
+
+	// convert state into serializable format for graph
+	kw := keywords.NGramsToKeywordsMap(s.NGrams)
+	notes := notes.DocumentsToNotes(s.Documents)
+
+	// serialize graph data
+	// TODO: pass in thresholds (with request? or with config?)
+	data, err := graph.SerializeGraph(kw, notes, 0.1, 80)
+	if err != nil {
+		http.Error(w, "Error serializing graph data", 500)
+	}
+
+	// encode graph data in reponse
+	if _, err := w.Write(data); err != nil {
+		http.Error(w, "Error encoding graph data", 500)
+		return
+	}
+	// if err := json.NewEncoder(w).Encode(data); err != nil {
+	// 	http.Error(w, "Error encoding graph data", 500)
+	// }
+}
diff --git a/internal/api/note_handlers.go b/internal/api/note_handlers.go
@@ -0,0 +1,135 @@
+package api
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/oolong-sh/oolong/internal/state"
+)
+
+type createUpdateRequest struct {
+	Path    string `json:"path"`
+	Content string `json:"content"`
+}
+
+// 'GET /notes' endpoint handler returns all available note paths
+func handleGetNotes(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+
+	s := state.State()
+	data := []string{}
+	for k := range s.Documents {
+		data = append(data, k)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}
+
+// 'GET /note?path=<path>' endpoint handler gets note contents corresponding to input path
+func handleGetNote(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+
+	path := r.URL.Query().Get("path")
+	if path == "" {
+		http.Error(w, "Path parameter not specified ", http.StatusBadRequest)
+		return
+	}
+
+	// read file contents
+	b, err := os.ReadFile(path)
+	if err != nil {
+		log.Println(err)
+		http.Error(w, "Could not read file '"+path+"'", 500)
+		return
+	}
+
+	// write file contents into response body
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(string(b)); err != nil {
+		log.Println(err)
+		http.Error(w, "Failed to encode file contents as JSON.\n", 500)
+	}
+}
+
+// 'POST /note' endpoint handler creates a note file (and any missing directories) corresponding to input path
+// Expected request body: { "path": "/path/to/note", "content", "full note contents to write" }
+func handleCreateNote(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+
+	// parse request body
+	var req createUpdateRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, "Failed to decode request body", 400)
+	}
+	log.Println("Request body: ", req)
+
+	// check if path before file exists, then check if file exists
+	if e, err := exists(req.Path); err != nil {
+		log.Println(err)
+		http.Error(w, "Error checking path", 500)
+		return
+	} else if e {
+		log.Printf("File %s already exists.\n", req.Path)
+		http.Error(w, "Note file already exists", 500)
+		return
+	}
+
+	// create directories and file
+	dir := filepath.Dir(req.Path)
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		log.Println(err)
+		http.Error(w, "Failed to create missing directories", 500)
+		return
+	}
+	if err := os.WriteFile(req.Path, []byte(req.Content), 0644); err != nil {
+		log.Println(err)
+		http.Error(w, "Failed to create file directories", 500)
+		return
+	}
+}
+
+// 'PUT /note' endpoint handler updates note contents corresponding to input path
+// It will create files that do not exist, but will not create directories
+// Expected request body: { "path": "/path/to/note", "content", "full note contents to write" }
+func handleUpdateNote(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+
+	// parse request body
+	var req createUpdateRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, "Failed to decode request body", 400)
+	}
+	log.Println("Request body: ", req)
+
+	// write contents to file
+	if err := os.WriteFile(req.Path, []byte(req.Content), 0666); err != nil {
+		log.Println(err)
+		http.Error(w, "Failed to write to note file", 500)
+		return
+	}
+}
+
+// 'Delete /note?path=/path/to/note' endpoint handler deletess a note file based on query input
+func handleDeleteNote(w http.ResponseWriter, r *http.Request) {
+	log.Println("Request received:", r.Method, r.URL, r.Host)
+
+	path := r.URL.Query().Get("path")
+	if path == "" {
+		http.Error(w, "Path parameter not specified ", http.StatusBadRequest)
+		return
+	}
+
+	// attempt to remove file
+	if err := os.Remove(path); err != nil {
+		log.Println("Failed to delete file", err)
+		http.Error(w, "Failed to remove file", 500)
+		return
+	}
+
+	// NOTE: this function may need to call the update function due to files no longer existing
+	// - check this case in state, this may require substantial logic missing there
+}
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -8,29 +8,29 @@ import (
 	"github.com/BurntSushi/toml"
 )
 
-var config OolongConfig
+var cfg OolongConfig
 
 type OolongConfig struct {
 	NotesDirPaths     []string `toml:"note_directories"`
 	NGramRange        []int    `toml:"ngram_range"`
 	AllowedExtensions []string `toml:"allowed_extensions"`
 	PluginPaths       []string `toml:"plugin_paths"`
 	IgnoreDirectories []string `toml:"ignored_directories"`
-	StopWords         []string `toml:"stopwords"`
+	StopWords         []string `toml:"stop_words"`
 }
 
-func Config() *OolongConfig { return &config }
+func Config() *OolongConfig { return &cfg }
 
-func NotesDirPaths() []string      { return config.NotesDirPaths }
-func NGramRange() []int            { return config.NGramRange }
-func AllowedExtensions() []string  { return config.AllowedExtensions }
-func PluginPaths() []string        { return config.PluginPaths }
-func IgnoredDirectories() []string { return config.IgnoreDirectories }
-func StopWords() []string          { return config.StopWords }
+func NotesDirPaths() []string      { return cfg.NotesDirPaths }
+func NGramRange() []int            { return cfg.NGramRange }
+func AllowedExtensions() []string  { return cfg.AllowedExtensions }
+func PluginPaths() []string        { return cfg.PluginPaths }
+func IgnoredDirectories() []string { return cfg.IgnoreDirectories }
+func StopWords() []string          { return cfg.StopWords }
 
 // TODO: file watcher for config file, reload on change
 
-func Setup(configPath string) (OolongConfig, error) {
+func Setup(configPath string) error {
 	var err error
 	configPath, err = expand(configPath)
 	if err != nil {
@@ -42,20 +42,20 @@ func Setup(configPath string) (OolongConfig, error) {
 		panic(err)
 	}
 
-	err = toml.Unmarshal(contents, &config)
+	err = toml.Unmarshal(contents, &cfg)
 	if err != nil {
 		panic(err)
 	}
 
-	for i, dir := range config.NotesDirPaths {
+	for i, dir := range cfg.NotesDirPaths {
 		d, err := expand(dir)
 		if err != nil {
 			panic(err)
 		}
-		config.NotesDirPaths[i] = d
+		cfg.NotesDirPaths[i] = d
 	}
 
-	return config, nil
+	return nil
 }
 
 func expand(path string) (string, error) {