feat: improved config structure with threshold values

examples/oolong.toml

@@ -9,16 +9,31 @@ note_directories = [
 # Subdirectory patterns to exclude from the file watcher
 ignored_directories = [
     ".git",
+    ".templates",
+    "target",
 ]
 
 # Whitelist of file extensions to use in linking
 allowed_extensions = [
     ".md",
+    ".mdx",
     ".tex",
     ".typ",
 ]
 
+# Additional stop words
+stop_words = [
+    "idx",
+]
+
 # List of plugins (lua files) to load
+[plugins]
 plugin_paths = [
     "./scripts/daily_note.lua",
 ]
+
+
+[graph]
+min_node_weight = 8.0
+max_node_weight = 80.0
+min_link_weight = 2.0

internal/api/api_utils.go

@@ -1,6 +1,30 @@
 package api
 
-import "os"
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"slices"
+)
+
+// TODO: whitelist native app and web urls
+var allowedOrigins = []string{
+	"http://localhost:8000",
+}
+
+func checkOrigin(w http.ResponseWriter, r *http.Request) error {
+	origin := r.Header.Get("Origin")
+
+	// check if the origin is whitelisted
+	if !slices.Contains(allowedOrigins, origin) {
+		http.Error(w, "Request origin not in allow list", http.StatusForbidden)
+		return fmt.Errorf("Requesting client not in allow list. Origin: %s\n", origin)
+	}
+
+	w.Header().Set("Access-Control-Allow-Origin", origin)
+
+	return nil
+}
 
 // exists returns whether the given file or directory exists
 func exists(path string) (bool, error) {

internal/api/graph_handlers.go

@@ -1,32 +1,25 @@
 package api
 
 import (
+	"fmt"
 	"log"
 	"net/http"
-	"slices"
 
 	"github.com/oolong-sh/oolong/internal/graph"
 	"github.com/oolong-sh/oolong/internal/keywords"
 	"github.com/oolong-sh/oolong/internal/notes"
 	"github.com/oolong-sh/oolong/internal/state"
 )
 
-var allowedOrigins = []string{
-	"http://localhost:8000",
-}
-
 func handleGetGraph(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 	w.Header().Set("Content-Type", "application/json")
-	origin := r.Header.Get("Origin")
 
-	// check if the origin is whitelisted
-	if !slices.Contains(allowedOrigins, origin) {
-		log.Println("Requesting client not in allow list. Origin:", origin)
-		http.Error(w, "Request origin not in allow list", http.StatusForbidden)
-		return
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
 	}
-	w.Header().Set("Access-Control-Allow-Origin", origin)
 
 	// get snapshot of current state
 	s := state.State()
@@ -37,7 +30,7 @@ func handleGetGraph(w http.ResponseWriter, r *http.Request) {
 
 	// serialize graph data
 	// TODO: pass in thresholds (with request? or with config?)
-	data, err := graph.SerializeGraph(kw, notes, 0.1, 80)
+	data, err := graph.SerializeGraph(kw, notes)
 	if err != nil {
 		http.Error(w, "Error serializing graph data", 500)
 	}

internal/api/note_handlers.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -19,6 +20,12 @@ type createUpdateRequest struct {
 func handleGetNotes(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
+	}
+
 	s := state.State()
 	data := []string{}
 	for k := range s.Documents {
@@ -33,6 +40,12 @@ func handleGetNotes(w http.ResponseWriter, r *http.Request) {
 func handleGetNote(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
+	}
+
 	path := r.URL.Query().Get("path")
 	if path == "" {
 		http.Error(w, "Path parameter not specified ", http.StatusBadRequest)
@@ -60,6 +73,12 @@ func handleGetNote(w http.ResponseWriter, r *http.Request) {
 func handleCreateNote(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
+	}
+
 	// parse request body
 	var req createUpdateRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -98,6 +117,12 @@ func handleCreateNote(w http.ResponseWriter, r *http.Request) {
 func handleUpdateNote(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
+	}
+
 	// parse request body
 	var req createUpdateRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -117,6 +142,12 @@ func handleUpdateNote(w http.ResponseWriter, r *http.Request) {
 func handleDeleteNote(w http.ResponseWriter, r *http.Request) {
 	log.Println("Request received:", r.Method, r.URL, r.Host)
 
+	// CORS handling
+	if err := checkOrigin(w, r); err != nil {
+		log.Println(err)
+		http.Error(w, fmt.Sprintln(err), 500)
+	}
+
 	path := r.URL.Query().Get("path")
 	if path == "" {
 		http.Error(w, "Path parameter not specified ", http.StatusBadRequest)

internal/config/config.go

@@ -14,19 +14,33 @@ type OolongConfig struct {
 	NotesDirPaths     []string `toml:"note_directories"`
 	NGramRange        []int    `toml:"ngram_range"`
 	AllowedExtensions []string `toml:"allowed_extensions"`
-	PluginPaths       []string `toml:"plugin_paths"`
 	IgnoreDirectories []string `toml:"ignored_directories"`
 	StopWords         []string `toml:"stop_words"`
+
+	PluginsConfig OolongPluginConfig `toml:"plugins"`
+	GraphConfig   OolongGraphConfig  `toml:"graph"`
+}
+
+type OolongPluginConfig struct {
+	PluginPaths []string `toml:"plugin_paths"`
+}
+
+type OolongGraphConfig struct {
+	MinNodeWeight float64 `toml:"min_node_weight"`
+	MaxNodeWeight float64 `toml:"max_node_weight"`
+	MinLinkWeight float64 `toml:"min_link_weight"`
+	// TODO: max link weight (call it a cap?)
 }
 
 func Config() *OolongConfig { return &cfg }
 
-func NotesDirPaths() []string      { return cfg.NotesDirPaths }
-func NGramRange() []int            { return cfg.NGramRange }
-func AllowedExtensions() []string  { return cfg.AllowedExtensions }
-func PluginPaths() []string        { return cfg.PluginPaths }
-func IgnoredDirectories() []string { return cfg.IgnoreDirectories }
-func StopWords() []string          { return cfg.StopWords }
+func NotesDirPaths() []string             { return cfg.NotesDirPaths }
+func NGramRange() []int                   { return cfg.NGramRange }
+func AllowedExtensions() []string         { return cfg.AllowedExtensions }
+func PluginPaths() []string               { return cfg.PluginsConfig.PluginPaths }
+func IgnoredDirectories() []string        { return cfg.IgnoreDirectories }
+func StopWords() []string                 { return cfg.StopWords }
+func WeightThresholds() OolongGraphConfig { return cfg.GraphConfig }
 
 // TODO: file watcher for config file, reload on change
 

internal/documents/corpus_test.go

@@ -16,7 +16,6 @@ var cfg = config.OolongConfig{
 	NotesDirPaths:     []string{"../../examples/data"},
 	NGramRange:        []int{1, 2, 3},
 	AllowedExtensions: []string{".md", ".tex", ".typ", ".txt"},
-	PluginPaths:       []string{},
 	IgnoreDirectories: []string{".templates", ".git"},
 }
 
@@ -29,7 +28,6 @@ func TestReadNotesDirs(t *testing.T) {
 	conf.NotesDirPaths = cfg.NotesDirPaths
 	conf.NGramRange = cfg.NGramRange
 	conf.AllowedExtensions = cfg.AllowedExtensions
-	conf.PluginPaths = cfg.PluginPaths
 	conf.IgnoreDirectories = cfg.IgnoreDirectories
 
 	// TODO: actual tests with an example data directory

internal/graph/graph.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"path/filepath"
 
+	"github.com/oolong-sh/oolong/internal/config"
 	"github.com/oolong-sh/oolong/internal/keywords"
 	"github.com/oolong-sh/oolong/internal/notes"
 )
@@ -37,15 +38,18 @@ func clamp(value, min, max float64) float64 {
 
 const NOTE_NODE_VAL = 1
 
-func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, lowerBound, upperBound float64) ([]byte, error) {
-	// func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, lowerBound, upperBound float64) (Graph, error) {
+func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note) ([]byte, error) {
+	thresholds := config.WeightThresholds()
+	minThresh := thresholds.MinNodeWeight
+	upperBound := thresholds.MaxNodeWeight
+
 	nodes := []NodeJSON{}
 	links := []LinkJSON{}
 
 	for _, keyword := range keywordMap {
 		// Only add nodes above the minimum threshold
-		if keyword.Weight >= lowerBound {
-			clampedWeight := clamp(keyword.Weight, lowerBound, upperBound)
+		if keyword.Weight >= minThresh {
+			clampedWeight := clamp(keyword.Weight, minThresh, upperBound)
 			nodes = append(nodes, NodeJSON{
 				ID:   keyword.Keyword,
 				Name: keyword.Keyword,
@@ -67,7 +71,7 @@ func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note,
 		// Link notes to keywords
 		for keywordID, wgt := range note.Weights {
 			keyword, exists := keywordMap[keywordID]
-			if exists && keyword.Weight >= lowerBound {
+			if exists && keyword.Weight >= minThresh {
 				links = append(links, LinkJSON{
 					Source: noteID,
 					Target: keyword.Keyword,

internal/keywords/keywords.go

@@ -3,6 +3,7 @@ package keywords
 import (
 	"encoding/json"
 
+	"github.com/oolong-sh/oolong/internal/config"
 	"github.com/oolong-sh/oolong/internal/linking/ngrams"
 )
 
@@ -23,38 +24,37 @@ func SerializeNGrams(ngmap map[string]*ngrams.NGram) ([]byte, error) {
 	return b, nil
 }
 
-// TODO: parameterize filtering threshold (maybe a percentage?)
-func NGramsToKeywords(ngmap map[string]*ngrams.NGram) []Keyword {
-	// keywords := make([]keyword, len(ngmap))
-	keywords := []Keyword{}
-	threshold := 8.0
+func NGramsToKeywordsMap(ngmap map[string]*ngrams.NGram) map[string]Keyword {
+	keywords := map[string]Keyword{}
+	minThresh := config.WeightThresholds().MinNodeWeight
 
 	for k, v := range ngmap {
 		w := v.Weight()
 
-		if w > threshold {
-			keywords = append(keywords, Keyword{
+		if w > minThresh {
+			keywords[k] = Keyword{
 				Keyword: k,
 				Weight:  w,
-			})
+			}
 		}
 	}
 
 	return keywords
 }
 
-func NGramsToKeywordsMap(ngmap map[string]*ngrams.NGram) map[string]Keyword {
-	keywords := map[string]Keyword{}
-	threshold := 8.0
+// Note: NGramsToKeywords is being used
+func NGramsToKeywords(ngmap map[string]*ngrams.NGram) []Keyword {
+	keywords := []Keyword{}
+	minThresh := config.WeightThresholds().MinNodeWeight
 
 	for k, v := range ngmap {
 		w := v.Weight()
 
-		if w > threshold {
-			keywords[k] = Keyword{
+		if w > minThresh {
+			keywords = append(keywords, Keyword{
 				Keyword: k,
 				Weight:  w,
-			}
+			})
 		}
 	}
 

internal/linking/ngrams/weighting.go

@@ -90,11 +90,3 @@ func FilterMeaningfulNGrams(ngmap map[string]*NGram, minDF int, maxDF int, minAv
 	}
 	return out
 }
-
-type Doc interface {
-	// get
-}
-
-// TODO:
-func NormalizeDocumentWeights() {
-}

internal/notes/notes.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"math"
 
+	"github.com/oolong-sh/oolong/internal/config"
 	"github.com/oolong-sh/oolong/internal/documents"
 )
 
@@ -24,18 +25,17 @@ func SerializeDocuments(documents map[string]*documents.Document) ([]byte, error
 	return b, nil
 }
 
-// TODO: parameterize filtering threshold (maybe as a percentage?)
 func DocumentsToNotes(documents map[string]*documents.Document) []Note {
 	notes := []Note{}
-	threshold := 2.0
+	minThresh := config.WeightThresholds().MinLinkWeight
 
 	for k, v := range documents {
 		weightSum := 0.0
 		weights := map[string]float64{}
 
 		// set weight values
 		for k, v := range v.Weights {
-			if v > threshold {
+			if v > minThresh {
 				weights[k] = v
 				weightSum += v * v
 			}