From 3c5e59145d3882ffe3d59e14e29965e50aab9f61 Mon Sep 17 00:00:00 2001 From: ptdewey Date: Sun, 24 Nov 2024 13:16:34 -0500 Subject: [PATCH] fix/feat: improved graph and updated api to allow fetch from webpage --- graph-2d.html | 28 --------------- graph-3d.html | 35 ------------------ index.html | 54 ++++++++++++++++++++-------- internal/api/graph_handlers.go | 21 +++++++++-- internal/api/note_handlers.go | 6 ++-- internal/daemon/watcher.go | 27 +++++++++----- internal/graph/graph.go | 12 ++++--- internal/linking/ngrams/weighting.go | 13 ++++--- internal/notes/notes.go | 14 ++++++++ internal/state/state.go | 25 +++++++------ 10 files changed, 123 insertions(+), 112 deletions(-) delete mode 100644 graph-2d.html delete mode 100644 graph-3d.html diff --git a/graph-2d.html b/graph-2d.html deleted file mode 100644 index ccf1814..0000000 --- a/graph-2d.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - -
- - - diff --git a/graph-3d.html b/graph-3d.html deleted file mode 100644 index 661df53..0000000 --- a/graph-3d.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - -
- - - - diff --git a/index.html b/index.html index 661df53..f3b9879 100644 --- a/index.html +++ b/index.html @@ -6,7 +6,6 @@ - @@ -16,20 +15,45 @@ diff --git a/internal/api/graph_handlers.go b/internal/api/graph_handlers.go index 7151d3e..bc156f2 100644 --- a/internal/api/graph_handlers.go +++ b/internal/api/graph_handlers.go @@ -1,9 +1,9 @@ package api import ( - "encoding/json" "log" "net/http" + "slices" "github.com/oolong-sh/oolong/internal/graph" "github.com/oolong-sh/oolong/internal/keywords" @@ -11,9 +11,22 @@ import ( "github.com/oolong-sh/oolong/internal/state" ) +var allowedOrigins = []string{ + "http://localhost:8000", +} + func handleGetGraph(w http.ResponseWriter, r *http.Request) { log.Println("Request received:", r.Method, r.URL, r.Host) w.Header().Set("Content-Type", "application/json") + origin := r.Header.Get("Origin") + + // check if the origin is whitelisted + if !slices.Contains(allowedOrigins, origin) { + log.Println("Requesting client not in allow list. Origin:", origin) + http.Error(w, "Request origin not in allow list", http.StatusForbidden) + return + } + w.Header().Set("Access-Control-Allow-Origin", origin) // get snapshot of current state s := state.State() @@ -30,7 +43,11 @@ func handleGetGraph(w http.ResponseWriter, r *http.Request) { } // encode graph data in reponse - if err := json.NewEncoder(w).Encode(data); err != nil { + if _, err := w.Write(data); err != nil { http.Error(w, "Error encoding graph data", 500) + return } + // if err := json.NewEncoder(w).Encode(data); err != nil { + // http.Error(w, "Error encoding graph data", 500) + // } } diff --git a/internal/api/note_handlers.go b/internal/api/note_handlers.go index 1bbdcb1..891b24e 100644 --- a/internal/api/note_handlers.go +++ b/internal/api/note_handlers.go @@ -58,13 +58,14 @@ func handleGetNote(w http.ResponseWriter, r *http.Request) { // 'POST /note' endpoint handler creates a note file (and any missing directories) corresponding to input path // Expected request body: { "path": "/path/to/note", "content", "full note contents to write" } func handleCreateNote(w http.ResponseWriter, r *http.Request) { - log.Println("Request received:", r.Method, r.URL, r.Host, r.Body) + log.Println("Request received:", r.Method, r.URL, r.Host) // parse request body var req createUpdateRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, "Failed to decode request body", 400) } + log.Println("Request body: ", req) // check if path before file exists, then check if file exists if e, err := exists(req.Path); err != nil { @@ -95,13 +96,14 @@ func handleCreateNote(w http.ResponseWriter, r *http.Request) { // It will create files that do not exist, but will not create directories // Expected request body: { "path": "/path/to/note", "content", "full note contents to write" } func handleUpdateNote(w http.ResponseWriter, r *http.Request) { - log.Println("Request received:", r.Method, r.URL, r.Host, r.Body) + log.Println("Request received:", r.Method, r.URL, r.Host) // parse request body var req createUpdateRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, "Failed to decode request body", 400) } + log.Println("Request body: ", req) // write contents to file if err := os.WriteFile(req.Path, []byte(req.Content), 0666); err != nil { diff --git a/internal/daemon/watcher.go b/internal/daemon/watcher.go index 5b22699..56888f1 100644 --- a/internal/daemon/watcher.go +++ b/internal/daemon/watcher.go @@ -4,6 +4,7 @@ import ( "errors" "io/fs" "log" + "os" "path/filepath" "slices" "time" @@ -50,7 +51,6 @@ func runNotesDirsWatcher(dirs ...string) error { } // watcher handler - // go func() { // running entire function as a goroutine, handler doesn't need to be one for { select { case event, ok := <-watcher.Events: @@ -58,11 +58,9 @@ func runNotesDirsWatcher(dirs ...string) error { log.Println("Watcher event channel returned bad result.") return errors.New("Invalid watcher errors channel value.") } - // log.Println("Event:", event) - // TODO: add new watcher in cases where new directories are created - - if event.Has(fsnotify.Write) { + switch { + case event.Has(fsnotify.Write): log.Println("Modified file:", event.Name) // write event is sent on write start, wait 500ms for write to finish @@ -72,6 +70,22 @@ func runNotesDirsWatcher(dirs ...string) error { documents.ReadDocuments(event.Name) // TODO: add dedup timer to prevent multi-write calls + + case event.Has(fsnotify.Remove): + log.Println("Removed file/directory", event.Name) + // TODO: remove from state + // - need to be careful with remove event as editors use it when writing files + // - state removal needs to also remove ngrams + // - should only trigger update on file deletions + + case event.Has(fsnotify.Create): + log.Println("Created file/directory", event.Name) + + if info, err := os.Stat(event.Name); err == nil { + if info.IsDir() { + watcher.Add(event.Name) + } + } } case err, ok := <-watcher.Errors: if !ok { @@ -80,7 +94,4 @@ func runNotesDirsWatcher(dirs ...string) error { log.Println("error:", err) } } - // }() - // <-make(chan struct{}) - // return nil } diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 6ea2420..0596b69 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -15,8 +15,9 @@ type NodeJSON struct { } type LinkJSON struct { - Source string `json:"source"` - Target string `json:"target"` + Source string `json:"source"` + Target string `json:"target"` + Value float64 `json:"strength"` } type Graph struct { @@ -34,9 +35,10 @@ func clamp(value, min, max float64) float64 { return value } -const NOTE_NODE_VAL = 10 +const NOTE_NODE_VAL = 1 func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, lowerBound, upperBound float64) ([]byte, error) { + // func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, lowerBound, upperBound float64) (Graph, error) { nodes := []NodeJSON{} links := []LinkJSON{} @@ -63,12 +65,13 @@ func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, }) // Link notes to keywords - for keywordID := range note.Weights { + for keywordID, wgt := range note.Weights { keyword, exists := keywordMap[keywordID] if exists && keyword.Weight >= lowerBound { links = append(links, LinkJSON{ Source: noteID, Target: keyword.Keyword, + Value: wgt, }) } } @@ -84,5 +87,6 @@ func SerializeGraph(keywordMap map[string]keywords.Keyword, notes []notes.Note, return nil, err } + // return graph, nil return jsonData, nil } diff --git a/internal/linking/ngrams/weighting.go b/internal/linking/ngrams/weighting.go index d2b66c2..38ee81c 100644 --- a/internal/linking/ngrams/weighting.go +++ b/internal/linking/ngrams/weighting.go @@ -46,12 +46,7 @@ var zoneB map[lexer.Zone]float64 = map[lexer.Zone]float64{ func CalcWeights(ngmap map[string]*NGram, N int) { idf(ngmap, N) // tfidf(ngmap) - // TODO: decide on k and b values (and allow them to be tweaked from config) bm25(ngmap) - // CHANGE: probably take n and word length into account - - // TODO: move adjustments to weights calculation function - // NOTE: these adjustments are much larger than the bm25 score and probably need to be scaled down for _, ng := range ngmap { ng.updateWeight() @@ -95,3 +90,11 @@ func FilterMeaningfulNGrams(ngmap map[string]*NGram, minDF int, maxDF int, minAv } return out } + +type Doc interface { + // get +} + +// TODO: +func NormalizeDocumentWeights() { +} diff --git a/internal/notes/notes.go b/internal/notes/notes.go index 24464f4..18fc5fa 100644 --- a/internal/notes/notes.go +++ b/internal/notes/notes.go @@ -2,6 +2,7 @@ package notes import ( "encoding/json" + "math" "github.com/oolong-sh/oolong/internal/documents" ) @@ -29,13 +30,20 @@ func DocumentsToNotes(documents map[string]*documents.Document) []Note { threshold := 2.0 for k, v := range documents { + weightSum := 0.0 weights := map[string]float64{} + + // set weight values for k, v := range v.Weights { if v > threshold { weights[k] = v + weightSum += v * v } } + // normalize resulting weights + normalizeWeights(weights, math.Sqrt(weightSum)) + notes = append(notes, Note{ Path: k, Weights: weights, @@ -44,3 +52,9 @@ func DocumentsToNotes(documents map[string]*documents.Document) []Note { return notes } + +func normalizeWeights(m map[string]float64, sum float64) { + for k, v := range m { + m[k] = v / sum + } +} diff --git a/internal/state/state.go b/internal/state/state.go index 86be0a9..3c7ced4 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -3,13 +3,9 @@ package state import ( "fmt" "log" - "os" "github.com/oolong-sh/oolong/internal/documents" - "github.com/oolong-sh/oolong/internal/graph" - "github.com/oolong-sh/oolong/internal/keywords" "github.com/oolong-sh/oolong/internal/linking/ngrams" - "github.com/oolong-sh/oolong/internal/notes" ) type OolongState struct { @@ -113,18 +109,21 @@ func (s *StateManager) updateState(docs []*documents.Document) { // // TODO: add threshold filtering params to these functions (use config) - kw := keywords.NGramsToKeywordsMap(s.state.NGrams) - notes := notes.DocumentsToNotes(s.state.Documents) + // kw := keywords.NGramsToKeywordsMap(s.state.NGrams) + // notes := notes.DocumentsToNotes(s.state.Documents) - dat, err := graph.SerializeGraph(kw, notes, 0.1, 80) - if err != nil { - panic(err) - } + // REFACTOR: store graph json in state field to make it available on request earlier? + // - this is probably a good idea + // + // dat, err := graph.SerializeGraph(kw, notes, 0.1, 80) + // if err != nil { + // panic(err) + // } // TEST: remove json output later - if err := os.WriteFile("graph.json", dat, 0644); err != nil { - panic(err) - } + // if err := os.WriteFile("graph.json", dat, 0644); err != nil { + // panic(err) + // } log.Println("State update complete.") }