From 9ca7463dda2cf143b9b7345dad78dd40650d5609 Mon Sep 17 00:00:00 2001 From: elraphty Date: Thu, 26 Oct 2023 16:59:24 +0100 Subject: [PATCH 01/28] started metrics --- db/structs.go | 1 + go.mod | 1 + go.sum | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/db/structs.go b/db/structs.go index 17a756379..e6dfcdb49 100644 --- a/db/structs.go +++ b/db/structs.go @@ -379,6 +379,7 @@ type Bounty struct { EstimatedSessionLength string `json:"estimated_session_length"` EstimatedCompletionDate string `json:"estimated_completion_date"` Updated *time.Time `json:"updated"` + CompletionDate *time.Time `json:"completion_date"` CodingLanguages pq.StringArray `gorm:"type:text[];not null default:'[]'" json:"coding_languages"` } diff --git a/go.mod b/go.mod index f0f837b05..5a517b123 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/miekg/dns v1.1.56 // indirect github.com/nbd-wtf/ln-decodepay v1.11.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/redis/go-redis/v9 v9.2.1 // indirect github.com/rs/cors v1.7.0 github.com/rs/xid v1.4.0 github.com/stretchr/testify v1.8.2 diff --git a/go.sum b/go.sum index 2ee26cb93..4c5fd8780 100644 --- a/go.sum +++ b/go.sum @@ -477,6 +477,8 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= @@ -565,6 +567,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -639,6 +642,8 @@ github.com/decred/dcrd/lru v1.1.2/go.mod h1:gEdCVgXs1/YoBvFWt7Scgknbhwik3FgVSzln github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dhui/dktest v0.3.16/go.mod h1:gYaA3LRmM8Z4vJl2MA0THIigJoZrwOansEOsp+kqxp0= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= @@ -1373,6 +1378,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= +github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From f9a74fec6c5b2d1bf08399046cafaa81212c614f Mon Sep 17 00:00:00 2001 From: elraphty Date: Fri, 27 Oct 2023 15:07:52 +0100 Subject: [PATCH 02/28] added redis --- db/redis.go | 33 +++++++++++++++++++++++++++++++++ main.go | 1 + 2 files changed, 34 insertions(+) create mode 100644 db/redis.go diff --git a/db/redis.go b/db/redis.go new file mode 100644 index 000000000..61ac56c66 --- /dev/null +++ b/db/redis.go @@ -0,0 +1,33 @@ +package db + +import ( + "fmt" + "os" + + "github.com/redis/go-redis/v9" + "github.com/stakwork/sphinx-tribes/utils" +) + +var RedisClient *redis.Client + +func InitRedis() { + redisURL := os.Getenv("REDIS_URL") + fmt.Println("redis url :", redisURL) + + if redisURL == "" { + dbInt, _ := utils.ConvertStringToInt(os.Getenv("REDIS_DB")) + RedisClient = redis.NewClient(&redis.Options{ + Addr: os.Getenv("REDIS_HOST"), + Username: os.Getenv("REDIS_USER"), + Password: os.Getenv("REDIS_PASS"), + DB: dbInt, + }) + } else { + opt, err := redis.ParseURL(redisURL) + if err != nil { + fmt.Println("REDIS URL CONNECTION ERROR ===", err) + } + + RedisClient = redis.NewClient(opt) + } +} diff --git a/main.go b/main.go index 5c49444b3..bb75d7cb9 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ func main() { } db.InitDB() + db.InitRedis() db.InitCache() db.InitRoles() // Config has to be inited before JWT, if not it will lead to NO JWT error From f8f6d2cd89738714a8d56470fc26ab2d0b7c4307 Mon Sep 17 00:00:00 2001 From: elraphty Date: Fri, 27 Oct 2023 17:46:09 +0100 Subject: [PATCH 03/28] added redis get and set functions --- db/redis.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/db/redis.go b/db/redis.go index 61ac56c66..89b2fdcd1 100644 --- a/db/redis.go +++ b/db/redis.go @@ -1,6 +1,7 @@ package db import ( + "context" "fmt" "os" @@ -8,6 +9,7 @@ import ( "github.com/stakwork/sphinx-tribes/utils" ) +var ctx = context.Background() var RedisClient *redis.Client func InitRedis() { @@ -31,3 +33,19 @@ func InitRedis() { RedisClient = redis.NewClient(opt) } } + +func SetValue(key string, value string) { + rerr := RedisClient.Set(ctx, key, value, 0).Err() + if rerr != nil { + panic(rerr) + } +} + +func GetValue(key string) string { + val, rerr := RedisClient.Get(ctx, key).Result() + if rerr != nil { + panic(rerr) + } + + return val +} From 3b5bc151181beaff08e4ecddaeeb2974e2ff2719 Mon Sep 17 00:00:00 2001 From: elraphty Date: Fri, 27 Oct 2023 20:11:29 +0100 Subject: [PATCH 04/28] added Redis map --- frontend/app/src/config/ModeDispatcher.tsx | 2 +- frontend/app/src/config/host.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/app/src/config/ModeDispatcher.tsx b/frontend/app/src/config/ModeDispatcher.tsx index 7072acfac..64633dc55 100644 --- a/frontend/app/src/config/ModeDispatcher.tsx +++ b/frontend/app/src/config/ModeDispatcher.tsx @@ -7,7 +7,7 @@ export enum AppMode { } const hosts: { [k: string]: AppMode } = { - 'localhost:3000': AppMode.TRIBES, + 'localhost:3005': AppMode.COMMUNITY, 'localhost:13000': AppMode.TRIBES, 'localhost:23000': AppMode.TRIBES, 'tribes.sphinx.chat': AppMode.TRIBES, diff --git a/frontend/app/src/config/host.ts b/frontend/app/src/config/host.ts index 51a0f3192..6b97a5cdb 100644 --- a/frontend/app/src/config/host.ts +++ b/frontend/app/src/config/host.ts @@ -3,7 +3,8 @@ const externalDockerHosts = ['localhost:23007', 'localhost:23000']; export function getHost(): string { const host = window.location.host.includes('localhost') ? 'localhost:5002' : window.location.host; - return host; + return "localhost:5005"; + // return host; } export function getHostIncludingDockerHosts() { @@ -19,5 +20,5 @@ export function getHostIncludingDockerHosts() { export const TribesURL = getHost().startsWith('localhost') ? `http://${getHost()}` : getHost().startsWith('http') - ? getHost() - : `https://${getHost()}`; + ? getHost() + : `https://${getHost()}`; From bd577168dd9a941085cdde519b591ae0d6e9734c Mon Sep 17 00:00:00 2001 From: elraphty Date: Sat, 28 Oct 2023 11:10:11 +0100 Subject: [PATCH 05/28] modified redis --- db/redis.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/db/redis.go b/db/redis.go index 89b2fdcd1..5d0815c1c 100644 --- a/db/redis.go +++ b/db/redis.go @@ -35,17 +35,31 @@ func InitRedis() { } func SetValue(key string, value string) { - rerr := RedisClient.Set(ctx, key, value, 0).Err() - if rerr != nil { - panic(rerr) + err := RedisClient.Set(ctx, key, value, 0).Err() + if err != nil { + fmt.Println("REDIS SET ERROR :", err) } } func GetValue(key string) string { - val, rerr := RedisClient.Get(ctx, key).Result() - if rerr != nil { - panic(rerr) + val, err := RedisClient.Get(ctx, key).Result() + if err != nil { + fmt.Println("REDIS GET ERROR :", err) } return val } + +func SetMap(key string, values map[string]string) { + for k, v := range values { + err := RedisClient.HSet(ctx, key, k, v).Err() + if err != nil { + fmt.Println("REDIS SET MAP ERROR :", err) + } + } +} + +func GetMap(key string) map[string]string { + values := RedisClient.HGetAll(ctx, key).Val() + return values +} From af51e07105e4f2c29f0fb16ed9b4e2d8b568ad98 Mon Sep 17 00:00:00 2001 From: elraphty Date: Mon, 30 Oct 2023 15:54:56 +0100 Subject: [PATCH 06/28] added endpoints for metrics --- db/metrics.go | 25 +++ db/structs.go | 12 +- frontend/app/src/config/ModeDispatcher.tsx | 2 +- frontend/app/src/config/host.ts | 3 +- handlers/invoiceCron.go | 244 +++++++++++++++++++++ handlers/metrics.go | 92 ++++++++ routes/index.go | 1 + routes/metrics.go | 20 ++ 8 files changed, 394 insertions(+), 5 deletions(-) create mode 100644 db/metrics.go create mode 100644 handlers/invoiceCron.go create mode 100644 handlers/metrics.go create mode 100644 routes/metrics.go diff --git a/db/metrics.go b/db/metrics.go new file mode 100644 index 000000000..02f2afd33 --- /dev/null +++ b/db/metrics.go @@ -0,0 +1,25 @@ +package db + +func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { + var sum uint + + db.db.Model(&PaymentHistory{}).Where("payment_type = ?", r.PaymentType).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) + + return sum +} + +func (db database) TotalPeopleByDateRange(r PaymentDateRange) int64 { + var count int64 + + db.db.Model(&Person{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + + return count +} + +func (db database) TotalOrganizationsByDateRange(r PaymentDateRange) int64 { + var count int64 + + db.db.Model(&Organization{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + + return count +} diff --git a/db/structs.go b/db/structs.go index cdc239663..f91af6a07 100644 --- a/db/structs.go +++ b/db/structs.go @@ -367,7 +367,6 @@ type Bounty struct { Price string `json:"price"` Title string `json:"title"` Tribe string `json:"tribe"` - Created int64 `json:"created"` Assignee string `json:"assignee"` TicketUrl string `json:"ticket_url"` OrgUuid string `json:"org_uuid"` @@ -378,8 +377,11 @@ type Bounty struct { OneSentenceSummary string `json:"one_sentence_summary"` EstimatedSessionLength string `json:"estimated_session_length"` EstimatedCompletionDate string `json:"estimated_completion_date"` + Created int64 `json:"created"` Updated *time.Time `json:"updated"` - CompletionDate *time.Time `json:"completion_date"` + AssignedDate *time.Time `json:"assigned_date,omitempty"` + CompletionDate *time.Time `json:"completion_date,omitempty"` + MarkAsPaidDate *time.Time `json:"mark_as_paid_date,omitempty"` CodingLanguages pq.StringArray `gorm:"type:text[];not null default:'[]'" json:"coding_languages"` } @@ -591,6 +593,12 @@ type WithdrawBudgetRequest struct { OrgUuid string `json:"org_uuid"` } +type PaymentDateRange struct { + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + PaymentType PaymentType `json:"payment_type,omitempty"` +} + func (Person) TableName() string { return "people" } diff --git a/frontend/app/src/config/ModeDispatcher.tsx b/frontend/app/src/config/ModeDispatcher.tsx index 64633dc55..7072acfac 100644 --- a/frontend/app/src/config/ModeDispatcher.tsx +++ b/frontend/app/src/config/ModeDispatcher.tsx @@ -7,7 +7,7 @@ export enum AppMode { } const hosts: { [k: string]: AppMode } = { - 'localhost:3005': AppMode.COMMUNITY, + 'localhost:3000': AppMode.TRIBES, 'localhost:13000': AppMode.TRIBES, 'localhost:23000': AppMode.TRIBES, 'tribes.sphinx.chat': AppMode.TRIBES, diff --git a/frontend/app/src/config/host.ts b/frontend/app/src/config/host.ts index 6b97a5cdb..19c7d2cc5 100644 --- a/frontend/app/src/config/host.ts +++ b/frontend/app/src/config/host.ts @@ -3,8 +3,7 @@ const externalDockerHosts = ['localhost:23007', 'localhost:23000']; export function getHost(): string { const host = window.location.host.includes('localhost') ? 'localhost:5002' : window.location.host; - return "localhost:5005"; - // return host; + return host; } export function getHostIncludingDockerHosts() { diff --git a/handlers/invoiceCron.go b/handlers/invoiceCron.go new file mode 100644 index 000000000..f1f578d8b --- /dev/null +++ b/handlers/invoiceCron.go @@ -0,0 +1,244 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strconv" + "time" + + "github.com/go-co-op/gocron" + "github.com/stakwork/sphinx-tribes/config" + "github.com/stakwork/sphinx-tribes/db" + "github.com/stakwork/sphinx-tribes/utils" +) + +// Keep for future reference +func InitInvoiceCron() { + s := gocron.NewScheduler(time.UTC) + msg := make(map[string]interface{}) + + s.Every(5).Seconds().Do(func() { + invoiceList, _ := db.Store.GetInvoiceCache() + invoiceCount := len(invoiceList) + + if invoiceCount > 0 { + for index, inv := range invoiceList { + url := fmt.Sprintf("%s/invoice?payment_request=%s", config.RelayUrl, inv.Invoice) + + client := &http.Client{} + req, err := http.NewRequest(http.MethodGet, url, nil) + + req.Header.Set("x-user-token", config.RelayAuthKey) + req.Header.Set("Content-Type", "application/json") + res, _ := client.Do(req) + + if err != nil { + log.Printf("Request Failed: %s", err) + return + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + + // Unmarshal result + invoiceRes := db.InvoiceResult{} + + err = json.Unmarshal(body, &invoiceRes) + + if err != nil { + log.Printf("Reading Invoice body failed: %s", err) + return + } + + if invoiceRes.Response.Settled { + if inv.Invoice == invoiceRes.Response.Payment_request { + /** + If the invoice is settled and still in store + make keysend payment + */ + msg["msg"] = "invoice_success" + msg["invoice"] = inv.Invoice + + socket, err := db.Store.GetSocketConnections(inv.Host) + + if err == nil { + socket.Conn.WriteJSON(msg) + } + + if inv.Type == "KEYSEND" { + url := fmt.Sprintf("%s/payment", config.RelayUrl) + + amount, _ := utils.ConvertStringToUint(inv.Amount) + + bodyData := utils.BuildKeysendBodyData(amount, inv.User_pubkey, inv.Route_hint) + + jsonBody := []byte(bodyData) + + client := &http.Client{} + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(jsonBody)) + + req.Header.Set("x-user-token", config.RelayAuthKey) + req.Header.Set("Content-Type", "application/json") + res, _ := client.Do(req) + + if err != nil { + log.Printf("Request Failed: %s", err) + return + } + + defer res.Body.Close() + + body, err = io.ReadAll(res.Body) + + if res.StatusCode == 200 { + // Unmarshal result + keysendRes := db.KeysendSuccess{} + err = json.Unmarshal(body, &keysendRes) + + dateInt, _ := strconv.ParseInt(inv.Created, 10, 32) + bounty, err := db.DB.GetBountyByCreated(uint(dateInt)) + + if err == nil { + bounty.Paid = true + } + + db.DB.UpdateBounty(bounty) + + // Delete the index from the store array list and reset the store + updateInvoiceCache(invoiceList, index) + + msg["msg"] = "keysend_success" + msg["invoice"] = inv.Invoice + + socket, err := db.Store.GetSocketConnections(inv.Host) + if err == nil { + socket.Conn.WriteJSON(msg) + } + } else { + // Unmarshal result + keysendError := db.KeysendError{} + err = json.Unmarshal(body, &keysendError) + log.Printf("Keysend Payment to %s Failed, with Error: %s", inv.User_pubkey, keysendError.Error) + + msg["msg"] = "keysend_error" + msg["invoice"] = inv.Invoice + + socket, err := db.Store.GetSocketConnections(inv.Host) + + if err == nil { + socket.Conn.WriteJSON(msg) + } + + updateInvoiceCache(invoiceList, index) + } + + if err != nil { + log.Printf("Reading body failed: %s", err) + return + } + } else { + dateInt, _ := strconv.ParseInt(inv.Created, 10, 32) + bounty, err := db.DB.GetBountyByCreated(uint(dateInt)) + + if err == nil { + bounty.Assignee = inv.User_pubkey + bounty.CommitmentFee = uint64(inv.Commitment_fee) + bounty.AssignedHours = uint8(inv.Assigned_hours) + bounty.BountyExpires = inv.Bounty_expires + } + + db.DB.UpdateBounty(bounty) + + // Delete the index from the store array list and reset the store + updateInvoiceCache(invoiceList, index) + + msg := make(map[string]interface{}) + msg["msg"] = "assign_success" + msg["invoice"] = inv.Invoice + + socket, err := db.Store.GetSocketConnections(inv.Host) + if err == nil { + socket.Conn.WriteJSON(msg) + } + } + } + } + } + } + }) + + s.Every(5).Seconds().Do(func() { + invoiceList, _ := db.Store.GetBudgetInvoiceCache() + invoiceCount := len(invoiceList) + + if invoiceCount > 0 { + for index, inv := range invoiceList { + url := fmt.Sprintf("%s/invoice?payment_request=%s", config.RelayUrl, inv.Invoice) + + client := &http.Client{} + req, err := http.NewRequest(http.MethodGet, url, nil) + + req.Header.Set("x-user-token", config.RelayAuthKey) + req.Header.Set("Content-Type", "application/json") + res, _ := client.Do(req) + + if err != nil { + log.Printf("Request Failed: %s", err) + return + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + + // Unmarshal result + invoiceRes := db.InvoiceResult{} + + err = json.Unmarshal(body, &invoiceRes) + + if err != nil { + log.Printf("Reading Organization Invoice body failed: %s", err) + return + } + + if invoiceRes.Response.Settled { + if inv.Invoice == invoiceRes.Response.Payment_request { + /** + If the invoice is settled and still in store + make keysend payment + */ + msg["msg"] = "budget_success" + msg["invoice"] = inv.Invoice + + socket, err := db.Store.GetSocketConnections(inv.Host) + + if err == nil { + socket.Conn.WriteJSON(msg) + } + + // db.DB.AddAndUpdateBudget(inv) + updateBudgetInvoiceCache(invoiceList, index) + } + } + } + + } + }) + + s.StartAsync() +} + +func updateInvoiceCache(invoiceList []db.InvoiceStoreData, index int) { + newInvoiceList := append(invoiceList[:index], invoiceList[index+1:]...) + db.Store.SetInvoiceCache(newInvoiceList) +} + +func updateBudgetInvoiceCache(invoiceList []db.BudgetStoreData, index int) { + newInvoiceList := append(invoiceList[:index], invoiceList[index+1:]...) + db.Store.SetBudgetInvoiceCache(newInvoiceList) +} diff --git a/handlers/metrics.go b/handlers/metrics.go new file mode 100644 index 000000000..c5012000d --- /dev/null +++ b/handlers/metrics.go @@ -0,0 +1,92 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/stakwork/sphinx-tribes/auth" + "github.com/stakwork/sphinx-tribes/db" +) + +func PaymentMetrics(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + if pubKeyFromAuth == "" { + fmt.Println("no pubkey from auth") + w.WriteHeader(http.StatusUnauthorized) + return + } + + request := db.PaymentDateRange{} + body, err := io.ReadAll(r.Body) + r.Body.Close() + + err = json.Unmarshal(body, &request) + if err != nil { + w.WriteHeader(http.StatusNotAcceptable) + json.NewEncoder(w).Encode("Request body not accepted") + return + } + + sumAmount := db.DB.TotalPaymentsByDateRange(request) + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(sumAmount) +} + +func OrganizationtMetrics(w http.ResponseWriter, r *http.Request) { + // ctx := r.Context() + // pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + // if pubKeyFromAuth == "" { + // fmt.Println("no pubkey from auth") + // w.WriteHeader(http.StatusUnauthorized) + // return + // } + + request := db.PaymentDateRange{} + body, err := io.ReadAll(r.Body) + r.Body.Close() + + err = json.Unmarshal(body, &request) + if err != nil { + w.WriteHeader(http.StatusNotAcceptable) + json.NewEncoder(w).Encode("Request body not accepted") + return + } + + sumAmount := db.DB.TotalOrganizationsByDateRange(request) + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(sumAmount) +} + +func PeopleMetrics(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + if pubKeyFromAuth == "" { + fmt.Println("no pubkey from auth") + w.WriteHeader(http.StatusUnauthorized) + return + } + + request := db.PaymentDateRange{} + body, err := io.ReadAll(r.Body) + r.Body.Close() + + err = json.Unmarshal(body, &request) + if err != nil { + w.WriteHeader(http.StatusNotAcceptable) + json.NewEncoder(w).Encode("Request body not accepted") + return + } + + sumAmount := db.DB.TotalPeopleByDateRange(request) + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(sumAmount) +} diff --git a/routes/index.go b/routes/index.go index bb96e5fb8..5b4e210b8 100644 --- a/routes/index.go +++ b/routes/index.go @@ -31,6 +31,7 @@ func NewRouter() *http.Server { r.Mount("/github_issue", GithubIssuesRoutes()) r.Mount("/gobounties", BountyRoutes()) r.Mount("/organizations", OrganizationRoutes()) + r.Mount("/metrics", MetricsRoutes()) r.Group(func(r chi.Router) { r.Get("/tribe_by_feed", handlers.GetFirstTribeByFeed) diff --git a/routes/metrics.go b/routes/metrics.go new file mode 100644 index 000000000..80175364e --- /dev/null +++ b/routes/metrics.go @@ -0,0 +1,20 @@ +package routes + +import ( + "github.com/go-chi/chi" + "github.com/stakwork/sphinx-tribes/auth" + "github.com/stakwork/sphinx-tribes/handlers" +) + +func MetricsRoutes() chi.Router { + r := chi.NewRouter() + + r.Group(func(r chi.Router) { + r.Use(auth.PubKeyContext) + + r.Post("/payment_metrics", handlers.PaymentMetrics) + r.Post("/people_created", handlers.PeopleMetrics) + r.Post("/organization_created", handlers.OrganizationtMetrics) + }) + return r +} From d27e8164d63f79eae4f16eb2c85f4cdcd02d6777 Mon Sep 17 00:00:00 2001 From: elraphty Date: Thu, 23 Nov 2023 20:27:18 +0100 Subject: [PATCH 07/28] added bounty metrics --- db/metrics.go | 52 +++++++++++++++++++++++++++++++++++++++++++++ db/structs.go | 9 ++++++++ handlers/metrics.go | 41 +++++++++++++++++++++++++++++++++++ routes/metrics.go | 1 + 4 files changed, 103 insertions(+) diff --git a/db/metrics.go b/db/metrics.go index 02f2afd33..9233fd4d7 100644 --- a/db/metrics.go +++ b/db/metrics.go @@ -1,5 +1,7 @@ package db +import "math" + func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { var sum uint @@ -8,6 +10,56 @@ func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { return sum } +func (db database) TotalSatsPaid(r PaymentDateRange) uint { + var sum uint + + db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) + + return sum +} + +func (db database) TotalSatsPosted(r PaymentDateRange) uint { + var sum uint + + db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) + + return sum +} + +func (db database) SatsPaidPercentage(r PaymentDateRange) uint { + satsPosted := DB.TotalSatsPosted(r) + satsPaid := DB.TotalSatsPaid(r) + + value := satsPosted * 100 / satsPaid + paidPercentage := math.Round(float64(value)) + return uint(paidPercentage) +} + +func (db database) TotalPaidBounties(r PaymentDateRange) int64 { + var count int64 + + db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Count(&count) + + return count +} + +func (db database) TotalBountiesPosted(r PaymentDateRange) int64 { + var count int64 + + db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + + return count +} + +func (db database) BountiesPaidPercentage(r PaymentDateRange) uint { + bountiesPosted := DB.TotalBountiesPosted(r) + bountiesPaid := DB.TotalPaidBounties(r) + + value := bountiesPaid * 100 / bountiesPosted + paidPercentage := math.Round(float64(value)) + return uint(paidPercentage) +} + func (db database) TotalPeopleByDateRange(r PaymentDateRange) int64 { var count int64 diff --git a/db/structs.go b/db/structs.go index 6b5f2b87c..8f648f67f 100644 --- a/db/structs.go +++ b/db/structs.go @@ -640,6 +640,15 @@ type Meme struct { Expiry *time.Time `json:"expiry"` } +type BountyMetrics struct { + BountiesPosted int64 + BountiesPaid int64 + BountiesPaidPercentage uint + SatsPosted uint + SatsPaid uint + SatsPaidPercentage uint +} + func (Person) TableName() string { return "people" } diff --git a/handlers/metrics.go b/handlers/metrics.go index 5f40808de..8bb823b6d 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -90,3 +90,44 @@ func PeopleMetrics(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(sumAmount) } + +func BountyMetrics(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + if pubKeyFromAuth == "" { + fmt.Println("no pubkey from auth") + w.WriteHeader(http.StatusUnauthorized) + return + } + + request := db.PaymentDateRange{} + body, err := io.ReadAll(r.Body) + r.Body.Close() + + err = json.Unmarshal(body, &request) + if err != nil { + w.WriteHeader(http.StatusNotAcceptable) + json.NewEncoder(w).Encode("Request body not accepted") + return + } + + totalBountiesPosted := db.DB.TotalBountiesPosted(request) + totalBountiesPaid := db.DB.TotalPaidBounties(request) + bountiesPaidPercentage := db.DB.BountiesPaidPercentage(request) + totalSatsPosted := db.DB.TotalSatsPosted(request) + totalSatsPaid := db.DB.TotalSatsPaid(request) + satsPaidPercentage := db.DB.SatsPaidPercentage(request) + + bountyMetrics := db.BountyMetrics{ + BountiesPosted: totalBountiesPosted, + BountiesPaid: totalBountiesPaid, + BountiesPaidPercentage: bountiesPaidPercentage, + SatsPosted: totalSatsPosted, + SatsPaid: totalSatsPaid, + SatsPaidPercentage: satsPaidPercentage, + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bountyMetrics) +} diff --git a/routes/metrics.go b/routes/metrics.go index 80175364e..79c1640fd 100644 --- a/routes/metrics.go +++ b/routes/metrics.go @@ -15,6 +15,7 @@ func MetricsRoutes() chi.Router { r.Post("/payment_metrics", handlers.PaymentMetrics) r.Post("/people_created", handlers.PeopleMetrics) r.Post("/organization_created", handlers.OrganizationtMetrics) + r.Post("/bounty_metrics", handlers.BountyMetrics) }) return r } From 153c154659aef68498eeede1c6825caba4551715 Mon Sep 17 00:00:00 2001 From: elraphty Date: Fri, 24 Nov 2023 12:36:30 +0100 Subject: [PATCH 08/28] added column for date difference --- db/structs.go | 13 +++++++------ go.mod | 4 ++-- handlers/bounty.go | 6 ++++++ utils/helpers.go | 8 ++++++++ utils/helpers_test.go | 16 ++++++++++++++++ 5 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 utils/helpers_test.go diff --git a/db/structs.go b/db/structs.go index 8f648f67f..64351234a 100644 --- a/db/structs.go +++ b/db/structs.go @@ -383,6 +383,7 @@ type Bounty struct { CompletionDate *time.Time `json:"completion_date,omitempty"` MarkAsPaidDate *time.Time `json:"mark_as_paid_date,omitempty"` PaidDate *time.Time `json:"paid_date,omitempty"` + PaidDateDifference int64 `json:"paid_date_difference,omitempty"` CodingLanguages pq.StringArray `gorm:"type:text[];not null default:'[]'" json:"coding_languages"` } @@ -641,12 +642,12 @@ type Meme struct { } type BountyMetrics struct { - BountiesPosted int64 - BountiesPaid int64 - BountiesPaidPercentage uint - SatsPosted uint - SatsPaid uint - SatsPaidPercentage uint + BountiesPosted int64 `json:"bounties_posted"` + BountiesPaid int64 `json:"bounties_paid"` + BountiesPaidPercentage uint `json:"bounties_paid_average"` + SatsPosted uint `json:"sats_posted"` + SatsPaid uint `json:"sats_paid"` + SatsPaidPercentage uint `json:"sats_paid_percentage"` } func (Person) TableName() string { diff --git a/go.mod b/go.mod index 5a517b123..db0ccd700 100644 --- a/go.mod +++ b/go.mod @@ -33,9 +33,9 @@ require ( github.com/lightningnetwork/lnd v0.17.0-beta.rc6 // indirect github.com/lightningnetwork/lnd/tor v1.1.3 // indirect github.com/miekg/dns v1.1.56 // indirect - github.com/nbd-wtf/ln-decodepay v1.11.1 // indirect + github.com/nbd-wtf/ln-decodepay v1.11.1 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/redis/go-redis/v9 v9.2.1 // indirect + github.com/redis/go-redis/v9 v9.2.1 github.com/rs/cors v1.7.0 github.com/rs/xid v1.4.0 github.com/stretchr/testify v1.8.2 diff --git a/handlers/bounty.go b/handlers/bounty.go index a580fc7b2..2c3139d8f 100644 --- a/handlers/bounty.go +++ b/handlers/bounty.go @@ -143,6 +143,11 @@ func CreateOrEditBounty(w http.ResponseWriter, r *http.Request) { return } + if bounty.Assignee != "" { + now := time.Now() + bounty.AssignedDate = &now + } + if bounty.Tribe == "" { bounty.Tribe = "None" } @@ -401,6 +406,7 @@ func MakeBountyPayment(w http.ResponseWriter, r *http.Request) { db.DB.AddPaymentHistory(paymentHistory) bounty.Paid = true bounty.PaidDate = &now + bounty.PaidDateDifference = utils.GetDateDaysDifference(bounty.Created, &now) db.DB.UpdateBounty(bounty) msg["msg"] = "keysend_success" diff --git a/utils/helpers.go b/utils/helpers.go index 7393b31c5..189b10ea4 100644 --- a/utils/helpers.go +++ b/utils/helpers.go @@ -5,6 +5,7 @@ import ( "encoding/base32" "fmt" "strconv" + "time" decodepay "github.com/nbd-wtf/ln-decodepay" ) @@ -52,3 +53,10 @@ func GetInvoiceAmount(paymentRequest string) uint { return amount } + +func GetDateDaysDifference(createdDate int64, paidDate *time.Time) int64 { + firstDate := time.Unix(createdDate, 0) + difference := paidDate.Sub(*&firstDate) + days := int64(difference.Hours() / 24) + return days +} diff --git a/utils/helpers_test.go b/utils/helpers_test.go new file mode 100644 index 000000000..7078b7325 --- /dev/null +++ b/utils/helpers_test.go @@ -0,0 +1,16 @@ +package utils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetDateDaysDifference(t *testing.T) { + created := 1700238956 + + testDate := time.Now() + days := GetDateDaysDifference(int64(created), &testDate) + assert.NotEqual(t, 0, days) +} From c1a27f9394a385ebaebe6369bc93b04049547ad5 Mon Sep 17 00:00:00 2001 From: elraphty Date: Fri, 24 Nov 2023 19:29:29 +0100 Subject: [PATCH 09/28] added completetion_date difference --- db/metrics.go | 58 ++++++++++++++++++++++++++++++----------- db/structs.go | 63 ++++++++++++++++++++++++--------------------- handlers/bounty.go | 23 ++++++++++++++--- handlers/metrics.go | 4 +++ 4 files changed, 100 insertions(+), 48 deletions(-) diff --git a/db/metrics.go b/db/metrics.go index 9233fd4d7..d4f24f530 100644 --- a/db/metrics.go +++ b/db/metrics.go @@ -4,32 +4,25 @@ import "math" func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { var sum uint - db.db.Model(&PaymentHistory{}).Where("payment_type = ?", r.PaymentType).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) - return sum } func (db database) TotalSatsPaid(r PaymentDateRange) uint { var sum uint - db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) - return sum } func (db database) TotalSatsPosted(r PaymentDateRange) uint { var sum uint - db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) - return sum } func (db database) SatsPaidPercentage(r PaymentDateRange) uint { satsPosted := DB.TotalSatsPosted(r) satsPaid := DB.TotalSatsPaid(r) - value := satsPosted * 100 / satsPaid paidPercentage := math.Round(float64(value)) return uint(paidPercentage) @@ -37,24 +30,19 @@ func (db database) SatsPaidPercentage(r PaymentDateRange) uint { func (db database) TotalPaidBounties(r PaymentDateRange) int64 { var count int64 - db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Count(&count) - return count } func (db database) TotalBountiesPosted(r PaymentDateRange) int64 { var count int64 - db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) - return count } func (db database) BountiesPaidPercentage(r PaymentDateRange) uint { bountiesPosted := DB.TotalBountiesPosted(r) bountiesPaid := DB.TotalPaidBounties(r) - value := bountiesPaid * 100 / bountiesPosted paidPercentage := math.Round(float64(value)) return uint(paidPercentage) @@ -62,16 +50,56 @@ func (db database) BountiesPaidPercentage(r PaymentDateRange) uint { func (db database) TotalPeopleByDateRange(r PaymentDateRange) int64 { var count int64 - db.db.Model(&Person{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) - return count } func (db database) TotalOrganizationsByDateRange(r PaymentDateRange) int64 { var count int64 - db.db.Model(&Organization{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + return count +} +func (db database) PaidDifferenceSum(r PaymentDateRange) uint { + var sum uint + db.db.Model(&Bounty{}).Where("paid_date_difference != ?", + "").Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(paid_date_difference)").Row().Scan(&sum) + return sum +} + +func (db database) PaidDifferenceCount(r PaymentDateRange) int64 { + var count int64 + db.db.Model(&Bounty{}).Where("paid_date_difference != ?", + "").Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) return count } + +func (db database) AveragePaidTime(r PaymentDateRange) uint { + paidSum := DB.PaidDifferenceSum(r) + paidCount := DB.PaidDifferenceCount(r) + avg := paidSum / uint(paidCount) + avgDays := math.Round(float64(avg)) + return uint(avgDays) +} + +func (db database) CompletedDifferenceSum(r PaymentDateRange) uint { + var sum uint + db.db.Model(&Bounty{}).Where("completion_date_difference != ?", + "").Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(completion_date_difference)").Row().Scan(&sum) + return sum +} + +func (db database) CompletesDifferenceCount(r PaymentDateRange) int64 { + var count int64 + db.db.Model(&Bounty{}).Where("completion_date_difference != ?", + "").Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + return count +} + +func (db database) AverageCompletedTime(r PaymentDateRange) uint { + paidSum := DB.PaidDifferenceSum(r) + paidCount := DB.PaidDifferenceCount(r) + avg := paidSum / uint(paidCount) + avgDays := math.Round(float64(avg)) + return uint(avgDays) +} diff --git a/db/structs.go b/db/structs.go index 64351234a..d61abd8a9 100644 --- a/db/structs.go +++ b/db/structs.go @@ -355,36 +355,37 @@ type Client struct { } type Bounty struct { - ID uint `json:"id"` - OwnerID string `json:"owner_id"` - Paid bool `json:"paid"` - Show bool `gorm:"default:false" json:"show"` - Type string `json:"type"` - Award string `json:"award"` - AssignedHours uint8 `json:"assigned_hours"` - BountyExpires string `json:"bounty_expires"` - CommitmentFee uint64 `json:"commitment_fee"` - Price string `json:"price"` - Title string `json:"title"` - Tribe string `json:"tribe"` - Assignee string `json:"assignee"` - TicketUrl string `json:"ticket_url"` - OrgUuid string `json:"org_uuid"` - Description string `json:"description"` - WantedType string `json:"wanted_type"` - Deliverables string `json:"deliverables"` - GithubDescription bool `json:"github_description"` - OneSentenceSummary string `json:"one_sentence_summary"` - EstimatedSessionLength string `json:"estimated_session_length"` - EstimatedCompletionDate string `json:"estimated_completion_date"` - Created int64 `json:"created"` - Updated *time.Time `json:"updated"` - AssignedDate *time.Time `json:"assigned_date,omitempty"` - CompletionDate *time.Time `json:"completion_date,omitempty"` - MarkAsPaidDate *time.Time `json:"mark_as_paid_date,omitempty"` - PaidDate *time.Time `json:"paid_date,omitempty"` - PaidDateDifference int64 `json:"paid_date_difference,omitempty"` - CodingLanguages pq.StringArray `gorm:"type:text[];not null default:'[]'" json:"coding_languages"` + ID uint `json:"id"` + OwnerID string `json:"owner_id"` + Paid bool `json:"paid"` + Show bool `gorm:"default:false" json:"show"` + Type string `json:"type"` + Award string `json:"award"` + AssignedHours uint8 `json:"assigned_hours"` + BountyExpires string `json:"bounty_expires"` + CommitmentFee uint64 `json:"commitment_fee"` + Price string `json:"price"` + Title string `json:"title"` + Tribe string `json:"tribe"` + Assignee string `json:"assignee"` + TicketUrl string `json:"ticket_url"` + OrgUuid string `json:"org_uuid"` + Description string `json:"description"` + WantedType string `json:"wanted_type"` + Deliverables string `json:"deliverables"` + GithubDescription bool `json:"github_description"` + OneSentenceSummary string `json:"one_sentence_summary"` + EstimatedSessionLength string `json:"estimated_session_length"` + EstimatedCompletionDate string `json:"estimated_completion_date"` + Created int64 `json:"created"` + Updated *time.Time `json:"updated"` + AssignedDate *time.Time `json:"assigned_date,omitempty"` + CompletionDate *time.Time `json:"completion_date,omitempty"` + MarkAsPaidDate *time.Time `json:"mark_as_paid_date,omitempty"` + PaidDate *time.Time `json:"paid_date,omitempty"` + PaidDateDifference int64 `gorm:"type:bigint;not null default:'0'" json:"paid_date_difference,omitempty"` + CompletionDateDifference int64 `gorm:"type:bigint;not null default:'0'" json:"completion_date_difference,omitempty"` + CodingLanguages pq.StringArray `gorm:"type:text[];not null default:'[]'" json:"coding_languages"` } type BountyData struct { @@ -648,6 +649,8 @@ type BountyMetrics struct { SatsPosted uint `json:"sats_posted"` SatsPaid uint `json:"sats_paid"` SatsPaidPercentage uint `json:"sats_paid_percentage"` + AveragePaid uint `json:"average_paid"` + AverageCompleted uint `json:"average_completed"` } func (Person) TableName() string { diff --git a/handlers/bounty.go b/handlers/bounty.go index 2c3139d8f..310d878c6 100644 --- a/handlers/bounty.go +++ b/handlers/bounty.go @@ -186,8 +186,15 @@ func CreateOrEditBounty(w http.ResponseWriter, r *http.Request) { } func DeleteBounty(w http.ResponseWriter, r *http.Request) { - //ctx := r.Context() - //pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + ctx := r.Context() + pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + if pubKeyFromAuth == "" { + fmt.Println("no pubkey from auth") + w.WriteHeader(http.StatusUnauthorized) + return + } + created := chi.URLParam(r, "created") pubkey := chi.URLParam(r, "pubkey") @@ -215,6 +222,14 @@ func UpdatePaymentStatus(w http.ResponseWriter, r *http.Request) { bounty, _ := db.DB.GetBountyByCreated(uint(created)) if bounty.ID != 0 && bounty.Created == int64(created) { bounty.Paid = !bounty.Paid + now := time.Now() + // if setting paid as true by mark as paid + // set completion date and mark as paid + if bounty.Paid { + bounty.CompletionDate = &now + bounty.MarkAsPaidDate = &now + bounty.CompletionDateDifference = utils.GetDateDaysDifference(bounty.Created, &now) + } db.DB.UpdateBountyPayment(bounty) } w.WriteHeader(http.StatusOK) @@ -402,11 +417,13 @@ func MakeBountyPayment(w http.ResponseWriter, r *http.Request) { Status: true, PaymentType: "payment", } - db.DB.AddPaymentHistory(paymentHistory) + bounty.Paid = true bounty.PaidDate = &now + bounty.CompletionDate = &now bounty.PaidDateDifference = utils.GetDateDaysDifference(bounty.Created, &now) + bounty.CompletionDateDifference = utils.GetDateDaysDifference(bounty.Created, &now) db.DB.UpdateBounty(bounty) msg["msg"] = "keysend_success" diff --git a/handlers/metrics.go b/handlers/metrics.go index 8bb823b6d..2c0d28956 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -118,6 +118,8 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { totalSatsPosted := db.DB.TotalSatsPosted(request) totalSatsPaid := db.DB.TotalSatsPaid(request) satsPaidPercentage := db.DB.SatsPaidPercentage(request) + avgPaidDays := db.DB.AveragePaidTime(request) + avgCompletedDays := db.DB.AverageCompletedTime(request) bountyMetrics := db.BountyMetrics{ BountiesPosted: totalBountiesPosted, @@ -126,6 +128,8 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { SatsPosted: totalSatsPosted, SatsPaid: totalSatsPaid, SatsPaidPercentage: satsPaidPercentage, + AveragePaid: avgPaidDays, + AverageCompleted: avgCompletedDays, } w.WriteHeader(http.StatusOK) From cfcc9c11f0987f11eb66ce39d3ec860d77249824 Mon Sep 17 00:00:00 2001 From: elraphty Date: Sun, 26 Nov 2023 11:33:28 +0100 Subject: [PATCH 10/28] added metrics to redis --- db/redis.go | 2 +- handlers/metrics.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/db/redis.go b/db/redis.go index 5d0815c1c..8bdf6f878 100644 --- a/db/redis.go +++ b/db/redis.go @@ -50,7 +50,7 @@ func GetValue(key string) string { return val } -func SetMap(key string, values map[string]string) { +func SetMap(key string, values map[string]interface{}) { for k, v := range values { err := RedisClient.HSet(ctx, key, k, v).Err() if err != nil { diff --git a/handlers/metrics.go b/handlers/metrics.go index 2c0d28956..03201d4ca 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -6,6 +6,7 @@ import ( "io" "net/http" + "github.com/ambelovsky/go-structs" "github.com/stakwork/sphinx-tribes/auth" "github.com/stakwork/sphinx-tribes/db" ) @@ -112,6 +113,19 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { return } + metricsKey := fmt.Sprintf("metrics - %s - %s", request.StartDate, request.EndDate) + /** + check redis if cache id available for the date range + or add to redis + */ + + redisMetrics := db.GetMap(metricsKey) + if len(redisMetrics) != 0 { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(redisMetrics) + return + } + totalBountiesPosted := db.DB.TotalBountiesPosted(request) totalBountiesPaid := db.DB.TotalPaidBounties(request) bountiesPaidPercentage := db.DB.BountiesPaidPercentage(request) @@ -132,6 +146,9 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { AverageCompleted: avgCompletedDays, } + metricsMap := structs.Map(bountyMetrics) + db.SetMap(metricsKey, metricsMap) + w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(bountyMetrics) } From 6b48054ed96a77282040691f1f6c9a333cb0214c Mon Sep 17 00:00:00 2001 From: elraphty Date: Sun, 26 Nov 2023 13:14:55 +0100 Subject: [PATCH 11/28] refactored redis --- db/redis.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db/redis.go b/db/redis.go index 8bdf6f878..ae0aca835 100644 --- a/db/redis.go +++ b/db/redis.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "time" "github.com/redis/go-redis/v9" "github.com/stakwork/sphinx-tribes/utils" @@ -34,8 +35,8 @@ func InitRedis() { } } -func SetValue(key string, value string) { - err := RedisClient.Set(ctx, key, value, 0).Err() +func SetValue(key string, value interface{}) { + err := RedisClient.Set(ctx, key, value, 6*time.Hour).Err() if err != nil { fmt.Println("REDIS SET ERROR :", err) } @@ -57,6 +58,7 @@ func SetMap(key string, values map[string]interface{}) { fmt.Println("REDIS SET MAP ERROR :", err) } } + RedisClient.Expire(ctx, key, 6*time.Hour) } func GetMap(key string) map[string]string { From f54d14221a65a35c63f09b2e7593e0e3970cf61c Mon Sep 17 00:00:00 2001 From: elraphty Date: Tue, 28 Nov 2023 19:26:36 +0100 Subject: [PATCH 12/28] added bounty by range --- db/metrics.go | 101 ++++++++++++++++++++++---------- db/redis.go | 1 - db/structs.go | 2 +- frontend/app/src/config/host.ts | 4 +- handlers/bounties.go | 4 +- handlers/bounty.go | 2 +- handlers/metrics.go | 27 ++++++++- 7 files changed, 103 insertions(+), 38 deletions(-) diff --git a/db/metrics.go b/db/metrics.go index d4f24f530..6b2586fc6 100644 --- a/db/metrics.go +++ b/db/metrics.go @@ -1,6 +1,24 @@ package db -import "math" +import ( + "fmt" + "math" + "net/http" + + "github.com/stakwork/sphinx-tribes/utils" +) + +func (db database) TotalPeopleByDateRange(r PaymentDateRange) int64 { + var count int64 + db.db.Model(&Person{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + return count +} + +func (db database) TotalOrganizationsByDateRange(r PaymentDateRange) int64 { + var count int64 + db.db.Model(&Organization{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) + return count +} func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { var sum uint @@ -8,29 +26,32 @@ func (db database) TotalPaymentsByDateRange(r PaymentDateRange) uint { return sum } -func (db database) TotalSatsPaid(r PaymentDateRange) uint { +func (db database) TotalSatsPosted(r PaymentDateRange) uint { var sum uint - db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) + db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(price)").Row().Scan(&sum) return sum } -func (db database) TotalSatsPosted(r PaymentDateRange) uint { +func (db database) TotalSatsPaid(r PaymentDateRange) uint { var sum uint - db.db.Model(&Bounty{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(amount)").Row().Scan(&sum) + db.db.Model(&Bounty{}).Where("paid = ?", true).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Select("SUM(price)").Row().Scan(&sum) return sum } func (db database) SatsPaidPercentage(r PaymentDateRange) uint { satsPosted := DB.TotalSatsPosted(r) satsPaid := DB.TotalSatsPaid(r) - value := satsPosted * 100 / satsPaid - paidPercentage := math.Round(float64(value)) - return uint(paidPercentage) + if satsPaid != 0 && satsPosted != 0 { + value := (satsPaid * 100) / satsPosted + paidPercentage := math.Round(float64(value)) + return uint(paidPercentage) + } + return 0 } func (db database) TotalPaidBounties(r PaymentDateRange) int64 { var count int64 - db.db.Model(&Bounty{}).Where("paid_date >= ?", r.StartDate).Where("paid_date <= ?", r.EndDate).Count(&count) + db.db.Model(&Bounty{}).Where("paid = ?", true).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) return count } @@ -43,21 +64,12 @@ func (db database) TotalBountiesPosted(r PaymentDateRange) int64 { func (db database) BountiesPaidPercentage(r PaymentDateRange) uint { bountiesPosted := DB.TotalBountiesPosted(r) bountiesPaid := DB.TotalPaidBounties(r) - value := bountiesPaid * 100 / bountiesPosted - paidPercentage := math.Round(float64(value)) - return uint(paidPercentage) -} - -func (db database) TotalPeopleByDateRange(r PaymentDateRange) int64 { - var count int64 - db.db.Model(&Person{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) - return count -} - -func (db database) TotalOrganizationsByDateRange(r PaymentDateRange) int64 { - var count int64 - db.db.Model(&Organization{}).Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) - return count + if bountiesPaid != 0 && bountiesPosted != 0 { + value := bountiesPaid * 100 / bountiesPosted + paidPercentage := math.Round(float64(value)) + return uint(paidPercentage) + } + return 0 } func (db database) PaidDifferenceSum(r PaymentDateRange) uint { @@ -77,9 +89,12 @@ func (db database) PaidDifferenceCount(r PaymentDateRange) int64 { func (db database) AveragePaidTime(r PaymentDateRange) uint { paidSum := DB.PaidDifferenceSum(r) paidCount := DB.PaidDifferenceCount(r) - avg := paidSum / uint(paidCount) - avgDays := math.Round(float64(avg)) - return uint(avgDays) + if paidCount != 0 && paidSum != 0 { + avg := paidSum / uint(paidCount) + avgDays := math.Round(float64(avg)) + return uint(avgDays) + } + return 0 } func (db database) CompletedDifferenceSum(r PaymentDateRange) uint { @@ -99,7 +114,33 @@ func (db database) CompletesDifferenceCount(r PaymentDateRange) int64 { func (db database) AverageCompletedTime(r PaymentDateRange) uint { paidSum := DB.PaidDifferenceSum(r) paidCount := DB.PaidDifferenceCount(r) - avg := paidSum / uint(paidCount) - avgDays := math.Round(float64(avg)) - return uint(avgDays) + if paidCount != 0 && paidSum != 0 { + avg := paidSum / uint(paidCount) + avgDays := math.Round(float64(avg)) + return uint(avgDays) + } + return 0 +} + +func (db database) GetBountiesByDateRange(r PaymentDateRange, re *http.Request) []Bounty { + offset, limit, sortBy, direction, _ := utils.GetPaginationParams(re) + + orderQuery := "" + limitQuery := "" + + if sortBy != "" && direction != "" { + orderQuery = "ORDER BY " + "body." + sortBy + " " + direction + } else { + orderQuery = " ORDER BY " + "body." + sortBy + "" + "DESC" + } + if limit != 0 { + limitQuery = fmt.Sprintf("LIMIT %d OFFSET %d", limit, offset) + } + + query := `SELECT * public.bounty WHERE created >= '` + r.StartDate + `' AND created <= '` + r.EndDate + `'` + allQuery := query + " " + " " + orderQuery + " " + limitQuery + + b := []Bounty{} + db.db.Raw(allQuery).Scan(&b) + return b } diff --git a/db/redis.go b/db/redis.go index ae0aca835..78dd76118 100644 --- a/db/redis.go +++ b/db/redis.go @@ -15,7 +15,6 @@ var RedisClient *redis.Client func InitRedis() { redisURL := os.Getenv("REDIS_URL") - fmt.Println("redis url :", redisURL) if redisURL == "" { dbInt, _ := utils.ConvertStringToInt(os.Getenv("REDIS_DB")) diff --git a/db/structs.go b/db/structs.go index d61abd8a9..6cf39217a 100644 --- a/db/structs.go +++ b/db/structs.go @@ -364,7 +364,7 @@ type Bounty struct { AssignedHours uint8 `json:"assigned_hours"` BountyExpires string `json:"bounty_expires"` CommitmentFee uint64 `json:"commitment_fee"` - Price string `json:"price"` + Price uint `json:"price"` Title string `json:"title"` Tribe string `json:"tribe"` Assignee string `json:"assignee"` diff --git a/frontend/app/src/config/host.ts b/frontend/app/src/config/host.ts index 19c7d2cc5..51a0f3192 100644 --- a/frontend/app/src/config/host.ts +++ b/frontend/app/src/config/host.ts @@ -19,5 +19,5 @@ export function getHostIncludingDockerHosts() { export const TribesURL = getHost().startsWith('localhost') ? `http://${getHost()}` : getHost().startsWith('http') - ? getHost() - : `https://${getHost()}`; + ? getHost() + : `https://${getHost()}`; diff --git a/handlers/bounties.go b/handlers/bounties.go index c452c9b9f..f2efb4e70 100644 --- a/handlers/bounties.go +++ b/handlers/bounties.go @@ -137,9 +137,9 @@ func MigrateBounties(w http.ResponseWriter, r *http.Request) { migrateBountyFinal.Award = Award } - Price, ok5 := migrateBounty["price"].(string) + Price, ok5 := migrateBounty["price"].(uint) if !ok5 { - migrateBountyFinal.Price = "0" + migrateBountyFinal.Price = 0 } else { migrateBountyFinal.Price = Price } diff --git a/handlers/bounty.go b/handlers/bounty.go index 310d878c6..d6c5bb379 100644 --- a/handlers/bounty.go +++ b/handlers/bounty.go @@ -334,7 +334,7 @@ func MakeBountyPayment(w http.ResponseWriter, r *http.Request) { } bounty := db.DB.GetBounty(id) - amount, _ := utils.ConvertStringToUint(bounty.Price) + amount := bounty.Price if bounty.ID != id { w.WriteHeader(http.StatusNotFound) diff --git a/handlers/metrics.go b/handlers/metrics.go index 03201d4ca..8bbea2475 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -118,7 +118,6 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { check redis if cache id available for the date range or add to redis */ - redisMetrics := db.GetMap(metricsKey) if len(redisMetrics) != 0 { w.WriteHeader(http.StatusOK) @@ -152,3 +151,29 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(bountyMetrics) } + +func MetricsBounties(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + pubKeyFromAuth, _ := ctx.Value(auth.ContextKey).(string) + + if pubKeyFromAuth == "" { + fmt.Println("no pubkey from auth") + w.WriteHeader(http.StatusUnauthorized) + return + } + + request := db.PaymentDateRange{} + body, err := io.ReadAll(r.Body) + r.Body.Close() + + err = json.Unmarshal(body, &request) + if err != nil { + w.WriteHeader(http.StatusNotAcceptable) + json.NewEncoder(w).Encode("Request body not accepted") + return + } + + metricBounties := db.DB.GetBountiesByDateRange(request, r) + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(metricBounties) +} From 9e8b87a7b487d2ca6d357aaee2d99cd6d0325eaf Mon Sep 17 00:00:00 2001 From: elraphty Date: Tue, 28 Nov 2023 19:30:49 +0100 Subject: [PATCH 13/28] enforce price in number --- frontend/app/src/people/main/FocusView.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/app/src/people/main/FocusView.tsx b/frontend/app/src/people/main/FocusView.tsx index 4ce16e3f3..72849fa97 100644 --- a/frontend/app/src/people/main/FocusView.tsx +++ b/frontend/app/src/people/main/FocusView.tsx @@ -186,6 +186,10 @@ function FocusedView(props: FocusViewProps) { newBody.description = description; } + if (newBody.price) { + newBody.price = Number(newBody.price); + } + // body.description = description; newBody.title = newBody.one_sentence_summary; From 2585fda8ef9b095f575c009c8175bf00fc028206 Mon Sep 17 00:00:00 2001 From: elraphty Date: Tue, 28 Nov 2023 19:36:45 +0100 Subject: [PATCH 14/28] added prettier info --- Contribution.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Contribution.md b/Contribution.md index 2e85c0cc6..d9fdba345 100644 --- a/Contribution.md +++ b/Contribution.md @@ -15,3 +15,7 @@ - Folders should be named in camel case i.e (peopleData) - Typescript files should be named in camel case also - Only the index.tsx files should be named in small letters + +### Prettier Fixing + +- Make sure you run ```yarn run prettier``` to fix prettier error before submitting From d4d56e5564d50ee3b92771f02551323a18ea1d92 Mon Sep 17 00:00:00 2001 From: elraphty Date: Tue, 28 Nov 2023 23:53:02 +0100 Subject: [PATCH 15/28] renamed routes --- routes/metrics.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/routes/metrics.go b/routes/metrics.go index 79c1640fd..aaaa603b9 100644 --- a/routes/metrics.go +++ b/routes/metrics.go @@ -12,10 +12,11 @@ func MetricsRoutes() chi.Router { r.Group(func(r chi.Router) { r.Use(auth.PubKeyContext) - r.Post("/payment_metrics", handlers.PaymentMetrics) - r.Post("/people_created", handlers.PeopleMetrics) - r.Post("/organization_created", handlers.OrganizationtMetrics) - r.Post("/bounty_metrics", handlers.BountyMetrics) + r.Post("/payment", handlers.PaymentMetrics) + r.Post("/people", handlers.PeopleMetrics) + r.Post("/organization", handlers.OrganizationtMetrics) + r.Post("/bounty", handlers.BountyMetrics) + r.Post("/bounties", handlers.MetricsBounties) }) return r } From e99671844e2c4ab6b09450396b291e80a1e5f40f Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 29 Nov 2023 07:10:55 +0100 Subject: [PATCH 16/28] added Redis to Readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index a917fffdc..89d273dc7 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,24 @@ Meme image upload works with Relay enabled, so a running Relay is required for M MEME_URL= ``` +### Add REDIS for cache + +- Create a Redis instance +- Create a .env file and populate the .env files with these variables + +If you have a Redis url add the REDIS_URL variable to .env + +```REDIS_URL = ``` + +else add these variables to the env to enable Redis + +``` + REDIS_HOST = + REDIS_DB = + REDIS_USER = + REDIS_PASS = +``` + ### For Contributions Read the contribution doc [here](./Contribution.md) From af9c997ef9f1b9f26cb311f205b5c91c9389a0c0 Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 29 Nov 2023 07:34:06 +0100 Subject: [PATCH 17/28] used completed functions --- db/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/metrics.go b/db/metrics.go index 6b2586fc6..cc235d8d1 100644 --- a/db/metrics.go +++ b/db/metrics.go @@ -104,7 +104,7 @@ func (db database) CompletedDifferenceSum(r PaymentDateRange) uint { return sum } -func (db database) CompletesDifferenceCount(r PaymentDateRange) int64 { +func (db database) CompletedDifferenceCount(r PaymentDateRange) int64 { var count int64 db.db.Model(&Bounty{}).Where("completion_date_difference != ?", "").Where("created >= ?", r.StartDate).Where("created <= ?", r.EndDate).Count(&count) @@ -112,8 +112,8 @@ func (db database) CompletesDifferenceCount(r PaymentDateRange) int64 { } func (db database) AverageCompletedTime(r PaymentDateRange) uint { - paidSum := DB.PaidDifferenceSum(r) - paidCount := DB.PaidDifferenceCount(r) + paidSum := DB.CompletedDifferenceSum(r) + paidCount := DB.CompletedDifferenceCount(r) if paidCount != 0 && paidSum != 0 { avg := paidSum / uint(paidCount) avgDays := math.Round(float64(avg)) From 85c3426160f085299167748dff0968389d79ccdc Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 29 Nov 2023 13:58:27 +0100 Subject: [PATCH 18/28] added more test for the difference function --- utils/helpers_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/helpers_test.go b/utils/helpers_test.go index 7078b7325..58932e4be 100644 --- a/utils/helpers_test.go +++ b/utils/helpers_test.go @@ -9,8 +9,20 @@ import ( func TestGetDateDaysDifference(t *testing.T) { created := 1700238956 + exactTime := time.Now().Unix() testDate := time.Now() + nextDate := time.Now().AddDate(0, 0, 2) days := GetDateDaysDifference(int64(created), &testDate) + assert.NotEqual(t, 0, days) + + daysEqual := GetDateDaysDifference(exactTime, &testDate) + assert.Equal(t, int64(0), daysEqual) + + daysNext := GetDateDaysDifference(exactTime, &nextDate) + assert.Equal(t, int64(2), daysNext) + + wrongDate := GetDateDaysDifference(0, &nextDate) + assert.Greater(t, wrongDate, int64(365)) } From 4f082eb10370b44eeaec091c4f52be0a6c3cb8f0 Mon Sep 17 00:00:00 2001 From: elraphty Date: Thu, 30 Nov 2023 07:52:29 +0100 Subject: [PATCH 19/28] added tests for all helpers --- utils/helpers_test.go | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/utils/helpers_test.go b/utils/helpers_test.go index 58932e4be..046dc7d9d 100644 --- a/utils/helpers_test.go +++ b/utils/helpers_test.go @@ -26,3 +26,49 @@ func TestGetDateDaysDifference(t *testing.T) { wrongDate := GetDateDaysDifference(0, &nextDate) assert.Greater(t, wrongDate, int64(365)) } + +func TestGetRandomToken(t *testing.T) { + randomToken32 := GetRandomToken(32) + assert.GreaterOrEqual(t, len(randomToken32), 32) + + randomToken64 := GetRandomToken(56) + assert.GreaterOrEqual(t, len(randomToken64), 56) +} + +func TestConvertStringToUint(t *testing.T) { + number := "20" + result, _ := ConvertStringToUint(number) + + assert.Equal(t, uint(20), result) + + wrongNum := "wrong" + result2, err := ConvertStringToUint(wrongNum) + assert.Equal(t, uint(0), result2) + + assert.NotEqual(t, err, nil) +} + +func TestConvertStringToInt(t *testing.T) { + number := "10" + result, _ := ConvertStringToInt(number) + + assert.Equal(t, int(10), result) + + wrongNum := "wrong" + result2, err := ConvertStringToInt(wrongNum) + assert.Equal(t, int(0), result2) + + assert.NotEqual(t, err, nil) +} + +func TestGetInvoiceAmount(t *testing.T) { + invoice := "lnbc15u1p3xnhl2pp5jptserfk3zk4qy42tlucycrfwxhydvlemu9pqr93tuzlv9cc7g3sdqsvfhkcap3xyhx7un8cqzpgxqzjcsp5f8c52y2stc300gl6s4xswtjpc37hrnnr3c9wvtgjfuvqmpm35evq9qyyssqy4lgd8tj637qcjp05rdpxxykjenthxftej7a2zzmwrmrl70fyj9hvj0rewhzj7jfyuwkwcg9g2jpwtk3wkjtwnkdks84hsnu8xps5vsq4gj5hs" + + amount := GetInvoiceAmount(invoice) + assert.Equal(t, uint(1500), amount) + + invalidInvoice := "lnbc15u1p3xnhl2pp5jptserfk3zk4qy42tlucycrfwxhydvlemu9pqr93tuzlv9cc7g3sdqsvfhkcap3xyhx7un8cqzpgxqzjcsp5f8c52y2stc300gl6s4xswtjpc37hrnnr3c9wvtgjfuvqmpm35evq9qyyssqy4lgd8tj637qcjp05rdpxxykjenthxftej7a2zzmwrmrl70fyj9hvj0rewhzj7jfyuwkwcg9g2jpwtk3wkjtwnkdks84hsnu8xps5vsq" + + amount2 := GetInvoiceAmount(invalidInvoice) + assert.Equal(t, uint(0), amount2) +} From 089d952a755509b22c57e405e950d237de18dd5d Mon Sep 17 00:00:00 2001 From: elraphty Date: Thu, 30 Nov 2023 09:19:50 +0100 Subject: [PATCH 20/28] added superadmin middleware --- auth/auth.go | 73 +++++++++++++++++++++++++++++++++++++++++++++++ config/config.go | 2 ++ routes/metrics.go | 2 +- 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/auth/auth.go b/auth/auth.go index 6a979088f..dd7adc5d9 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -83,6 +83,79 @@ func PubKeyContext(next http.Handler) http.Handler { }) } +// PubKeyContext parses pukey from signed timestamp +func PubKeyContextSuperAdmin(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.URL.Query().Get("token") + if token == "" { + token = r.Header.Get("x-jwt") + } + + if token == "" { + fmt.Println("[auth] no token") + http.Error(w, http.StatusText(401), 401) + return + } + + isJwt := strings.Contains(token, ".") && !strings.HasPrefix(token, ".") + + if isJwt { + claims, err := DecodeJwt(token) + + if err != nil { + fmt.Println("Failed to parse JWT") + http.Error(w, http.StatusText(401), 401) + return + } + + if claims.VerifyExpiresAt(time.Now().UnixNano(), true) { + fmt.Println("Token has expired") + http.Error(w, http.StatusText(401), 401) + return + } + + pubkey := fmt.Sprintf("%v", claims["pubkey"]) + if !AdminCheck(pubkey) { + fmt.Println("Not a super admin") + http.Error(w, http.StatusText(401), 401) + return + } + + ctx := context.WithValue(r.Context(), ContextKey, claims["pubkey"]) + next.ServeHTTP(w, r.WithContext(ctx)) + } else { + pubkey, err := VerifyTribeUUID(token, true) + + if pubkey == "" || err != nil { + fmt.Println("[auth] no pubkey || err != nil") + if err != nil { + fmt.Println(err) + } + http.Error(w, http.StatusText(401), 401) + return + } + + if !AdminCheck(pubkey) { + fmt.Println("Not a super admin") + http.Error(w, http.StatusText(401), 401) + return + } + + ctx := context.WithValue(r.Context(), ContextKey, pubkey) + next.ServeHTTP(w, r.WithContext(ctx)) + } + }) +} + +func AdminCheck(pubkey string) bool { + for _, val := range config.SuperAdmins { + if val == pubkey { + return true + } + } + return false +} + // VerifyTribeUUID takes base64 uuid and returns hex pubkey func VerifyTribeUUID(uuid string, checkTimestamp bool) (string, error) { diff --git a/config/config.go b/config/config.go index ac2844eeb..5062c637a 100644 --- a/config/config.go +++ b/config/config.go @@ -49,6 +49,8 @@ func InitConfig() { } +var SuperAdmins []string = []string{""} + func GenerateRandomString() string { const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" diff --git a/routes/metrics.go b/routes/metrics.go index aaaa603b9..22082f243 100644 --- a/routes/metrics.go +++ b/routes/metrics.go @@ -10,7 +10,7 @@ func MetricsRoutes() chi.Router { r := chi.NewRouter() r.Group(func(r chi.Router) { - r.Use(auth.PubKeyContext) + r.Use(auth.PubKeyContextSuperAdmin) r.Post("/payment", handlers.PaymentMetrics) r.Post("/people", handlers.PeopleMetrics) From 0de702d992c1f5b44ff92502d6982a183329779b Mon Sep 17 00:00:00 2001 From: elraphty Date: Thu, 30 Nov 2023 15:35:12 +0100 Subject: [PATCH 21/28] added superadmin middleware functionality --- config/config.go | 27 +++++++++++++++++++++++++-- config/config_test.go | 24 +++++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 5062c637a..131832945 100644 --- a/config/config.go +++ b/config/config.go @@ -18,6 +18,7 @@ var RelayUrl string var MemeUrl string var RelayAuthKey string var RelayNodeKey string +var SuperAdmins []string = []string{""} // these are constants for the store var InvoiceList = "INVOICELIST" @@ -29,6 +30,9 @@ func InitConfig() { RelayUrl = os.Getenv("RELAY_URL") MemeUrl = os.Getenv("MEME_URL") RelayAuthKey = os.Getenv("RELAY_AUTH_KEY") + AdminStrings := os.Getenv("SUPER_ADMINS") + // Add to super admins + SuperAdmins = StripSuperAdmins(AdminStrings) // only make this call if there is a Relay auth key if RelayAuthKey != "" { @@ -46,10 +50,29 @@ func InitConfig() { if JwtKey == "" { JwtKey = GenerateRandomString() } - } -var SuperAdmins []string = []string{""} +func StripSuperAdmins(adminStrings string) []string { + superAdmins := []string{} + if adminStrings != "" { + if strings.Contains(adminStrings, ",") { + splitArray := strings.Split(adminStrings, ",") + splitLength := len(splitArray) + + for i := 0; i < splitLength; i++ { + // append indexes, and skip all the commas + if splitArray[i] == "," { + continue + } else { + superAdmins = append(superAdmins, splitArray[i]) + } + } + } else { + superAdmins = append(superAdmins, adminStrings) + } + } + return superAdmins +} func GenerateRandomString() string { const charset = "abcdefghijklmnopqrstuvwxyz" + diff --git a/config/config_test.go b/config/config_test.go index 7ec021d9b..1ffab775b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,10 @@ package config -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestInitConfig(t *testing.T) { InitConfig() @@ -17,3 +21,21 @@ func TestInitConfig(t *testing.T) { t.Error("Could not load random jwtKey") } } + +func TestStripSuperAdmins(t *testing.T) { + testAdminList := "hello, hi, yes, now" + admins := StripSuperAdmins(testAdminList) + assert.Equal(t, len(admins), 4) + + testAdminNocomma := "hello" + adminsNoComma := StripSuperAdmins(testAdminNocomma) + assert.Equal(t, len(adminsNoComma), 1) + + testNoAdmins := "" + noAdmins := StripSuperAdmins(testNoAdmins) + assert.Equal(t, len(noAdmins), 0) + + test2Admins := "hello, hi" + admins2 := StripSuperAdmins(test2Admins) + assert.Equal(t, len(admins2), 2) +} From 1792647c6d3b3d5480b694f5ffc8559b1f3cfe50 Mon Sep 17 00:00:00 2001 From: elraphty Date: Sun, 3 Dec 2023 12:13:38 +0100 Subject: [PATCH 22/28] added bounty data for metrics --- handlers/metrics.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/handlers/metrics.go b/handlers/metrics.go index 8bbea2475..780a4665d 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -174,6 +174,36 @@ func MetricsBounties(w http.ResponseWriter, r *http.Request) { } metricBounties := db.DB.GetBountiesByDateRange(request, r) + var metricBountiesData []db.BountyData + + for _, bounty := range metricBounties { + bountyOwner := db.DB.GetPersonByPubkey(bounty.OwnerID) + bountyAssignee := db.DB.GetPersonByPubkey(bounty.Assignee) + organization := db.DB.GetOrganizationByUuid(bounty.OrgUuid) + + bountyData := db.BountyData{ + Bounty: bounty, + BountyId: bounty.ID, + Person: bountyOwner, + BountyCreated: bounty.Created, + BountyDescription: bounty.Description, + BountyUpdated: bounty.Updated, + AssigneeId: bountyAssignee.ID, + AssigneeAlias: bountyAssignee.OwnerAlias, + AssigneeDescription: bountyAssignee.Description, + AssigneeRouteHint: bountyAssignee.OwnerRouteHint, + BountyOwnerId: bountyOwner.ID, + OwnerUuid: bountyOwner.Uuid, + OwnerDescription: bountyOwner.Description, + OwnerUniqueName: bountyOwner.UniqueName, + OwnerImg: bountyOwner.Img, + OrganizationName: organization.Name, + OrganizationImg: organization.Img, + OrganizationUuid: organization.Uuid, + } + + metricBountiesData = append(metricBountiesData, bountyData) + } w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(metricBounties) + json.NewEncoder(w).Encode(metricBountiesData) } From 42ebd4e4d78bcb89c3ea7eefba0a7f76bff5f677 Mon Sep 17 00:00:00 2001 From: elraphty Date: Sun, 3 Dec 2023 19:22:05 +0100 Subject: [PATCH 23/28] added SUPER_ADMINS to README --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 89d273dc7..b26041015 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,16 @@ else add these variables to the env to enable Redis REDIS_PASS = ``` +### Add SuperAdmins to access admin dashboard + +Add comma separated public keys to the SUPER_ADMINS env var in the .env file, +any user public key added to this comaa separated strings will have access to the admin dashboard +e.g '{pubkey}, {pubkey}, {pubkey}' + +``` +SUPER_ADMINS +``` + ### For Contributions Read the contribution doc [here](./Contribution.md) From 7fb0e886e76b81c4f72e6cd06e7832fe2961c1f0 Mon Sep 17 00:00:00 2001 From: elraphty Date: Mon, 4 Dec 2023 23:02:51 +0100 Subject: [PATCH 24/28] slight changes --- db/redis.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db/redis.go b/db/redis.go index 78dd76118..450da315d 100644 --- a/db/redis.go +++ b/db/redis.go @@ -12,6 +12,7 @@ import ( var ctx = context.Background() var RedisClient *redis.Client +var expireTime = 6 * time.Hour func InitRedis() { redisURL := os.Getenv("REDIS_URL") @@ -35,7 +36,7 @@ func InitRedis() { } func SetValue(key string, value interface{}) { - err := RedisClient.Set(ctx, key, value, 6*time.Hour).Err() + err := RedisClient.Set(ctx, key, value, expireTime).Err() if err != nil { fmt.Println("REDIS SET ERROR :", err) } @@ -57,7 +58,7 @@ func SetMap(key string, values map[string]interface{}) { fmt.Println("REDIS SET MAP ERROR :", err) } } - RedisClient.Expire(ctx, key, 6*time.Hour) + RedisClient.Expire(ctx, key, expireTime) } func GetMap(key string) map[string]string { From 7c7465355ad54ed9c44d98af2aa77c110c4fbe5b Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 6 Dec 2023 18:11:36 +0100 Subject: [PATCH 25/28] uncommented redis --- handlers/metrics.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/handlers/metrics.go b/handlers/metrics.go index 5a793604e..780a4665d 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -118,12 +118,12 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { check redis if cache id available for the date range or add to redis */ - // redisMetrics := db.GetMap(metricsKey) - // if len(redisMetrics) != 0 { - // w.WriteHeader(http.StatusOK) - // json.NewEncoder(w).Encode(redisMetrics) - // return - // } + redisMetrics := db.GetMap(metricsKey) + if len(redisMetrics) != 0 { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(redisMetrics) + return + } totalBountiesPosted := db.DB.TotalBountiesPosted(request) totalBountiesPaid := db.DB.TotalPaidBounties(request) From 28b709266b51a9eb4f14c5987820b6246966dea1 Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 13 Dec 2023 19:31:29 +0100 Subject: [PATCH 26/28] added Redis error clause to metrics code --- config/config.go | 4 ++-- db/redis.go | 7 ++++++- handlers/metrics.go | 18 +++++++++++------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 131832945..9440b8368 100644 --- a/config/config.go +++ b/config/config.go @@ -64,11 +64,11 @@ func StripSuperAdmins(adminStrings string) []string { if splitArray[i] == "," { continue } else { - superAdmins = append(superAdmins, splitArray[i]) + superAdmins = append(superAdmins, strings.TrimSpace(splitArray[i])) } } } else { - superAdmins = append(superAdmins, adminStrings) + superAdmins = append(superAdmins, strings.TrimSpace(adminStrings)) } } return superAdmins diff --git a/db/redis.go b/db/redis.go index 450da315d..b5e1871cc 100644 --- a/db/redis.go +++ b/db/redis.go @@ -12,6 +12,7 @@ import ( var ctx = context.Background() var RedisClient *redis.Client +var RedisError interface{} var expireTime = 6 * time.Hour func InitRedis() { @@ -28,11 +29,15 @@ func InitRedis() { } else { opt, err := redis.ParseURL(redisURL) if err != nil { + RedisError = err fmt.Println("REDIS URL CONNECTION ERROR ===", err) } - RedisClient = redis.NewClient(opt) } + if err := RedisClient.Ping(ctx); err != nil { + RedisError = err + fmt.Println("Could Not Connect To Redis", err) + } } func SetValue(key string, value interface{}) { diff --git a/handlers/metrics.go b/handlers/metrics.go index 780a4665d..a78c5e1e1 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -118,11 +118,13 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { check redis if cache id available for the date range or add to redis */ - redisMetrics := db.GetMap(metricsKey) - if len(redisMetrics) != 0 { - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(redisMetrics) - return + if db.RedisError == nil { + redisMetrics := db.GetMap(metricsKey) + if len(redisMetrics) != 0 { + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(redisMetrics) + return + } } totalBountiesPosted := db.DB.TotalBountiesPosted(request) @@ -145,8 +147,10 @@ func BountyMetrics(w http.ResponseWriter, r *http.Request) { AverageCompleted: avgCompletedDays, } - metricsMap := structs.Map(bountyMetrics) - db.SetMap(metricsKey, metricsMap) + if db.RedisError == nil { + metricsMap := structs.Map(bountyMetrics) + db.SetMap(metricsKey, metricsMap) + } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(bountyMetrics) From 78c81b97093df81fab6a3983817d8267ed90a20a Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 13 Dec 2023 19:44:09 +0100 Subject: [PATCH 27/28] fixed config test error --- config/config_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 6cc26c433..f6e943dd2 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,8 +1,11 @@ package config import ( + "fmt" + "os" "testing" + "github.com/h2non/gock" "github.com/stretchr/testify/assert" ) @@ -45,8 +48,6 @@ func TestGenerateRandomString(t *testing.T) { func TestGetNodePubKey(t *testing.T) { defer gock.Off() - //response := map[string]string{"identity_pubkey": "1234"} - //success := map[string]bool{"success": true} response := NodeGetInfoResponse{IdentityPubkey: "1234"} nodeGetInfo := NodeGetInfo{Success: true, Response: response} From 1844321b00d2dd6ad3addb4bd77094f23319955a Mon Sep 17 00:00:00 2001 From: elraphty Date: Wed, 13 Dec 2023 21:41:06 +0100 Subject: [PATCH 28/28] change SUPER_ADMINS to ADMINS --- README.md | 2 +- config/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b26041015..584437d3a 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ any user public key added to this comaa separated strings will have access to th e.g '{pubkey}, {pubkey}, {pubkey}' ``` -SUPER_ADMINS +ADMINS ``` ### For Contributions diff --git a/config/config.go b/config/config.go index bb88abfe7..2ace1fa25 100644 --- a/config/config.go +++ b/config/config.go @@ -30,7 +30,7 @@ func InitConfig() { RelayUrl = os.Getenv("RELAY_URL") MemeUrl = os.Getenv("MEME_URL") RelayAuthKey = os.Getenv("RELAY_AUTH_KEY") - AdminStrings := os.Getenv("SUPER_ADMINS") + AdminStrings := os.Getenv("ADMINS") // Add to super admins SuperAdmins = StripSuperAdmins(AdminStrings)