From 3c14ac28bd260dc2f88f14ad63a8f397a33d5844 Mon Sep 17 00:00:00 2001 From: Ned Palacios Date: Mon, 15 Apr 2024 09:52:35 +0800 Subject: [PATCH] fix: remaining bugs in log analyzer --- server/cli/main.go | 50 +++++++++-- server/go.mod | 17 +++- server/go.sum | 26 ++++++ .../analyzer/error_quotient/error_quotient.go | 89 +++++++++++++++---- server/logger/analyzer/internal/structures.go | 8 +- server/logger/logger.go | 53 ++++++++--- 6 files changed, 205 insertions(+), 38 deletions(-) diff --git a/server/cli/main.go b/server/cli/main.go index 70688ce..2af5523 100644 --- a/server/cli/main.go +++ b/server/cli/main.go @@ -25,7 +25,7 @@ import ( "github.com/nedpals/bugbuddy/server/runner" "github.com/nedpals/errgoengine" "github.com/spf13/cobra" - "github.com/tealeg/xlsx" + "github.com/tealeg/xlsx/v3" "golang.org/x/exp/maps" ) @@ -293,16 +293,31 @@ var analyzerCellNames = map[string]string{ "tts": "Time To Solve", } +func adjustToTextWidth(s string) float64 { + return float64(len(s)) +} + var analyzeLogCmd = &cobra.Command{ Use: "analyze-log", Short: "Analyzes a set of log files. The results will be saved to an excel file.", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + afterFlag, _ := cmd.Flags().GetString("after") + afterDate := time.Time{} + + if len(afterFlag) != 0 { + var err error + afterDate, err = time.Parse("01/02/2006", afterFlag) + if err != nil { + log.Fatalln(err) + } + } + selectedAnalyzers, _ := cmd.Flags().GetStringSlice("metrics") for _, analyzerName := range selectedAnalyzers { if _, ok := supportedAnalyzers[analyzerName]; !ok { - return fmt.Errorf("invalid analyzer: %s. only %s were allowed", analyzerName, strings.Join(maps.Keys(supportedAnalyzers), ", ")) + log.Fatalf("invalid analyzer: %s. only %s were allowed\n", analyzerName, strings.Join(maps.Keys(supportedAnalyzers), ", ")) } } @@ -341,6 +356,10 @@ var analyzeLogCmd = &cobra.Command{ } } + if len(loggerLoaders) == 0 { + log.Fatalln("no log files were loaded") + } + results := analyzerResult{} for _, lgLoader := range loggerLoaders { @@ -349,6 +368,10 @@ var analyzeLogCmd = &cobra.Command{ log.Fatalln(err) } + if !afterDate.IsZero() { + lg.After = afterDate + } + loader := func() (*logger.Logger, error) { return lg, nil } @@ -386,14 +409,25 @@ var analyzeLogCmd = &cobra.Command{ aCell := row.AddCell() aCell.SetValue(name) analyzerCellLocations[analyzerName] = aCellRow + 1 + + if analyzerName == "tts" { + row.AddCell().SetValue("Time To Solve (HH:MM:SS)") + sheet.SetColAutoWidth(aCellRow+3, adjustToTextWidth) + } + + sheet.SetColAutoWidth(aCellRow+2, adjustToTextWidth) } for fileIdx, filePath := range result.Filenames { - row := sheet.Row(fileIdx + 1) + if len(strings.TrimSpace(filePath)) == 0 { + continue + } + + row, _ := sheet.Row(fileIdx + 1) row.AddCell().SetValue(filePath) for _, analyzerName := range selectedAnalyzers { - cell := sheet.Cell(fileIdx+1, analyzerCellLocations[analyzerName]) + cell, _ := sheet.Cell(fileIdx+1, analyzerCellLocations[analyzerName]) switch analyzerName { case "eq": @@ -401,10 +435,15 @@ var analyzeLogCmd = &cobra.Command{ case "red": cell.SetValue(result.RepeatedErrorDensity[fileIdx]) case "tts": - cell.SetValue(formatDuration(result.TimeToSolve[fileIdx])) + cell.SetValue(result.TimeToSolve[fileIdx].Seconds()) + + hhMmSsCell, _ := sheet.Cell(fileIdx+1, analyzerCellLocations[analyzerName]+1) + hhMmSsCell.SetValue(formatDuration(result.TimeToSolve[fileIdx])) } } } + + sheet.SetColAutoWidth(1, xlsx.DefaultAutoWidth) } if err := wb.Save(outputPath); err != nil { @@ -438,6 +477,7 @@ func init() { daemonCmd.PersistentFlags().String("data-dir", "", "the directory to use for the daemon. To override the default directory, set the BUGBUDDY_DIR environment variable.") analyzeLogCmd.PersistentFlags().StringP("output", "o", "results.xlsx", "the output file to save the results") analyzeLogCmd.PersistentFlags().StringSliceP("metrics", "m", []string{"eq", "red", "tts"}, "the analyzers to use") + analyzeLogCmd.PersistentFlags().String("after", "", "the date to start analyzing the logs") } func main() { diff --git a/server/go.mod b/server/go.mod index 521cb5a..ede700c 100644 --- a/server/go.mod +++ b/server/go.mod @@ -17,7 +17,20 @@ require ( modernc.org/sqlite v1.28.0 ) -require golang.org/x/text v0.9.0 // indirect +require ( + github.com/frankban/quicktest v1.14.6 // indirect + github.com/google/btree v1.0.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/peterbourgon/diskv/v3 v3.0.1 // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect + golang.org/x/text v0.9.0 // indirect +) require ( github.com/dustin/go-humanize v1.0.1 // indirect @@ -39,6 +52,7 @@ require ( ) require ( + github.com/Masterminds/squirrel v1.5.4 github.com/agnivade/levenshtein v1.1.1 github.com/carlmjohnson/versioninfo v0.22.5 github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -50,6 +64,7 @@ require ( github.com/sourcegraph/jsonrpc2 v0.2.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 // indirect + github.com/tealeg/xlsx/v3 v3.3.6 go.lsp.dev/jsonrpc2 v0.10.0 // indirect go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2 // indirect go.lsp.dev/uri v0.3.0 diff --git a/server/go.sum b/server/go.sum index 7b64a3b..3754f51 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,17 +1,24 @@ +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -29,10 +36,16 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/liamg/memoryfs v1.6.0 h1:jAFec2HI1PgMTem5gR7UT8zi9u4BfG5jorCRlLH06W8= github.com/liamg/memoryfs v1.6.0/go.mod h1:z7mfqXFQS8eSeBBsFjYLlxYRMRyiPktytvYCYTb3BSk= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -52,10 +65,17 @@ github.com/nedpals/errgoengine v0.0.0-20240325041034-7b0546a12c1f h1:aK4qaPcr4up github.com/nedpals/errgoengine v0.0.0-20240325041034-7b0546a12c1f/go.mod h1:uYC7Mfw/No0v7VerEzJMPt6MdqgS6OYlkWdhan7cnz0= github.com/nedpals/errgoengine v0.0.0-20240401074707-66b671a5f075 h1:UBecmIjXmkSXcgMuT7kAbMQPysWk1hba3Jh/x9QRa7Y= github.com/nedpals/errgoengine v0.0.0-20240401074707-66b671a5f075/go.mod h1:uYC7Mfw/No0v7VerEzJMPt6MdqgS6OYlkWdhan7cnz0= +github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= +github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -63,6 +83,8 @@ github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOV github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa h1:2cO3RojjYl3hVTbEvJVqrMaFmORhL6O06qdW42toftk= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa/go.mod h1:Yjr3bdWaVWyME1kha7X0jsz3k2DgXNa1Pj3XGyUAbx8= github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8 h1:DxgjlvWYsb80WEN2Zv3WqJFAg2DKjUQJO6URGdf1x6Y= github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE= github.com/sourcegraph/jsonrpc2 v0.2.0 h1:KjN/dC4fP6aN9030MZCJs9WQbTOjWHhrtKVpzzSrr/U= @@ -73,6 +95,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -80,6 +103,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= +github.com/tealeg/xlsx/v3 v3.3.6 h1:b0SPORnNa8BDbFEujljp2IpTDVse3D+Ad5IaMz7KUL8= +github.com/tealeg/xlsx/v3 v3.3.6/go.mod h1:KV4FTFtvGy0TBlOivJLZu/YNZk6e0Qtk7eOSglWksuA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.lsp.dev/jsonrpc2 v0.10.0 h1:Pr/YcXJoEOTMc/b6OTmcR1DPJ3mSWl/SWiU1Cct6VmI= go.lsp.dev/jsonrpc2 v0.10.0/go.mod h1:fmEzIdXPi/rf6d4uFcayi8HpFP1nBF99ERP1htC72Ac= @@ -128,6 +153,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= diff --git a/server/logger/analyzer/error_quotient/error_quotient.go b/server/logger/analyzer/error_quotient/error_quotient.go index 5a9b3f8..f3c6593 100644 --- a/server/logger/analyzer/error_quotient/error_quotient.go +++ b/server/logger/analyzer/error_quotient/error_quotient.go @@ -1,6 +1,7 @@ package errorquotient import ( + "database/sql" "fmt" "strings" @@ -77,21 +78,58 @@ func ErrorTypeConversion(errorCode int) int { } // Function to calculate CharDelta by comparing two versions of the same file -func CalculateCharDeltaAndLocation(log *logger.Logger, filepath string, version1, version2 *logger.LogEntry) (charDelta int, location string, err error) { +func CalculateCharDeltaAndLocation(log *logger.Logger, filepath string, version1, version2 logger.LogEntry) (charDelta int, location string, err error) { if version1.FileVersion == 0 && strings.HasPrefix(version1.GeneratedOutput, "# UnknownError") { // This is a special case where the file is not found, and the error message is "# UnknownError" return 0, "", nil } // open the files first - content1, err := log.OpenVersionedFile(filepath, version1.FileVersion) + content1, err := log.OpenVersionedFileFromPID(version1.ParticipantId, filepath, version1.FileVersion) if err != nil { - return 0, "", err + if err == sql.ErrNoRows { + if version1.ErrorCode != 0 { + return 0, "", fmt.Errorf("(%s) file not found: %s (version: %d)", version1.ParticipantId, filepath, version1.FileVersion) + } + + // Just ignore and use the contents from version2 + } else { + return 0, "", err + } } - content2, err := log.OpenVersionedFile(filepath, version2.FileVersion) + + content2, err := log.OpenVersionedFileFromPID(version2.ParticipantId, filepath, version2.FileVersion) if err != nil { - return 0, "", err + if err == sql.ErrNoRows { + if version2.ErrorCode != 0 { + return 0, "", fmt.Errorf("file not found: %s (version: %d)", filepath, version2.FileVersion) + } + + // Use latest version if the file is not found and error code is 0 + latestVersion, verErr := log.LatestVersionFromFile(filepath) + if latestVersion == -1 { + return 0, "", verErr + } + + newContent2, err := log.OpenVersionedFileFromPID(version2.ParticipantId, filepath, latestVersion) + if err != nil { + if err == sql.ErrNoRows { + return 0, "", fmt.Errorf("(2nd) file not found: %s (version: %d)", filepath, latestVersion) + } + return 0, "", err + } + + // Use the content from version2 if version1 is not found + if version1.FileVersion == 0 { + content1 = content2 + } + + content2 = newContent2 + } else { + return 0, "", err + } } + err = nil // diff the files dmp := diffmatchpatch.New() @@ -133,6 +171,15 @@ type Analyzer struct { ErrorsByParticipant map[string]ErrorQuotientAnalysisResult } +func getLastVersionNumberFromIdx(entries []logger.LogEntry, filePath string, idx int) int { + for i := idx; i >= 0; i-- { + if entries[i].FilePath == filePath && entries[i].FileVersion != 0 { + return entries[i].FileVersion + } + } + return 0 +} + func (e *Analyzer) Analyze(writer analyzer.KVWriter, loaders ...analyzer.LoggerLoader) error { results := map[string]ErrorQuotientAnalysisResult{} @@ -169,12 +216,9 @@ func (e *Analyzer) Analyze(writer analyzer.KVWriter, loaders ...analyzer.LoggerL logEntries[participantId] = internal.NewResultStore[[]logger.LogEntry]() } - filePath := entry.FilePath - logEntries[participantId].Set( - filePath, - append( - logEntries[participantId].GetOr(filePath, []logger.LogEntry{}), - entry)) + filePath := logEntries[participantId].FilenameNearest(entry.FilePath) + existing := logEntries[participantId].GetOr(filePath, []logger.LogEntry{}) + logEntries[participantId].Set(filePath, append(existing, entry)) } for participantId, logEntries := range logEntries { @@ -182,14 +226,27 @@ func (e *Analyzer) Analyze(writer analyzer.KVWriter, loaders ...analyzer.LoggerL compilationEvents := internal.NewResultStore[[]CompilationEvent]() for filePathIdx, entries := range logEntries.Values { - filePath := logEntries.Filenames[filePathIdx] + filePath := logEntries.FilenameNearest(logEntries.Filenames[filePathIdx]) for i := 0; i < len(entries)-1; i++ { - entry1 := &entries[i] - entry1.FilePath = logEntries.FilenameNearest(entry1.FilePath) + entry1 := entries[i] + + // Because the filePath uses the nearestFilename, some entries may have the wrong version number + // because the original file path is not found in the logEntries. + if entry1.FileVersion == 0 { + entry1.FileVersion = getLastVersionNumberFromIdx(entries, filePath, i) + } + + entry2 := entries[i+1] - entry2 := &entries[i+1] - entry2.FilePath = logEntries.FilenameNearest(entry2.FilePath) + // Same as above, but for the second entry + if entry2.FileVersion == 0 { + if entry1.FileVersion != 0 { + entry2.FileVersion = entry1.FileVersion + } else { + entry2.FileVersion = getLastVersionNumberFromIdx(entries, filePath, i+1) + } + } // Calculate CharDelta between file versions charDelta, location, err := CalculateCharDeltaAndLocation(log, filePath, entry1, entry2) diff --git a/server/logger/analyzer/internal/structures.go b/server/logger/analyzer/internal/structures.go index 44ef083..baa8566 100644 --- a/server/logger/analyzer/internal/structures.go +++ b/server/logger/analyzer/internal/structures.go @@ -36,17 +36,15 @@ func (r *ResultStore[T]) FilenameNearest(filePath string) string { } func (r *ResultStore[T]) checkAndUpdateFilename(filePath string) string { + filePath = strings.TrimSpace(filePath) + if alias, ok := r.FilenameAliases[filePath]; ok { // do not mutate the original file path return alias } // nearest based on levenstein distance - nearest := r.FilenameNearest(filePath) - - if nearest != filePath && strings.HasPrefix(filePath, nearest) { - // fmt.Println("SETTING NEAREST", nearest, filePath, r.FilenamesIndices[nearest]) - + if nearest := r.FilenameNearest(filePath); nearest != filePath && strings.HasPrefix(filePath, nearest) { // if it is, replace the found path with the file path r.Filenames[r.FilenamesIndices[nearest]] = filePath r.FilenameAliases[nearest] = filePath diff --git a/server/logger/logger.go b/server/logger/logger.go index c221515..7b1bb20 100644 --- a/server/logger/logger.go +++ b/server/logger/logger.go @@ -9,6 +9,7 @@ import ( "strconv" "time" + "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" "github.com/lucasepe/codename" @@ -48,6 +49,7 @@ var initScript string type Logger struct { participantId string + After time.Time db *sqlx.DB } @@ -281,17 +283,22 @@ func (it *LogEntryIterator) List() ([]LogEntry, error) { } func (log *Logger) Entries() (*LogEntryIterator, error) { - rows, err := log.db.Queryx("SELECT * FROM logs") - if err != nil { - defer rows.Close() - return nil, err - } - return &LogEntryIterator{rows: rows}, nil + return log.EntriesByParticipantId(log.ParticipantId()) } // EntriesDescending returns log entries in descending order (latest first) func (log *Logger) EntriesDescending() (*LogEntryIterator, error) { - rows, err := log.db.Queryx("SELECT * FROM logs ORDER BY created_at DESC") + query := squirrel.Select("*").From("logs").OrderBy("created_at DESC") + if !log.After.IsZero() { + query = query.Where(squirrel.GtOrEq{"created_at": log.After}) + } + + sql, args, err := query.ToSql() + if err != nil { + return nil, err + } + + rows, err := log.db.Queryx(sql, args...) if err != nil { defer rows.Close() return nil, err @@ -300,7 +307,17 @@ func (log *Logger) EntriesDescending() (*LogEntryIterator, error) { } func (log *Logger) EntriesByParticipantId(participantId string) (*LogEntryIterator, error) { - rows, err := log.db.Queryx("SELECT * FROM logs WHERE participant_id = ?", participantId) + query := squirrel.Select("*").From("logs").Where(squirrel.Eq{"participant_id": participantId}) + if !log.After.IsZero() { + query = query.Where(squirrel.GtOrEq{"created_at": log.After}) + } + + sql, args, err := query.ToSql() + if err != nil { + return nil, err + } + + rows, err := log.db.Queryx(sql, args...) if err != nil { defer rows.Close() return nil, err @@ -310,12 +327,22 @@ func (log *Logger) EntriesByParticipantId(participantId string) (*LogEntryIterat func (log *Logger) Reset() error { // delete logs - if _, err := log.db.Exec("DELETE FROM logs WHERE participant_id = ?", log.ParticipantId()); err != nil { + logQuery := squirrel.Delete("logs").Where(squirrel.Eq{"participant_id": log.ParticipantId()}) + if !log.After.IsZero() { + logQuery = logQuery.Where(squirrel.GtOrEq{"created_at": log.After}) + } + + if _, err := logQuery.RunWith(log.db).Exec(); err != nil { return err } // delete files - if _, err := log.db.Exec("DELETE FROM files WHERE participant_id = ?", log.ParticipantId()); err != nil { + filesQuery := squirrel.Delete("files").Where(squirrel.Eq{"participant_id": log.ParticipantId()}) + if !log.After.IsZero() { + filesQuery = filesQuery.Where(squirrel.GtOrEq{"created_at": log.After}) + } + + if _, err := filesQuery.RunWith(log.db).Exec(); err != nil { return err } @@ -330,11 +357,15 @@ func (log *Logger) OpenFile(filepath string) ([]byte, error) { } func (log *Logger) OpenVersionedFile(filepath string, file_version int) ([]byte, error) { + return log.OpenVersionedFileFromPID(log.ParticipantId(), filepath, file_version) +} + +func (log *Logger) OpenVersionedFileFromPID(pid string, filepath string, file_version int) ([]byte, error) { var content []byte // get the recent file by file_version err := log.db.QueryRow( "SELECT content FROM files WHERE participant_id = ? AND file_path = ? AND file_version = ?", - log.ParticipantId(), + pid, filepath, file_version).Scan(&content)