From 6832631fa863d09919cfee6741add1779a2eaf9d Mon Sep 17 00:00:00 2001 From: "Dr. Gernot Starke" Date: Mon, 27 Nov 2023 19:19:00 +0100 Subject: [PATCH] migrated to zerolog for logging and output, fixing #46 --- go.mod | 6 +++- go.sum | 17 ++++++++++ internal/api/apiGateway.go | 31 ++++++++++-------- internal/domain/domain.go | 3 +- internal/github/issuesAndBugs.go | 9 +++--- internal/plausible/vpvStatistics.go | 40 ++++++++++++++--------- main.go | 50 +++++++++++++++++++++++++++-- readme.md | 8 +++++ 8 files changed, 125 insertions(+), 39 deletions(-) create mode 100644 readme.md diff --git a/go.mod b/go.mod index 6250d63..5d37f48 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.21 require ( github.com/andrerfcsantos/go-plausible v0.3.1 + github.com/rs/zerolog v1.31.0 github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278 + golang.org/x/net v0.18.0 golang.org/x/oauth2 v0.14.0 ) @@ -12,10 +14,12 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/klauspost/compress v1.16.3 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.50.0 // indirect - golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index b2db026..0732734 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -16,6 +18,16 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278 h1:kdEGVAV4sO46DPtb8k793jiecUEhaX9ixoIBt41HEGU= github.com/shurcooL/githubv4 v0.0.0-20230704064427-599ae7bbf278/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= @@ -39,6 +51,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/internal/api/apiGateway.go b/internal/api/apiGateway.go index 789419a..439f800 100644 --- a/internal/api/apiGateway.go +++ b/internal/api/apiGateway.go @@ -2,9 +2,8 @@ package api import ( "embed" - "fmt" + "github.com/rs/zerolog/log" "html/template" - "log" "net/http" "os" "path/filepath" @@ -23,6 +22,10 @@ const TemplatesDir = "" const HtmlTableTmpl = "arc42statistics.gohtml" const PingTmpl = "ping.gohtml" +func init() { + log.Debug().Msg("apiGateway initialized ") +} + // embed templates into compiled binary, so we don't need to read from file system // embeds the templates folder into variable embeddedTemplatesFolder // === DON'T REMOVE THE COMMENT BELOW @@ -37,7 +40,7 @@ var embeddedTemplatesFolder embed.FS // 4. renders the output via HtmlTableTmpl func statsHTMLTableHandler(w http.ResponseWriter, r *http.Request) { - fmt.Printf("received statsTable request \n") + log.Debug().Msg("received statsTable request") // 1. set timer var startOfProcessing = time.Now() @@ -67,8 +70,8 @@ func pingHandler(w http.ResponseWriter, r *http.Request) { var Host string = r.Host var RequestURI string = r.RequestURI - fmt.Printf("Host = %s\n", Host) - fmt.Printf("RequestURI = %s\n", RequestURI) + log.Debug().Msgf("Host = %s\n", Host) + log.Debug().Msgf("RequestURI = %s\n", RequestURI) executeTemplate(w, filepath.Join(TemplatesDir, PingTmpl), r) } @@ -81,7 +84,7 @@ func SendCORSHeaders(w *http.ResponseWriter, r *http.Request) { var origin string origin = r.Host - fmt.Printf("received request from host: %s\n", origin) + log.Debug().Msgf("received request from host: %s\n", origin) // TODO: don't always allow origin, restrict to known hosts //(*w).Header().Set("Access-Control-Allow-Origin", origin) @@ -108,13 +111,13 @@ func executeTemplate(w http.ResponseWriter, templatePath string, data any) { tpl, err := template.ParseFS(embeddedTemplatesFolder, templatePath) if err != nil { - log.Printf("Error parsing template: %v", err) + log.Error().Msgf("Error parsing template: %v", err) http.Error(w, "There was an error parsing the template "+err.Error(), http.StatusInternalServerError) return } err = tpl.Execute(w, data) if err != nil { - log.Printf("Error executing template: %v", err) + log.Error().Msgf("Error executing template: %v", err) http.Error(w, "There was an error executing the template "+err.Error(), http.StatusInternalServerError) return } @@ -124,22 +127,22 @@ func logRequestHandler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() h.ServeHTTP(w, r) - log.Printf("%s %s %v", r.Method, r.URL, time.Since(start)) + log.Info().Msgf("%s %s %v", r.Method, r.URL, time.Since(start)) }) } -// PrintServerDetails displays a few details about this program, +// LogServerDetails displays a few details about this program, // mainly to give admins some idea what version is currently running // and where in the fly.io cloud the service is deployed. -func PrintServerDetails(appVersion string) { +func LogServerDetails(appVersion string) { - fmt.Printf("Starting API server, version %s on Port %s at %s\n\n", appVersion, getPort(), time.Now().Format("2. January 2006, 15:04h")) + log.Info().Msgf("Starting API server, version %s on Port %s at %s", appVersion, getPort(), time.Now().Format("2. January 2006, 15:04h")) // assumes we're running this program within the fly.io cloud. // There, the env variable FLY_REGION should be set. // If this variable is empty, we assume we're running locally region, location := fly.RegionAndLocation() - fmt.Printf("Server region is %s/%s", region, location) + log.Info().Msgf("Server region is %s/%s", region, location) } // StartAPIServer creates an http ServeMux with a few predefined routes. @@ -160,7 +163,7 @@ func StartAPIServer() { err := http.ListenAndServe(homeIP+getPort(), loggedMux) if err != nil { - log.Fatalf("API server failed to start: %v", err) + log.Fatal().Msgf("API server failed to start: %v", err) } } diff --git a/internal/domain/domain.go b/internal/domain/domain.go index cea470a..d0dc54b 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -1,6 +1,7 @@ package domain import ( + "github.com/rs/zerolog/log" "site-usage-statistics/internal/badge" "site-usage-statistics/internal/github" "site-usage-statistics/internal/plausible" @@ -15,7 +16,7 @@ var SumOfStats types.TotalsForAllSites func SetAppVersion(appVersion string) { AppVersion = appVersion - + log.Debug().Msg("App version set to " + appVersion) } func setServerMetaInfo(a42s *types.Arc42Statistics) { diff --git a/internal/github/issuesAndBugs.go b/internal/github/issuesAndBugs.go index e675af1..308b213 100644 --- a/internal/github/issuesAndBugs.go +++ b/internal/github/issuesAndBugs.go @@ -2,10 +2,10 @@ package github import ( + "github.com/rs/zerolog/log" "github.com/shurcooL/githubv4" "golang.org/x/net/context" "golang.org/x/oauth2" - "log" "os" ) @@ -59,15 +59,14 @@ func IssuesAndBugsCountForSite(thisSite string) (nrOfIssues int, nrOfBugs int) { // Perform the query err := client.Query(context.Background(), &query, variables) if err != nil { - log.Fatal(err, query) + log.Error().Msgf(err.Error(), query) } nrOfBugs = int(query.Repository.Bugs.TotalCount) nrOfIssues = int(query.Repository.Issues.TotalCount) - // Output the result if running on localhost - //fmt.Printf("Number of open issues: %d\n", nrOfIssues) - //fmt.Printf("Number of open bugs: %d\n", nrOfBugs) + log.Debug().Msgf("Number of open issues: %d\n", nrOfIssues) + log.Debug().Msgf("Number of open bugs: %d\n", nrOfBugs) // this kind of return takes the named result parameters and returns those... return diff --git a/internal/plausible/vpvStatistics.go b/internal/plausible/vpvStatistics.go index 70e02e0..b6d4316 100644 --- a/internal/plausible/vpvStatistics.go +++ b/internal/plausible/vpvStatistics.go @@ -1,4 +1,4 @@ -// plausible interacts with the https://plausible.io web statistics service, +// Package plausible interacts with the https://plausible.io web statistics service, // that counts visitors and pageviews of the arc42 sites. package plausible @@ -8,9 +8,8 @@ package plausible // ============================================== import ( - "fmt" "github.com/andrerfcsantos/go-plausible/plausible" - "log" + "github.com/rs/zerolog/log" "os" "site-usage-statistics/internal/types" "strconv" @@ -18,31 +17,42 @@ import ( // plausibleClient wraps the plausible API. // The required (secret) API key is set within the initialization. -var plausibleClient = initPlausibleHandler() +var plausibleClient *plausible.Client = nil // initPlausibleHandler gets the plausible API key // and creates a handler (NewClient) to perform queries upon func initPlausibleHandler() *plausible.Client { - // APIKEY is a personal key for https://plausible.io - // It needs to be set via environment variable - var APIKEY string = os.Getenv("PLAUSIBLE_API_KEY") + if plausibleClient == nil { - if APIKEY == "" { - // no value, no API calls, no results. - // we exit here, as we have no chance of recovery - fmt.Printf("CRITICAL ERROR: required plausible API key not set.\n") - fmt.Printf("You need to set the 'PLAUSIBLE_API_KEY' environment variable prior to launching this application.\n") + // APIKEY is a personal key for https://plausible.io + // It needs to be set via environment variable + var APIKEY string = os.Getenv("PLAUSIBLE_API_KEY") - os.Exit(13) + if APIKEY == "" { + // no value, no API calls, no results. + // we exit here, as we have no chance of recovery + log.Error().Msgf("CRITICAL ERROR: required plausible API key not set.\n" + + "You need to set the 'PLAUSIBLE_API_KEY' environment variable prior to launching this application.\n") + + os.Exit(13) + } else { + log.Debug().Msg("PLAUSIBLE_API_KEY set") + } + return plausible.NewClient(APIKEY) + } else { + return plausibleClient } - return plausible.NewClient(APIKEY) } // StatsForSite collects all relevant statistics for a given site // (currently 7D, 30D and 12M), and updates the Sums accordingly func StatsForSite(thisSite string, stats *types.SiteStats, totals *types.TotalsForAllSites) { + // init the required handler + // the function ensures it's initialized only once. + plausibleClient = initPlausibleHandler() + // Get a handler to perform queries for a given site siteHandler := plausibleClient.Site(thisSite) @@ -86,7 +96,7 @@ func SiteMetrics(siteHandler *plausible.Site, period plausible.TimePeriod) types // Execute query to plausible.io result, err := siteHandler.Aggregate(siteMetricsQuery) if err != nil { - log.Println("Error performing query to plausible.io: %v", err) + log.Error().Msgf("Error performing query to plausible.io: %v", err) // in this case, we don't add anything to the Sums vApvs.Pageviews = "n/a" vApvs.PageViewNr = 0 diff --git a/main.go b/main.go index 1b6446c..fb81dd4 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,58 @@ package main import ( + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "os" "site-usage-statistics/internal/api" "site-usage-statistics/internal/domain" + "strings" ) -const AppVersion = "0.3.3" +const AppVersion = "0.3.4" + +// version history +// 0.3.4 removed all fmt.print*, migrated to zerolog +// 0.3.3 fixed issue #46 +// 0.3.1 slight refactoring +// 0.2.0 integrated GitHub issues +// 0.1.0 made it work + +// init is called AFTER all imported packages have been initialized. +func init() { + // Configure the global logger to write to console/stdout and add file and line number + output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: zerolog.TimeFormatUnix} + log.Logger = zerolog.New(output).With().Timestamp().Caller().Logger() + +} func main() { + // log levels for zerolog: + + // zerolog allows for logging at the following levels (from highest to lowest): + + // panic (zerolog.PanicLevel, 5) + // fatal (zerolog.FatalLevel, 4) + // error (zerolog.ErrorLevel, 3) + // warn (zerolog.WarnLevel, 2) + // info (zerolog.InfoLevel, 1) + // debug (zerolog.DebugLevel, 0) + // trace (zerolog.TraceLevel, -1) + + // find out runtime environment: + // PROD or PRODUCTION -> fly.io, running in the cloud + // DEV or DEVELOPMENT -> localhost, running on local machine + environment := strings.ToUpper(os.Getenv("ENVIRONMENT")) + if strings.HasPrefix(environment, "PROD") { + log.Info().Msg("Running on fly.io") + // Set the minimum level to WarnLevel to log only warnings and errors + zerolog.SetGlobalLevel(zerolog.WarnLevel) + + } else { + log.Info().Msg("Running on localhost") + } + // as the main package cannot be imported, constants defined here // cannot directly be used in internal/* packages, therefore we // set the AppVersion via a func. @@ -16,7 +60,7 @@ func main() { // Start a server which runs in background and waits for http requests to arrive // on predefined routes. - // THIS IS A BLOCKING CALL! - api.PrintServerDetails(AppVersion) + // THIS IS A BLOCKING CALL, therefore server details are printed prior to starting the server + api.LogServerDetails(AppVersion) api.StartAPIServer() } diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..da36470 --- /dev/null +++ b/readme.md @@ -0,0 +1,8 @@ +# site-usage statistics + + +This is the repo for the statistics and status backend service +for the https://status.arc42.org website. + +The documentation is contained in the [status-repository](https://github.com/arc42/status.arc42.org-site) repository. +