Merge pull request #2 from arc42/parallel_plausible

Parallel Goroutines

cmd/githubGraphQL/main.go

@@ -0,0 +1,9 @@
+package main
+
+import "site-usage-statistics/internal/github"
+
+func main() {
+
+	github.StatsForRepo("faq.arc42.org-site")
+	github.StatsForRepo("arc42.org-site")
+}

cmd/goroutines/main.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+)
+
+var mutex sync.Mutex
+
+// ApiResponse represents the structure of the API response
+type ApiResponse struct {
+	Value1 string
+	Type   string
+}
+
+type ApiCalls struct {
+	MyValue string
+	MyType  string
+}
+
+// this func called in parallel channels
+func callAPI(url string, myType string, res *ApiResponse, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	res.Value1 = url
+	res.Type = myType
+
+	println("API called with type", myType)
+}
+
+// try out Goroutines
+func main() {
+	var wg sync.WaitGroup
+	results := make([]ApiResponse, 3)
+
+	urls := []ApiCalls{
+		{"endpoint1", "D7"},
+		{"endpoint2", "D30"},
+		{"endpoint3", "M12"}}
+
+	for index, url := range urls {
+		wg.Add(1)
+		/*		results[index].Value1 = ""
+				results[index].Type = ""
+		*/
+		go callAPI(url.MyValue, url.MyType, &results[index], &wg)
+
+	}
+
+	// wait for goroutines to finish
+	wg.Wait()
+
+	fmt.Println("Results:")
+	fmt.Println("==========")
+
+	for _, r := range results {
+		println(r.Value1, r.Type)
+	}
+}

internal/api/apiGateway.go

@@ -84,7 +84,7 @@ func SendCORSHeaders(w *http.ResponseWriter, r *http.Request) {
 
 	var origin string
 	origin = r.Host
-	log.Debug().Msgf("received request from host: %s\n", origin)
+	log.Debug().Msgf("received request from host: %s", origin)
 
 	// TODO: don't always allow origin, restrict to known hosts
 	//(*w).Header().Set("Access-Control-Allow-Origin", origin)
@@ -142,7 +142,7 @@ func LogServerDetails(appVersion string) {
 	// There, the env variable FLY_REGION should be set.
 	// If this variable is empty, we assume we're running locally
 	region, location := fly.RegionAndLocation()
-	log.Info().Msgf("Server region is %s/%s", region, location)
+	log.Info().Msgf("Server region is%s %s", region, location)
 }
 
 // StartAPIServer creates an http ServeMux with a few predefined routes.

internal/api/arc42statistics.gohtml

@@ -17,7 +17,7 @@
         <th>PageViews</th>
     </tr>
 
-    {{ range .Stats4Site}}
+    {{ range .Stats4Site }}
         <tr>
             <td><a href="https://{{.Site }}">{{.Site}}</a></td>
             <td style="border-left: 2px solid black;">{{ .Visitors7d}}</td>
@@ -50,8 +50,10 @@
 <div style="font-size: 12px; padding-bottom: 14px;">
     Data collected in {{ .HowLongDidItTake }} msecs by arc42 statistics service
     (v. {{.AppVersion}}) at {{.LastUpdatedString }}
-    {{if .FlyRegion}}
-        running on <a href="https://fly.io" target="_blank"> fly.io </a>
+    running {{if .FlyRegion}}
+        on <a href="https://fly.io" target="_blank"> fly.io </a>
         in {{.WhereDoesItRun}} (region {{.FlyRegion}})
-    {{end }}
+    {{ else }}
+        {{.WhereDoesItRun}}
+    {{ end}}
 </div>

internal/domain/domain.go

@@ -6,13 +6,16 @@ import (
 	"site-usage-statistics/internal/github"
 	"site-usage-statistics/internal/plausible"
 	"site-usage-statistics/internal/types"
+	"sync"
 	"time"
 )
 
 var AppVersion string
 
+// ArcStats collects all data
 var ArcStats types.Arc42Statistics
-var SumOfStats types.TotalsForAllSites
+
+var mutex sync.Mutex
 
 func SetAppVersion(appVersion string) {
 	AppVersion = appVersion
@@ -31,46 +34,95 @@ func setServerMetaInfo(a42s *types.Arc42Statistics) {
 	a42s.LastUpdatedString = bielefeldTime.Format("2. January 2006, 15:04:03h")
 }
 
-// LoadStats4AllSites calls the various wrapper APIs to retrieve all site and repo statistics.
-// 1.) set some meta info: App-version plus current time
-// 2.) for all sites:
-// 2.1) get data from plausible.io
-// 2.2) get issues and bug counts from GitHub.com
-// 2.3) set the URLs for the issue and bug badges
-
+// LoadStats4AllSites calls the plausible wrapper package to retrieve site statistics.
 func LoadStats4AllSites() types.Arc42Statistics {
 
-	a42s := types.Arc42Statistics{}
+	// the WaitGroup synchronises the parallel goroutines
+	var wg sync.WaitGroup
+
+	var a42s = types.Arc42Statistics{}
+
+	var Stats4Sites = make([]types.SiteStats, len(types.Arc42sites))
+	var Stats4Repos = make([]types.RepoStats, len(types.Arc42sites))
 
+	// var IssuesAndBugs4Sites = make([]types.)
 	// 1.) set meta info
 	setServerMetaInfo(&a42s)
 
-	// 2.) for all sites...
-	// this could be done in Goroutines to improve performance
+	// retrieve usage statistics (visitors and pageviews)
 	for index, site := range types.Arc42sites {
+		wg.Add(1)
 
-		// to avoid repeating the expression, introduce local var
-		thisSite := &a42s.Stats4Site[index]
-
-		// previously this was: a42s.Stats4Site[index].Site = site
-		thisSite.Site = site
-		// all arc42 website repos follow this naming convention, e.g. arc42.org-site
-		thisSite.Repo = github.GithubArc42URL + site + "-site"
+		go getUsageStatisticsForSite(site, &Stats4Sites[index], &wg)
+	}
 
-		// 2.1 set the statistic data from plausible.io
-		plausible.StatsForSite(site, &a42s.Stats4Site[index], &a42s.Totals)
+	// retrieve repo statistics
+	// currently:  number of open bugs and issues from GitHub
+	for index, site := range types.Arc42sites {
+		wg.Add(1)
 
-		// 2.2 query the number of open bugs and issues from GitHub
-		//a42s.Stats4Site[index].NrOfOpenIssues, a42s.Stats4Site[index].NrOfOpenBugs = github.IssuesAndBugsCountForSite(site)
-		thisSite.NrOfOpenIssues, thisSite.NrOfOpenBugs = github.IssuesAndBugsCountForSite(site + "-site")
+		go getRepoStatisticsForSite(site+"-site", &Stats4Repos[index], &wg)
+	}
 
-		// set some URLs so the templates get smaller
-		setIssuesAndBugBadgeURLsForSite(thisSite)
+	wg.Wait()
 
+	// get results from Goroutines
+	log.Debug().Msgf("transferring results in LoadStats4Site")
+	for index := range types.Arc42sites {
+		a42s.Stats4Site[index] = Stats4Sites[index]
+		a42s.Stats4Site[index].NrOfOpenIssues = Stats4Repos[index].NrOfOpenIssues
+		a42s.Stats4Site[index].NrOfOpenBugs = Stats4Repos[index].NrOfOpenBugs
 	}
+
+	// now calculate totals
+	a42s.Totals = calculateTotals(Stats4Sites)
+
 	return a42s
 }
 
+func calculateTotals(stats []types.SiteStats) types.TotalsForAllSites {
+	var totals types.TotalsForAllSites
+
+	for index := range types.Arc42sites {
+		totals.SumOfVisitors7d += stats[index].Visitors7dNr
+		totals.SumOfPageviews7d += stats[index].Pageviews7dNr
+		totals.SumOfVisitors30d += stats[index].Visitors30dNr
+		totals.SumOfPageviews30d += stats[index].Pageviews30dNr
+		totals.SumOfVisitors12m += stats[index].Visitors12mNr
+		totals.SumOfPageviews12m += stats[index].Pageviews12mNr
+	}
+	log.Debug().Msgf("Total visits and pageviews (V/PV, 7d, 30d, 12m)= %d/%d, %d/%d, %d/%d", totals.SumOfVisitors7d, totals.SumOfPageviews7d, totals.SumOfVisitors30d, totals.SumOfPageviews30d, totals.SumOfVisitors12m, totals.SumOfPageviews12m)
+
+	return totals
+}
+
+// getUsageStatisticsForSite retrieves the statistics for a single site from plausible.io.
+// This func is called as Goroutine.
+func getUsageStatisticsForSite(site string, thisSiteStats *types.SiteStats, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	// it seems we don't need to lock here:
+	//mutex.Lock()
+	//defer mutex.Unlock()
+
+	// to avoid repeating the expression, introduce local var
+	thisSiteStats.Site = site
+
+	// get statistic data from plausible.io
+	plausible.StatsForSite(site, thisSiteStats)
+
+}
+
+func getRepoStatisticsForSite(site string, thisRepoStats *types.RepoStats, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	thisRepoStats.Site = site
+	thisRepoStats.Repo = github.GithubArc42URL + site + "-site"
+
+	github.StatsForRepo(site, thisRepoStats)
+
+}
+
 // bugBadgeURL returns a shields.io bug badge URL,
 // if the bug-count is >= 0. Otherwise, NO bug badge
 // shall be shown.
@@ -91,7 +143,7 @@ func bugBadgeURL(site string, nrOfBugs int) string {
 // if the number of bugs==0, then this URL remains empty, so no badge will be shown
 // if the number of issues==0, then a special "hurray" badge shall be shown.
 
-func setIssuesAndBugBadgeURLsForSite(stats *types.SiteStats) {
+func setIssuesAndBugBadgeURLsForSite(stats *types.RepoStats) {
 
 	// shields.io issues URLS look like that: https://img.shields.io/github/issues-raw/arc42/arc42.org-site
 	stats.IssueBadgeURL = badge.ShieldsGithubIssuesURL + stats.Site + "-site"

internal/fly/flyHosting.go

@@ -20,19 +20,19 @@ func RegionAndLocation() (string, string) {
 func flyRegion() string {
 	region := os.Getenv("FLY_REGION")
 	if region == "" {
-		return "???"
+		return ""
 	} else {
 		return region
 	}
 }
 
 // flyRegionCodeToLocation converts a 3-letter fly.io region
-// code to a location name, hopefully beeing compatible
+// code to a location name, hopefully being compatible
 // with https://fly.io/docs/reference/regions/
 // e.g. ams -> Amsterdam
 func flyRegionCodeToLocation(regionCode string) string {
 	switch strings.ToUpper(regionCode) {
-	case "???":
+	case "":
 		return "on premise (likely: localhost)"
 	case "AMS":
 		return "Amsterdam, Netherlands"

internal/github/issuesAndBugs.go

@@ -7,6 +7,7 @@ import (
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"os"
+	"site-usage-statistics/internal/types"
 )
 
 const GithubArc42URL = "https://github.com/arc42/"
@@ -42,7 +43,7 @@ func initGitHubGraphQLClient() *githubv4.Client {
 
 }
 
-func IssuesAndBugsCountForSite(thisSite string) (nrOfIssues int, nrOfBugs int) {
+func StatsForRepo(thisSite string, stats *types.RepoStats) {
 
 	// Initialize GitHub GraphQL client
 	client := initGitHubGraphQLClient()
@@ -62,12 +63,9 @@ func IssuesAndBugsCountForSite(thisSite string) (nrOfIssues int, nrOfBugs int) {
 		log.Error().Msgf(err.Error(), query)
 	}
 
-	nrOfBugs = int(query.Repository.Bugs.TotalCount)
-	nrOfIssues = int(query.Repository.Issues.TotalCount)
+	stats.NrOfOpenBugs = int(query.Repository.Bugs.TotalCount)
+	stats.NrOfOpenIssues = int(query.Repository.Issues.TotalCount)
 
-	log.Debug().Msgf("Number of open issues on %s: %d\n", thisSite, nrOfIssues)
-	log.Debug().Msgf("Number of open bugs on %s: %d\n", thisSite, nrOfBugs)
+	log.Debug().Msgf("%s has %d open issues and %d bugs", thisSite, stats.NrOfOpenIssues, stats.NrOfOpenBugs)
 
-	// this kind of return takes the named result parameters and returns those...
-	return
 }