Merge pull request #70 from flanksource/moshloop

add severity and category levels

.gitignore

@@ -3,5 +3,5 @@
 _DS_Store
 .bin/
 vendor
-confighub
-scraped/
+config-db
+scraped/

README.md

@@ -28,7 +28,7 @@ To explicitly run scrapping.
 
 ```bash
 ./bin/config-db run <scrapper-config.yaml> -vvv
-confighub serve
+config-db serve
 ```
 
 See fixtures/ for example scrape configs.

analyzers/patches.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/flanksource/commons/logger"
-	v1 "github.com/flanksource/confighub/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 )
 
 // PatchAnalyzer ...

api/v1/file.go

@@ -21,5 +21,4 @@ func (f File) RedactedString() string {
 	}
 
 	return url.Redacted()
-
 }

api/v1/interface.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/flanksource/commons/logger"
-	fs "github.com/flanksource/confighub/filesystem"
+	fs "github.com/flanksource/config-db/filesystem"
 	"github.com/flanksource/kommons"
 )
 
@@ -192,5 +192,5 @@ func (ctx ScrapeContext) GetNamespace() string {
 
 // IsTrace ...
 func (ctx ScrapeContext) IsTrace() bool {
-	return logger.IsTraceEnabled()
+	return ctx.Scraper != nil && ctx.Scraper.IsTrace()
 }

api/v1/types.go

@@ -2,6 +2,7 @@ package v1
 
 // ConfigScraper ...
 type ConfigScraper struct {
+	LogLevel string `json:"logLevel,omitempty"`
 	Schedule string `json:"schedule,omitempty"`
 	AWS      []AWS  `json:"aws,omitempty" yaml:"aws,omitempty"`
 	File     []File `json:"file,omitempty" yaml:"file,omitempty"`
@@ -11,3 +12,7 @@ type ConfigScraper struct {
 func (c ConfigScraper) IsEmpty() bool {
 	return len(c.AWS) == 0 && len(c.File) == 0
 }
+
+func (c ConfigScraper) IsTrace() bool {
+	return c.LogLevel == "trace"
+}

cmd/analyze.go

@@ -5,9 +5,9 @@ import (
 	"io/ioutil"
 
 	"github.com/flanksource/commons/logger"
-	"github.com/flanksource/confighub/analyzers"
-	v1 "github.com/flanksource/confighub/api/v1"
-	"github.com/flanksource/confighub/scrapers/aws"
+	"github.com/flanksource/config-db/analyzers"
+	v1 "github.com/flanksource/config-db/api/v1"
+	"github.com/flanksource/config-db/scrapers/aws"
 	"github.com/spf13/cobra"
 )
 

cmd/config.go

@@ -10,7 +10,7 @@ import (
 	gotemplate "text/template"
 
 	"github.com/flanksource/commons/text"
-	v1 "github.com/flanksource/confighub/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/pkg/errors"
 
 	"gopkg.in/flanksource/yaml.v3"

cmd/offline.go

@@ -2,14 +2,14 @@ package cmd
 
 import (
 	"github.com/flanksource/commons/logger"
-	"github.com/flanksource/confighub/db"
+	"github.com/flanksource/config-db/db"
 	"github.com/spf13/cobra"
 )
 
 // GoOffline ...
 var GoOffline = &cobra.Command{
 	Use:  "go-offline",
-	Long: "Download all dependencies so that confighub can work without an internet connection",
+	Long: "Download all dependencies so that config-db can work without an internet connection",
 	Run: func(cmd *cobra.Command, args []string) {
 		if err := db.GoOffline(); err != nil {
 			logger.Fatalf("Failed to go offline: %+v", err)

cmd/root.go

@@ -5,8 +5,8 @@ import (
 	"os"
 
 	"github.com/flanksource/commons/logger"
-	"github.com/flanksource/confighub/db"
-	"github.com/flanksource/confighub/utils/kube"
+	"github.com/flanksource/config-db/db"
+	"github.com/flanksource/config-db/utils/kube"
 	"github.com/flanksource/kommons"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -34,7 +34,7 @@ func readFromEnv(v string) string {
 
 // Root ...
 var Root = &cobra.Command{
-	Use: "confighub",
+	Use: "config-db",
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		count, _ := cmd.Flags().GetCount("loglevel")
 		// logger.StandardLogger().(logsrusapi.Logger).Out = os.Stderr
@@ -75,7 +75,7 @@ func init() {
 	}
 	Root.AddCommand(&cobra.Command{
 		Use:   "version",
-		Short: "Print the version of confighub",
+		Short: "Print the version of config-db",
 		Args:  cobra.MinimumNArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
 			fmt.Println(version)

cmd/run.go

@@ -8,10 +8,10 @@ import (
 	"path"
 
 	"github.com/flanksource/commons/logger"
-	v1 "github.com/flanksource/confighub/api/v1"
-	"github.com/flanksource/confighub/db"
-	fs "github.com/flanksource/confighub/filesystem"
-	"github.com/flanksource/confighub/scrapers"
+	v1 "github.com/flanksource/config-db/api/v1"
+	"github.com/flanksource/config-db/db"
+	fs "github.com/flanksource/config-db/filesystem"
+	"github.com/flanksource/config-db/scrapers"
 	"github.com/spf13/cobra"
 )
 
@@ -43,21 +43,23 @@ var Run = &cobra.Command{
 		if err != nil {
 			logger.Fatalf(err.Error())
 		}
-		logger.Infof("Found %d resources", len(results))
 
 		if db.ConnectionString != "" {
+			logger.Infof("Exporting %d resources to DB", len(results))
 			if err = db.Update(ctx, results); err != nil {
 				logger.Errorf("Failed to update db: %+v", err)
 			}
 		} else if outputDir != "" {
+			logger.Infof("Exporting %d resources to %s", outputDir)
+
 			for _, result := range results {
 				if err := exportResource(result, filename, outputDir); err != nil {
 					logger.Fatalf("failed to export results %v", err)
 				}
 			}
 
 		} else {
-			logger.Infof("skipping export: neither --output-dir or --db is specified")
+			logger.Fatalf("skipping export: neither --output-dir or --db is specified")
 		}
 
 	},

cmd/server.go

@@ -6,12 +6,12 @@ import (
 	"net/url"
 
 	"github.com/flanksource/commons/logger"
-	v1 "github.com/flanksource/confighub/api/v1"
-	"github.com/flanksource/confighub/db"
-	fs "github.com/flanksource/confighub/filesystem"
-	"github.com/flanksource/confighub/query"
+	v1 "github.com/flanksource/config-db/api/v1"
+	"github.com/flanksource/config-db/db"
+	fs "github.com/flanksource/config-db/filesystem"
+	"github.com/flanksource/config-db/query"
 
-	"github.com/flanksource/confighub/scrapers"
+	"github.com/flanksource/config-db/scrapers"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/robfig/cron/v3"

db/ext.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 
 	"github.com/flanksource/commons/logger"
-	v1 "github.com/flanksource/confighub/api/v1"
-	"github.com/flanksource/confighub/db/models"
+	v1 "github.com/flanksource/config-db/api/v1"
+	"github.com/flanksource/config-db/db/models"
 )
 
 func GetJSON(ci models.ConfigItem) []byte {

db/init.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/flanksource/commons/logger"
-	repoimpl "github.com/flanksource/confighub/db/repository"
+	repoimpl "github.com/flanksource/config-db/db/repository"
 	"github.com/jackc/pgx/v4/log/logrusadapter"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/jackc/pgx/v4/stdlib"
@@ -38,6 +38,9 @@ func Flags(flags *pflag.FlagSet) {
 //go:embed migrations/*.sql
 var embedMigrations embed.FS
 
+//go:embed migrations/_always/*.sql
+var embedScripts embed.FS
+
 // Pool ...
 var Pool *pgxpool.Pool
 var repository repoimpl.Database
@@ -119,8 +122,26 @@ func Migrate() error {
 	}
 	defer db.Close()
 
-	if err := goose.Up(db, "migrations", goose.WithAllowMissing()); err != nil {
-		return err
+	for {
+		err = goose.UpByOne(db, "migrations", goose.WithAllowMissing())
+		if err == goose.ErrNoNextVersion {
+			break
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	scripts, _ := embedScripts.ReadDir("migrations/_always")
+
+	for _, file := range scripts {
+		script, err := embedScripts.ReadFile("migrations/_always/" + file.Name())
+		if err != nil {
+			return err
+		}
+		if _, err := Pool.Exec(context.TODO(), string(script)); err != nil {
+			return err
+		}
 	}
 	return nil
 }

db/migrations/003_seed.sql

@@ -7,6 +7,7 @@ CREATE TABLE config_scrapers (
   description TEXT NULL,
   scraper_type text NOT NULL,
   spec jsonb,
+  created_by UUID null,
   created_at timestamp NOT NULL DEFAULT now(),
   updated_at timestamp NOT NULL DEFAULT now()
 );
@@ -28,6 +29,7 @@ CREATE TABLE config_items (
   config jsonb null,
   source TEXT null,
   tags jsonb null,
+  created_by UUID null,
   created_at timestamp NOT NULL DEFAULT now(),
   updated_at timestamp NOT NULL DEFAULT now(),
   FOREIGN KEY (scraper_id) REFERENCES config_scrapers(id)
@@ -52,12 +54,14 @@ CREATE TABLE config_changes (
   id UUID DEFAULT generate_ulid() PRIMARY KEY,
   config_id UUID NOT NULL,
   external_change_id text NULL,
+  external_created_by TEXT NULL,
   change_type text NULL,
   severity text  NULL,
   source text  NULL,
   summary text,
   patches jsonb null,
   details jsonb null,
+  created_by UUID null,
   created_at timestamp NOT NULL DEFAULT now(),
   FOREIGN KEY (config_id) REFERENCES config_items(id)
 );
@@ -89,8 +93,6 @@ CREATE TABLE saved_query (
   updated_at timestamp NOT NULL DEFAULT now()
 );
 
--- INSERT INTO config_db_version (version_id,is_applied,tstamp) values ('3',true, now())
-
 
 -- +goose StatementEnd
 -- +goose Down

db/migrations/099_post_seed.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+
+
+INSERT INTO config_db_version(version_id, tstamp, is_applied) (
+  SELECT  version_id, now() as tstamp, true as is_applied
+  FROM    generate_series(100, 101) version_id
+);
+

db/migrations/101_created_by.sql

@@ -0,0 +1,6 @@
+-- +goose Up
+
+ALTER TABLE config_scrapers ADD COLUMN created_by UUID NULL;
+ALTER TABLE config_items ADD COLUMN     created_by UUID NULL;
+ALTER TABLE config_changes ADD COLUMN   IF NOT EXISTS created_by UUID NULL;
+ALTER TABLE config_changes ADD COLUMN   IF NOT EXISTS external_created_by TEXT NULL;

db/migrations/_always/views.sql

@@ -0,0 +1,37 @@
+DROP VIEW IF EXISTS configs;
+
+CREATE or REPLACE VIEW configs AS
+  SELECT
+    ci.*,
+    analysis,
+    changes
+  FROM config_items as ci
+    full join (
+      SELECT config_id,
+        json_agg(json_build_object('analyzer',analyzer,'analysis_type',analysis_type,'severity',severity)) as analysis
+      FROM config_analysis
+      GROUP BY  config_id
+    ) as ca on ca.config_id = ci.id
+    full join (
+      SELECT config_id,
+        json_agg(total) as changes
+      FROM
+      (SELECT config_id,json_build_object('change_type',change_type, 'severity', severity, 'total', count(*)) as total FROM config_changes GROUP BY config_id, change_type, severity) as config_change_types
+      GROUP BY  config_id
+    ) as cc on cc.config_id = ci.id;
+
+
+CREATE or REPLACE VIEW config_names AS
+  SELECT id, config_type, external_id, name FROM config_items;
+
+CREATE or REPLACE VIEW config_types AS
+  SELECT DISTINCT config_type FROM config_items;
+
+CREATE or REPLACE VIEW analyzer_types AS
+  SELECT DISTINCT analyzer FROM config_analysis;
+
+CREATE or REPLACE VIEW analysis_types AS
+  SELECT DISTINCT analysis_type FROM config_analysis;
+
+CREATE or REPLACE VIEW change_types AS
+  SELECT DISTINCT change_type FROM config_changes;

db/models/analysis.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"time"
 
-	v1 "github.com/flanksource/confighub/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 )
 
 type Analysis struct {

db/models/config_change.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	v1 "github.com/flanksource/confighub/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 )
 
 // ConfigChange represents the config change database table

db/models/config_item.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	v1 "github.com/flanksource/confighub/api/v1"
+	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/lib/pq"
 )