Merge pull request #25 from Anondo/feature/file-scrapper

feat(scrapper): add new file scrapper

.gitignore

@@ -1,4 +1,4 @@
-*.json
+/*.json
 .vscode
 _DS_Store
 .bin/

analyzers/patches.go

@@ -9,6 +9,7 @@ import (
 	v1 "github.com/flanksource/confighub/api/v1"
 )
 
+// PatchAnalyzer ...
 func PatchAnalyzer(configs []v1.ScrapeResult) v1.AnalysisResult {
 
 	result := v1.AnalysisResult{

api/v1/aws.go

@@ -1,5 +1,6 @@
 package v1
 
+// AWS ...
 type AWS struct {
 	*AWSConnection
 	PatchStates         bool `json:"patch_states,omitempty"`

api/v1/common.go

@@ -6,23 +6,28 @@ import (
 	"github.com/flanksource/kommons"
 )
 
+// Authentication ...
 type Authentication struct {
 	Username kommons.EnvVar `yaml:"username" json:"username"`
 	Password kommons.EnvVar `yaml:"password" json:"password"`
 }
 
+// IsEmpty ...
 func (auth Authentication) IsEmpty() bool {
 	return auth.Username.IsEmpty() && auth.Password.IsEmpty()
 }
 
+// GetUsername ...
 func (auth Authentication) GetUsername() string {
 	return auth.Username.Value
 }
 
+// GetPassword ...
 func (auth Authentication) GetPassword() string {
 	return auth.Password.Value
 }
 
+// GetDomain ...
 func (auth Authentication) GetDomain() string {
 	parts := strings.Split(auth.GetUsername(), "@")
 	if len(parts) == 2 {
@@ -31,6 +36,7 @@ func (auth Authentication) GetDomain() string {
 	return ""
 }
 
+// AWSConnection ...
 type AWSConnection struct {
 	AccessKey     kommons.EnvVar `yaml:"accessKey,omitempty" json:"accessKey,omitempty"`
 	SecretKey     kommons.EnvVar `yaml:"secretKey,omitempty" json:"secretKey,omitempty"`
@@ -40,6 +46,7 @@ type AWSConnection struct {
 	AssumeRole    string         `yaml:"assumeRole,omitempty" json:"assumeRole,omitempty"`
 }
 
+// GCPConnection ...
 type GCPConnection struct {
 	Endpoint    string          `yaml:"endpoint" json:"endpoint,omitempty"`
 	Credentials *kommons.EnvVar `yaml:"credentials" json:"credentials,omitempty"`

api/v1/data.go

@@ -1,5 +1,6 @@
 package v1
 
+// Host ...
 type Host interface {
 	GetHostname() string
 	GetPlatform() string
@@ -8,6 +9,7 @@ type Host interface {
 	GetPatches() []Patch
 }
 
+// Patch ...
 type Patch interface {
 	GetName() string
 	GetVersion() string
@@ -18,7 +20,10 @@ type Patch interface {
 	IsFailed() bool
 }
 
+// Properties ...
 type Properties []Property
+
+// Property ...
 type Property struct {
 	Name string `json:"name"`
 	// Line comments or description associated with this property
@@ -31,19 +36,22 @@ type Property struct {
 	OpenAPI *OpenAPIFieldRef `json:"openapiRef,omitempty"`
 }
 
+// FileLocation ...
 type FileLocation struct {
 	Host       string `json:"host,omitempty"`
 	FilePath   string `json:"filePath"`
 	LineNumber int    `json:"lineNumber"`
 }
 
+// GitLocation ...
 type GitLocation struct {
 	Repository string `json:"repository"`
 	FilePath   string `json:"filePath"`
 	LineNumber int    `json:"lineNumber"`
 	GitRef     string `json:"gitRef"`
 }
 
+// OpenAPIFieldRef ...
 type OpenAPIFieldRef struct {
 	// Location of the OpenAPI spec
 	Location string `json:"location,omitempty"`

api/v1/file.go

@@ -0,0 +1,8 @@
+package v1
+
+// File ...
+type File struct {
+	ID   string   `json:"id"`
+	Type string   `json:"type"`
+	Glob []string `json:"path"`
+}

api/v1/interface.go

@@ -6,20 +6,30 @@ import (
 	"time"
 
 	"github.com/flanksource/commons/logger"
+	fs "github.com/flanksource/confighub/filesystem"
 	"github.com/flanksource/kommons"
 )
 
+// Scraper ...
 type Scraper interface {
-	Scrape(ctx ScrapeContext, config ConfigScraper) []ScrapeResult
+	Scrape(ctx ScrapeContext, config ConfigScraper, manager Manager) []ScrapeResult
 }
 
+// Analyzer ...
 type Analyzer func(configs []ScrapeResult) AnalysisResult
 
+// AnalysisResult ...
 type AnalysisResult struct {
 	Analyzer string
 	Messages []string
 }
 
+// Manager ...
+type Manager struct {
+	Finder fs.Finder
+}
+
+// ScrapeResult ...
 type ScrapeResult struct {
 	LastModified time.Time   `json:"last_modified,omitempty"`
 	Type         string      `json:"type,omitempty"`
@@ -30,46 +40,53 @@ type ScrapeResult struct {
 	Zone         string      `json:"zone,omitempty"`
 	Name         string      `json:"name,omitempty"`
 	Namespace    string      `json:"namespace,omitempty"`
-	Id           string      `json:"id,omitempty"`
+	ID           string      `json:"id,omitempty"`
 	Config       interface{} `json:"config,omitempty"`
 }
 
 func (s ScrapeResult) String() string {
-	return fmt.Sprintf("%s/%s", s.Type, s.Id)
+	return fmt.Sprintf("%s/%s", s.Type, s.ID)
 }
 
+// QueryColumn ...
 type QueryColumn struct {
 	Name string `json:"name"`
 	Type string `json:"type"`
 }
 
+// QueryResult ...
 type QueryResult struct {
 	Count   int                      `json:"count"`
 	Columns []QueryColumn            `json:"columns"`
 	Results []map[string]interface{} `json:"results"`
 }
 
+// QueryRequest ...
 type QueryRequest struct {
 	Query string `json:"query"`
 }
 
+// ScrapeContext ...
 type ScrapeContext struct {
 	context.Context
 	Namespace string
 	Kommons   *kommons.Client
 	Scraper   *ConfigScraper
 }
 
+// WithScraper ...
 func (ctx ScrapeContext) WithScraper(config *ConfigScraper) ScrapeContext {
 	ctx.Scraper = config
 	return ctx
 
 }
 
+// GetNamespace ...
 func (ctx ScrapeContext) GetNamespace() string {
 	return ctx.Namespace
 }
 
+// IsTrace ...
 func (ctx ScrapeContext) IsTrace() bool {
 	return logger.IsTraceEnabled()
 }

api/v1/types.go

@@ -1,10 +1,13 @@
 package v1
 
+// ConfigScraper ...
 type ConfigScraper struct {
 	Schedule string `json:"schedule,omitempty"`
 	AWS      []AWS  `json:"aws,omitempty" yaml:"aws,omitempty"`
+	File     []File `json:"file,omitempty" yaml:"file,omitempty"`
 }
 
+// IsEmpty ...
 func (c ConfigScraper) IsEmpty() bool {
 	return len(c.AWS) == 0
 }

cmd/analyze.go

@@ -40,7 +40,7 @@ var Analyze = &cobra.Command{
 				nested, _ := json.Marshal(obj.Config)
 				instance := aws.Instance{}
 				if err := json.Unmarshal(nested, &instance); err != nil {
-					logger.Fatalf("Failed to unmarshal object into ec2 instance %s", obj.Id)
+					logger.Fatalf("Failed to unmarshal object into ec2 instance %s", obj.ID)
 				}
 				obj.Config = instance
 			}

cmd/config.go

@@ -89,6 +89,7 @@ func ParseConfig(configfile string, datafile string) ([]v1.ConfigScraper, error)
 	}
 
 	var scrapers []v1.ConfigScraper
+
 	re := regexp.MustCompile(`(?m)^---\n`)
 	for _, chunk := range re.Split(configs, -1) {
 		if strings.TrimSpace(chunk) == "" {

cmd/run.go

@@ -13,6 +13,7 @@ import (
 	"github.com/flanksource/commons/text"
 	v1 "github.com/flanksource/confighub/api/v1"
 	"github.com/flanksource/confighub/db"
+	fs "github.com/flanksource/confighub/filesystem"
 	"github.com/flanksource/confighub/scrapers"
 	"github.com/spf13/cobra"
 )
@@ -34,7 +35,11 @@ var Run = &cobra.Command{
 
 		ctx := v1.ScrapeContext{Context: context.Background(), Kommons: kommonsClient}
 
-		results, err := scrapers.Run(ctx, scraperConfigs...)
+		manager := v1.Manager{
+			Finder: fs.NewFileFinder(),
+		}
+
+		results, err := scrapers.Run(ctx, manager, scraperConfigs...)
 		if err != nil {
 			logger.Fatalf(err.Error())
 		}

cmd/server.go

@@ -8,6 +8,7 @@ import (
 	"github.com/flanksource/commons/logger"
 	v1 "github.com/flanksource/confighub/api/v1"
 	"github.com/flanksource/confighub/db"
+	fs "github.com/flanksource/confighub/filesystem"
 	"github.com/flanksource/confighub/query"
 
 	"github.com/flanksource/confighub/scrapers"
@@ -72,7 +73,10 @@ func serve(configFiles []string) {
 		_scraper := scraper
 		fn := func() {
 			ctx := v1.ScrapeContext{Context: context.Background(), Kommons: kommonsClient, Scraper: &_scraper}
-			if results, err := scrapers.Run(ctx, _scraper); err != nil {
+			manager := v1.Manager{
+				Finder: fs.NewFileFinder(),
+			}
+			if results, err := scrapers.Run(ctx, manager, _scraper); err != nil {
 				logger.Errorf("Failed to run scraper %s: %v", _scraper, err)
 			} else if err = db.Update(ctx, results); err != nil {
 				//FIXME cache results to save to db later

db/ext.go

@@ -17,19 +17,27 @@ func GetJSON(ci models.ConfigItem) []byte {
 	return data
 }
 
+// GetConfigItem ...
 func GetConfigItem(id string) (*models.ConfigItem, error) {
 	return repository.GetConfigItem(id)
 }
 
+// CreateConfigItem ...
 func CreateConfigItem(item *models.ConfigItem) error {
 	return repository.CreateConfigItem(item)
 }
+
+// UpdateConfigItem ...
 func UpdateConfigItem(item *models.ConfigItem) error {
 	return repository.UpdateConfigItem(item)
 }
+
+// CreateConfigChange ...
 func CreateConfigChange(change *models.ConfigChange) error {
 	return repository.CreateConfigChange(change)
 }
+
+// QueryConfigItems ...
 func QueryConfigItems(request v1.QueryRequest) (*v1.QueryResult, error) {
 	return repository.QueryConfigItems(request)
 }

db/repository/db.go

@@ -58,6 +58,7 @@ func (d *DBRepo) CreateConfigChange(cc *models.ConfigChange) error {
 	return nil
 }
 
+// QueryConfigItems ...
 func (d *DBRepo) QueryConfigItems(request v1.QueryRequest) (*v1.QueryResult, error) {
 	results := d.db.Raw(request.Query)
 	logger.Tracef(request.Query)
@@ -69,24 +70,25 @@ func (d *DBRepo) QueryConfigItems(request v1.QueryRequest) (*v1.QueryResult, err
 		Results: make([]map[string]interface{}, 0),
 	}
 
-	if rows, err := results.Rows(); err != nil {
+	rows, err := results.Rows()
+	if err != nil {
 		return nil, fmt.Errorf("failed to run query: %s -> %s", request.Query, err)
-	} else {
-
-		columns, err := rows.Columns()
-		if err != nil {
-			logger.Errorf("failed to get column details: %v", err)
-		}
-		rows.Next()
-		if err := results.ScanRows(rows, &response.Results); err != nil {
-			return nil, fmt.Errorf("failed to scan rows: %s -> %s", request.Query, err)
-		}
-		for _, col := range columns {
-			response.Columns = append(response.Columns, v1.QueryColumn{
-				Name: col,
-			})
-		}
 	}
+
+	columns, err := rows.Columns()
+	if err != nil {
+		logger.Errorf("failed to get column details: %v", err)
+	}
+	rows.Next()
+	if err := results.ScanRows(rows, &response.Results); err != nil {
+		return nil, fmt.Errorf("failed to scan rows: %s -> %s", request.Query, err)
+	}
+	for _, col := range columns {
+		response.Columns = append(response.Columns, v1.QueryColumn{
+			Name: col,
+		})
+	}
+
 	response.Count = len(response.Results)
 	return &response, nil
 }

db/ulid/ulid.go

@@ -10,8 +10,10 @@ import (
 	v2 "github.com/oklog/ulid/v2"
 )
 
+// ULID ...
 type ULID [16]byte
 
+// AsUUID ...
 func (u ULID) AsUUID() string {
 	return uuid.UUID(u).String()
 }
@@ -22,13 +24,15 @@ var pool = sync.Pool{
 	},
 }
 
+// New ...
 func New() (ULID, error) {
 	entropy := pool.Get()
 	result, err := v2.New(v2.Timestamp(time.Now()), entropy.(io.Reader))
 	pool.Put(entropy)
 	return ULID(result), err
 }
 
+// MustNew ...
 func MustNew() ULID {
 	entropy := pool.Get()
 	defer pool.Put(entropy)