Merge pull request #63 from flanksource/moshloop2

Improve AWS support

.github/workflows/release.yml

@@ -101,3 +101,28 @@ jobs:
           branch: gh-pages
           repository: ./charts
 
+  update-incident-commander:
+    runs-on: ubuntu-latest
+    needs: [helm, semantic-release]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: "${{ github.repository_owner }}/incident-commander"
+          token: ${{ secrets.FLANKBOT }}
+          path: ./incident-commander
+      - name: Install yq
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\
+          tar xz && sudo mv ${BINARY} /usr/bin/yq
+        env:
+          VERSION: v4.25.1
+          BINARY: yq_linux_amd64
+      - name: Update config-db version in Incident-commander
+        run: |
+          cd incident-commander
+          yq eval-all -i  '(.dependencies[] | select(.name == "config-db")) ref $d | $d.version = "${{ needs.semantic-release.outputs.release-version }}"' chart/Chart.yaml
+      - name: Push changes to chart repo
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "chore: update config-db chart dependency to ${{ needs.semantic-release.outputs.release-version }}"
+          repository: ./incident-commander

api/v1/aws.go

@@ -1,11 +1,49 @@
 package v1
 
+import (
+	"strings"
+	"time"
+)
+
 // AWS ...
 type AWS struct {
 	*AWSConnection
-	PatchStates         bool `json:"patch_states,omitempty"`
-	PatchDetails        bool `json:"patch_details,omitempty"`
-	Inventory           bool `json:"inventory,omitempty"`
-	Compliance          bool `json:"compliance,omitempty"`
-	TrustedAdvisorCheck bool `json:"trusted_advisor_check,omitempty"`
+	PatchStates         bool       `json:"patch_states,omitempty"`
+	PatchDetails        bool       `json:"patch_details,omitempty"`
+	Inventory           bool       `json:"inventory,omitempty"`
+	Compliance          bool       `json:"compliance,omitempty"`
+	CloudTrail          CloudTrail `json:"cloudtrail,omitempty"`
+	TrustedAdvisorCheck bool       `json:"trusted_advisor_check,omitempty"`
+	Include             []string   `json:"include,omitempty"`
+	Exclude             []string   `json:"exclude,omitempty"`
+	BaseScraper         `json:",inline"`
+}
+
+type CloudTrail struct {
+	Exclude []string       `json:"exclude,omitempty"`
+	MaxAge  *time.Duration `json:"max_age,omitempty"`
+}
+
+func (aws AWS) Includes(resource string) bool {
+	if len(aws.Include) == 0 {
+		return true
+	}
+	for _, include := range aws.Include {
+		if strings.EqualFold(include, resource) {
+			return true
+		}
+	}
+	return false
+}
+
+func (aws AWS) Excludes(resource string) bool {
+	if len(aws.Exclude) == 0 {
+		return false
+	}
+	for _, exclude := range aws.Exclude {
+		if strings.EqualFold(exclude, resource) {
+			return true
+		}
+	}
+	return false
 }

api/v1/common.go

@@ -6,6 +6,30 @@ import (
 	"github.com/flanksource/kommons"
 )
 
+type Filter struct {
+	JSONPath string `json:"jsonpath,omitempty"`
+}
+
+type Transform struct {
+	Include []Filter `json:"include,omitempty"`
+	// Fields to remove from the config, useful for removing sensitive data and fields
+	// that change often without a material impact i.e. Last Scraped Time
+	Exclude []Filter `json:"exclude,omitempty"`
+}
+
+type BaseScraper struct {
+	// A static value or JSONPath expression to use as the ID for the resource.
+	ID string `json:"id,omitempty"`
+	// A static value or JSONPath expression to use as the ID for the resource.
+	Name string `json:"name,omitempty"`
+	// A JSONPath expression to use to extract individual items from the resource,
+	// items are extracted first and then the ID,Name,Type and transformations are applied for each item.
+	Items string `json:"items,omitempty"`
+	// A static value or JSONPath expression to use as the type for the resource.
+	Type      string    `json:"type,omitempty"`
+	Transform Transform `json:"transform,omitempty"`
+}
+
 // Authentication ...
 type Authentication struct {
 	Username kommons.EnvVar `yaml:"username" json:"username"`
@@ -40,7 +64,7 @@ func (auth Authentication) GetDomain() string {
 type AWSConnection struct {
 	AccessKey     kommons.EnvVar `yaml:"accessKey,omitempty" json:"accessKey,omitempty"`
 	SecretKey     kommons.EnvVar `yaml:"secretKey,omitempty" json:"secretKey,omitempty"`
-	Region        string         `yaml:"region,omitempty" json:"region"`
+	Region        []string       `yaml:"region,omitempty" json:"region"`
 	Endpoint      string         `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
 	SkipTLSVerify bool           `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
 	AssumeRole    string         `yaml:"assumeRole,omitempty" json:"assumeRole,omitempty"`
@@ -51,3 +75,11 @@ type GCPConnection struct {
 	Endpoint    string          `yaml:"endpoint" json:"endpoint,omitempty"`
 	Credentials *kommons.EnvVar `yaml:"credentials" json:"credentials,omitempty"`
 }
+
+type Template struct {
+	Template   string `yaml:"template,omitempty" json:"template,omitempty"`
+	JSONPath   string `yaml:"jsonPath,omitempty" json:"jsonPath,omitempty"`
+	GSONPath   string `yaml:"gsonPath,omitempty" json:"gsonPath,omitempty"`
+	Expression string `yaml:"expr,omitempty" json:"expr,omitempty"`
+	Javascript string `yaml:"javascript,omitempty" json:"javascript,omitempty"`
+}

api/v1/file.go

@@ -1,9 +1,25 @@
 package v1
 
+import "net/url"
+
 // File ...
 type File struct {
-	ID    string   `json:"id,omitempty"`
-	Type  string   `json:"type,omitempty"`
-	URL   string   `json:"url,omitempty"`
-	Paths []string `json:"paths,omitempty"`
+	BaseScraper `json:",inline"`
+	URL         string   `json:"url,omitempty"`
+	Paths       []string `json:"paths,omitempty"`
+	Ignore      []string `json:"ignore,omitempty"`
+}
+
+func (f File) RedactedString() string {
+	if f.URL == "" {
+		return f.URL
+	}
+
+	url, err := url.Parse(f.URL)
+	if err != nil {
+		return f.URL
+	}
+
+	return url.Redacted()
+
 }

api/v1/interface.go

@@ -12,42 +12,142 @@ import (
 
 // Scraper ...
 type Scraper interface {
-	Scrape(ctx ScrapeContext, config ConfigScraper, manager Manager) []ScrapeResult
+	Scrape(ctx ScrapeContext, config ConfigScraper, manager Manager) ScrapeResults
 }
 
 // Analyzer ...
 type Analyzer func(configs []ScrapeResult) AnalysisResult
 
 // AnalysisResult ...
 type AnalysisResult struct {
-	Analyzer string
-	Messages []string
+	ExternalID    string
+	ExternalType  string
+	Summary       string
+	Analysis      map[string]string
+	AnalysisType  string
+	Status        string
+	Severity      string
+	FirstObserved *time.Time
+	LastObserved  *time.Time
+	Analyzer      string
+	Messages      []string
+	Error         error
 }
 
+type ChangeResult struct {
+	ExternalID   string
+	ExternalType string
+	ChangeType   string
+	Patches      string
+	Summary      string
+	Severity     string
+	Source       string
+	CreatedAt    *time.Time
+	Details      map[string]string
+}
+
+func (result AnalysisResult) String() string {
+	return fmt.Sprintf("%s: %s", result.Analyzer, result.Messages)
+}
+func (result *AnalysisResult) Message(msg string) *AnalysisResult {
+	if msg == "" {
+		return result
+	}
+	result.Messages = append(result.Messages, msg)
+	return result
+}
+
+type AnalysisResults []AnalysisResult
+
 // Manager ...
 type Manager struct {
 	Finder fs.Finder
 }
 
+type ScrapeResults []ScrapeResult
+
+func (s *ScrapeResults) AddChange(change ChangeResult) *ScrapeResults {
+	*s = append(*s, ScrapeResult{
+		ChangeResult: &change,
+	})
+	return s
+}
+
+func (s *ScrapeResults) Analysis(analyzer string, externalType string, id string) *AnalysisResult {
+	result := AnalysisResult{
+		Analyzer:     analyzer,
+		ExternalType: externalType,
+		ExternalID:   id,
+	}
+	*s = append(*s, ScrapeResult{
+		AnalysisResult: &result,
+	})
+	return &result
+}
+
+func (s *ScrapeResults) Errorf(e error, msg string, args ...interface{}) ScrapeResults {
+	logger.Errorf(msg, args...)
+	*s = append(*s, ScrapeResult{Error: e})
+	return *s
+}
+
 // ScrapeResult ...
 type ScrapeResult struct {
-	LastModified time.Time     `json:"last_modified,omitempty"`
-	Type         string        `json:"type,omitempty"`
-	Account      string        `json:"account,omitempty"`
-	Network      string        `json:"network,omitempty"`
-	Subnet       string        `json:"subnet,omitempty"`
-	Region       string        `json:"region,omitempty"`
-	Zone         string        `json:"zone,omitempty"`
-	Name         string        `json:"name,omitempty"`
-	Namespace    string        `json:"namespace,omitempty"`
-	ID           string        `json:"id,omitempty"`
-	Source       string        `json:"source,omitempty"`
-	Config       interface{}   `json:"config,omitempty"`
-	Tags         JSONStringMap `json:"tags,omitempty"`
+	LastModified   time.Time       `json:"last_modified,omitempty"`
+	Type           string          `json:"type,omitempty"`
+	ExternalType   string          `json:"external_type,omitempty"`
+	Account        string          `json:"account,omitempty"`
+	Network        string          `json:"network,omitempty"`
+	Subnet         string          `json:"subnet,omitempty"`
+	Region         string          `json:"region,omitempty"`
+	Zone           string          `json:"zone,omitempty"`
+	Name           string          `json:"name,omitempty"`
+	Namespace      string          `json:"namespace,omitempty"`
+	ID             string          `json:"id,omitempty"`
+	Aliases        []string        `json:"aliases,omitempty"`
+	Source         string          `json:"source,omitempty"`
+	Config         interface{}     `json:"config,omitempty"`
+	Tags           JSONStringMap   `json:"tags,omitempty"`
+	BaseScraper    BaseScraper     `json:"-"`
+	Error          error           `json:"-"`
+	AnalysisResult *AnalysisResult `json:"analysis,omitempty"`
+	ChangeResult   *ChangeResult   `json:"change,omitempty"`
+}
+
+func (s ScrapeResult) Success(config interface{}) ScrapeResult {
+	s.Config = config
+	return s
+}
+
+func (s ScrapeResult) Errorf(msg string, args ...interface{}) ScrapeResult {
+	s.Error = fmt.Errorf(msg, args...)
+	return s
+}
+
+func (s ScrapeResult) Clone(config interface{}) ScrapeResult {
+	clone := ScrapeResult{
+		LastModified: s.LastModified,
+		Aliases:      s.Aliases,
+		Type:         s.Type,
+		Account:      s.Account,
+		Network:      s.Network,
+		Subnet:       s.Subnet,
+		Region:       s.Region,
+		Zone:         s.Zone,
+		Name:         s.Name,
+		Namespace:    s.Namespace,
+		ID:           s.ID,
+		Source:       s.Source,
+		Config:       config,
+		Tags:         s.Tags,
+		BaseScraper:  s.BaseScraper,
+		Error:        s.Error,
+	}
+	return clone
 }
 
 func (s ScrapeResult) String() string {
-	return fmt.Sprintf("%s/%s", s.Type, s.ID)
+	return fmt.Sprintf("%s/%s (%s)", s.Type, s.Name, s.ID)
 }
 
 // QueryColumn ...

cmd/root.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/confighub/db"
-	"github.com/flanksource/confighub/kube"
+	"github.com/flanksource/confighub/utils/kube"
 	"github.com/flanksource/kommons"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"