Skip to content

Commit

Permalink
Merge pull request #639 from overmindtech/source_metadata
Browse files Browse the repository at this point in the history 
(feat) sources return metadata now
  • Loading branch information
@tphoney
tphoney authored Oct 11, 2024
2 parents 66e3ef3 + 6fabc4d commit 50dd833
Show file tree
Hide file tree
Showing 364 changed files with 8,933 additions and 6,025 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
fi
- name: Vet
run: go vet
run: go vet ./...

# get .golangci.yml from github.com/overmindtech/golangci-lint_config
- name: Get .golangci.yml from github.com/overmindtech/golangci-lint_configs
Expand Down
67 changes: 36 additions & 31 deletions sources/always_get_source.go → adapters/always_get_source.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package sources
package adapters

import (
"context"
Expand All @@ -25,18 +25,19 @@ func (m MaxParallel) Value() int {
return int(m)
}

// AlwaysGetSource This source is designed for AWS APIs that have separate List
// AlwaysGetAdapter This adapter is designed for AWS APIs that have separate List
// and Get functions. It also assumes that the results of the list function
// cannot be converted directly into items as they do not contain enough
// information, and therefore they always need to be passed to the Get function
// before returning. An example is the `ListClusters` API in EKS which returns a
// list of cluster names.
type AlwaysGetSource[ListInput InputType, ListOutput OutputType, GetInput InputType, GetOutput OutputType, ClientStruct ClientStructType, Options OptionsType] struct {
ItemType string // The type of items to return
Client ClientStruct // The AWS API client
AccountID string // The AWS account ID
Region string // The AWS region this is related to
MaxParallel MaxParallel // How many Get request to run in parallel for a single List request
type AlwaysGetAdapter[ListInput InputType, ListOutput OutputType, GetInput InputType, GetOutput OutputType, ClientStruct ClientStructType, Options OptionsType] struct {
ItemType string // The type of items to return
Client ClientStruct // The AWS API client
AccountID string // The AWS account ID
Region string // The AWS region this is related to
MaxParallel MaxParallel // How many Get request to run in parallel for a single List request
AdapterMetadata sdp.AdapterMetadata

// Disables List(), meaning all calls will return empty results. This does
// not affect Search()
Expand All @@ -60,12 +61,12 @@ type AlwaysGetSource[ListInput InputType, ListOutput OutputType, GetInput InputT

// Maps search terms from an SDP Search request into the relevant input for
// the ListFunc. If this is not set, Search() will handle ARNs like most AWS
// sources. Note that this and `SearchGetInputMapper` are mutually exclusive
// adapters. Note that this and `SearchGetInputMapper` are mutually exclusive
SearchInputMapper func(scope, query string) (ListInput, error)

// Maps search terms from an SDP Search request into the relevant input for
// the GetFunc. If this is not set, Search() will handle ARNs like most AWS
// sources. Note that this and `SearchInputMapper` are mutually exclusive
// adapters. Note that this and `SearchInputMapper` are mutually exclusive
SearchGetInputMapper func(scope, query string) (GetInput, error)

// A function that returns a paginator for the ListFunc
Expand All @@ -77,19 +78,19 @@ type AlwaysGetSource[ListInput InputType, ListOutput OutputType, GetInput InputT
ListFuncOutputMapper func(output ListOutput, input ListInput) ([]GetInput, error)

CacheDuration time.Duration // How long to cache items for
cache *sdpcache.Cache // The sdpcache of this source
cache *sdpcache.Cache // The sdpcache of this adapter
cacheInitMu sync.Mutex // Mutex to ensure cache is only initialised once
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) cacheDuration() time.Duration {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) cacheDuration() time.Duration {
if s.CacheDuration == 0 {
return DefaultCacheDuration
}

return s.CacheDuration
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) ensureCache() {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) ensureCache() {
s.cacheInitMu.Lock()
defer s.cacheInitMu.Unlock()

Expand All @@ -98,13 +99,13 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
}
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Cache() *sdpcache.Cache {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Cache() *sdpcache.Cache {
s.ensureCache()
return s.cache
}

// Validate Checks that the source has been set up correctly
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Validate() error {
// Validate Checks that the adapter has been set up correctly
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Validate() error {
if !s.DisableList {
if s.ListFuncPaginatorBuilder == nil {
return errors.New("ListFuncPaginatorBuilder is nil")
Expand All @@ -130,27 +131,31 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
return nil
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Type() string {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Type() string {
return s.ItemType
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Name() string {
return fmt.Sprintf("%v-source", s.ItemType)
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Name() string {
return fmt.Sprintf("%v-adapter", s.ItemType)
}

// List of scopes that this source is capable of find items for. This will be
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Metadata() *sdp.AdapterMetadata {
return &s.AdapterMetadata
}

// List of scopes that this adapter is capable of find items for. This will be
// in the format {accountID}.{region}
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Scopes() []string {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Scopes() []string {
return []string{
FormatScope(s.AccountID, s.Region),
}
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Get(ctx context.Context, scope string, query string, ignoreCache bool) (*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Get(ctx context.Context, scope string, query string, ignoreCache bool) (*sdp.Item, error) {
if scope != s.Scopes()[0] {
return nil, &sdp.QueryError{
ErrorType: sdp.QueryError_NOSCOPE,
ErrorString: fmt.Sprintf("requested scope %v does not match source scope %v", scope, s.Scopes()[0]),
ErrorString: fmt.Sprintf("requested scope %v does not match adapter scope %v", scope, s.Scopes()[0]),
}
}

Expand Down Expand Up @@ -192,11 +197,11 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc

// List Lists all available items. This is done by running the ListFunc, then
// passing these results to GetFunc in order to get the details
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) List(ctx context.Context, scope string, ignoreCache bool) ([]*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) List(ctx context.Context, scope string, ignoreCache bool) ([]*sdp.Item, error) {
if scope != s.Scopes()[0] {
return nil, &sdp.QueryError{
ErrorType: sdp.QueryError_NOSCOPE,
ErrorString: fmt.Sprintf("requested scope %v does not match source scope %v", scope, s.Scopes()[0]),
ErrorString: fmt.Sprintf("requested scope %v does not match adapter scope %v", scope, s.Scopes()[0]),
}
}

Expand Down Expand Up @@ -232,7 +237,7 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
}

// listInternal Accepts a ListInput and runs the List logic against it
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) listInternal(ctx context.Context, scope string, input ListInput) ([]*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) listInternal(ctx context.Context, scope string, input ListInput) ([]*sdp.Item, error) {
var output ListOutput
var err error
items := make([]*sdp.Item, 0)
Expand Down Expand Up @@ -327,11 +332,11 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
}

// Search Searches for AWS resources by ARN
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Search(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Search(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
if scope != s.Scopes()[0] {
return nil, &sdp.QueryError{
ErrorType: sdp.QueryError_NOSCOPE,
ErrorString: fmt.Sprintf("requested scope %v does not match source scope %v", scope, s.Scopes()[0]),
ErrorString: fmt.Sprintf("requested scope %v does not match adapter scope %v", scope, s.Scopes()[0]),
}
}

Expand Down Expand Up @@ -362,7 +367,7 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc

// SearchCustom Searches using custom mapping logic. The SearchInputMapper is
// used to create an input for ListFunc, at which point the usual logic is used
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) SearchCustom(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) SearchCustom(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
s.ensureCache()
cacheHit, ck, cachedItems, qErr := s.cache.Lookup(ctx, s.Name(), sdp.QueryMethod_SEARCH, scope, s.ItemType, query, ignoreCache)
if qErr != nil {
Expand Down Expand Up @@ -420,7 +425,7 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
return items, nil
}

func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) SearchARN(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) SearchARN(ctx context.Context, scope string, query string, ignoreCache bool) ([]*sdp.Item, error) {
// Parse the ARN
a, err := ParseARN(query)

Expand Down Expand Up @@ -448,6 +453,6 @@ func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruc
// This is used to resolve conflicts where two sources of the same type
// return an item for a GET request. In this instance only one item can be
// seen on, so the one with the higher weight value will win.
func (s *AlwaysGetSource[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Weight() int {
func (s *AlwaysGetAdapter[ListInput, ListOutput, GetInput, GetOutput, ClientStruct, Options]) Weight() int {
return 100
}
32 changes: 16 additions & 16 deletions sources/always_get_source_test.go → adapters/always_get_source_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package sources
package adapters

import (
"context"
Expand All @@ -19,7 +19,7 @@ func TestMaxParallel(t *testing.T) {
}

func TestAlwaysGetSourceType(t *testing.T) {
lgs := AlwaysGetSource[any, any, any, any, any, any]{
lgs := AlwaysGetAdapter[any, any, any, any, any, any]{
ItemType: "foo",
}

Expand All @@ -29,17 +29,17 @@ func TestAlwaysGetSourceType(t *testing.T) {
}

func TestAlwaysGetSourceName(t *testing.T) {
lgs := AlwaysGetSource[any, any, any, any, any, any]{
lgs := AlwaysGetAdapter[any, any, any, any, any, any]{
ItemType: "foo",
}

if lgs.Name() != "foo-source" {
t.Errorf("expected name to be foo-source, got %v", lgs.Name())
if lgs.Name() != "foo-adapter" {
t.Errorf("expected name to be foo-adapter, got %v", lgs.Name())
}
}

func TestAlwaysGetSourceScopes(t *testing.T) {
lgs := AlwaysGetSource[any, any, any, any, any, any]{
lgs := AlwaysGetAdapter[any, any, any, any, any, any]{
AccountID: "foo",
Region: "bar",
}
Expand All @@ -51,7 +51,7 @@ func TestAlwaysGetSourceScopes(t *testing.T) {

func TestAlwaysGetSourceGet(t *testing.T) {
t.Run("with no errors", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -83,7 +83,7 @@ func TestAlwaysGetSourceGet(t *testing.T) {
})

t.Run("with an error", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -117,7 +117,7 @@ func TestAlwaysGetSourceGet(t *testing.T) {

func TestAlwaysGetSourceList(t *testing.T) {
t.Run("with no errors", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -154,7 +154,7 @@ func TestAlwaysGetSourceList(t *testing.T) {
})

t.Run("with a failing output mapper", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -196,7 +196,7 @@ func TestAlwaysGetSourceList(t *testing.T) {
})

t.Run("with a failing GetFunc", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -236,7 +236,7 @@ func TestAlwaysGetSourceList(t *testing.T) {

func TestAlwaysGetSourceSearch(t *testing.T) {
t.Run("with ARN search", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -291,7 +291,7 @@ func TestAlwaysGetSourceSearch(t *testing.T) {
})

t.Run("with Custom & ARN search", func(t *testing.T) {
lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -352,7 +352,7 @@ func TestAlwaysGetSourceSearch(t *testing.T) {
t.Run("with custom search logic", func(t *testing.T) {
var searchMapperCalled bool

lgs := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
lgs := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -392,7 +392,7 @@ func TestAlwaysGetSourceSearch(t *testing.T) {
})

t.Run("with SearchGetInputMapper", func(t *testing.T) {
ags := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
ags := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "bar",
Expand Down Expand Up @@ -440,7 +440,7 @@ func TestAlwaysGetSourceSearch(t *testing.T) {
func TestAlwaysGetSourceCaching(t *testing.T) {
ctx := context.Background()
generation := 0
s := AlwaysGetSource[string, string, string, string, struct{}, struct{}]{
s := AlwaysGetAdapter[string, string, string, string, struct{}, struct{}]{
ItemType: "test",
AccountID: "foo",
Region: "eu-west-2",
Expand Down
34 changes: 26 additions & 8 deletions sources/apigateway/resource.go → adapters/apigateway/resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

"github.com/aws/aws-sdk-go-v2/service/apigateway"
"github.com/aws/aws-sdk-go-v2/service/apigateway/types"
"github.com/overmindtech/aws-source/sources"
"github.com/overmindtech/aws-source/adapters"
"github.com/overmindtech/sdp-go"
)

Expand Down Expand Up @@ -38,7 +38,7 @@ func resourceOutputMapper(query, scope string, awsItem *types.Resource) (*sdp.It
}
}

attributes, err := sources.ToAttributesWithExclude(awsItem, "tags")
attributes, err := adapters.ToAttributesWithExclude(awsItem, "tags")
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -66,12 +66,13 @@ func resourceOutputMapper(query, scope string, awsItem *types.Resource) (*sdp.It
// +overmind:group AWS
// +overmind:terraform:queryMap aws_api_gateway_resource.id

func NewResourceSource(client *apigateway.Client, accountID string, region string) *sources.GetListSource[*types.Resource, *apigateway.Client, *apigateway.Options] {
return &sources.GetListSource[*types.Resource, *apigateway.Client, *apigateway.Options]{
ItemType: "apigateway-resource",
Client: client,
AccountID: accountID,
Region: region,
func NewResourceAdapter(client *apigateway.Client, accountID string, region string) *adapters.GetListAdapter[*types.Resource, *apigateway.Client, *apigateway.Options] {
return &adapters.GetListAdapter[*types.Resource, *apigateway.Client, *apigateway.Options]{
ItemType: "apigateway-resource",
Client: client,
AccountID: accountID,
Region: region,
AdapterMetadata: APIGatewayMetadata(),
GetFunc: func(ctx context.Context, client *apigateway.Client, scope, query string) (*types.Resource, error) {
f := strings.Split(query, "/")
if len(f) != 2 {
Expand Down Expand Up @@ -112,3 +113,20 @@ func NewResourceSource(client *apigateway.Client, accountID string, region strin
},
}
}

func APIGatewayMetadata() sdp.AdapterMetadata {
return sdp.AdapterMetadata{
Type: "apigateway-resource",
DescriptiveName: "API Gateway",
Category: sdp.AdapterCategory_ADAPTER_CATEGORY_COMPUTE_APPLICATION,
SupportedQueryMethods: &sdp.AdapterSupportedQueryMethods{
Get: true,
Search: true,
GetDescription: "Get a Resource by rest-api-id/resource-id",
SearchDescription: "Search Resources by REST API ID",
},
TerraformMappings: []*sdp.TerraformMapping{
{TerraformQueryMap: "aws_api_gateway_resource.id"},
},
}
}
Loading

0 comments on commit 50dd833

Please sign in to comment.