Skip to content

Commit

Permalink
add support for defining TTL for image credentials cache
Browse files Browse the repository at this point in the history
  • Loading branch information
dejanzele committed Jul 12, 2024
1 parent 3b1a4b9 commit eb231cd
Show file tree
Hide file tree
Showing 12 changed files with 393 additions and 49 deletions.
3 changes: 2 additions & 1 deletion cmd/api-server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ func main() {
inspector := imageinspector.NewInspector(
cfg.TestkubeRegistry,
imageinspector.NewSkopeoFetcher(),
imageinspector.NewSecretFetcher(secretClient),
imageinspector.NewSecretFetcher(secretClient, imageinspector.WithSecretCacheTTL(cfg.TestkubeImageCredentialsCacheTTL)),
inspectorStorages...,
)

Expand Down Expand Up @@ -515,6 +515,7 @@ func main() {
features,
cfg.TestkubeDefaultStorageClassName,
cfg.WhitelistedContainers,
cfg.TestkubeImageCredentialsCacheTTL,
)
if err != nil {
exitOnError("Creating container executor", err)
Expand Down
1 change: 1 addition & 0 deletions cmd/tcl/testworkflow-toolkit/spawn/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ func CreateBaseMachine() expressions.Machine {
"images.toolkit": env.Config().Images.Toolkit,
"images.persistence.enabled": strconv.FormatBool(env.Config().Images.InspectorPersistenceEnabled),
"images.persistence.key": env.Config().Images.InspectorPersistenceCacheKey,
"images.cache.ttl": env.Config().Images.ImageCredentialsCacheTTL.String(),
}),
)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/testworkflow-toolkit/env/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func ImageInspector() imageinspector.Inspector {
return imageinspector.NewInspector(
Config().System.DefaultRegistry,
imageinspector.NewSkopeoFetcher(),
imageinspector.NewSecretFetcher(secretClient),
imageinspector.NewSecretFetcher(secretClient, imageinspector.WithSecretCacheTTL(Config().Images.ImageCredentialsCacheTTL)),
inspectorStorages...,
)
}
Expand Down
9 changes: 5 additions & 4 deletions cmd/testworkflow-toolkit/env/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ type envSystemConfig struct {
}

type envImagesConfig struct {
Init string `envconfig:"TESTKUBE_TW_INIT_IMAGE"`
Toolkit string `envconfig:"TESTKUBE_TW_TOOLKIT_IMAGE"`
InspectorPersistenceEnabled bool `envconfig:"TK_IMG_P" default:"false"`
InspectorPersistenceCacheKey string `envconfig:"TK_IMG_PK"`
Init string `envconfig:"TESTKUBE_TW_INIT_IMAGE"`
Toolkit string `envconfig:"TESTKUBE_TW_TOOLKIT_IMAGE"`
InspectorPersistenceEnabled bool `envconfig:"TK_IMG_P" default:"false"`
InspectorPersistenceCacheKey string `envconfig:"TK_IMG_PK"`
ImageCredentialsCacheTTL time.Duration `envconfig:"TK_IMG_CRED_TTL"`
}

type featuresConfig struct {
Expand Down
58 changes: 30 additions & 28 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,36 +82,38 @@ type Config struct {
TestkubeProTLSSecret string `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""`
TestkubeProRunnerCustomCASecret string `envconfig:"TESTKUBE_PRO_RUNNER_CUSTOM_CA_SECRET" default:""`
TestkubeWatcherNamespaces string `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""`
GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""`
TestkubePodStartTimeout time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"`
CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""`
TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"`
TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"`
TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
EnableListingAllSecrets bool `envconfig:"ENABLE_LISTING_ALL_SECRETS" default:"false"`
SecretCreationPrefix string `envconfig:"SECRET_CREATION_PREFIX" default:"testkube-"`
DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
Debug bool `envconfig:"DEBUG" default:"false"`
Trace bool `envconfig:"TRACE" default:"false"`
EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"`
LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""`
LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""`
LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""`
DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
TestkubeDefaultStorageClassName string `envconfig:"TESTKUBE_DEFAULT_STORAGE_CLASS_NAME" default:""`
GlobalWorkflowTemplateName string `envconfig:"TESTKUBE_GLOBAL_WORKFLOW_TEMPLATE_NAME" default:""`
EnableK8sEvents bool `envconfig:"ENABLE_K8S_EVENTS" default:"true"`
// TestkubeImageCredentialsCacheTTL is the duration for which the image pull credentials should be cached.
TestkubeImageCredentialsCacheTTL time.Duration `envconfig:"TESTKUBE_IMAGE_CREDENTIALS_CACHE_TTL" default:"30m"`
GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""`
TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
DisableReconciler bool `envconfig:"DISABLE_RECONCILER" default:"false"`
TestkubeClusterName string `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
CompressArtifacts bool `envconfig:"COMPRESSARTIFACTS" default:"false"`
TestkubeHelmchartVersion string `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
DebugListenAddr string `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
EnableDebugServer bool `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
EnableSecretsEndpoint bool `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
EnableListingAllSecrets bool `envconfig:"ENABLE_LISTING_ALL_SECRETS" default:"false"`
SecretCreationPrefix string `envconfig:"SECRET_CREATION_PREFIX" default:"testkube-"`
DisableMongoMigrations bool `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
Debug bool `envconfig:"DEBUG" default:"false"`
Trace bool `envconfig:"TRACE" default:"false"`
EnableImageDataPersistentCache bool `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
ImageDataPersistentCacheKey string `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
LogServerGrpcAddress string `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
LogServerSecure bool `envconfig:"LOG_SERVER_SECURE" default:"false"`
LogServerSkipVerify bool `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
LogServerCertFile string `envconfig:"LOG_SERVER_CERT_FILE" default:""`
LogServerKeyFile string `envconfig:"LOG_SERVER_KEY_FILE" default:""`
LogServerCAFile string `envconfig:"LOG_SERVER_CA_FILE" default:""`
DisableSecretCreation bool `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
TestkubeExecutionNamespaces string `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
TestkubeDefaultStorageClassName string `envconfig:"TESTKUBE_DEFAULT_STORAGE_CLASS_NAME" default:""`
GlobalWorkflowTemplateName string `envconfig:"TESTKUBE_GLOBAL_WORKFLOW_TEMPLATE_NAME" default:""`
EnableK8sEvents bool `envconfig:"ENABLE_K8S_EVENTS" default:"true"`

// DEPRECATED: Use TestkubeProAPIKey instead
TestkubeCloudAPIKey string `envconfig:"TESTKUBE_CLOUD_API_KEY" default:""`
Expand Down
30 changes: 30 additions & 0 deletions pkg/cache/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package cache

import (
"context"
"time"

"github.com/pkg/errors"
)

var (
ErrStaleCache = errors.New("item's ttl has expired")
ErrNotFound = errors.New("item not found")
)

type Option func(*Cache)

type Cache interface {
// Get retrieves the cached value for the given key.
// If the key is not found, it returns ErrNotFound or ErrStaleCache if the item's ttl has expired.
Get(ctx context.Context, key string) (any, error)
// Set stores the value in the cache with the given key.
// If ttl is 0, the item will be cached indefinitely.
Set(ctx context.Context, key string, value any, ttl time.Duration) error
}

// IsCacheMiss returns true if the error is a cache miss error.
// This is a helper function to determine so users don't have to compare errors manually.
func IsCacheMiss(err error) bool {
return errors.Is(err, ErrNotFound) || errors.Is(err, ErrStaleCache)
}
64 changes: 64 additions & 0 deletions pkg/cache/inmem.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package cache

import (
"context"
"github.com/pkg/errors"
"sync"
"time"
)

type item struct {
value any
expiresAt *time.Time
}

// timeGetter is a function that returns the current time.
type timeGetter func() time.Time

type InMemoryCache struct {
cache sync.Map
timeGetter timeGetter
}

// NewInMemoryCache creates a new in-memory cache.
// The underlying cache implementation uses a sync.Map so it is thread-safe.
func NewInMemoryCache() *InMemoryCache {
return &InMemoryCache{
timeGetter: time.Now,
}
}

func (c *InMemoryCache) Get(ctx context.Context, key string) (any, error) {
rawItem, ok := c.cache.Load(key)
if !ok {
return nil, ErrNotFound
}
i, ok := rawItem.(*item)
if !ok {
return nil, errors.New("unexpected item type found in cache")
}

if i.expiresAt != nil && i.expiresAt.Before(time.Now()) {
c.cache.Delete(key)
return nil, ErrStaleCache
}

return i.value, nil
}

func (c *InMemoryCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
if ttl < 0 {
return errors.New("ttl must be greater than 0")
}

i := &item{
value: value,
}
if ttl > 0 {
expiresAt := c.timeGetter().Add(ttl)
i.expiresAt = &expiresAt
}
c.cache.Store(key, i)

return nil
}
Loading

0 comments on commit eb231cd

Please sign in to comment.