Skip to content

Commit

Permalink
refactor: configurations (#1018)
Browse files Browse the repository at this point in the history
  • Loading branch information
@m8vago
m8vago authored Jan 6, 2025
1 parent 17a52f8 commit 50c0759
Show file tree
Hide file tree
Showing 218 changed files with 10,779 additions and 7,442 deletions.
83 changes: 49 additions & 34 deletions golang/internal/grpc/grpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type ClientLoop struct {

type (
DeployFunc func(context.Context, *dogger.DeploymentLogger, *v1.DeployImageRequest, *v1.VersionData) error
DeploySharedSecretsFunc func(context.Context, string, map[string]string) error
WatchContainerStatusFunc func(context.Context, string, bool) (*ContainerStatusStream, error)
DeleteFunc func(context.Context, string, string) error
SecretListFunc func(context.Context, string, string) ([]string, error)
Expand All @@ -94,6 +95,7 @@ type (

type WorkerFunctions struct {
Deploy DeployFunc
DeploySharedSecrets DeploySharedSecretsFunc
WatchContainerStatus WatchContainerStatusFunc
Delete DeleteFunc
SecretList SecretListFunc
Expand Down Expand Up @@ -186,7 +188,13 @@ func fetchCertificatesFromURL(ctx context.Context, addr string) (*x509.CertPool,
func (cl *ClientLoop) grpcProcessCommand(command *agent.AgentCommand) {
switch {
case command.GetDeploy() != nil:
go executeVersionDeployRequest(cl.Ctx, command.GetDeploy(), cl.WorkerFuncs.Deploy, cl.AppConfig)
go executeDeployRequest(
cl.Ctx,
command.GetDeploy(),
cl.WorkerFuncs.Deploy,
cl.WorkerFuncs.DeploySharedSecrets,
cl.AppConfig,
)
case command.GetContainerState() != nil:
go executeWatchContainerStatus(cl.Ctx, command.GetContainerState(), cl.WorkerFuncs.WatchContainerStatus)
case command.GetContainerDelete() != nil:
Expand Down Expand Up @@ -455,9 +463,9 @@ func (cl *ClientLoop) handleGrpcTokenError(err error, token *config.ValidJWT) {
}
}

func executeVersionDeployRequest(
ctx context.Context, req *agent.VersionDeployRequest,
deploy DeployFunc, appConfig *config.CommonConfiguration,
func executeDeployRequest(
ctx context.Context, req *agent.DeployRequest,
deploy DeployFunc, deploySecrets DeploySharedSecretsFunc, appConfig *config.CommonConfiguration,
) {
if deploy == nil {
log.Error().Msg("Deploy function not implemented")
Expand Down Expand Up @@ -486,10 +494,27 @@ func executeVersionDeployRequest(
return
}

failed := false
var deployStatus common.DeploymentStatus
deployStatus := common.DeploymentStatus_FAILED
defer func() {
dog.WriteDeploymentStatus(deployStatus)

err = statusStream.CloseSend()
if err != nil {
log.Error().Stack().Err(err).Str("deployment", req.Id).Msg("Status close error")
}
}()

if len(req.Secrets) > 0 {
dog.WriteInfo("Deploying secrets")
err = deploySecrets(ctx, req.Prefix, req.Secrets)
if err != nil {
dog.WriteError(err.Error())
return
}
}

for i := range req.Requests {
imageReq := mapper.MapDeployImage(req.Requests[i], appConfig)
imageReq := mapper.MapDeployImage(req.Prefix, req.Requests[i], appConfig)
dog.SetRequestID(imageReq.RequestID)

var versionData *v1.VersionData
Expand All @@ -498,24 +523,12 @@ func executeVersionDeployRequest(
}

if err = deploy(ctx, dog, imageReq, versionData); err != nil {
failed = true
dog.WriteError(err.Error())
return
}
}

if failed {
deployStatus = common.DeploymentStatus_FAILED
} else {
deployStatus = common.DeploymentStatus_SUCCESSFUL
}

dog.WriteDeploymentStatus(deployStatus)

err = statusStream.CloseSend()
if err != nil {
log.Error().Stack().Err(err).Str("deployment", req.Id).Msg("Status close error")
return
}
deployStatus = common.DeploymentStatus_SUCCESSFUL
}

func streamContainerStatus(
Expand Down Expand Up @@ -636,13 +649,10 @@ func executeDeleteMultipleContainers(
req *common.DeleteContainersRequest,
deleteFn DeleteContainersFunc,
) *AgentGrpcError {
var prefix, name string
if req.GetContainer() != nil {
prefix = req.GetContainer().Prefix
name = req.GetContainer().Name
} else {
prefix = req.GetPrefix()
name = ""
prefix, name, err := mapper.MapContainerOrPrefixToPrefixName(req.Target)
if err != nil {
log.Error().Err(err).Msg("Failed to delete multiple containers")
return agentError(ctx, err)
}

ctx = metadata.AppendToOutgoingContext(ctx, "dyo-container-prefix", prefix, "dyo-container-name", name)
Expand All @@ -653,7 +663,7 @@ func executeDeleteMultipleContainers(

log.Info().Msg("Deleting multiple containers")

err := deleteFn(ctx, req)
err = deleteFn(ctx, req)
if err != nil {
log.Error().Stack().Err(err).Msg("Failed to delete multiple containers")
return agentError(ctx, err)
Expand Down Expand Up @@ -741,8 +751,15 @@ func executeSecretList(
listFunc SecretListFunc,
appConfig *config.CommonConfiguration,
) *AgentGrpcError {
prefix := command.Container.Prefix
name := command.Container.Name
var prefix string
name := ""

if command.Target.GetContainer() != nil {
prefix = command.Target.GetContainer().Prefix
name = command.Target.GetContainer().Name
} else {
prefix = command.Target.GetPrefix()
}

ctx = metadata.AppendToOutgoingContext(ctx, "dyo-container-prefix", prefix, "dyo-container-name", name)

Expand All @@ -765,10 +782,8 @@ func executeSecretList(
}

resp := &common.ListSecretsResponse{
Prefix: prefix,
Name: name,
Target: command.Target,
PublicKey: publicKey,
HasKeys: keys != nil,
Keys: keys,
}

Expand Down
60 changes: 29 additions & 31 deletions golang/internal/mapper/grpc.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package mapper

import (
"errors"
"fmt"
"strings"
"time"
Expand All @@ -25,33 +26,18 @@ import (
corev1 "k8s.io/api/core/v1"
)

func mapInstanceConfig(in *agent.InstanceConfig) v1.InstanceConfig {
instanceConfig := v1.InstanceConfig{
ContainerPreName: in.Prefix,
Name: in.Prefix,
SharedEnvironment: map[string]string{},
}

if in.RepositoryPrefix != nil {
instanceConfig.RepositoryPreName = *in.RepositoryPrefix
}

if in.MountPath != nil {
instanceConfig.MountPath = *in.MountPath
}
var ErrNoTargetContainerOrPrefix = errors.New("no target container or prefix")

if in.Environment != nil {
instanceConfig.Environment = in.Environment
}

return instanceConfig
}

func MapDeployImage(req *agent.DeployRequest, appConfig *config.CommonConfiguration) *v1.DeployImageRequest {
func MapDeployImage(prefix string, req *agent.DeployWorkloadRequest, appConfig *config.CommonConfiguration) *v1.DeployImageRequest {
res := &v1.DeployImageRequest{
RequestID: req.Id,
InstanceConfig: mapInstanceConfig(req.InstanceConfig),
ContainerConfig: mapContainerConfig(req),
RequestID: req.Id,
InstanceConfig: v1.InstanceConfig{
UseSharedEnvs: false,
Environment: map[string]string{},
SharedEnvironment: map[string]string{},
ContainerPreName: prefix,
},
ContainerConfig: mapContainerConfig(prefix, req),
ImageName: req.ImageName,
Tag: req.Tag,
Registry: req.Registry,
Expand All @@ -68,22 +54,18 @@ func MapDeployImage(req *agent.DeployRequest, appConfig *config.CommonConfigurat

v1.SetDeploymentDefaults(res, appConfig)

if req.RuntimeConfig != nil {
res.RuntimeConfig = v1.Base64JSONBytes(*req.RuntimeConfig)
}

if req.Registry != nil {
res.Registry = req.Registry
}
return res
}

func mapContainerConfig(in *agent.DeployRequest) v1.ContainerConfig {
func mapContainerConfig(prefix string, in *agent.DeployWorkloadRequest) v1.ContainerConfig {
cc := in.Common

containerConfig := v1.ContainerConfig{
Container: cc.Name,
ContainerPreName: in.InstanceConfig.Prefix,
ContainerPreName: prefix,
Ports: MapPorts(cc.Ports),
PortRanges: mapPortRanges(cc.PortRanges),
Volumes: mapVolumes(cc.Volumes),
Expand Down Expand Up @@ -692,3 +674,19 @@ func MapDockerContainerEventToContainerState(event string) common.ContainerState
return common.ContainerState_CONTAINER_STATE_UNSPECIFIED
}
}

func MapContainerOrPrefixToPrefixName(target *common.ContainerOrPrefix) (prefix, name string, err error) {
if target == nil {
return "", "", ErrNoTargetContainerOrPrefix
}

if target.GetContainer() != nil {
return target.GetContainer().GetPrefix(), target.GetContainer().Name, nil
}

if target.GetPrefix() == "" {
return "", "", ErrNoTargetContainerOrPrefix
}

return target.GetPrefix(), "", nil
}
53 changes: 19 additions & 34 deletions golang/internal/mapper/grpc_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestMapDeployImageRequest(t *testing.T) {
req := testDeployRequest()
cfg := testAppConfig()

res := MapDeployImage(req, cfg)
res := MapDeployImage("", req, cfg)
expected := testExpectedCommon(req)

assert.Equal(t, expected, res)
Expand Down Expand Up @@ -52,12 +52,12 @@ func TestMapDeployImageRequestRestartPolicies(t *testing.T) {
for _, tC := range cases {
req.Dagent.RestartPolicy = tC.policy
expected.ContainerConfig.RestartPolicy = tC.dockerType
res := MapDeployImage(req, cfg)
res := MapDeployImage("", req, cfg)
assert.Equal(t, expected, res)
}
}

func testExpectedCommon(req *agent.DeployRequest) *v1.DeployImageRequest {
func testExpectedCommon(req *agent.DeployWorkloadRequest) *v1.DeployImageRequest {
return &v1.DeployImageRequest{
RequestID: "testID",
RegistryAuth: &image.RegistryAuth{
Expand All @@ -67,17 +67,13 @@ func testExpectedCommon(req *agent.DeployRequest) *v1.DeployImageRequest {
Password: "test-pass",
},
InstanceConfig: v1.InstanceConfig{
ContainerPreName: "test-prefix",
MountPath: "/path/to/mount",
Name: "test-prefix",
Environment: map[string]string{"Evn1": "Val1", "Env2": "Val2"},
Registry: "",
RepositoryPreName: "repo-prefix",
SharedEnvironment: map[string]string{},
UseSharedEnvs: false,
Environment: map[string]string{},
SharedEnvironment: map[string]string{},
ContainerPreName: "",
},
ContainerConfig: v1.ContainerConfig{
ContainerPreName: "test-prefix",
ContainerPreName: "",
Container: "test-common-config",
Ports: []builder.PortBinding{{ExposedPort: 0x4d2, PortBinding: pointer.ToUint16(0x1a85)}},
PortRanges: []builder.PortRangeBinding{{Internal: builder.PortRange{From: 0x0, To: 0x18}, External: builder.PortRange{From: 0x40, To: 0x80}}},
Expand Down Expand Up @@ -153,7 +149,7 @@ func testExpectedCommon(req *agent.DeployRequest) *v1.DeployImageRequest {
UseLoadBalancer: true,
ExtraLBAnnotations: map[string]string{"annotation1": "value1"},
},
RuntimeConfig: v1.Base64JSONBytes{0x6b, 0x65, 0x79, 0x31, 0x3d, 0x76, 0x61, 0x6c, 0x31, 0x2c, 0x6b, 0x65, 0x79, 0x32, 0x3d, 0x76, 0x61, 0x6c, 0x32}, // encoded string: a2V5MT12YWwxLGtleTI9dmFsMg==
RuntimeConfig: nil,
Registry: req.Registry,
ImageName: "test-image",
Tag: "test-tag",
Expand Down Expand Up @@ -210,24 +206,19 @@ func TestMapDockerContainerEventToContainerState(t *testing.T) {
assert.Equal(t, common.ContainerState_EXITED, MapDockerContainerEventToContainerState("die"))
}

func testDeployRequest() *agent.DeployRequest {
func testDeployRequest() *agent.DeployWorkloadRequest {
registry := "https://my-registry.com"
runtimeCfg := "key1=val1,key2=val2"
var uid int64 = 777
upLimit := "5Mi"
mntPath := "/path/to/mount"
repoPrefix := "repo-prefix"
strategy := common.ExposeStrategy_EXPOSE_WITH_TLS
b := true
return &agent.DeployRequest{
Id: "testID",
ContainerName: "test-container",
ImageName: "test-image",
Tag: "test-tag",
Registry: &registry,
RuntimeConfig: &runtimeCfg,
Dagent: testDagentConfig(),
Crane: testCraneConfig(),
return &agent.DeployWorkloadRequest{
Id: "testID",
ImageName: "test-image",
Tag: "test-tag",
Registry: &registry,
Dagent: testDagentConfig(),
Crane: testCraneConfig(),
Common: &agent.CommonContainerConfig{
Name: "test-common-config",
Commands: []string{"make", "test"},
Expand Down Expand Up @@ -274,12 +265,6 @@ func testDeployRequest() *agent.DeployRequest {
Password: "test-pass",
Url: "https://test-url.com",
},
InstanceConfig: &agent.InstanceConfig{
Prefix: "test-prefix",
MountPath: &mntPath,
RepositoryPrefix: &repoPrefix,
Environment: map[string]string{"Evn1": "Val1", "Env2": "Val2"},
},
}
}

Expand Down Expand Up @@ -411,7 +396,7 @@ func testAppConfig() *config.CommonConfiguration {
}
}

func testDeployRequestWithLogDriver(driverType common.DriverType) *agent.DeployRequest {
func testDeployRequestWithLogDriver(driverType common.DriverType) *agent.DeployWorkloadRequest {
request := testDeployRequest()
if driverType == common.DriverType_NODE_DEFAULT {
request.Dagent.LogConfig = nil
Expand All @@ -421,7 +406,7 @@ func testDeployRequestWithLogDriver(driverType common.DriverType) *agent.DeployR
return request
}

func testExpectedCommonWithLogConfigType(req *agent.DeployRequest, logConfigType string) *v1.DeployImageRequest {
func testExpectedCommonWithLogConfigType(req *agent.DeployWorkloadRequest, logConfigType string) *v1.DeployImageRequest {
expected := testExpectedCommon(req)
if req.Dagent.LogConfig == nil {
expected.ContainerConfig.LogConfig = nil
Expand Down Expand Up @@ -508,7 +493,7 @@ func TestMapDeployImageLogConfig(t *testing.T) {
req := testDeployRequestWithLogDriver(tC.driver)
cfg := testAppConfig()

res := MapDeployImage(req, cfg)
res := MapDeployImage("", req, cfg)
expected := testExpectedCommonWithLogConfigType(req, tC.want)

assert.Equal(t, expected, res)
Expand Down
1 change: 1 addition & 0 deletions golang/pkg/crane/crane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func Serve(cfg *config.Configuration, secretStore commonConfig.SecretStore) {
grpcContext := grpc.WithGRPCConfig(context.Background(), cfg)
grpc.Init(grpcContext, &cfg.CommonConfiguration, secretStore, &grpc.WorkerFunctions{
Deploy: k8s.Deploy,
DeploySharedSecrets: k8s.DeploySharedSecrets,
WatchContainerStatus: crux.WatchDeploymentsByPrefix,
Delete: k8s.Delete,
ContainerCommand: crux.DeploymentCommand,
Expand Down
Loading

0 comments on commit 50c0759

Please sign in to comment.