diff --git a/.mapping.json b/.mapping.json index 944b1fbc..37cd4878 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3064,7 +3064,6 @@ "pkg/abstract/coordinator/status_message.go":"transfer_manager/go/pkg/abstract/coordinator/status_message.go", "pkg/abstract/coordinator/transfer.go":"transfer_manager/go/pkg/abstract/coordinator/transfer.go", "pkg/abstract/coordinator/transfer_state.go":"transfer_manager/go/pkg/abstract/coordinator/transfer_state.go", - "pkg/abstract/dedicated_runtime.go":"transfer_manager/go/pkg/abstract/dedicated_runtime.go", "pkg/abstract/dterrors/error.go":"transfer_manager/go/pkg/abstract/dterrors/error.go", "pkg/abstract/dterrors/error_test.go":"transfer_manager/go/pkg/abstract/dterrors/error_test.go", "pkg/abstract/dterrors/errors_test_helper.go":"transfer_manager/go/pkg/abstract/dterrors/errors_test_helper.go", @@ -3078,8 +3077,6 @@ "pkg/abstract/gotest/canondata/result.json":"transfer_manager/go/pkg/abstract/gotest/canondata/result.json", "pkg/abstract/homo_valuer.go":"transfer_manager/go/pkg/abstract/homo_valuer.go", "pkg/abstract/includeable.go":"transfer_manager/go/pkg/abstract/includeable.go", - "pkg/abstract/k8s_runtime.go":"transfer_manager/go/pkg/abstract/k8s_runtime.go", - "pkg/abstract/local_runtime.go":"transfer_manager/go/pkg/abstract/local_runtime.go", "pkg/abstract/metrics.go":"transfer_manager/go/pkg/abstract/metrics.go", "pkg/abstract/middleware.go":"transfer_manager/go/pkg/abstract/middleware.go", "pkg/abstract/model.go":"transfer_manager/go/pkg/abstract/model.go", @@ -3106,8 +3103,6 @@ "pkg/abstract/model/transfer_status.go":"transfer_manager/go/pkg/abstract/model/transfer_status.go", "pkg/abstract/model/transformation.go":"transfer_manager/go/pkg/abstract/model/transformation.go", "pkg/abstract/movable.go":"transfer_manager/go/pkg/abstract/movable.go", - "pkg/abstract/multi_yt_runtime.go":"transfer_manager/go/pkg/abstract/multi_yt_runtime.go", - "pkg/abstract/nirvactor_runtime.go":"transfer_manager/go/pkg/abstract/nirvactor_runtime.go", "pkg/abstract/operations.go":"transfer_manager/go/pkg/abstract/operations.go", "pkg/abstract/operations_test.go":"transfer_manager/go/pkg/abstract/operations_test.go", "pkg/abstract/parsers.go":"transfer_manager/go/pkg/abstract/parsers.go", @@ -3138,9 +3133,6 @@ "pkg/abstract/typesystem/schema_doc.go":"transfer_manager/go/pkg/abstract/typesystem/schema_doc.go", "pkg/abstract/typesystem/values/type_checkers.go":"transfer_manager/go/pkg/abstract/typesystem/values/type_checkers.go", "pkg/abstract/validator.go":"transfer_manager/go/pkg/abstract/validator.go", - "pkg/abstract/yc_runtime.go":"transfer_manager/go/pkg/abstract/yc_runtime.go", - "pkg/abstract/yt_runtime.go":"transfer_manager/go/pkg/abstract/yt_runtime.go", - "pkg/abstract/yt_runtime_test.go":"transfer_manager/go/pkg/abstract/yt_runtime_test.go", "pkg/auth/permissions/permissions.go":"transfer_manager/go/pkg/auth/permissions/permissions.go", "pkg/base/adapter/legacy_table_adapter.go":"transfer_manager/go/pkg/base/adapter/legacy_table_adapter.go", "pkg/base/eventbatch_test.go":"transfer_manager/go/pkg/base/eventbatch_test.go", @@ -3191,25 +3183,6 @@ "pkg/config/aws_creds.go":"transfer_manager/go/pkg/config/aws_creds.go", "pkg/config/cloud_creds.go":"transfer_manager/go/pkg/config/cloud_creds.go", "pkg/config/config.go":"transfer_manager/go/pkg/config/config.go", - "pkg/config/controlplane/config.go":"transfer_manager/go/pkg/config/controlplane/config.go", - "pkg/config/controlplane/config_test.go":"transfer_manager/go/pkg/config/controlplane/config_test.go", - "pkg/config/controlplane/devconfigs/external_preprod.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/external_preprod.yaml", - "pkg/config/controlplane/devconfigs/external_prod.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/external_prod.yaml", - "pkg/config/controlplane/devconfigs/internal.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/internal.yaml", - "pkg/config/controlplane/devconfigs/internal_minimal.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/internal_minimal.yaml", - "pkg/config/controlplane/devconfigs/ovandriyanov/external/preprod.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/ovandriyanov/external/preprod.yaml", - "pkg/config/controlplane/devconfigs/ovandriyanov/internal/prod.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/ovandriyanov/internal/prod.yaml", - "pkg/config/controlplane/devconfigs/ovandriyanov/internal/testing.yaml":"transfer_manager/go/pkg/config/controlplane/devconfigs/ovandriyanov/internal/testing.yaml", - "pkg/config/controlplane/installations/external_kz.yaml":"transfer_manager/go/pkg/config/controlplane/installations/external_kz.yaml", - "pkg/config/controlplane/installations/external_preprod.k8s.yaml":"transfer_manager/go/pkg/config/controlplane/installations/external_preprod.k8s.yaml", - "pkg/config/controlplane/installations/external_preprod.yaml":"transfer_manager/go/pkg/config/controlplane/installations/external_preprod.yaml", - "pkg/config/controlplane/installations/external_prod.yaml":"transfer_manager/go/pkg/config/controlplane/installations/external_prod.yaml", - "pkg/config/controlplane/installations/internal_prod.yaml":"transfer_manager/go/pkg/config/controlplane/installations/internal_prod.yaml", - "pkg/config/controlplane/installations/internal_testing.yaml":"transfer_manager/go/pkg/config/controlplane/installations/internal_testing.yaml", - "pkg/config/controlplane/loader.go":"transfer_manager/go/pkg/config/controlplane/loader.go", - "pkg/config/controlplane/parts/common_local.yaml":"transfer_manager/go/pkg/config/controlplane/parts/common_local.yaml", - "pkg/config/controlplane/parts/dc_preprod_local.yaml":"transfer_manager/go/pkg/config/controlplane/parts/dc_preprod_local.yaml", - "pkg/config/controlplane/parts/dc_prod_local.yaml":"transfer_manager/go/pkg/config/controlplane/parts/dc_prod_local.yaml", "pkg/config/dataplane/config.go":"transfer_manager/go/pkg/config/dataplane/config.go", "pkg/config/dataplane/config_test.go":"transfer_manager/go/pkg/config/dataplane/config_test.go", "pkg/config/dataplane/devconfigs/dc_local.yaml":"transfer_manager/go/pkg/config/dataplane/devconfigs/dc_local.yaml", @@ -3236,7 +3209,6 @@ "pkg/config/env.go":"transfer_manager/go/pkg/config/env.go", "pkg/config/env/common.go":"transfer_manager/go/pkg/config/env/common.go", "pkg/config/env/environment.go":"transfer_manager/go/pkg/config/env/environment.go", - "pkg/config/env/environment_provider.go":"transfer_manager/go/pkg/config/env/environment_provider.go", "pkg/config/environment.go":"transfer_manager/go/pkg/config/environment.go", "pkg/config/events.go":"transfer_manager/go/pkg/config/events.go", "pkg/config/fine_grained_ssa.go":"transfer_manager/go/pkg/config/fine_grained_ssa.go", @@ -3462,8 +3434,6 @@ "pkg/middlewares/table_temporator_test.go":"transfer_manager/go/pkg/middlewares/table_temporator_test.go", "pkg/middlewares/transformation.go":"transfer_manager/go/pkg/middlewares/transformation.go", "pkg/middlewares/type_strictness_tracker.go":"transfer_manager/go/pkg/middlewares/type_strictness_tracker.go", - "pkg/oplock/optimistic_lock.go":"transfer_manager/go/pkg/oplock/optimistic_lock.go", - "pkg/oplock/optimistic_lock_test.go":"transfer_manager/go/pkg/oplock/optimistic_lock_test.go", "pkg/parsequeue/parsequeue.go":"transfer_manager/go/pkg/parsequeue/parsequeue.go", "pkg/parsequeue/parsequeue_test.go":"transfer_manager/go/pkg/parsequeue/parsequeue_test.go", "pkg/parsequeue/waitable_parsequeue.go":"transfer_manager/go/pkg/parsequeue/waitable_parsequeue.go", @@ -3571,10 +3541,9 @@ "pkg/parsers/registry/tskv/parser_tskv.go":"transfer_manager/go/pkg/parsers/registry/tskv/parser_tskv.go", "pkg/parsers/resource_wrapper.go":"transfer_manager/go/pkg/parsers/resource_wrapper.go", "pkg/parsers/resources/abstract.go":"transfer_manager/go/pkg/parsers/resources/abstract.go", + "pkg/parsers/resources/dummy.go":"transfer_manager/go/pkg/parsers/resources/dummy.go", "pkg/parsers/resources/embedded_resources.go":"transfer_manager/go/pkg/parsers/resources/embedded_resources.go", "pkg/parsers/resources/no_resources.go":"transfer_manager/go/pkg/parsers/resources/no_resources.go", - "pkg/parsers/resources/yt_resources.go":"transfer_manager/go/pkg/parsers/resources/yt_resources.go", - "pkg/parsers/resources/yt_resources_test.go":"transfer_manager/go/pkg/parsers/resources/yt_resources_test.go", "pkg/parsers/scanner/donotsplit_scanner.go":"transfer_manager/go/pkg/parsers/scanner/donotsplit_scanner.go", "pkg/parsers/scanner/donotsplit_scanner_test.go":"transfer_manager/go/pkg/parsers/scanner/donotsplit_scanner_test.go", "pkg/parsers/scanner/event_scanner.go":"transfer_manager/go/pkg/parsers/scanner/event_scanner.go", @@ -4275,7 +4244,6 @@ "pkg/providers/yt/model_yt_copy_destination.go":"transfer_manager/go/pkg/providers/yt/model_yt_copy_destination.go", "pkg/providers/yt/model_yt_destination.go":"transfer_manager/go/pkg/providers/yt/model_yt_destination.go", "pkg/providers/yt/model_yt_source.go":"transfer_manager/go/pkg/providers/yt/model_yt_source.go", - "pkg/providers/yt/prepare_job_spec.go":"transfer_manager/go/pkg/providers/yt/prepare_job_spec.go", "pkg/providers/yt/provider.go":"transfer_manager/go/pkg/providers/yt/provider.go", "pkg/providers/yt/provider/batch.go":"transfer_manager/go/pkg/providers/yt/provider/batch.go", "pkg/providers/yt/provider/dataobjects/objectpresharded.go":"transfer_manager/go/pkg/providers/yt/provider/dataobjects/objectpresharded.go", @@ -4353,13 +4321,6 @@ "pkg/schemaregistry/confluent/ysr_test.go":"transfer_manager/go/pkg/schemaregistry/confluent/ysr_test.go", "pkg/schemaregistry/format/common.go":"transfer_manager/go/pkg/schemaregistry/format/common.go", "pkg/schemaregistry/format/json_schema_format.go":"transfer_manager/go/pkg/schemaregistry/format/json_schema_format.go", - "pkg/schemaregistry/ytschemaregistry/parsers.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/parsers.go", - "pkg/schemaregistry/ytschemaregistry/parsers_cache.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/parsers_cache.go", - "pkg/schemaregistry/ytschemaregistry/parsers_test.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/parsers_test.go", - "pkg/schemaregistry/ytschemaregistry/util.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/util.go", - "pkg/schemaregistry/ytschemaregistry/yt_schema_registry.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/yt_schema_registry.go", - "pkg/schemaregistry/ytschemaregistry/yt_schema_registry_impl.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_impl.go", - "pkg/schemaregistry/ytschemaregistry/yt_schema_registry_mock.go":"transfer_manager/go/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_mock.go", "pkg/serializer/batch.go":"transfer_manager/go/pkg/serializer/batch.go", "pkg/serializer/batch_test.go":"transfer_manager/go/pkg/serializer/batch_test.go", "pkg/serializer/csv.go":"transfer_manager/go/pkg/serializer/csv.go", @@ -4606,12 +4567,11 @@ "pkg/worker/tasks/load_snapshot_test.go":"transfer_manager/go/pkg/worker/tasks/load_snapshot_test.go", "pkg/worker/tasks/load_snapshot_v2.go":"transfer_manager/go/pkg/worker/tasks/load_snapshot_v2.go", "pkg/worker/tasks/load_snapshot_with_transformers_test.go":"transfer_manager/go/pkg/worker/tasks/load_snapshot_with_transformers_test.go", + "pkg/worker/tasks/oss_job.go":"transfer_manager/go/pkg/worker/tasks/oss_job.go", "pkg/worker/tasks/remove_tables.go":"transfer_manager/go/pkg/worker/tasks/remove_tables.go", "pkg/worker/tasks/reupload.go":"transfer_manager/go/pkg/worker/tasks/reupload.go", "pkg/worker/tasks/snapshot_table_metrics_tracker.go":"transfer_manager/go/pkg/worker/tasks/snapshot_table_metrics_tracker.go", "pkg/worker/tasks/snapshot_table_progress_tracker.go":"transfer_manager/go/pkg/worker/tasks/snapshot_table_progress_tracker.go", - "pkg/worker/tasks/start_job.go":"transfer_manager/go/pkg/worker/tasks/start_job.go", - "pkg/worker/tasks/stop_job.go":"transfer_manager/go/pkg/worker/tasks/stop_job.go", "pkg/worker/tasks/task_visitor.go":"transfer_manager/go/pkg/worker/tasks/task_visitor.go", "pkg/worker/tasks/test_endpoint.go":"transfer_manager/go/pkg/worker/tasks/test_endpoint.go", "pkg/worker/tasks/transformation.go":"transfer_manager/go/pkg/worker/tasks/transformation.go", diff --git a/pkg/abstract/dedicated_runtime.go b/pkg/abstract/dedicated_runtime.go deleted file mode 100644 index b7b4f07e..00000000 --- a/pkg/abstract/dedicated_runtime.go +++ /dev/null @@ -1,227 +0,0 @@ -package abstract - -import ( - "os" - "strconv" - "strings" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - "github.com/dustin/go-humanize" -) - -var _ ShardingTaskRuntime = (*DedicatedRuntime)(nil) - -type DedicatedRuntimeFlavor string -type DedicatedCloudType string - -const ( - DedicatedTinyRuntime = DedicatedRuntimeFlavor("tiny") - DedicatedSmallRuntime = DedicatedRuntimeFlavor("small") - DedicatedMediumRuntime = DedicatedRuntimeFlavor("medium") - DedicatedLargeRuntime = DedicatedRuntimeFlavor("large") -) - -const ( - DedicatedAWSCloudType = DedicatedCloudType("aws") - DedicatedGCPCloudType = DedicatedCloudType("gcp") -) - -type Ec2Region string - -type DedicatedRuntime struct { - Flavor DedicatedRuntimeFlavor `json:"flavor"` - ImageID string `json:"ImageID"` - Disposition *Disposition - LogLevel log.Level - GCPercentageLimit int `json:"GCPercentageLimit"` - ShardingUpload ShardUploadParams - // JobCount is used by incremental transfer - JobCount int64 -} - -func (c *DedicatedRuntime) isShardingEnabled() {} - -func (c *DedicatedRuntime) WorkersNum() int { - return c.ShardingUpload.JobCount -} - -func (c *DedicatedRuntime) ThreadsNumPerWorker() int { - return c.ShardingUpload.ProcessCount -} - -func (c *DedicatedRuntime) CurrentJobIndex() int { - name, err := os.Hostname() - if err != nil { - return 0 - } - parts := strings.Split(name, "-") - if len(parts) <= 1 { - return 0 - } - i, _ := strconv.Atoi(parts[len(parts)-1]) - return i -} - -func (c *DedicatedRuntime) IsMain() bool { - return c.CurrentJobIndex() == 0 -} - -func (c *DedicatedRuntime) FinalImageID() (string, error) { - if c.ImageID != "" { - return c.ImageID, nil - } - if c.ImageID == "" { - imageID := string(cpconfig.Common.DataplaneK8SImage) - if imageID != "" { - return imageID, nil - } - } - return "", NewFatalError(xerrors.New("dataplane version is not configured with DataplaneK8SImage env variable")) -} - -func (*DedicatedRuntime) Type() RuntimeType { - return DedicatedRuntimeType -} - -type Disposition struct { - Region Ec2Region - SubnetID string - SecurityGroupID string - NetworkID string - AWSBYOA *BYOASpecAWS - GCPBYOA *BYOASpecGCP - CloudType DedicatedCloudType -} - -type BYOASpecAWS struct { - IAMRole, AccountID string -} - -type BYOASpecGCP struct { - ProjectName, ServiceAccountEmail string -} - -func (c *Disposition) IsBYOA() bool { - return c.GCPBYOA != nil -} - -func (c *DedicatedRuntime) isRuntime() { -} - -func (c *DedicatedRuntime) FlavorName() string { - return string(c.Flavor) -} - -func (c *DedicatedRuntime) NeedRestart(runtime Runtime) bool { - switch rnt := runtime.(type) { - case *DedicatedRuntime: - if c.Flavor != rnt.Flavor || c.ImageID != rnt.ImageID { - return true - } - } - return false -} - -func (c *DedicatedRuntime) GetAWSFlavour() string { - switch c.Flavor { - case DedicatedTinyRuntime: - return "t3.small" - case DedicatedSmallRuntime: - return "c5.large" - case DedicatedMediumRuntime: - return "c5.xlarge" - case DedicatedLargeRuntime: - return "c5.2xlarge" - default: - return "c5.large" - } -} - -func (c *DedicatedRuntime) WithDefaults() { - if c.Flavor == "" { - c.Flavor = DedicatedSmallRuntime - } - if c.LogLevel == 0 { - c.LogLevel = log.InfoLevel - } - if c.GCPercentageLimit == 0 { - c.GCPercentageLimit = 25 - } - if c.ShardingUpload.JobCount == 0 { - c.ShardingUpload = *DefaultShardUploadParams() - } - if c.JobCount == 0 { - c.JobCount = 1 - } -} - -func (c *DedicatedRuntime) Validate() error { - if c.LogLevel > log.WarnLevel { - return xerrors.New("Log must be at least warn level") - } - if c.Disposition != nil { - d := c.Disposition - if d.NetworkID == "" { - return xerrors.New("network ID is required for manual ec2 disposition") - } - } - return nil -} - -func (c *DedicatedRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - if runtimeSpecificVersion != "" { - c.ImageID = runtimeSpecificVersion - } - return nil -} - -func (c *DedicatedRuntime) RAMGuarantee() uint64 { - var total uint64 - switch c.Flavor { - case DedicatedTinyRuntime: - total = 2 * humanize.GiByte - case DedicatedSmallRuntime: - total = 4 * humanize.GiByte - case DedicatedMediumRuntime: - total = 8 * humanize.GiByte - case DedicatedLargeRuntime: - total = 16 * humanize.GiByte - default: - return 0 - } - // Reserve some memory for OS and system daemons, docker, osquery, etc. - return total - 512*humanize.MiByte -} - -func (c *DedicatedRuntime) GCPercentage() int { - return c.GCPercentageLimit -} - -// ResourceLimiterEnabled for ec2 always on. -func (c *DedicatedRuntime) ResourceLimiterEnabled() bool { - return true -} - -func (c *DedicatedRuntime) GCPBYOASA() string { - if c.Disposition == nil || c.Disposition.GCPBYOA == nil { - return "" - } - return c.Disposition.GCPBYOA.ServiceAccountEmail -} - -func (c *DedicatedRuntime) GCPFlavor() string { - switch c.Flavor { - case DedicatedTinyRuntime: - return "e2-small" - case DedicatedSmallRuntime: - return "e2-standard-2" - case DedicatedMediumRuntime: - return "e2-standard-4" - case DedicatedLargeRuntime: - return "e2-standard-8" - default: - return "e2-standard-2" - } -} diff --git a/pkg/abstract/k8s_runtime.go b/pkg/abstract/k8s_runtime.go deleted file mode 100644 index ab74dfa9..00000000 --- a/pkg/abstract/k8s_runtime.go +++ /dev/null @@ -1,82 +0,0 @@ -package abstract - -import ( - "os" - "strconv" - "strings" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/runtime/shared/pod" -) - -type K8SRuntime struct { - Image string - ClusterID string - LogLevel log.Level - ShardingUpload ShardUploadParams - JobCount int64 - GCPercentageLimit int `json:"GCPercentageLimit"` -} - -func (k *K8SRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - k.Image = runtimeSpecificVersion - return nil -} - -func (*K8SRuntime) Type() RuntimeType { - return K8SRuntimeType -} - -func (k *K8SRuntime) isRuntime() { -} - -func (k *K8SRuntime) NeedRestart(runtime Runtime) bool { - return false -} - -func (k *K8SRuntime) WithDefaults() { - if k.LogLevel == 0 { - k.LogLevel = log.InfoLevel - } - if k.GCPercentageLimit == 0 { - k.GCPercentageLimit = 25 - } -} - -func (k *K8SRuntime) isShardingEnabled() {} -func (k *K8SRuntime) WorkersNum() int { return k.ShardingUpload.JobCount } -func (k *K8SRuntime) ThreadsNumPerWorker() int { return k.ShardingUpload.ProcessCount } -func (k *K8SRuntime) IsMain() bool { return k.CurrentJobIndex() == 0 } -func (k *K8SRuntime) CurrentJobIndex() int { - name, err := os.Hostname() - if err != nil { - return 0 - } - parts := strings.Split(name, "-") - if len(parts) <= 1 { - return 0 - } - i, _ := strconv.Atoi(parts[len(parts)-1]) - return i -} - -func (k *K8SRuntime) Validate() error { - if k.LogLevel > log.WarnLevel { - return xerrors.New("Log must be at least warn leve") - } - return nil -} - -func (k *K8SRuntime) GCPercentage() int { - return k.GCPercentageLimit -} - -func (k *K8SRuntime) RAMGuarantee() uint64 { - return uint64(pod.RAMMax.Value()) -} - -// ResourceLimiterEnabled for k8s always on. -func (k *K8SRuntime) ResourceLimiterEnabled() bool { - return true -} diff --git a/pkg/abstract/local_runtime.go b/pkg/abstract/local_runtime.go deleted file mode 100644 index 51ab9119..00000000 --- a/pkg/abstract/local_runtime.go +++ /dev/null @@ -1,34 +0,0 @@ -package abstract - -type LocalRuntime struct { - Host string - CurrentJob int - ShardingUpload ShardUploadParams -} - -func (*LocalRuntime) Type() RuntimeType { - return LocalRuntimeType -} - -func (l *LocalRuntime) isRuntime() { -} - -func (l *LocalRuntime) NeedRestart(runtime Runtime) bool { - return false -} - -func (l *LocalRuntime) WithDefaults() { -} - -func (l *LocalRuntime) Validate() error { - return nil -} - -func (l *LocalRuntime) isShardingEnabled() {} -func (l *LocalRuntime) WorkersNum() int { return l.ShardingUpload.JobCount } -func (l *LocalRuntime) ThreadsNumPerWorker() int { return l.ShardingUpload.ProcessCount } -func (l *LocalRuntime) CurrentJobIndex() int { return l.CurrentJob } -func (l *LocalRuntime) IsMain() bool { return l.CurrentJob == 0 } -func (l *LocalRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - return nil -} diff --git a/pkg/abstract/multi_yt_runtime.go b/pkg/abstract/multi_yt_runtime.go deleted file mode 100644 index 3d7445c3..00000000 --- a/pkg/abstract/multi_yt_runtime.go +++ /dev/null @@ -1,87 +0,0 @@ -package abstract - -import ( - "fmt" - "strings" - - "github.com/doublecloud/tross/internal/core/xerrors" -) - -type CrossDcYtRuntime map[YtCluster]YtRuntime - -func CreateDefaultCrossDcYtRuntime() CrossDcYtRuntime { - return CrossDcYtRuntime{} -} - -func (*CrossDcYtRuntime) Type() RuntimeType { - return MultiYtRuntimeType -} - -func (c *CrossDcYtRuntime) isRuntime() { -} - -func (c *CrossDcYtRuntime) NeedRestart(runtime Runtime) bool { - switch r := runtime.(type) { - case *CrossDcYtRuntime: - if len(*r) != len(*c) { - // One or more yt runtimes were added or deleted, need to restart - return true - } - - for cluster, rRuntime := range *r { - cRuntime, ok := (*c)[cluster] - if ok { - if cRuntime.NeedRestart(&rRuntime) { - return true - } - } else { - // One or more yt runtimes were replaced, need to restart - return true - } - } - - return false - default: - // Runtime type changed, need to restart - return true - } -} - -func (c *CrossDcYtRuntime) WithDefaults() { - if len(*c) > 0 { - return - } - - var ytRuntime YtRuntime - ytRuntime.WithDefaults() - (*c)[ytRuntime.Cluster] = ytRuntime -} - -func (c *CrossDcYtRuntime) Validate() error { - builder := strings.Builder{} - for cluster, runtime := range *c { - if cluster == "" { - builder.WriteString("Empty cluster; ") - continue - } - err := runtime.Validate() - if err == nil { - continue - } - builder.WriteString(fmt.Sprintf("Incorrect settings for cluster '%v': %v", cluster, err)) - } - if builder.Len() > 0 { - return xerrors.New(builder.String()) - } - return nil -} - -func (c *CrossDcYtRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - for cluster, runtime := range *c { - if err := runtime.SetVersion(runtimeSpecificVersion, versionProperties); err != nil { - return xerrors.Errorf("cluster %s: %w", cluster, err) - } - (*c)[cluster] = runtime - } - return nil -} diff --git a/pkg/abstract/nirvactor_runtime.go b/pkg/abstract/nirvactor_runtime.go deleted file mode 100644 index 8814f023..00000000 --- a/pkg/abstract/nirvactor_runtime.go +++ /dev/null @@ -1,27 +0,0 @@ -package abstract - -type NirvactorRuntime struct { - Stub int `json:"Stub"` -} - -func (*NirvactorRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - return nil -} - -func (*NirvactorRuntime) Type() RuntimeType { - return NirvactorRuntimeType -} - -func (*NirvactorRuntime) isRuntime() { -} - -func (*NirvactorRuntime) NeedRestart(runtime Runtime) bool { - return false -} - -func (*NirvactorRuntime) WithDefaults() { -} - -func (*NirvactorRuntime) Validate() error { - return nil -} diff --git a/pkg/abstract/runtime.go b/pkg/abstract/runtime.go index 827772a9..fa7f2ff3 100644 --- a/pkg/abstract/runtime.go +++ b/pkg/abstract/runtime.go @@ -5,17 +5,12 @@ import ( "encoding/json" "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/yt/go/mapreduce" ) +var knownRuntimes = map[RuntimeType]func(spec string) (Runtime, error){} + func init() { - mapreduce.RegisterJobPart(new(YtRuntime)) - mapreduce.RegisterJobPart(new(YcRuntime)) - mapreduce.RegisterJobPart(new(CrossDcYtRuntime)) - mapreduce.RegisterJobPart(new(LocalRuntime)) - mapreduce.RegisterJobPart(new(K8SRuntime)) - mapreduce.RegisterJobPart(new(NirvactorRuntime)) - gob.RegisterName("*abstract.Ec2Runtime", new(DedicatedRuntime)) + gob.RegisterName("*abstract.LocalRuntime", new(LocalRuntime)) } type YtCluster string @@ -23,15 +18,7 @@ type YtCluster string type RuntimeType string const ( - YtRuntimeType = RuntimeType("yt") - MultiYtRuntimeType = RuntimeType("multi_yt") - YcRuntimeType = RuntimeType("yc") - NirvactorRuntimeType = RuntimeType("nirvactor") - K8SRuntimeType = RuntimeType("k8s") - LocalRuntimeType = RuntimeType("local") - EC2RuntimeType = RuntimeType("ec2") // TODO: rename to dedicated - GCPRuntimeType = RuntimeType("gcp") - DedicatedRuntimeType = RuntimeType("dedicated") + LocalRuntimeType = RuntimeType("local") ) type Runtime interface { @@ -45,50 +32,18 @@ type Runtime interface { func NewRuntime(runtime RuntimeType, runtimeSpec string) (Runtime, error) { switch runtime { - case YtRuntimeType: - var res YtRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("YT: %w", err) - } - return &res, nil - case MultiYtRuntimeType: - var res CrossDcYtRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("Multi YT: %w", err) - } - return &res, nil - case YcRuntimeType: - var res YcRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("YC: %w", err) - } - return &res, nil - case NirvactorRuntimeType: - var res NirvactorRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("Nirvactor: %w", err) - } - return &res, nil - case K8SRuntimeType: - var res K8SRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("k8s: %w", err) - } - return &res, nil case LocalRuntimeType: var res LocalRuntime if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { return nil, xerrors.Errorf("local: %w", err) } return &res, nil - case EC2RuntimeType, DedicatedRuntimeType, GCPRuntimeType: - var res DedicatedRuntime - if err := json.Unmarshal([]byte(runtimeSpec), &res); err != nil { - return nil, xerrors.Errorf("dedicated: %w", err) - } - return &res, nil default: - return nil, xerrors.Errorf("unable to parse runtime of type: %v", runtime) + f, ok := knownRuntimes[runtime] + if !ok { + return nil, xerrors.Errorf("unable to parse runtime of type: %v", runtime) + } + return f(runtimeSpec) } } diff --git a/pkg/abstract/yc_runtime.go b/pkg/abstract/yc_runtime.go deleted file mode 100644 index dace6fb0..00000000 --- a/pkg/abstract/yc_runtime.go +++ /dev/null @@ -1,154 +0,0 @@ -package abstract - -import ( - "encoding/json" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/instanceutil" - "github.com/dustin/go-humanize" -) - -type YcRuntimeFlavor string - -const ( - YcSmallRuntime = YcRuntimeFlavor("small") - YcMediumRuntime = YcRuntimeFlavor("medium") - YcLargeRuntime = YcRuntimeFlavor("large") -) - -type YcRuntimeVersionProperties struct { - UseFineGrainedSSA bool `json:"use_fine_grained_ssa"` -} - -type YcRuntime struct { - JobCount int `json:"jobCount"` - Flavor YcRuntimeFlavor `json:"flavor"` - Token string - ImageID string - FolderID string - InstanceGroupID string - LogLevel log.Level - UploadShardParams ShardUploadParams - VersionProperties *YcRuntimeVersionProperties `json:"version_properties"` - GCPercentageLimit int `json:"GCPercentageLimit"` -} - -func (c *YcRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - if versionProperties != nil { - c.VersionProperties = new(YcRuntimeVersionProperties) - if err := json.Unmarshal([]byte(*versionProperties), c.VersionProperties); err != nil { - return err - } - } - if runtimeSpecificVersion != "" { - c.ImageID = runtimeSpecificVersion - } - return nil -} - -func (*YcRuntime) Type() RuntimeType { - return YcRuntimeType -} - -func (c *YcRuntime) isRuntime() { -} - -func (c *YcRuntime) FlavorName() string { - return string(c.Flavor) -} - -func (c *YcRuntime) NeedRestart(runtime Runtime) bool { - switch rnt := runtime.(type) { - case *YcRuntime: - if c.Flavor != rnt.Flavor || c.ImageID != rnt.ImageID || c.JobCount != rnt.JobCount { - return true - } - } - return false -} - -func (c *YcRuntime) WithDefaults() { - if c.JobCount == 0 { - c.JobCount = 1 - } - if c.Flavor == "" { - c.Flavor = YcSmallRuntime - } - if c.LogLevel == 0 { - c.LogLevel = log.InfoLevel - } - if c.GCPercentageLimit == 0 { - c.GCPercentageLimit = 25 - } -} - -func (c *YcRuntime) Validate() error { - if c.LogLevel > log.WarnLevel { - return xerrors.New("Log must be at least warn level") - } - if c.JobCount < 1 || c.JobCount > 8 { - return xerrors.New("Incorrect job count, allowed: [1; 8]") - } - return nil -} - -func (c *YcRuntime) isShardingEnabled() { -} - -func (c *YcRuntime) WorkersNum() int { - return c.UploadShardParams.JobCount -} - -// YcJobsCount returns the total number of jobs (including the main one) set for YC runtime. -// The result does not depend on whether this is the number of replications set using a dedicated parameter or a standardized number of jobs for sharded transfer -func (c *YcRuntime) YcJobsCount() int { - if c.UploadShardParams.JobCount > 1 { - return c.UploadShardParams.JobCount + 1 - } - return 1 -} - -func (c *YcRuntime) ThreadsNumPerWorker() int { - return c.UploadShardParams.ProcessCount -} - -func (c *YcRuntime) IsMain() bool { - return c.CurrentJobIndex() == 0 -} - -func (c *YcRuntime) CurrentJobIndex() int { - cji, err := instanceutil.JobIndex() - if err != nil { - logger.Log.Warn("Failed to obtain current job index", log.Error(err)) - return cji - } - - return cji -} - -func (c *YcRuntime) GCPercentage() int { - return c.GCPercentageLimit -} - -func (c *YcRuntime) RAMGuarantee() uint64 { - var total uint64 - switch c.Flavor { - case YcSmallRuntime: - total = 4 * humanize.GiByte - case YcMediumRuntime: - total = 8 * humanize.GiByte - case YcLargeRuntime: - total = 16 * humanize.GiByte - default: - return 0 - } - // Reserve some memory for OS and system daemons, docker, osquery, etc. - return total - 512*humanize.MiByte -} - -// ResourceLimiterEnabled for yc always on. -func (c *YcRuntime) ResourceLimiterEnabled() bool { - return true -} diff --git a/pkg/abstract/yt_runtime.go b/pkg/abstract/yt_runtime.go deleted file mode 100644 index 5dce83a3..00000000 --- a/pkg/abstract/yt_runtime.go +++ /dev/null @@ -1,206 +0,0 @@ -package abstract - -import ( - "encoding/json" - "fmt" - "os" - "strconv" - "strings" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/dustin/go-humanize" -) - -const ( - minRAM = 100 * humanize.MiByte // 100 Mb - minCPU = 0.1 - minJobCount = 1 - // Soft limits are applied if user doesn't specify its own quota (YT Pool) - maxRAMSoft = 10 * humanize.GiByte - maxRAMHard = 100 * humanize.GiByte - maxCPUSoft = 8 - maxJobSoft = 8 - maxShardedJobsSoft = 8 -) - -type YtRuntimeVersionProperties struct { - DefaultNetworkProject *string `json:"default_network_project"` -} - -type YtRuntime struct { - JobCount int `json:"jobCount"` - CPU float32 `json:"cpu"` - RAM uint64 `json:"ram"` - Cluster YtCluster `json:"cluster"` - Pool string `json:"pool"` - Operation string `json:"operation"` // Is never set by the client, modified and used only by control plane - Version string `json:"version"` - VersionProperties *YtRuntimeVersionProperties `json:"version_properties"` - ShardingUpload ShardUploadParams `json:"shardingUpload"` - NetworkProject string `json:"networkProject"` - LogLevel log.Level `json:"LogLevel"` - DisableCPUReclaim bool `json:"disableCPUReclaim"` - CoreTablePath string `json:"coreTablePath"` - GCPercentageLimit int `json:"GCPercentageLimit"` - ResourceLimiterDisabled bool `json:"ResourceLimiterDisabled"` -} - -func (*YtRuntime) Type() RuntimeType { - return YtRuntimeType -} - -func (y *YtRuntime) SetVersion(runtimeSpecificVersion string, versionProperties *string) error { - if versionProperties != nil { - y.VersionProperties = new(YtRuntimeVersionProperties) - if err := json.Unmarshal([]byte(*versionProperties), y.VersionProperties); err != nil { - return err - } - } else { - y.VersionProperties = nil - } - if runtimeSpecificVersion != "" { - y.Version = runtimeSpecificVersion - } - return nil -} - -var ( - DefaultYtCluster YtCluster -) - -func (y *YtRuntime) isRuntime() { -} - -func stringPtrDiffer(l, r *string) bool { - if l == nil && r == nil { - return false - } - if l == nil || r == nil { - return true - } - return *l != *r -} - -func versionPropertiesDiffer(l, r *YtRuntimeVersionProperties) bool { - if l == nil && r == nil { - return false - } - if l == nil || r == nil { - return true - } - return stringPtrDiffer(l.DefaultNetworkProject, r.DefaultNetworkProject) -} - -func (y *YtRuntime) NeedRestart(runtime Runtime) bool { - switch r := runtime.(type) { - case *YtRuntime: - return y.JobCount != r.JobCount || - y.CPU != r.CPU || - y.RAM != r.RAM || - y.Cluster != r.Cluster || - y.Pool != r.Pool || - y.Version != r.Version || - versionPropertiesDiffer(y.VersionProperties, r.VersionProperties) || - y.DisableCPUReclaim != r.DisableCPUReclaim - default: - // Runtime type changed, need to restart - return true - } -} - -func (y *YtRuntime) WithDefaults() { - if y.JobCount == 0 { - y.JobCount = 1 - } - if y.CPU == 0 { - y.CPU = 0.2 - } - if y.RAM == 0 { - y.RAM = uint64(y.CPU * 10 * 1000 * 1000 * 1000) - if y.RAM < minRAM { - y.RAM = minRAM - } - if y.RAM > maxRAMSoft { - y.RAM = maxRAMSoft - } - } - if y.Cluster == "" { - y.Cluster = DefaultYtCluster - } - if y.LogLevel == 0 { - y.LogLevel = log.InfoLevel - } - if y.ShardingUpload.JobCount < 1 { - y.ShardingUpload.JobCount = 0 - } - if y.GCPercentageLimit == 0 { - y.GCPercentageLimit = 25 - } -} - -func (y *YtRuntime) Validate() error { - builder := strings.Builder{} - if y.JobCount < minJobCount { - builder.WriteString(fmt.Sprintf("Incorrect job count, min. %v; ", minJobCount)) - } - if y.CPU < minCPU { - builder.WriteString(fmt.Sprintf("Incorrect CPU count, min. %v; ", minCPU)) - } - if y.RAM < minRAM || y.RAM > maxRAMHard { - builder.WriteString(fmt.Sprintf("Incorrect RAM size, min. %v, max. %v; ", humanize.Bytes(minRAM), humanize.Bytes(maxRAMHard))) - } - if y.Cluster == "" { - builder.WriteString("Empty cluster; ") - } - if y.Pool == "" { - if y.JobCount > maxJobSoft { - builder.WriteString(fmt.Sprintf("Requested more than %v job count and YT pool is not specified; ", maxJobSoft)) - } - if y.CPU > maxCPUSoft { - builder.WriteString(fmt.Sprintf("Requested more than %v CPU and YT pool is not specified; ", maxCPUSoft)) - } - if y.RAM > maxRAMSoft { - builder.WriteString(fmt.Sprintf("Requested more than %v RAM and YT pool is not specified; ", humanize.Bytes(maxRAMSoft))) - } - if y.ShardingUpload.JobCount > maxShardedJobsSoft { - builder.WriteString(fmt.Sprintf("Requested more than %v upload jobs and YT pool is not specified; ", maxShardedJobsSoft)) - } - if y.Cluster != DefaultYtCluster { - builder.WriteString(fmt.Sprintf("YT Cluster is not specified, only %s is allowed as runtime cluster; ", DefaultYtCluster)) - } - } - if builder.Len() > 0 { - return xerrors.New(builder.String()) - } - if y.LogLevel > log.WarnLevel { - return xerrors.New("Log must be at least warn level") - } - return nil -} - -func (y *YtRuntime) isShardingEnabled() {} -func (y *YtRuntime) WorkersNum() int { return y.ShardingUpload.JobCount } -func (y *YtRuntime) ThreadsNumPerWorker() int { return y.ShardingUpload.ProcessCount } -func (y *YtRuntime) IsMain() bool { return y.CurrentJobIndex() == 0 } -func (y *YtRuntime) CurrentJobIndex() int { - i, _ := strconv.Atoi(os.Getenv("YT_JOB_COOKIE")) - return i -} - -func (y *YtRuntime) EnableCPUReclaim() *bool { - enableCPUReclaim := !y.DisableCPUReclaim - return &enableCPUReclaim -} - -func (y *YtRuntime) RAMGuarantee() uint64 { - return y.RAM -} - -func (y *YtRuntime) GCPercentage() int { - return y.GCPercentageLimit -} - -func (y *YtRuntime) ResourceLimiterEnabled() bool { - return !y.ResourceLimiterDisabled -} diff --git a/pkg/abstract/yt_runtime_test.go b/pkg/abstract/yt_runtime_test.go deleted file mode 100644 index 7d7fa163..00000000 --- a/pkg/abstract/yt_runtime_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package abstract - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNeedRestart(t *testing.T) { - runtimeOld := YtRuntime{} - runtimeOld.WithDefaults() - - runtimeNew := YtRuntime{} - runtimeNew.WithDefaults() - - t.Run("JobCount", func(t *testing.T) { - runtimeNew.JobCount = 2 - - // Different JobCount's - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.JobCount = 2 - }) - - t.Run("CPU", func(t *testing.T) { - runtimeNew.CPU = 0.3 - - // Different CPU's - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.CPU = 0.3 - - // Same CPU's, no need to restart - assert.False(t, runtimeOld.NeedRestart(&runtimeNew)) - }) - - t.Run("RAM", func(t *testing.T) { - runtimeOld.RAM = 42_000_000 - runtimeNew.RAM = 42_000_000 - - assert.False(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.RAM = 0 - runtimeNew.RAM = 100_000_000 - - // Different RAM's - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.RAM = 100_000_000 - }) - - t.Run("Version", func(t *testing.T) { - // old.Version == "" && new.Version == "" => should not restart - assert.False(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.Version = "111000" - - // old.Version != "" && new.Version == "" => need to restart with the current version of Data Plane - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeNew.Version = "111001" - - // old.Version != "" && new.Version != "" && old.Version != new.Version => need to restart with new version - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.Version = "111001" - - // old.Version != "" && new.Version != "" && old.Version == new.Version => should not restart - assert.False(t, runtimeOld.NeedRestart(&runtimeNew)) - - runtimeOld.Version = "" - runtimeNew.Version = "111100" - - // Different Version's - assert.True(t, runtimeOld.NeedRestart(&runtimeNew)) - }) -} - -func TestValidation(t *testing.T) { - DefaultYtCluster = "vanga" - t.Run("EmptyPoolValid", func(t *testing.T) { - var rt YtRuntime - rt.WithDefaults() - rt.JobCount = 3 - rt.CPU = 1 - rt.RAM = 1 * 1000 * 1000 * 1000 - rt.ShardingUpload = ShardUploadParams{ - JobCount: 4, - ProcessCount: 4, - } - require.NoError(t, rt.Validate()) - }) - t.Run("EmptyPoolInvalid", func(t *testing.T) { - var rt YtRuntime - rt.WithDefaults() - rt.JobCount = 30 - rt.CPU = 100 - rt.RAM = 20 * 1000 * 1000 * 1000 - rt.ShardingUpload = ShardUploadParams{ - JobCount: 40, - ProcessCount: 4, - } - err := rt.Validate() - require.ErrorContains(t, err, "Requested more than") - require.ErrorContains(t, err, "CPU") - require.ErrorContains(t, err, "RAM") - require.ErrorContains(t, err, "upload jobs") - require.ErrorContains(t, err, "job count") - - rt = YtRuntime{} - rt.WithDefaults() - rt.Cluster = "hahn" - require.Errorf(t, rt.Validate(), "runtime cluster") - }) - t.Run("NonEmptyPoolValid", func(t *testing.T) { - var rt YtRuntime - rt.WithDefaults() - rt.JobCount = 30 - rt.CPU = 100 - rt.RAM = 20 * 1000 * 1000 * 1000 - rt.ShardingUpload = ShardUploadParams{ - JobCount: 40, - ProcessCount: 4, - } - rt.Cluster = "hahn" - rt.Pool = "some_pool" - require.NoError(t, rt.Validate()) - }) -} diff --git a/pkg/config/controlplane/config.go b/pkg/config/controlplane/config.go deleted file mode 100644 index ff39e0b7..00000000 --- a/pkg/config/controlplane/config.go +++ /dev/null @@ -1,372 +0,0 @@ -package cpconfig - -import ( - "crypto/tls" - _ "embed" - "io" - - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/pkg/config" - "github.com/doublecloud/tross/pkg/xtls" -) - -var ( - // Not nil for controlplane - Common *CommonConfig - - // Not nil when either Internal != nil or Public != nil - Cloud *CloudConfig - - // Exactly one of these is not nil - InternalCloud *InternalCloudConfig - ExternalCloud *ExternalCloudConfig - AWS *AWSConfig -) - -var ( - //go:embed installations/external_preprod.yaml - externalPreprodYaml []byte - - //go:embed installations/external_prod.yaml - externalProdYaml []byte - - //go:embed installations/external_kz.yaml - externalKzYaml []byte - - //go:embed installations/internal_prod.yaml - internalProdYaml []byte - - //go:embed installations/internal_testing.yaml - internalTestingYaml []byte - - //go:embed installations/external_preprod.k8s.yaml - externalPreprodK8SYaml []byte - - //go:embed parts/common_local.yaml - commonLocaParts []byte - - //go:embed parts/dc_preprod_local.yaml - dcPreprodLocalParts []byte - - //go:embed parts/dc_prod_local.yaml - dcProdLocalParts []byte - - EmbeddedConfigs = map[string][]byte{ - "parts/dc_preprod_local.yaml": dcPreprodLocalParts, - "parts/dc_prod_local.yaml": dcProdLocalParts, - "external_preprod.yaml": externalPreprodYaml, - "external_preprod.k8s.yaml": externalPreprodK8SYaml, - "external_prod.yaml": externalProdYaml, - "external_kz.yaml": externalKzYaml, - "internal_prod.yaml": internalProdYaml, - "internal_testing.yaml": internalTestingYaml, - "parts/common_local.yaml": commonLocaParts, - } -) - -type CommonConfig struct { - DBDiscovery config.DBDiscovery `mapstructure:"db_discovery"` - DBUser string `mapstructure:"db_user"` - DBPassword config.Secret `mapstructure:"db_password"` - DBName string `mapstructure:"db_name"` - DBSchema string `mapstructure:"db_schema"` - GRPCPort int `mapstructure:"grpc_port"` - PubGRPCPort int `mapstructure:"public_grpc_port"` - RESTPort int `mapstructure:"rest_port"` - ExportedMetricsHTTPPort int `mapstructure:"exported_metrics_http_port"` - Log config.Log `mapstructure:"log"` - Profiler config.Profiler `mapstructure:"profiler"` - PushMetrics bool `mapstructure:"push_metrics"` - SkipAuth bool `mapstructure:"skip_auth"` - SyncTasks bool `mapstructure:"sync_tasks,local_only"` - LocalReplication bool `mapstructure:"local_replication,local_only"` - DisableDataplane bool `mapstructure:"disable_dataplane"` - DataplaneConfigVariableName string `mapstructure:"data_plane_config_variable_name"` - DataplaneConfigOptionValue string `mapstructure:"data_plane_config_variable_value"` - DataplaneVersion config.DataplaneVersion `mapstructure:"dataplane_version"` - DataplaneK8SImage config.Secret `mapstructure:"dataplane_image_k8s_image"` - Environment config.Environment `mapstructure:"environment"` - HealthProviders []config.HealthProvider `mapstructure:"health_providers"` - MetricsProvider config.MetricsProvider `mapstructure:"metrics_provider"` - Search config.Search `mapstructure:"search"` - Events config.Events `mapstructure:"events"` -} - -type CloudConfig struct { - CloudCreds config.CloudCreds `mapstructure:"cloud_creds"` - ConsoleHostname string `mapstructure:"console_hostname"` - YdbLogsInstance string `mapstructure:"ydb_logs_instance"` - YdbLogsDatabase string `mapstructure:"ydb_logs_database"` - RootCACertPaths []string `mapstructure:"root_cert_paths"` - YDBDataplaneLogDir string `mapstructure:"ydb_dataplane_log_dir"` - MDBAPIURLBase string `mapstructure:"mdb_api_url_base"` - IAMAPIURLBase string `mapstructure:"iam_api_url_base"` - CloudAPIEndpoint string `mapstructure:"cloud_api_endpoint"` - IAMAccessServiceEndpoint string `mapstructure:"iam_access_service_endpoint"` - LogSourceID string `mapstructure:"log_source_id,local_only"` -} - -type ExternalCloudConfig struct { - SolomonURL string `mapstructure:"solomon_url"` - ServiceAccountID string `mapstructure:"service_account_id"` - ServiceCloudID string `mapstructure:"service_cloud_id"` - ServiceFolderID string `mapstructure:"service_folder_id"` - KMSKeyID string `mapstructure:"kms_key_id"` - KMSDiscoveryEndpoint string `mapstructure:"kms_discovery_endpoint"` - NetID string `mapstructure:"net_id"` - TransferDashboardID string `mapstructure:"transfer_dashboard_id"` - YDSServerlessEndpoint string `mapstructure:"yds_serverless_endpoint"` - S3Endpoint string `mapstructure:"s3_endpoint"` - IsProd bool `mapstructure:"is_prod"` - YavToken config.Secret `mapstructure:"yav_token,local_only"` - ServerlessRuntimeConfig config.Secret `mapstructure:"serverless_runtime_kubeconfig"` - DataPlaneVMUsers VMUsers `mapstructure:"data_plane_vm_users"` - DataPlaneComputePlatformID string `mapstructure:"data_plane_compute_platform_id"` - NTPServers string `mapstructure:"ntp_servers"` - FallbackNTP string `mapstructure:"fallback_ntp"` - MetrikaAPIConfig config.MetrikaAPIConfig `mapstructure:"metrika_api_config"` - FineGrainedSSA config.FineGrainedSSA `mapstructure:"fine_grained_ssa"` -} - -type InternalCloudConfig struct { - YdbToken config.Secret `mapstructure:"ydb_token"` - MDBToken config.Secret `mapstructure:"mdb_token"` - YavToken config.Secret `mapstructure:"yav_token"` - YtToken config.Secret `mapstructure:"yt_token"` - LogfellerYtToken config.Secret `mapstructure:"logfeller_yt_token"` - LogfellerNirvactorToken config.Secret `mapstructure:"logfeller_nirvactor_token"` - LogfellerLogbrokerToken config.Secret `mapstructure:"logfeller_logbroker_token"` - LogfellerLfDescriberLogbrokerToken config.Secret `mapstructure:"logfeller_lf_describer_logbroker_token"` - LogbrokerToken config.Secret `mapstructure:"logbroker_token"` - SolomonToken config.Secret `mapstructure:"solomon_token"` - TLSServerPrivateCertificate config.Secret `mapstructure:"tls_server_private_certificate"` - DataplaneToken config.Secret `mapstructure:"dataplane_token"` - SolomonCluster string `mapstructure:"solomon_cluster"` - GRPCSSLPort int `mapstructure:"grpc_ssl_port"` - PubGRPCSSLPort int `mapstructure:"public_grpc_ssl_port"` - YtProxy string `mapstructure:"yt_proxy"` - YtOperationPool string `mapstructure:"yt_operation_pool"` - UDFDir string `mapstructure:"udf_dir"` - TaskMonitorLockPath string `mapstructure:"task_monitor_lock_path"` - SolomonURL string `mapstructure:"solomon_url"` - SolomonProject string `mapstructure:"solomon_project"` - DisableJobBinCache bool `mapstructure:"disable_job_bin_cache,local_only"` - DisableSchemaRegistry bool `mapstructure:"disable_schema_registry,local_only"` - // NotificationsAPI is the address of the HTTP(s) API of the Notify service. Empty or invalid value redirects notifications to log. - NotificationsAPI string `mapstructure:"notifications_api"` -} - -type AWSConfig struct { - Ignore bool `mapstructure:"ignore,local_only"` - Profile string `mapstructure:"profile,local_only"` - KeyID string `mapstructure:"key_id"` - GCP GCPConfig `mapstructure:"gcp"` - AmiConfig AWSAmiConfig `mapstructure:"ami_config"` - Dataplanes []AWSDataplane `mapstructure:"dataplanes"` - CloudCreds config.CloudCreds `mapstructure:"cloud_creds"` - IAM IAMApis `mapstructure:"iam"` - MDBService string `mapstructure:"mdb_service"` - MDBServicePlaintext bool `mapstructure:"mdb_service_plaintext,local_only"` - MdbVpcService string `mapstructure:"mdb_vpc_service"` - DataplaneSA DataplaneSA `mapstructure:"dataplane_sa"` - DataplaneSpecS3Bucket string `mapstructure:"dataplane_spec_s3_bucket"` - AssumeRoleTarget string `mapstructure:"assume_role_target"` - // NotificationsAPI is the address of the HTTP(s) API of the Notify service. Empty or invalid value redirects notifications to log. - NotificationsAPI string `mapstructure:"notifications_api"` - - // AWS-managed Prometheus Service (i.e. AMP) - AMPRegion string `mapstructure:"amp_region"` - AMPURL string `mapstructure:"amp_url"` - AMPAssumeRoleTarget string `mapstructure:"amp_assume_role_target"` - - ClustersCPAccountID string `mapstructure:"clusters_cp_account_id"` - CPAccountID string `mapstructure:"cp_account_id"` - DPAccountID string `mapstructure:"dp_account_id"` - - Teleport config.Teleport `mapstructure:"teleport"` - - OverrideDPConfig string `mapstructure:"override_dp_config,local_only"` -} - -type GCPConfig struct { - ByoaCommonAccount string `mapstructure:"byoa_common_account"` - ProjectID string `mapstructure:"project_id"` - ImageID string `mapstructure:"image_id"` - ServiceAccount string `mapstructure:"service_account"` - - // Creds is json config for OAUTH2 authorization see here: https://cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds#go - // to generate this creds you must call gcloud-cli: - // gcloud iam workload-identity-pools create-cred-config \ - // projects/mdb-dp-preprod/locations/global/workloadIdentityPools/mdb-cp/providers/aws \ - // --service-account=main-dt-sa-preprod@mdb-dp-preprod.iam.gserviceaccount.com \ - // --service-account-token-lifetime-seconds=3600 \ - // --aws \ - // --output-file=gcp_creds.json - // this creds is not secret, it's just config. It's use aws auth to gain temporal secret value in runtime. - Creds string `mapstructure:"creds"` -} - -type DataplaneSA struct { - ID string `mapstructure:"id"` - FolderID string `mapstructure:"folder_id"` -} - -type IAMApis struct { - AccessService string `mapstructure:"access_service"` - TokenService string `mapstructure:"token_service"` - PrivateServices string `mapstructure:"private_service"` -} - -type AWSAmiConfig struct { - InstanceProfile string `mapstructure:"instance_profile"` - RegionToAmi map[string]string `mapstructure:"region_to_ami"` -} - -type AWSRegionToAmi struct { - Region string `mapstructure:"region"` - Ami string `mapstructure:"ami"` -} - -type AWSDataplane struct { - Secret config.Secret `mapstructure:"secret"` - Region string `mapstructure:"region"` - Name string `mapstructure:"name"` -} - -type VMUsers struct { - SSHKey string `mapstructure:"ssh_key"` - Users []string `mapstructure:"users"` -} - -func IsControlPlane() bool { - return Common != nil -} - -func IsCloud() bool { - return Cloud != nil -} - -func IsExternalCloud() bool { - return ExternalCloud != nil -} - -func IsInternalCloud() bool { - return InternalCloud != nil -} - -func IsAWS() bool { - return AWS != nil -} - -func GetAmiAndIamRole(region string) (ami, iamRole string, err error) { - if AWS == nil { - return ami, iamRole, xerrors.Errorf("AWS config is empty, check CP config.") - } - for reg, amiID := range AWS.AmiConfig.RegionToAmi { - if reg == region { - return amiID, AWS.AmiConfig.InstanceProfile, nil - } - } - return ami, iamRole, xerrors.Errorf("AMI image for region '%s' is not configured, check CP config.", region) -} - -func DBSchema() string { - if Common != nil { - return Common.DBSchema - } - return "" -} - -func LoadF(reader io.Reader, secretResolverFactory config.SecretResolverFactory) (processedLocalOnlyFields map[string]struct{}, err error) { - configReadClosers, err := config.PreprocessIncludes(reader, EmbeddedConfigs) - if err != nil { - return nil, xerrors.Errorf("Cannot preprocess include configs: %w", err) - } - - defer func() { - for _, configReadCloser := range configReadClosers { - _ = configReadCloser.ReadCloser.Close() - } - }() - - configReaders := make([]config.ReaderWithID, len(configReadClosers)) - for i, configReadCloser := range configReadClosers { - configReaders[i] = config.ReaderWithID{ID: configReadCloser.ID, Reader: configReadCloser.ReadCloser} - } - - processedLocalOnlyFields = map[string]struct{}{} - err = config.LoadBundle(configReaders, config.ConfigBundle{ - Common: &Common, - Cloud: &Cloud, - ExternalCloud: &ExternalCloud, - InternalCloud: &InternalCloud, - AWS: &AWS, - SecretResolverFactory: secretResolverFactory, - ProcessedLocalOnlyFields: processedLocalOnlyFields, - }) - return processedLocalOnlyFields, err -} - -func Load() error { - return config.Load(new(Loader)) -} - -func (c *CloudConfig) TLSConfig() (*tls.Config, error) { - return xtls.FromPath(c.RootCACertPaths) -} - -var ( - _ config.SecretResolverFactory = (*SecretResolverFactory)(nil) -) - -type SecretResolverFactory struct{} - -func (SecretResolverFactory) NewAWSSecretManagerResolver(configBundle config.ConfigBundle) (config.AWSSecretManagerResolver, error) { - awsConfig := (*configBundle.AWS.(**AWSConfig)) - return config.NewAWSSecretManagerResolver(awsConfig.Profile, "") -} - -func (SecretResolverFactory) NewLockboxResolver(configBundle config.ConfigBundle) (config.LockboxSecretResolver, error) { - creds := (*configBundle.Cloud.(**CloudConfig)).CloudCreds - yandexCloudEndpoint := (*configBundle.Cloud.(**CloudConfig)).CloudAPIEndpoint - return config.NewLockboxResolver(creds, yandexCloudEndpoint) -} - -func (f SecretResolverFactory) NewAWSResolver(configBundle config.ConfigBundle) (config.AWSSecretResolver, error) { - awsConfig := *configBundle.AWS.(**AWSConfig) - if awsConfig == nil { - return nil, xerrors.New("aws config is not set") - } - - keyID := awsConfig.KeyID - if keyID == "" { - return nil, xerrors.New("key_id not given") - } - - return config.NewAWSResolver(keyID, awsConfig.Profile, "") -} - -func (SecretResolverFactory) NewYavResolver(configBundle config.ConfigBundle) (config.YavSecretResolver, error) { - var yavToken string - if *configBundle.InternalCloud.(**InternalCloudConfig) != nil { - yavToken = string((*configBundle.InternalCloud.(**InternalCloudConfig)).YavToken) - } else if *configBundle.ExternalCloud.(**ExternalCloudConfig) != nil { - yavToken = string((*configBundle.ExternalCloud.(**ExternalCloudConfig)).YavToken) - } - if yavToken == "" { - return nil, xerrors.New("yav_token not given") - } - return config.NewYavResolver(yavToken) -} - -func (SecretResolverFactory) NewKMSResolver(configBundle config.ConfigBundle) (config.KMSSecretResolver, error) { - creds := (*configBundle.Cloud.(**CloudConfig)).CloudCreds - yandexCloudEndpoint := (*configBundle.Cloud.(**CloudConfig)).CloudAPIEndpoint - kmsKeyID := (*configBundle.ExternalCloud.(**ExternalCloudConfig)).KMSKeyID - kmsDiscoveryEndpoint := (*configBundle.ExternalCloud.(**ExternalCloudConfig)).KMSDiscoveryEndpoint - return config.NewKMSResolver(creds, yandexCloudEndpoint, kmsKeyID, kmsDiscoveryEndpoint) -} - -func (SecretResolverFactory) NewMetadataResolver(configBundle config.ConfigBundle) (config.MetadataSecretResolver, error) { - return config.NewMetadataResolver() -} diff --git a/pkg/config/controlplane/config_test.go b/pkg/config/controlplane/config_test.go deleted file mode 100644 index 4fa81ac9..00000000 --- a/pkg/config/controlplane/config_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package cpconfig - -import ( - "bytes" - _ "embed" - "os" - "strings" - "testing" - "time" - - "github.com/doublecloud/tross/pkg/config" - "github.com/stretchr/testify/require" -) - -var ( - //go:embed devconfigs/external_preprod.yaml - devExternalPreprodConfig []byte - - //go:embed devconfigs/external_prod.yaml - devExternalProdConfig []byte - - //go:embed devconfigs/internal.yaml - devInternalConfig []byte - - //go:embed devconfigs/internal_minimal.yaml - devInternalMinimalConfig []byte - - //go:embed devconfigs/ovandriyanov/external/preprod.yaml - devOvandriyanovExternalPreprod []byte - - //go:embed devconfigs/ovandriyanov/internal/prod.yaml - devOvandriyanovInternalProd []byte - - //go:embed devconfigs/ovandriyanov/internal/testing.yaml - devOvandriyanovInternalTesting []byte - - devConfigs = map[string][]byte{ - "external_preprod.yaml": devExternalPreprodConfig, - "external_prod.yaml": devExternalProdConfig, - "internal.yaml": devInternalConfig, - "internal_minimal.yaml": devInternalMinimalConfig, - "ovandriyanov/external/preprod.yaml": devOvandriyanovExternalPreprod, - "ovandriyanov/internal/prod.yaml": devOvandriyanovInternalProd, - "ovandriyanov/internal/testing.yaml": devOvandriyanovInternalTesting, - } -) - -func clearConfigs() { - Common = nil - Cloud = nil - InternalCloud = nil - ExternalCloud = nil - AWS = nil -} - -func init() { - config.UserData = map[string]interface{}{ - "dataplane_image_id": "x", - } -} - -func TestInstallationsConfigsSanity(t *testing.T) { - _ = os.Setenv("DATA_PLANE_K8S_IMAGE", "x") - for configName, configContent := range EmbeddedConfigs { - t.Run(configName, func(t *testing.T) { - if strings.HasPrefix(configName, "parts/") { - t.Skip() - return - } - configReader := bytes.NewReader(configContent) - if configName == "internal_prod.yaml" || configName == "internal_testing.yaml" { - _ = os.Setenv("YAV_TOKEN", "x") - } - processedLocalOnlyFields, err := LoadF(configReader, config.MockSecretResolverFactory{T: t}) - _ = os.Unsetenv("YAV_TOKEN") - require.NoErrorf(t, err, "Error occurred while checking sanity of installations/%s config file", configName) - require.Emptyf(t, processedLocalOnlyFields, "No local-only fields must be specified in application configuration") - clearConfigs() - }) - } - _ = os.Setenv("YAV_TOKEN", "x") - // For devconfigs/internal_minimal - _ = os.Setenv("YC_TOKEN", "x") - defer os.Unsetenv("YC_TOKEN") - _ = os.Setenv("CP_DB_PASSWORD", "x") - defer os.Unsetenv("CP_DB_PASSWORD") - for configName, configContent := range devConfigs { - t.Run("devconfig/"+configName, func(t *testing.T) { - _, err := LoadF(bytes.NewReader(configContent), config.MockSecretResolverFactory{T: t}) - require.NoErrorf(t, err, "Error occurred while checking sanity of devconfigs/%s config file", configName) - clearConfigs() - }) - } - _ = os.Unsetenv("YAV_TOKEN") - _ = os.Unsetenv("DATA_PLANE_K8S_IMAGE") -} - -func TestComputeCertificate(t *testing.T) { - certificate, err := config.ParseCertificate("-----BEGIN CERTIFICATE-----\nMIIC4TCCAcmgAwIBAgIUP0zcGO1MeRwze8VdSMEt/OdBXoIwDQYJKoZIhvcNAQEL\nBQAwADAeFw0yMzA2MDcwNjU4MTBaFw0zMzA2MDQwNjU4MTBaMAAwggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDw6TvAvrbJvY4tzIIuDnLEVfRUW4BZJD3y\nK8fyyxXrYDvC69RKCKk9+TQhnUOLhZNlDST4HFfSPlakOjXUduyJE5M1EmoLAstN\n81aP3TejseDavxmaNijXRsa9E731T5H+zo44PgAHfQJmiD7rtcr+QOIosKUB2dwp\nF2acp9hLKd389BfNctziG0Oxq7hlISTDBnhzBg7eKuqWtShjVW5RqQvp3bARfUPa\nRWdYjmZvR+AnmozV1SGnpAnatzhnF6tNAb5XSEw49tumsX1D4A11J6mtrafO6bsP\nwdIPwy9W15iCszUNlFcdBaZhESc34VbyCyLMvA5T0Uj1FJHz1RFlAgMBAAGjUzBR\nMB0GA1UdDgQWBBQq0z6Vcmjcn8wnRTwKGSm5YGas9TAfBgNVHSMEGDAWgBQq0z6V\ncmjcn8wnRTwKGSm5YGas9TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\nA4IBAQBplippQ/Pxn7AkuwOTSwSTeJ7S+rMSb6iSL9chNHetanft0Ikr5BDsSrd6\nTeHV0sEMilDIjX0EjSNHwYtYrDPk6cGjkzDTYb6/U10c5Xhwi0g7/lMH/RPihPz5\nco80VEqXWlgfgHuE7/cAiTJ61PiFD9oI494bQcIISQNDfbUUiYfn32+8nK20rn8C\nw7PbGoIv6zz6A0c6DJT7yXJF5sAHgX4M03Oi9edzQ077ZOboXSuUKe4VfHIpjTjZ\n0sM/NbG5BFstyetVc3FZOGWGukTRb0C0GSASOm6hCyh5ctmpwlS4menc/OAx9BYO\nr9ZBjEa0oLFVV0pP5Tj4Gf1DDpuJ\n-----END CERTIFICATE-----\n") - require.NoError(t, err) - require.True(t, certificate.NotBefore.Before(time.Now())) - twoMonths := 2 * 30 * 24 * time.Hour - require.True(t, certificate.NotAfter.After(time.Now().Add(twoMonths))) -} diff --git a/pkg/config/controlplane/devconfigs/external_preprod.yaml b/pkg/config/controlplane/devconfigs/external_preprod.yaml deleted file mode 100644 index 2ea8bac3..00000000 --- a/pkg/config/controlplane/devconfigs/external_preprod.yaml +++ /dev/null @@ -1,153 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: e4uo49d89n34h1o6jkl6 -db_user: db_user -db_password: - type: yav - secret_id: sec-01f7txh34cez2shwbgyg89z3a2 - version: ver-01f7tyyf761hck4nsksz4402n6 - key: db_password_preprod -db_name: db -db_schema: dev -grpc_port: 8787 -disable_dataplane: false -public_grpc_port: 9797 -rest_port: 8088 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: stdout -push_metrics: false -sync_tasks: true -skip_auth: false -local_replication: true -dataplane_version: - type: literal - value: x -health_providers: - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: yc_profile - profile_name: tm-preprod -console_hostname: console-preprod.cloud.yandex.ru -ydb_logs_instance: lb.cc803hd4j8tmhgidh880.ydb.mdb.cloud-preprod.yandex.net:2135 -ydb_logs_database: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880 -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt - -ydb_dataplane_log_dir: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880/log/dataplane -mdb_api_url_base: https://mdb.api.cloud-preprod.yandex.net/ -iam_api_url_base: https://iam.api.cloud-preprod.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud-preprod.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud-preprod.yandex.net:4286 -events: - type: disabled -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXlNVEUxTWpBME5Wb1hEVE14TURZeE9URTFNakEwTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEI4CjJTQ29aZlU3aGlCbWZjTkpEVTl6TnhCbnZoM1ZIYldLVTdpaHg4L2RhaC9ueEpMTkNDQlcwUjNjd3FQYWtRZlYKeU1rQVlHRHBlbUkwbVdZMWhKV3lsUlBocjVaSm1ucXJqZ0ZvdGF1WEU3SEp1RGpXTVA1NGFha250NmFjUDQ4ZwpSMkp4cklaZG9FUmtSTEtBeUxOckJtelZQR1liQ3VscEhORTc0b09XMTU5RDhGZUhhRlU4NlAxblZPQ3Nsb2pqCmttZ2JUL1JkNk9NY1EvN2Y0UnVXcGdVSk9WZlZ0b1BvYWxYVnFNQS94UU1pSXo0b3NMNGFYeHhZd1BpWC9aK0oKdlRNazFMLzN1Nm5XcldtUmVvdXU5bkhrdlhuS1NMUzF0Q0VQZllha21tRi80RWI0c0NveVdxQUI1SmZUYVQ0NApUVGlmTUdidGtDQi85Q1o1MDNjQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJNG9Ib1ZzNkp4NElXbkVjVFdxa2QzT2ZpTGhNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBdGVXSUhucnY0dEZHRzhNakNXTk5mbzBSZjExNTdpdGtNTU9KdXpXWUhvb04xeDlkbApXREtKU2pLdXZrNmF4YnlTZHR0UTdJYllFSXNNT0Y4TXZPUzVKamQzVEhKdE9FQjhiYyt5MVRsR3VUdkhFcFdNCm1nU0dwd2FwU3ZUWElzSE1VZUdIVEw4WmYwMDV5eU5sR1hoaXM3OWpZd01SMFcybGxRZFdKMnNyN0ZnNkVxcXIKTHJUamZadWY2QkVObGtXaHNLdGhIRjc4dVlNdlUrSWp2NzM2REVjVGJ2azhVVjZrMmZFdUlXRE5weEswbW11bworK2hweE9pZlFleVRoTnB3OXJ1SEFEeXN0dVRxMm8rU0JsN216YUpxU090ek9DbVppS1M0QTNtT2pxY1JBNFVrCllzY0xZckM4cGNmS2FXTGVQSms4L3F5NkhFaW5JNWZVbVdGNQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://130.193.32.8 - name: yc-managed-k8s-c49dvuuspe14vq0553ve - contexts: - - context: - cluster: yc-managed-k8s-c49dvuuspe14vq0553ve - user: yc-managed-k8s-c49dvuuspe14vq0553ve - name: yc-prestable - current-context: yc-prestable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-c49dvuuspe14vq0553ve - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - - --profile=tm-preprod - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud-preprod.yandex-team.ru -service_account_id: bfbgnr9a209qmtjepb55 -service_cloud_id: aoerc1it1q975rso0ia8 -service_folder_id: aoenp6537r7gn90vb4al -kms_key_id: e10jg948rdlerghctsv3 -kms_discovery_endpoint: kms.cloud-preprod.yandex.net:8443 -net_id: c640ek4eqbrmoaf2t9u3 -transfer_dashboard_id: monkgqmtip9skkdv9d3l -yds_serverless_endpoint: lb.cc802lcdcldbsh7b5fd1.ydb.mdb.cloud-preprod.yandex.net -s3_endpoint: https://storage.cloud-preprod.yandex.net -is_prod: false -yav_token: - type: environment_variable - variable_name: YAV_TOKEN # Get your token at https://oauth.yandex-team.ru/authorize?response_type=token&client_id=ce68fbebc76c4ffda974049083729982 - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_preprod.yaml - -dataplane_image_k8s_image: unused -data_plane_compute_platform_id: standard-v2 -metrika_api_config: - endpoint: internalapi.test.metrika.yandex.net - tvm_metrika_id: 2000269 - tvm_self_id: 2038887 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABCD9bsphJMKF3bjK7X5DiHBAAAADF/mVtsY/aEm3mBEaR5ApOQf9QU7ZdIrfEae6PgfAQNaFRyK1fGAAcVPxueHUC8F6FYObg== -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud-preprod.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud-preprod.yandex.net:9051 - diff --git a/pkg/config/controlplane/devconfigs/external_prod.yaml b/pkg/config/controlplane/devconfigs/external_prod.yaml deleted file mode 100644 index 76d75223..00000000 --- a/pkg/config/controlplane/devconfigs/external_prod.yaml +++ /dev/null @@ -1,151 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: c9qnb04en018asm4bktv -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRhYmo4bHBnMGhwYjRyOWo5cDlkbgAAABCK41PbtP/ehs93biF8QQUtAAAADM04Je4w - phoys2Ii/SUY+yiWSPdL0ut3ChNraPeojPrQ3GWaCQ== -db_name: db -db_schema: dev -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: stdout -skip_auth: true -push_metrics: true -dataplane_version: - type: literal - value: x -health_providers: - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: yc_profile - profile_name: tm-prod -console_hostname: console.cloud.yandex.ru -ydb_logs_instance: lb.etn01pp55otetd3u805d.ydb.mdb.yandexcloud.net:2135 -ydb_logs_database: /global/b1g9jhnga9ji23ii4dks/etn01pp55otetd3u805d -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /global/b1g9jhnga9ji23ii4dks/etn01pp55otetd3u805d/log/dataplane -mdb_api_url_base: https://mdb.api.cloud.yandex.net/ -iam_api_url_base: https://iam.api.cloud.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud.yandex.net:4286 -events: - type: disabled -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EY3dNVEV4TVRBek1Wb1hEVE14TURZeU9URXhNVEF6TVZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEZCCnhWNVVQaytKdmNRZ2lZY25tS3dlRDg1K0pDZ2NlMmZIbWsvaHV4L3ZhOFlMOUVtbXREajJwcTRrS3ZaVzEwcGgKMzdxSCtEQ2s2UzZOMSsvcDFvNThZK3ZPUmxWOGtuZFNnb3lHMk1IcWkyL1B1Z1FDVC9rSUgrRU1vZnBrYk8rawpZRmRyYlg3cG5ySmlzcFIrS2M5MHJ5RTVDMnhmN3RVNEFuYzl5VVBWS2g5eCtKMGNFdTZtdzlpVllWdGd6VUw4CklSV2w4ZXIzTldtOGdKV1RLeHpDd3B4NUwxWE1CRnQ1aEp1eVBRY1FYVmN3U1Q3MkNFN1lMU0hGMXJoRk4wYmMKZHFTbVZ2ZUZ4ODBnMUJ6R0JhMElXcS8xeHA1clR5TEpzNndDZHJsYnI2TlI0bHJoa0NqNXREOUhaQmV4Nkc0VQpMdW9QR1N3aVZYUjZDWWZHMGJFQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZONGdOOUVjWEt2RnFnK25LclNQcGcrL2Rzb0FNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBYW9CbUhZWG8wZWp6Y3k3K2tjK2VIaDR1bzlsY3lmWG44ak5UTHVBU0RMcHZ2ZkRwUwpud3V5aUlQSkF0azd1bk02Q2pVbFU4bFZGa1RCTll3SEJEdXZoWnMydTVYR2x0VmF2cU1Gc1pqQ0g3RFdWVlNqCkdpQmVtNHJXVHFHNkRwWUZWdVZOT0J6UkRmODJtNmxtRU1CYTVwVjZsMFV0NmV6OXBzYndmYVJtcGlpdWNNQVEKYXJvU0tWUVVtbi9CMmIvbDJTaEFNSlhmdEdXbWpPc29DYVhVVjlhdmpNTHl0NkRXMzhNbmVQSDkvTFJJdVFxRAp2VUpwbU1PVkprcVEwR0VuNndlL2s3SlZSaTdsM2tOcXUrN2lCai81dTYrNTY0bVJZUXp5akdiNklSMDc3QmF2CjRpbzgwUzlnQzI3c2ZQMHhISE5sdkd3TWZYNEoveklqVy8zLwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://217.28.229.218 - name: yc-managed-k8s-catddso5crfcum2pkkdn - contexts: - - context: - cluster: yc-managed-k8s-catddso5crfcum2pkkdn - user: yc-managed-k8s-catddso5crfcum2pkkdn - name: yc-stable - current-context: yc-stable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-catddso5crfcum2pkkdn - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud.yandex-team.ru -service_account_id: ajet5egu65m66slf9tn0 -service_cloud_id: b1g9jhnga9ji23ii4dks -service_folder_id: b1gds2039chq7kv8pmvv -kms_key_id: abjvp62i676fpehfd3vj -kms_discovery_endpoint: kms.yandex:8443 -net_id: enpl4ftghn8vt5oad2a7 -transfer_dashboard_id: monr9n46l9kjjllugubn -yds_serverless_endpoint: yds.serverless.cloud.yandex.net -s3_endpoint: https://storage.yandexcloud.net -is_prod: true - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh - -yav_token: - type: environment_variable - variable_name: YAV_TOKEN - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_prod.yaml - -dataplane_image_k8s_image: - type: environment_variable - variable_name: DATA_PLANE_K8S_IMAGE -data_plane_compute_platform_id: standard-v2 -metrika_api_config: - endpoint: internalapi.metrika.yandex.net - tvm_metrika_id: 2000270 - tvm_self_id: 2038889 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRhYmo4bHBnMGhwYjRyOWo5cDlkbgAAABAoofCTpx29gJknk38dGA8MAAAADAFr6+HCQyVDnIj6KelYHW8KSuSn3V3EGeO6hzK2QUnGFjfmBZcWxZjqV9HgpzBAdMng2w== -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud.yandex.net:9051 diff --git a/pkg/config/controlplane/devconfigs/internal.yaml b/pkg/config/controlplane/devconfigs/internal.yaml deleted file mode 100644 index 9bed122d..00000000 --- a/pkg/config/controlplane/devconfigs/internal.yaml +++ /dev/null @@ -1,160 +0,0 @@ -type: internal_cloud - -# Common -environment: - type: internal -db_discovery: - type: mdb - cluster_id: mdblc14b5vs4onea1hfi -db_user: db_user -db_password: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f7bvkkt4cgckdd6t72wmerdf - key: db_password -db_name: db -db_schema: dev -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 8088 -exported_metrics_http_port: 9092 -log: - type: stdout -profiler: - type: logbroker - endpoint: logbroker.yandex.net - creds: - type: oauth_token - token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb - database: '' - topic: cdc/prod/profiler - tls_mode: - type: disabled - use_topic_api: false -push_metrics: false -sync_tasks: true -skip_auth: false -local_replication: true -dataplane_version: - type: arcadia_source_revision -health_providers: - - type: solomon - alert_id: data-transfer_testing_dta_01_dataplane_is_alive - name: dta_01_dataplane_is_alive - - type: solomon - alert_id: data-transfer_testing_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: data-transfer_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: data-transfer_testing_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: data-transfer_testing_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message -metrics_provider: - type: solomon - -# Cloud -solomon_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01eewths5arkqh0k8d0pnan8vx - key: solomon -logbroker_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb -cloud_creds: - type: passport_oauth_token - token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -console_hostname: yc-test.yandex-team.ru -ydb_logs_instance: ydb-ru-prestable.yandex.net:2135 -ydb_logs_database: /ru-prestable/home/tserakhau/mydb -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /ru-prestable/home/tserakhau/mydb/log/dataplane -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru:443/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -events: - type: disabled -search: - type: disabled - -# Internal cloud -ydb_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01e5wrf991geg3h02pe4n2fcv4 - key: ydb -mdb_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -yav_token: - type: environment_variable - variable_name: YAV_TOKEN # Get your token at https://oauth.yandex-team.ru/authorize?response_type=token&client_id=ce68fbebc76c4ffda974049083729982 -yt_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f8t8hy3699hrb2050s680gbt - key: yt -logfeller_yt_token: - type: yav - secret_id: sec-01e4xz5vc007whwy8pvax96bz2 - version: ver-01fxsne874kxnvaj19y2x6be32 - key: nirvana-secret -logfeller_nirvactor_token: - type: yav - secret_id: sec-01e4xzjqs0a61e3y576ek5c7zw - version: ver-01fxstmtn3mvyyamy050xfkbg5 - key: nirvana-secret -logfeller_logbroker_token: - type: yav - secret_id: sec-01gdn7wkcgc3awnvvtxcpk7rkw - version: ver-01gdn7wkcse36m7xk4xsr99czv - key: logbroker-token -logfeller_lf_describer_logbroker_token: - type: yav - secret_id: sec-01h4k2mwcva6xq9w5p90kkwnpc - version: ver-01h52qv834xj62dvmbfsx1d3f6 - key: logbroker-oauth-token -tls_server_private_certificate: - type: yav - secret_id: sec-01eb0yxja1r9p8zj9qmxzd6b5d - version: ver-01eca79g4xg1js9z0wcnhn70qr - key: b26qyhko6jcler4s.man.yp-c.yandex.net.pem -dataplane_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -solomon_cluster: rtc-local -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: internal_testing.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/internal_prod_dev -notifications_api: '' -dataplane_image_k8s_image: '' -disable_job_bin_cache: false -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer diff --git a/pkg/config/controlplane/devconfigs/internal_minimal.yaml b/pkg/config/controlplane/devconfigs/internal_minimal.yaml deleted file mode 100644 index fa2815f4..00000000 --- a/pkg/config/controlplane/devconfigs/internal_minimal.yaml +++ /dev/null @@ -1,136 +0,0 @@ -type: internal_cloud - -environment: - type: internal - -# === Not about secrets -# You may store various secrets used in config in YAV -# To use such secrets you need to set yav token. -# Get it here https://oauth.yandex-team.ru/authorize?response_type=token&client_id=ce68fbebc76c4ffda974049083729982 -yav_token: - type: empty - -# === Control plane postgresql DB connection settings -# Control plane uses postgresql to store transfers, endpoints, etc -# You may use any PG DB (including localhost/docker/etc), you need just empty schema -# Control plane creates all needed tables and other entities on start -# With default settings you should pass DB password via CP_DB_PASSWORD environment variable -db_discovery: - type: mdb # this is the example for using MDB cluster - cluster_id: mdbd894fa7kkpk84vft3 # replace the cluster ID with your own -# To specify host/port connection use the following lines as an example: -# type: on_premise -# host: localhost -# port: 5432 -# use_tls: false -db_user: user1 -db_password: - type: environment_variable - variable_name: CP_DB_PASSWORD -db_name: db1 -db_schema: dev # Specify empty postgres schema, all tables are created automatically - -# === Where to run dataplane processes -# The settings are extremely useful for local debug -# Works only if the binary is build with -DDATA_TRANSFER_ALL_IN_ONE_BINARY flag -sync_tasks: true # if enabled, all tasks including activation/deactivation run inside the controlplane process using local runtime -local_replication: true # if enabled, replication is run inside the controlplane process using local runtime -dataplane_version: - type: arcadia_source_revision - -# Y.Cloud integration -# Get YC token here # Get YC token here https://oauth.yandex-team.ru/authorize?response_type=token&client_id=8cdb2f6a0dca48398c6880312ee2f78d -cloud_creds: - type: passport_oauth_token - token: - type: environment_variable - variable_name: YC_TOKEN -# The same as above, but used specifically for MDB requests -mdb_token: - type: environment_variable - variable_name: YC_TOKEN -skip_auth: false # You may disable auth for all incoming API requests, but you probably don't need it -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru:443/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -console_hostname: yc-test.yandex-team.ru -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -dataplane_token: - type: environment_variable - variable_name: YC_TOKEN - -# allow to run background data planes -disable_dataplane: false -# === Various network ports -# This port is used for private API and thus is used by dataplane instances and frontend farms. -# You may find it useful to send requests to this port via grpcurl -grpc_port: 8787 -rest_port: 8088 # Use this port to access Swagger UI. It's http without ssl -public_grpc_port: 9797 -exported_metrics_http_port: 9092 -# TLS services is usually useless in dev and thus are disabled in this config -tls_server_private_certificate: - type: empty -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 - -# === Logs-related settings -log: - type: stdout -logbroker_token: # You need this token if you are willing to send anything (logs/metering/search items/etc) to LB topics. - type: empty -profiler: # Enable to send profiling samples to LB topic (useless for local environment) - type: disabled -# YDB is used to store dataplane logs to later show it in UI -# But you need to set up a transfer to deliver logs from LB topic to YDB -ydb_token: - type: empty -ydb_logs_instance: ydb-ru-prestable.yandex.net:2135 -ydb_logs_database: /ru-prestable/home/tserakhau/mydb -ydb_dataplane_log_dir: /ru-prestable/home/tserakhau/mydb/log/dataplane - -# === Monitoring metrics settings -push_metrics: false # Enable to send metrics to solomon. Be sure to specify correct project/cluster/token -solomon_token: - type: empty -solomon_cluster: rtc-local -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer - -# Various services -health_providers: # Provide info about transfer health - - type: fake -metrics_provider: - type: fake -events: # Send cloud events to LB topic - type: disabled -search: # Send updates to search LB topic - type: disabled -notifications_api: '' - -# Touch this section if you decide to run dataplane in YT (to be documented later) -yt_token: - type: empty -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: internal_testing.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/internal_prod_dev -# For app built in single binary mode upload job binary each time even if arc revision hasn't changed -disable_job_bin_cache: false - -# Logfeller integration (не надо вскрывать эту тему) -logfeller_yt_token: - type: empty -logfeller_nirvactor_token: - type: empty -logfeller_logbroker_token: - type: empty -logfeller_lf_describer_logbroker_token: - type: empty - -# Not used actually -dataplane_image_k8s_image: '' diff --git a/pkg/config/controlplane/devconfigs/ovandriyanov/external/preprod.yaml b/pkg/config/controlplane/devconfigs/ovandriyanov/external/preprod.yaml deleted file mode 100644 index 350c7bf2..00000000 --- a/pkg/config/controlplane/devconfigs/ovandriyanov/external/preprod.yaml +++ /dev/null @@ -1,166 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: e4uo49d89n34h1o6jkl6 -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABAHOBaAcedZGEWWF1kJtFpWAAAADLRGMybs - gZXMIjItAnMOVkFJgXi1TaGXz25DuFVH7B2sFQ1PmA== -db_name: db -db_schema: ovandriyanov1 -grpc_port: 9876 -public_grpc_port: 9877 -rest_port: 8088 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: logbroker - endpoint: lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net - creds: - type: use_cloud_creds - database: /pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3 - topic: /aoerc1it1q975rso0ia8/data-transfer/control-plane/log - tls_mode: - type: enabled - ca_certificate_paths: - - /etc/ssl/certs/YandexInternalCA.pem - use_topic_api: false -skip_auth: false -push_metrics: false -local_replication: false -sync_tasks: false -disable_dataplane: false -dataplane_version: - type: literal - value: fdv6gi43s06s5vffrf0f # dataplane-r9365693-b207 -health_providers: - - type: solomon - alert_id: dt_alert_combined - name: dt_alert_combined - - type: status_message -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: service_account_key - id: bfb1t32ubmouil47690i - service_account_id: bfbgnr9a209qmtjepb55 - key_algorithm: RSA_2048 - private_key: - type: yav - secret_id: sec-01f7rct6h4zm2m22kcas8k7zxg - version: ver-01f7rct6hk6xsenwnzjxympvwy - key: preprod - public_key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoaEfA9epQ0AN4xOkhxok - s7TexwJ3diDuD54A9Z1zJRI55/yLFcd0Eb9V2MC/MEfRRoO/UJovXyW7X9fufuYJ - IWWsHm0TtkQ6EINE6MnKTtaUOBhlczq3kRbn+aU0svL6sqxe2Wxejus8ExKtS9o6 - OPEE5W7U4qGdLr8RhsGUFq5WyVrU/6DYN5HYTZV4JR3zyVQP6ZQIYyPg+F8ebwyA - yq+3420Db6LK2LsQEo/ui5huYHFdBqwE2P7g/gW9mBM2NcE9rA689TnRlhpK+hGp - tVzRSrtqN26mRCdVsVgDcEKKbrdtw6L7YmwfeUaOYt7RNPh2CnopH22Lw55t+6LH - uQIDAQAB - -----END PUBLIC KEY----- -console_hostname: console-preprod.cloud.yandex.ru -ydb_logs_instance: lb.cc803hd4j8tmhgidh880.ydb.mdb.cloud-preprod.yandex.net:2135 -ydb_logs_database: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880 -root_cert_paths: - - /etc/ssl/certs/YandexInternalCA.pem -ydb_dataplane_log_dir: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880/log/dataplane -mdb_api_url_base: https://mdb.api.cloud-preprod.yandex.net/ -iam_api_url_base: https://iam.api.cloud-preprod.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud-preprod.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud-preprod.yandex.net:4286 -events: - type: disabled -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXlNVEUxTWpBME5Wb1hEVE14TURZeE9URTFNakEwTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEI4CjJTQ29aZlU3aGlCbWZjTkpEVTl6TnhCbnZoM1ZIYldLVTdpaHg4L2RhaC9ueEpMTkNDQlcwUjNjd3FQYWtRZlYKeU1rQVlHRHBlbUkwbVdZMWhKV3lsUlBocjVaSm1ucXJqZ0ZvdGF1WEU3SEp1RGpXTVA1NGFha250NmFjUDQ4ZwpSMkp4cklaZG9FUmtSTEtBeUxOckJtelZQR1liQ3VscEhORTc0b09XMTU5RDhGZUhhRlU4NlAxblZPQ3Nsb2pqCmttZ2JUL1JkNk9NY1EvN2Y0UnVXcGdVSk9WZlZ0b1BvYWxYVnFNQS94UU1pSXo0b3NMNGFYeHhZd1BpWC9aK0oKdlRNazFMLzN1Nm5XcldtUmVvdXU5bkhrdlhuS1NMUzF0Q0VQZllha21tRi80RWI0c0NveVdxQUI1SmZUYVQ0NApUVGlmTUdidGtDQi85Q1o1MDNjQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJNG9Ib1ZzNkp4NElXbkVjVFdxa2QzT2ZpTGhNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBdGVXSUhucnY0dEZHRzhNakNXTk5mbzBSZjExNTdpdGtNTU9KdXpXWUhvb04xeDlkbApXREtKU2pLdXZrNmF4YnlTZHR0UTdJYllFSXNNT0Y4TXZPUzVKamQzVEhKdE9FQjhiYyt5MVRsR3VUdkhFcFdNCm1nU0dwd2FwU3ZUWElzSE1VZUdIVEw4WmYwMDV5eU5sR1hoaXM3OWpZd01SMFcybGxRZFdKMnNyN0ZnNkVxcXIKTHJUamZadWY2QkVObGtXaHNLdGhIRjc4dVlNdlUrSWp2NzM2REVjVGJ2azhVVjZrMmZFdUlXRE5weEswbW11bworK2hweE9pZlFleVRoTnB3OXJ1SEFEeXN0dVRxMm8rU0JsN216YUpxU090ek9DbVppS1M0QTNtT2pxY1JBNFVrCllzY0xZckM4cGNmS2FXTGVQSms4L3F5NkhFaW5JNWZVbVdGNQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://130.193.32.8 - name: yc-managed-k8s-c49dvuuspe14vq0553ve - contexts: - - context: - cluster: yc-managed-k8s-c49dvuuspe14vq0553ve - user: yc-managed-k8s-c49dvuuspe14vq0553ve - name: yc-prestable - current-context: yc-prestable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-c49dvuuspe14vq0553ve - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud-preprod.yandex-team.ru -service_account_id: bfbgnr9a209qmtjepb55 -service_cloud_id: aoerc1it1q975rso0ia8 -service_folder_id: aoenp6537r7gn90vb4al -kms_key_id: e10jg948rdlerghctsv3 -kms_discovery_endpoint: kms.cloud-preprod.yandex.net:8443 -net_id: c640ek4eqbrmoaf2t9u3 -transfer_dashboard_id: monkgqmtip9skkdv9d3l -yds_serverless_endpoint: yds.serverless.cloud-preprod.yandex.net -s3_endpoint: https://storage.cloud-preprod.yandex.net -is_prod: false - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_prod.yaml -dataplane_image_k8s_image: i don't give a crap about k8s -yav_token: - type: environment_variable - variable_name: YAV_TOKEN - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh -data_plane_compute_platform_id: standard-v2 -metrika_api_config: - endpoint: internalapi.test.metrika.yandex.net - tvm_metrika_id: 2000269 - tvm_self_id: 2038887 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABCD9bsphJMKF3bjK7X5DiHBAAAADF/mVtsY/aEm3mBEaR5ApOQf9QU7ZdIrfEae6PgfAQNaFRyK1fGAAcVPxueHUC8F6FYObg== -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud-preprod.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud-preprod.yandex.net:9051 diff --git a/pkg/config/controlplane/devconfigs/ovandriyanov/internal/prod.yaml b/pkg/config/controlplane/devconfigs/ovandriyanov/internal/prod.yaml deleted file mode 100644 index 8cab75c6..00000000 --- a/pkg/config/controlplane/devconfigs/ovandriyanov/internal/prod.yaml +++ /dev/null @@ -1,135 +0,0 @@ -type: internal_cloud - -# Common -environment: - type: internal -db_discovery: - type: mdb - cluster_id: mdblc14b5vs4onea1hfi -db_user: db_user -db_password: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f7bvkkt4cgckdd6t72wmerdf - key: db_password -db_name: db -db_schema: ovandriyanov -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 8088 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: stdout -skip_auth: false -push_metrics: false -local_replication: false -sync_tasks: false -disable_dataplane: false -dataplane_version: - type: arcadia_source_revision -health_providers: - - type: solomon - alert_id: dt_alert_combined - name: dt_alert_combined - - type: status_message -metrics_provider: - type: solomon - -# Cloud -solomon_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01eewths5arkqh0k8d0pnan8vx - key: solomon -logbroker_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb -cloud_creds: - type: passport_oauth_token - token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -console_hostname: yc.yandex-team.ru -ydb_logs_instance: ydb-ru.yandex.net:2135 -ydb_logs_database: /ru/transfer_manager/prod/data-transfer-yt -root_cert_paths: - - /etc/ssl/certs/YandexInternalCA.pem -ydb_dataplane_log_dir: /ru/transfer_manager/prod/data-transfer-yt/log/dataplane -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru:443/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -events: - type: disabled -search: - type: disabled - -# Internal cloud -ydb_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01e5wrf991geg3h02pe4n2fcv4 - key: ydb -mdb_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -yav_token: - type: environment_variable - variable_name: YAV_TOKEN -yt_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f8t8hy3699hrb2050s680gbt - key: yt -logfeller_yt_token: - type: yav - secret_id: sec-01e4xz5vc007whwy8pvax96bz2 - version: ver-01fxsne874kxnvaj19y2x6be32 - key: nirvana-secret -logfeller_nirvactor_token: - type: yav - secret_id: sec-01e4xzjqs0a61e3y576ek5c7zw - version: ver-01fxstmtn3mvyyamy050xfkbg5 - key: nirvana-secret -logfeller_logbroker_token: - type: yav - secret_id: sec-01gdn7wkcgc3awnvvtxcpk7rkw - version: ver-01gdn7wkcse36m7xk4xsr99czv - key: logbroker-token -logfeller_lf_describer_logbroker_token: - type: yav - secret_id: sec-01h4k2mwcva6xq9w5p90kkwnpc - version: ver-01h52qv834xj62dvmbfsx1d3f6 - key: logbroker-oauth-token -tls_server_private_certificate: - type: yav - secret_id: sec-01eb0yxja1r9p8zj9qmxzd6b5d - version: ver-01eca79g4xg1js9z0wcnhn70qr - key: b26qyhko6jcler4s.man.yp-c.yandex.net.pem -dataplane_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -solomon_cluster: rtc-prod -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_URL -data_plane_config_variable_value: http://ovandriyanov.sas.yp-c.yandex.net/dataplane/devconfigs/ovandriyanov/internal/prod.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/ovandriyanov_internal_prod -notifications_api: '' -dataplane_image_k8s_image: '' -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer - diff --git a/pkg/config/controlplane/devconfigs/ovandriyanov/internal/testing.yaml b/pkg/config/controlplane/devconfigs/ovandriyanov/internal/testing.yaml deleted file mode 100644 index 30a428c8..00000000 --- a/pkg/config/controlplane/devconfigs/ovandriyanov/internal/testing.yaml +++ /dev/null @@ -1,130 +0,0 @@ -type: internal_cloud - -# Common -environment: - type: internal -db_discovery: - type: on_premise - host: localhost - port: 6002 - use_tls: false -db_user: postgres -db_password: '123' -db_name: postgres -db_schema: dev -grpc_port: 9876 -public_grpc_port: 9877 -rest_port: 8088 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: stdout -skip_auth: true -push_metrics: false -local_replication: false -sync_tasks: false -disable_dataplane: false -dataplane_version: - type: arcadia_source_revision -health_providers: - - type: fake -metrics_provider: - type: fake - -# Cloud -solomon_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01eewths5arkqh0k8d0pnan8vx - key: solomon -logbroker_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb -cloud_creds: - type: passport_oauth_token - token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -console_hostname: yc-test.yandex-team.ru -ydb_logs_instance: ydb-ru-prestable.yandex.net:2135 -ydb_logs_database: /ru-prestable/home/tserakhau/mydb -root_cert_paths: - - /etc/ssl/certs/YandexInternalCA.pem -ydb_dataplane_log_dir: /ru-prestable/home/tserakhau/mydb/log/dataplane -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru:443/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -events: - type: disabled -search: - type: disabled - -# Internal cloud -ydb_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01e5wrf991geg3h02pe4n2fcv4 - key: ydb -mdb_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -yav_token: - type: environment_variable - variable_name: YAV_TOKEN -yt_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f8t8hy3699hrb2050s680gbt - key: yt -logfeller_yt_token: - type: yav - secret_id: sec-01e4xz5vc007whwy8pvax96bz2 - version: ver-01fxsne874kxnvaj19y2x6be32 - key: nirvana-secret -logfeller_nirvactor_token: - type: yav - secret_id: sec-01e4xzjqs0a61e3y576ek5c7zw - version: ver-01fxstmtn3mvyyamy050xfkbg5 - key: nirvana-secret -logfeller_logbroker_token: - type: yav - secret_id: sec-01gdn7wkcgc3awnvvtxcpk7rkw - version: ver-01gdn7wkcse36m7xk4xsr99czv - key: logbroker-token -logfeller_lf_describer_logbroker_token: - type: yav - secret_id: sec-01h4k2mwcva6xq9w5p90kkwnpc - version: ver-01h52qv834xj62dvmbfsx1d3f6 - key: logbroker-oauth-token -tls_server_private_certificate: - type: yav - secret_id: sec-01eb0yxja1r9p8zj9qmxzd6b5d - version: ver-01eca79g4xg1js9z0wcnhn70qr - key: b26qyhko6jcler4s.man.yp-c.yandex.net.pem -dataplane_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -solomon_cluster: rtc-testing -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_URL -data_plane_config_variable_value: http://ovandriyanov.sas.yp-c.yandex.net/dataplane/devconfigs/ovandriyanov/internal/testing.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/ovandriyanov_internal_testing -notifications_api: '' -dataplane_image_k8s_image: '' -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer - diff --git a/pkg/config/controlplane/installations/external_kz.yaml b/pkg/config/controlplane/installations/external_kz.yaml deleted file mode 100644 index 5b5d3df0..00000000 --- a/pkg/config/controlplane/installations/external_kz.yaml +++ /dev/null @@ -1,130 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: a0d5t9ego1lhcuc3qf37 -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRhcGRub2dzZDQ4ZTNsOWQ1aDR2cQAAABA/zigXordw9bK/bpcHQB2TAAAADHPIZl8d - 49TWBQsDtL3QDn8Zrpm1gkYCji/w/pMg62icmaU1xSC5XLbwVbeL -db_name: db -db_schema: public -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: logbroker - endpoint: lb.ahid59uhdmk1n56tdq09.ydb.yacloudkz.tech - creds: - type: use_cloud_creds - database: /kz_global/yc.data-transfer/ahin5nt05f2jscbpc9qj - topic: /kz_global/yc.data-transfer/ahin5nt05f2jscbpc9qj/control-plane-log - tls_mode: - type: enabled - ca_certificate_paths: - - /usr/local/share/ca-certificates/YCKZInternalRootCA.crt - use_topic_api: false -skip_auth: false -push_metrics: true -dataplane_version: - type: from_database -# TODO: fix alert IDs -health_providers: - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: instance_metadata -console_hostname: kz.console.yandex.cloud -ydb_logs_instance: lb.ahiosvv2p1l5pvr31el1.ydb.yacloudkz.tech:2135 -ydb_logs_database: /kz_global/yc.data-transfer/ahiosvv2p1l5pvr31el1 -root_cert_paths: - - /usr/local/share/ca-certificates/YCKZInternalRootCA.crt -ydb_dataplane_log_dir: /kz_global/yc.data-transfer/ahiosvv2p1l5pvr31el1/log/dataplane -mdb_api_url_base: https://mdb.api.yandexcloud.kz/ -iam_api_url_base: https://UNUSED -cloud_api_endpoint: api.yandexcloud.kz:443 -iam_access_service_endpoint: as.private-api.yacloudkz.tech:14286 -events: - type: audit_trails -serverless_runtime_kubeconfig: UNUSED -search: - type: disabled - -# External cloud -solomon_url: https://solomon.yacloudkz.tech -service_account_id: yc.datatransfer.dt -service_cloud_id: yc.data-transfer -service_folder_id: ao7p33vbaatr5ar4ls8b # Data plane -kms_key_id: apdtc557a3tm4ku59v5p -kms_discovery_endpoint: dpl.kms.private-api.yacloudkz.tech:8443 -net_id: dbp3slie4lkfnvdtijje -transfer_dashboard_id: biirkhmb694ph4irfuln -yds_serverless_endpoint: lb.ahid59uhdmk1n56tdq09.ydb.yacloudkz.tech # https://logbroker.yandex-team.ru/docs/concepts/clusters_and_installations#installations -s3_endpoint: https://storage.yandexcloud.kz -is_prod: true - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_kz.yaml - -dataplane_image_k8s_image: - type: metadata - user_data_field: dataplane_image_k8s_image -data_plane_compute_platform_id: standard-v3 -ntp_servers: kz01.einebox.net -fallback_ntp: ru.pool.ntp.org -metrika_api_config: # Metrika source is disabled in KZ grants - endpoint: UNUSED - tvm_metrika_id: 0 - tvm_self_id: 0 - tvm_secret: - type: empty - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.yacloudkz.tech:14282 - instance_service_endpoint: compute.private-api.yacloudkz.tech:19051 diff --git a/pkg/config/controlplane/installations/external_preprod.k8s.yaml b/pkg/config/controlplane/installations/external_preprod.k8s.yaml deleted file mode 100644 index 144aaa8f..00000000 --- a/pkg/config/controlplane/installations/external_preprod.k8s.yaml +++ /dev/null @@ -1,146 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: e4uo49d89n34h1o6jkl6 -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABAHOBaAcedZGEWWF1kJtFpWAAAADLRGMybs - gZXMIjItAnMOVkFJgXi1TaGXz25DuFVH7B2sFQ1PmA== -db_name: db -db_schema: k8s # TODO: migrate to public when lb will be ready -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: logbroker - endpoint: lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net - creds: - type: use_cloud_creds - database: /pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3 - topic: /aoerc1it1q975rso0ia8/data-transfer/control-plane/log - tls_mode: - type: enabled - ca_certificate_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt - use_topic_api: false -skip_auth: false -push_metrics: true -dataplane_version: - type: literal # TODO: pass from env - value: 1cfe40d2a2f96edb87ac97b07daba19873235015821405b2c8a246fcc7dda935 -health_providers: - - type: solomon - alert_id: dt_alert_combined - name: dt_alert_combined - - type: status_message -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: instance_metadata -console_hostname: console-preprod.cloud.yandex.ru -ydb_logs_instance: lb.cc803hd4j8tmhgidh880.ydb.mdb.cloud-preprod.yandex.net:2135 -ydb_logs_database: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880 -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880/log/dataplane -mdb_api_url_base: https://mdb.api.cloud-preprod.yandex.net/ -iam_api_url_base: https://iam.api.cloud-preprod.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud-preprod.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud-preprod.yandex.net:4286 -events: - type: audit_trails -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXlNVEUxTWpBME5Wb1hEVE14TURZeE9URTFNakEwTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEI4CjJTQ29aZlU3aGlCbWZjTkpEVTl6TnhCbnZoM1ZIYldLVTdpaHg4L2RhaC9ueEpMTkNDQlcwUjNjd3FQYWtRZlYKeU1rQVlHRHBlbUkwbVdZMWhKV3lsUlBocjVaSm1ucXJqZ0ZvdGF1WEU3SEp1RGpXTVA1NGFha250NmFjUDQ4ZwpSMkp4cklaZG9FUmtSTEtBeUxOckJtelZQR1liQ3VscEhORTc0b09XMTU5RDhGZUhhRlU4NlAxblZPQ3Nsb2pqCmttZ2JUL1JkNk9NY1EvN2Y0UnVXcGdVSk9WZlZ0b1BvYWxYVnFNQS94UU1pSXo0b3NMNGFYeHhZd1BpWC9aK0oKdlRNazFMLzN1Nm5XcldtUmVvdXU5bkhrdlhuS1NMUzF0Q0VQZllha21tRi80RWI0c0NveVdxQUI1SmZUYVQ0NApUVGlmTUdidGtDQi85Q1o1MDNjQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJNG9Ib1ZzNkp4NElXbkVjVFdxa2QzT2ZpTGhNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBdGVXSUhucnY0dEZHRzhNakNXTk5mbzBSZjExNTdpdGtNTU9KdXpXWUhvb04xeDlkbApXREtKU2pLdXZrNmF4YnlTZHR0UTdJYllFSXNNT0Y4TXZPUzVKamQzVEhKdE9FQjhiYyt5MVRsR3VUdkhFcFdNCm1nU0dwd2FwU3ZUWElzSE1VZUdIVEw4WmYwMDV5eU5sR1hoaXM3OWpZd01SMFcybGxRZFdKMnNyN0ZnNkVxcXIKTHJUamZadWY2QkVObGtXaHNLdGhIRjc4dVlNdlUrSWp2NzM2REVjVGJ2azhVVjZrMmZFdUlXRE5weEswbW11bworK2hweE9pZlFleVRoTnB3OXJ1SEFEeXN0dVRxMm8rU0JsN216YUpxU090ek9DbVppS1M0QTNtT2pxY1JBNFVrCllzY0xZckM4cGNmS2FXTGVQSms4L3F5NkhFaW5JNWZVbVdGNQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://130.193.32.8 - name: yc-managed-k8s-c49dvuuspe14vq0553ve - contexts: - - context: - cluster: yc-managed-k8s-c49dvuuspe14vq0553ve - user: yc-managed-k8s-c49dvuuspe14vq0553ve - name: yc-prestable - current-context: yc-prestable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-c49dvuuspe14vq0553ve - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud-preprod.yandex-team.ru -service_account_id: bfbgnr9a209qmtjepb55 -service_cloud_id: aoerc1it1q975rso0ia8 -service_folder_id: aoenp6537r7gn90vb4al -kms_key_id: e10jg948rdlerghctsv3 -kms_discovery_endpoint: kms.cloud-preprod.yandex.net:8443 -net_id: c640ek4eqbrmoaf2t9u3 -transfer_dashboard_id: monkgqmtip9skkdv9d3l -yds_serverless_endpoint: yds.serverless.cloud-preprod.yandex.net -s3_endpoint: https://storage.cloud-preprod.yandex.net -is_prod: false - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_preprod.yaml - -dataplane_image_k8s_image: - type: environment_variable - variable_name: DATA_PLANE_K8S_IMAGE - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh -data_plane_compute_platform_id: standard-v2 -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org -metrika_api_config: - endpoint: internalapi.test.metrika.yandex.net - tvm_metrika_id: 2000269 - tvm_self_id: 2038887 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABCD9bsphJMKF3bjK7X5DiHBAAAADF/mVtsY/aEm3mBEaR5ApOQf9QU7ZdIrfEae6PgfAQNaFRyK1fGAAcVPxueHUC8F6FYObg== - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud-preprod.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud-preprod.yandex.net:9051 diff --git a/pkg/config/controlplane/installations/external_preprod.yaml b/pkg/config/controlplane/installations/external_preprod.yaml deleted file mode 100644 index fd8ef19e..00000000 --- a/pkg/config/controlplane/installations/external_preprod.yaml +++ /dev/null @@ -1,155 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: e4uo49d89n34h1o6jkl6 -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABAHOBaAcedZGEWWF1kJtFpWAAAADLRGMybs - gZXMIjItAnMOVkFJgXi1TaGXz25DuFVH7B2sFQ1PmA== -db_name: db -db_schema: public -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: logbroker - endpoint: lb.cc8035oc71oh9um52mv3.ydb.mdb.cloud-preprod.yandex.net:2135 - creds: - type: use_cloud_creds - database: /pre-prod_global/aoeb66ftj1tbt1b2eimn/cc8035oc71oh9um52mv3 - topic: /aoerc1it1q975rso0ia8/data-transfer/control-plane/log - tls_mode: - type: enabled - ca_certificate_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt - use_topic_api: true -skip_auth: false -push_metrics: true -dataplane_version: - type: from_database -health_providers: - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: aoerc1it1q975rso0ia8_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: instance_metadata -console_hostname: console-preprod.cloud.yandex.ru -ydb_logs_instance: lb.cc803hd4j8tmhgidh880.ydb.mdb.cloud-preprod.yandex.net:2135 -ydb_logs_database: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880 -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /pre-prod_global/aoerc1it1q975rso0ia8/cc803hd4j8tmhgidh880/log/dataplane -mdb_api_url_base: https://mdb.api.cloud-preprod.yandex.net/ -iam_api_url_base: https://iam.api.cloud-preprod.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud-preprod.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud-preprod.yandex.net:4286 -events: - type: audit_trails -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXlNVEUxTWpBME5Wb1hEVE14TURZeE9URTFNakEwTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEI4CjJTQ29aZlU3aGlCbWZjTkpEVTl6TnhCbnZoM1ZIYldLVTdpaHg4L2RhaC9ueEpMTkNDQlcwUjNjd3FQYWtRZlYKeU1rQVlHRHBlbUkwbVdZMWhKV3lsUlBocjVaSm1ucXJqZ0ZvdGF1WEU3SEp1RGpXTVA1NGFha250NmFjUDQ4ZwpSMkp4cklaZG9FUmtSTEtBeUxOckJtelZQR1liQ3VscEhORTc0b09XMTU5RDhGZUhhRlU4NlAxblZPQ3Nsb2pqCmttZ2JUL1JkNk9NY1EvN2Y0UnVXcGdVSk9WZlZ0b1BvYWxYVnFNQS94UU1pSXo0b3NMNGFYeHhZd1BpWC9aK0oKdlRNazFMLzN1Nm5XcldtUmVvdXU5bkhrdlhuS1NMUzF0Q0VQZllha21tRi80RWI0c0NveVdxQUI1SmZUYVQ0NApUVGlmTUdidGtDQi85Q1o1MDNjQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJNG9Ib1ZzNkp4NElXbkVjVFdxa2QzT2ZpTGhNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBdGVXSUhucnY0dEZHRzhNakNXTk5mbzBSZjExNTdpdGtNTU9KdXpXWUhvb04xeDlkbApXREtKU2pLdXZrNmF4YnlTZHR0UTdJYllFSXNNT0Y4TXZPUzVKamQzVEhKdE9FQjhiYyt5MVRsR3VUdkhFcFdNCm1nU0dwd2FwU3ZUWElzSE1VZUdIVEw4WmYwMDV5eU5sR1hoaXM3OWpZd01SMFcybGxRZFdKMnNyN0ZnNkVxcXIKTHJUamZadWY2QkVObGtXaHNLdGhIRjc4dVlNdlUrSWp2NzM2REVjVGJ2azhVVjZrMmZFdUlXRE5weEswbW11bworK2hweE9pZlFleVRoTnB3OXJ1SEFEeXN0dVRxMm8rU0JsN216YUpxU090ek9DbVppS1M0QTNtT2pxY1JBNFVrCllzY0xZckM4cGNmS2FXTGVQSms4L3F5NkhFaW5JNWZVbVdGNQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://130.193.32.8 - name: yc-managed-k8s-c49dvuuspe14vq0553ve - contexts: - - context: - cluster: yc-managed-k8s-c49dvuuspe14vq0553ve - user: yc-managed-k8s-c49dvuuspe14vq0553ve - name: yc-prestable - current-context: yc-prestable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-c49dvuuspe14vq0553ve - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud-preprod.yandex-team.ru -service_account_id: bfbgnr9a209qmtjepb55 -service_cloud_id: aoerc1it1q975rso0ia8 -service_folder_id: aoenp6537r7gn90vb4al -kms_key_id: e10jg948rdlerghctsv3 -kms_discovery_endpoint: kms.cloud-preprod.yandex.net:8443 -net_id: c640ek4eqbrmoaf2t9u3 -transfer_dashboard_id: monkgqmtip9skkdv9d3l -yds_serverless_endpoint: yds.serverless.cloud-preprod.yandex.net -s3_endpoint: https://storage.cloud-preprod.yandex.net -is_prod: false - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_preprod.yaml - -dataplane_image_k8s_image: - type: metadata - user_data_field: dataplane_image_k8s_image -data_plane_compute_platform_id: standard-v2 -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org -metrika_api_config: - endpoint: internalapi.test.metrika.yandex.net - tvm_metrika_id: 2000269 - tvm_self_id: 2038887 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRlMTBrN2Jndm1xcGh2YnVja2NxYgAAABCD9bsphJMKF3bjK7X5DiHBAAAADF/mVtsY/aEm3mBEaR5ApOQf9QU7ZdIrfEae6PgfAQNaFRyK1fGAAcVPxueHUC8F6FYObg== - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud-preprod.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud-preprod.yandex.net:9051 diff --git a/pkg/config/controlplane/installations/external_prod.yaml b/pkg/config/controlplane/installations/external_prod.yaml deleted file mode 100644 index 74ce480e..00000000 --- a/pkg/config/controlplane/installations/external_prod.yaml +++ /dev/null @@ -1,155 +0,0 @@ -type: external_cloud - -# Common -environment: - type: yc -db_discovery: - type: mdb - cluster_id: c9qnb04en018asm4bktv -db_user: db_user -db_password: - type: kms - ciphertext: !!binary | - AAAAAQAAABRhYmo4bHBnMGhwYjRyOWo5cDlkbgAAABCK41PbtP/ehs93biF8QQUtAAAADM04Je4w - phoys2Ii/SUY+yiWSPdL0ut3ChNraPeojPrQ3GWaCQ== -db_name: db -db_schema: public -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: logbroker - endpoint: lb.etn03iai600jur7pipla.ydb.mdb.yandexcloud.net - creds: - type: use_cloud_creds - database: /global/b1gvcqr959dbmi1jltep/etn03iai600jur7pipla - topic: /b1g9jhnga9ji23ii4dks/data-transfer/control-plane/log - tls_mode: - type: enabled - ca_certificate_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt - use_topic_api: false -skip_auth: false -push_metrics: true -dataplane_version: - type: from_database -health_providers: - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: b1g9jhnga9ji23ii4dks_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon - -# Cloud -cloud_creds: - type: instance_metadata -console_hostname: console.cloud.yandex.ru -ydb_logs_instance: lb.etn01pp55otetd3u805d.ydb.mdb.yandexcloud.net:2135 -ydb_logs_database: /global/b1g9jhnga9ji23ii4dks/etn01pp55otetd3u805d -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /global/b1g9jhnga9ji23ii4dks/etn01pp55otetd3u805d/log/dataplane -mdb_api_url_base: https://mdb.api.cloud.yandex.net/ -iam_api_url_base: https://iam.api.cloud.yandex.net/iam/v1/ -cloud_api_endpoint: api.cloud.yandex.net:443 -iam_access_service_endpoint: as.private-api.cloud.yandex.net:4286 -events: - type: audit_trails -serverless_runtime_kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EY3dNakV6TVRjMU5Wb1hEVE14TURZek1ERXpNVGMxTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTJpCmtQZUhCWWF1cHA2OFladWFuWTVlWHVEcU1LRjh4MytkTlpnVTZYakh2UzFLVTVXc2I4eVVnL1lFODd6a1d6OGsKTVp4M1RIODVWUGpINm9mWmlzYnJYR1V5M0szMm9saklCZTJieFplb0NSRllNd0RsYm81NE1oVG1hemhpaUtVNApseHB5SGhTeTVaanpMSEFCQ1VWWWI5a291eHhmVFl2ckU2T3ROOTVvSTVVNzhJYVdkaS9SWlc5L1pLeThvZjNCCkZxVWVIZWFIN0g4RTlHS2xZbjdYMWZVMWFJdStEL01xTUx1Ty9jSEZBVHVTbHMveWVXdU5QOUdwYzVlU244TXIKMGM2K21TZ2hNYzYwMlg0aVJ4R2Zxa0RlK1U2ZHNrS3gxQlJHZE42YjZuc2dGSm44cjNiMjB0b0ROVUlFSWlUbwpyenZuWk1vU0RYNVhlWk9rdDVFQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZDeEh5RlZCWHZMY01UV2djQkVrQ0JFbUhkUDJNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDOFVEaVhLQ3pVK2xqN1NZUVVvblEyaUhiTVkyTWErV3BTNXBpQWxwSkVTNXErdHpSUQpGWk01ME51WVRFYnJsNjM1UmRRZXVMTW4yc0FaNXRTa0ZuSFdyQkIweWJFRzdibWExNzlHdHh6NTN6cDFEd0w0CnFXS3REakVqOURqS2l5MlpBV01rSWpHOGhGUkpFVXpZVkxveWY5aDVjbEVxcEVvMVdnaW5LWjRoOTNGU2hUa0oKam9iYjBkMHlnZWd6N2swTERZQjZMcVZnRnJNNnRwU25SNi9TTWcrUHdKcy8vL3drRjlHZkpOZHgrbzNOUzVvbwphd1M1K3c0cUJVcE9OWXhPOXpPMWVSNlpaRzJzWlVncTNjRWRMUnpsL3JGWEZVVlZuRXNyRnJ4NTZPOWovL0p2CjdxQXBWRGhzdGlqODlDYnRPUjlJK2Nzc0lkMzZXV1ZpckFMdAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - server: https://178.154.228.81 - name: yc-managed-k8s-cat9vrf8bhq5j9jdam95 - contexts: - - context: - cluster: yc-managed-k8s-cat9vrf8bhq5j9jdam95 - user: yc-managed-k8s-cat9vrf8bhq5j9jdam95 - name: yc-stable - current-context: yc-stable - kind: Config - preferences: {} - users: - - name: yc-managed-k8s-cat9vrf8bhq5j9jdam95 - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - k8s - - create-token - command: yc - env: null -search: - type: disabled - -# External cloud -solomon_url: https://solomon.cloud.yandex-team.ru -service_account_id: ajet5egu65m66slf9tn0 -service_cloud_id: b1g9jhnga9ji23ii4dks -service_folder_id: b1gds2039chq7kv8pmvv -kms_key_id: abjvp62i676fpehfd3vj -kms_discovery_endpoint: kms.yandex:8443 -net_id: enpl4ftghn8vt5oad2a7 -transfer_dashboard_id: monr9n46l9kjjllugubn -yds_serverless_endpoint: yds.serverless.yandexcloud.net -s3_endpoint: https://storage.yandexcloud.net -is_prod: true - -data_plane_vm_users: - ssh_key: cert-authority ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAFxDHbZHZCx72sztvWsFSJVUhW+fngANgR5iiIWy9gicOdjYOy0pxpb1U8PqWmJYJYOWLwtfm/f9qnSV3Q/wAX/igCAwdYuwsWtx/0eBp2G3ECIUJzEQRme+TrtECpJHyLFCJonE/Cg24JlQ8N6mVjrQZwLiVjCdVwRFGoXxb3F1N4SHg== bastion - users: - - svemarch - - marufkaraev - - ovandriyanov - - kateleb - - wiftwift - - unikoid - - timmyb32r - - booec - - denisevdkmv - - kosovgrigorii - - kry127 - - daniilmarukh - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: external_prod.yaml - -dataplane_image_k8s_image: - type: metadata - user_data_field: dataplane_image_k8s_image -data_plane_compute_platform_id: standard-v2 -ntp_servers: ntp1.yandex.net ntp2.yandex.net ntp3.yandex.net ntp4.yandex.net -fallback_ntp: ru.pool.ntp.org -metrika_api_config: - endpoint: internalapi.metrika.yandex.net - tvm_metrika_id: 2000270 - tvm_self_id: 2038889 - tvm_secret: - type: kms - ciphertext: !!binary | - AAAAAQAAABRhYmo4bHBnMGhwYjRyOWo5cDlkbgAAABAoofCTpx29gJknk38dGA8MAAAADAFr6+HCQyVDnIj6KelYHW8KSuSn3V3EGeO6hzK2QUnGFjfmBZcWxZjqV9HgpzBAdMng2w== - -fine_grained_ssa: - type: enabled - system_service_account_id: yc.datatransfer.controlplane-api - certificate_path: /usr/local/share/certificates/compute-certificate.crt - service_id: data-transfer - microservice_id: internal-api - token_service_endpoint: ts.private-api.cloud.yandex.net:4282 - instance_service_endpoint: compute-api.private-api.compute.cloud.yandex.net:9051 diff --git a/pkg/config/controlplane/installations/internal_prod.yaml b/pkg/config/controlplane/installations/internal_prod.yaml deleted file mode 100644 index 0b6cf64c..00000000 --- a/pkg/config/controlplane/installations/internal_prod.yaml +++ /dev/null @@ -1,171 +0,0 @@ -type: internal_cloud - -# Common -environment: - type: internal -db_discovery: - type: mdb - cluster_id: mdbn9rt3ij3n0p93r701 -db_user: db_user -db_password: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f7bvkkt4cgckdd6t72wmerdf - key: db_password -db_name: db -db_schema: public -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -log: - type: logbroker - endpoint: logbroker.yandex.net - creds: - type: oauth_token - token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb - database: '' - topic: /cdc/prod/control-plane/log - tls_mode: - type: disabled - use_topic_api: false -skip_auth: false -push_metrics: true -profiler: - type: logbroker - endpoint: logbroker.yandex.net - creds: - type: oauth_token - token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb - database: '' - topic: cdc/prod/profiler - tls_mode: - type: disabled - use_topic_api: false - -# Cloud -solomon_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01eewths5arkqh0k8d0pnan8vx - key: solomon -logbroker_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb -cloud_creds: - type: passport_oauth_token - token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -console_hostname: yc.yandex-team.ru -ydb_logs_instance: ydb-ru.yandex.net:2135 -ydb_logs_database: /ru/transfer_manager/prod/data-transfer-yt -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /ru/transfer_manager/prod/data-transfer-yt/log/dataplane -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -events: - type: disabled -dataplane_version: - type: from_database -health_providers: -# - type: solomon TODO: decide whether this alert should be fixed or dropped -# alert_id: 7fd353ed-025a-4a9d-8e48-3be6a300e96f -# name: dta_01_dataplane_is_alive - - type: solomon - alert_id: b806e979-48e7-4faf-8876-0e92148c71b7 - name: dta_02_no_items_in_memory - - type: solomon - alert_id: bb31436f-11be-4264-a873-59f67b39ae23 - name: dta_03_transfer_time_sound - - type: solomon - alert_id: 0f149b88-1662-40a6-b34d-fc6856c728e2 - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: data-transfer_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: solomon -search: - type: disabled - -# Internal cloud -ydb_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01e5wrf991geg3h02pe4n2fcv4 - key: ydb -mdb_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -yav_token: - type: environment_variable - variable_name: YAV_TOKEN -yt_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f8t8hy3699hrb2050s680gbt - key: yt -logfeller_yt_token: - type: yav - secret_id: sec-01e4xz5vc007whwy8pvax96bz2 - version: ver-01fxsne874kxnvaj19y2x6be32 - key: nirvana-secret -logfeller_nirvactor_token: - type: yav - secret_id: sec-01e4xzjqs0a61e3y576ek5c7zw - version: ver-01fxstmtn3mvyyamy050xfkbg5 - key: nirvana-secret -logfeller_logbroker_token: - type: yav - secret_id: sec-01gdn7wkcgc3awnvvtxcpk7rkw - version: ver-01gdn7wkcse36m7xk4xsr99czv - key: logbroker-token -logfeller_lf_describer_logbroker_token: - type: yav - secret_id: sec-01h4k2mwcva6xq9w5p90kkwnpc - version: ver-01h52qv834xj62dvmbfsx1d3f6 - key: logbroker-oauth-token -tls_server_private_certificate: - type: yav - secret_id: sec-01hzh7xb2s83m03ydhndm6qmhk - version: ver-01hzh7xb45rwavz8npkmpkd88a - key: 7F002F06818CD05B3D1129DACF0002002F0681_key_cert -dataplane_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -solomon_cluster: rtc-prod -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: internal_prod.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/internal_prod.yaml -notifications_api: https://notify.cloud.yandex-team.ru/v2/send -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer -dataplane_image_k8s_image: '' # TODO: move this into the dataplane_stage_versions table in postgres diff --git a/pkg/config/controlplane/installations/internal_testing.yaml b/pkg/config/controlplane/installations/internal_testing.yaml deleted file mode 100644 index 8bfd5b75..00000000 --- a/pkg/config/controlplane/installations/internal_testing.yaml +++ /dev/null @@ -1,158 +0,0 @@ -type: internal_cloud - -# Common -environment: - type: internal -db_discovery: - type: mdb - cluster_id: mdblc14b5vs4onea1hfi -db_user: db_user -db_password: - type: yav - secret_id: sec-01f7m29728abwy1248m9rntzst - version: ver-01f7m2972jv8vkb2tp6g2h4zm7 - key: cdc_pg_password -db_name: db -db_schema: testing -disable_dataplane: false -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 80 -exported_metrics_http_port: 9092 -log: - type: stdout -profiler: - type: logbroker - endpoint: logbroker.yandex.net - creds: - type: oauth_token - token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb - database: '' - topic: cdc/prod/profiler - tls_mode: - type: disabled - use_topic_api: false -skip_auth: false -push_metrics: true -dataplane_version: - type: from_database -health_providers: - - type: solomon - alert_id: data-transfer_testing_dta_01_dataplane_is_alive - name: dta_01_dataplane_is_alive - - type: solomon - alert_id: data-transfer_testing_dta_02_no_items_in_memory - name: dta_02_no_items_in_memory - - type: solomon - alert_id: data-transfer_dta_03_transfer_time_sound - name: dta_03_transfer_time_sound - - type: solomon - alert_id: data-transfer_testing_dta_04_row_max_lag_constant - name: dta_04_row_max_lag_constant - - type: solomon - alert_id: data-transfer_testing_dta_05_replication_restarts - name: dta_05_replication_restarts - - type: status_message - - type: operation_status -metrics_provider: - type: fake - -# Cloud -solomon_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01eewths5arkqh0k8d0pnan8vx - key: solomon -logbroker_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01dpnps7ggve70nrybv5ap5taw - key: lb -cloud_creds: - type: passport_oauth_token - token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -console_hostname: yc-test.yandex-team.ru -ydb_logs_instance: ydb-ru-prestable.yandex.net:2135 -ydb_logs_database: /ru-prestable/transfer_manager/prestable/data-transfer-yt -root_cert_paths: - - /usr/local/share/ca-certificates/YandexInternalRootCA.crt -ydb_dataplane_log_dir: /ru-prestable/transfer_manager/prestable/data-transfer-yt/log/dataplane -mdb_api_url_base: https://gw.db.yandex-team.ru/ -iam_api_url_base: https://gw.db.yandex-team.ru/iam/v1/ -cloud_api_endpoint: gw.db.yandex-team.ru:443 -iam_access_service_endpoint: as.cloud.yandex-team.ru:4286 -events: - type: disabled -search: - type: disabled - -# Internal cloud -ydb_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01e5wrf991geg3h02pe4n2fcv4 - key: ydb -mdb_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -yav_token: - type: environment_variable - variable_name: YAV_TOKEN -yt_token: - type: yav - secret_id: sec-01dpnps7g3jkwwsfhzvdm2ybw4 - version: ver-01f8t8hy3699hrb2050s680gbt - key: yt -logfeller_yt_token: - type: yav - secret_id: sec-01e4xz5vc007whwy8pvax96bz2 - version: ver-01fxsne874kxnvaj19y2x6be32 - key: nirvana-secret -logfeller_nirvactor_token: - type: yav - secret_id: sec-01e4xzjqs0a61e3y576ek5c7zw - version: ver-01fxstmtn3mvyyamy050xfkbg5 - key: nirvana-secret -logfeller_logbroker_token: - type: yav - secret_id: sec-01gdn7wkcgc3awnvvtxcpk7rkw - version: ver-01gdn7wkcse36m7xk4xsr99czv - key: logbroker-token -logfeller_lf_describer_logbroker_token: - type: yav - secret_id: sec-01h4k2mwcva6xq9w5p90kkwnpc - version: ver-01h52qv834xj62dvmbfsx1d3f6 - key: logbroker-oauth-token -tls_server_private_certificate: - type: yav - secret_id: sec-01hzh7xb2s83m03ydhndm6qmhk - version: ver-01hzh7xb45rwavz8npkmpkd88a - key: 7F002F06818CD05B3D1129DACF0002002F0681_key_cert -dataplane_token: - type: yav - secret_id: sec-01ddj4pn4eg5hmyd3kjkczhm0w - version: ver-01fg9r96zjq7hzyzs2bvx1xnj4 - key: CDC_DBAAS_TOKEN -solomon_cluster: rtc-testing -grpc_ssl_port: 8443 -public_grpc_ssl_port: 9443 -yt_proxy: vanga -yt_operation_pool: transfer_manager -udf_dir: /usr/local/bin/udfs -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: internal_testing.yaml -task_monitor_lock_path: //home/transfer-manager/task_monitor/internal_testing.yaml -notifications_api: https://notify.cloud-test.yandex-team.ru/v2/send -solomon_url: https://solomon.yandex-team.ru -solomon_project: data-transfer -dataplane_image_k8s_image: '' # TODO: move this into the dataplane_stage_versions table in postgres diff --git a/pkg/config/controlplane/loader.go b/pkg/config/controlplane/loader.go deleted file mode 100644 index 2567aea1..00000000 --- a/pkg/config/controlplane/loader.go +++ /dev/null @@ -1,28 +0,0 @@ -package cpconfig - -import ( - "flag" - "io" - - "github.com/doublecloud/tross/pkg/config" -) - -type Loader struct{} - -func (l *Loader) EmbeddedConfigs() map[string][]byte { - return EmbeddedConfigs -} - -func (l *Loader) Setup(flagSet *flag.FlagSet, envVarMap map[string]*string, configSource *config.ConfigSource) { - config.SetupControlPlane(flagSet, envVarMap, configSource) - if config.AllInOneBinary { - // Allow data plane flags to be passed on the command line - var dummyConfigSource config.ConfigSource - config.SetupDataPlane(flagSet, envVarMap, &dummyConfigSource) - } -} - -func (l *Loader) Load(reader io.Reader) error { - _, err := LoadF(reader, SecretResolverFactory{}) - return err -} diff --git a/pkg/config/controlplane/parts/common_local.yaml b/pkg/config/controlplane/parts/common_local.yaml deleted file mode 100644 index 249fa7a3..00000000 --- a/pkg/config/controlplane/parts/common_local.yaml +++ /dev/null @@ -1,43 +0,0 @@ -type: internal_cloud - -db_discovery: - type: on_premise - host: postgres - port: 5432 - use_tls: false -db_user: root -db_password: 'secret' -db_name: transferdb -db_schema: public -grpc_port: 8787 -public_grpc_port: 9797 -rest_port: 8088 -exported_metrics_http_port: 9092 -profiler: - type: disabled -log: - type: stdout - -push_metrics: true -sync_tasks: false -local_replication: false -disable_dataplane: true -skip_auth: true # skip for now, we should wait till IAM create folders for us - -dataplane_version: - type: arcadia_source_revision -health_providers: - - type: fake -metrics_provider: - type: fake -search: - type: disabled -events: - type: disabled - -dataplane_image_k8s_image: - type: environment_variable - variable_name: DATA_PLANE_K8S_IMAGE -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: 'x' - diff --git a/pkg/config/controlplane/parts/dc_preprod_local.yaml b/pkg/config/controlplane/parts/dc_preprod_local.yaml deleted file mode 100644 index a62432c4..00000000 --- a/pkg/config/controlplane/parts/dc_preprod_local.yaml +++ /dev/null @@ -1,152 +0,0 @@ -type: aws -environment: - type: aws - -db_name: transferdb -db_user: root -db_password: secret -db_schema: public - -dataplane_sa: - id: yc.data-transfer.agent - folder_id: yc.data-transfer.service -assume_role_target: arn:aws:iam::867394682372:role/data-transfer-mdb-cp -iam: - access_service: preprod.iam.internal.yadc.tech:4286 - token_service: preprod.iam.internal.yadc.tech:4282 - private_service: preprod.iam.internal.yadc.tech:4283 -mdb_service: preprod.mdb.internal.yadc.tech:443 -mdb_vpc_service: vpc.preprod.mdb.internal.yadc.tech:443 -cloud_creds: - type: service_account_key - id: yc.data-transfer.agent - service_account_id: yc.data-transfer.agent - key_algorithm: RSA_4096 - private_key: - type: aws_secret_manager - secret_id: arn:aws:secretsmanager:eu-central-1:840525340941:secret:control-plane - key: sa_private_key - public_key: | - -----BEGIN PUBLIC KEY----- - MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9cSvRBYEEWopnhw8mrwM - ERtBxp8ukJ/nqTUGBR9u6+Boo+pHGY4C3xqFy1dyZxrAcl8eRekMdQEAHmFBo9tG - DuYTsO5u3WTdyQkFawLJ/RF3xQOjCSsJMiZrB4N94auHzdIXryzgsK13u3t+X3EA - bc+UMGd6Y+3BTmhZMqbDwGkAjufju0f1+R/oNj91Cwu5a4qDbCp4RqRJ3OTWeXEE - SZ+a8QIwqkNygrTes5g2SOFPowUbe5GaJmqGMKOR9ZyMUs7CS8Jjhd1/gxoimKoS - soT3/oyokwbljOtVeq7owc5ScAdkEhSqh3ZF2O91jU1gEoWZsdia0NTR2hfKR2cI - DxUM/Pk0VkZlnbna0J44tU39x/FiaLDZF4yurjs8dOyd/oVFta3gfipft+WoGQr+ - +SylhfNSgqUu1a3rwsRmOQwJFiFzYnhMOToaQ2KOoM1crW3plQaR9sypWw8tG3gK - lxXKnwORHyMpOiX2odmjFiz50LgW07Kc7ruNjk864evQLMGi3LK3TW7eD/hLaqQx - rIg4xD99DjOlKU1CrugjrXtJ8cJjVFp+kJJiJ3qmvkjkbPbHc0vSfuo6kYOj4a64 - O8hwYUVN8xBNkeIXqBr9TJl3d+yZ1Izm+EZHmDI5KEXGQdx03f6hN24x2VSQhDGV - G5mGTUfYGy3CBKq6e4rhTisCAwEAAQ== - -----END PUBLIC KEY----- -key_id: 'arn:aws:kms:eu-central-1:840525340941:key/c4b45158-9287-4187-bcff-7ace91df54d2' -ami_config: - instance_profile: data-transfer-mdb-dp - region_to_ami: # Dataplane EC2 AMI per-region - ap-northeast-1: ami-0e37cf3c46ceb50a6 - ap-northeast-2: ami-0da0f7dd80a8e3f00 - ap-northeast-3: ami-0e118b9baecff1d1c - ap-south-1: ami-01821392009f8b19f - ap-southeast-1: ami-0dcca38f125cf27f3 - ap-southeast-2: ami-07e1076fba13804ea - ca-central-1: ami-047031eed206d251c - eu-central-1: ami-009288f3b708bb17f - eu-north-1: ami-022e403d440b95b93 - eu-west-1: ami-0e3baebc0319ec8e0 - eu-west-2: ami-03dc9dbd59b21f18b - eu-west-3: ami-093c1599ac69831d6 - me-south-1: ami-07439796dcab53531 - sa-east-1: ami-00e4ae0231e59cbae - us-east-1: ami-0ba25297cb72dc55d - us-east-2: ami-007be952d54c37388 - us-west-2: ami-0c5df8193cf8bae9b -dataplanes: # named list of kubeconfig-s - - name: data-transfer-dp-frankfurt-ipv6 # default data plane k8s cluster - region: 'eu-central-1' - secret: | - apiVersion: v1 - preferences: {} - kind: Config - - clusters: - - cluster: - server: https://66D27AAE164C19A874D59AFE2569313A.gr7.eu-central-1.eks.amazonaws.com - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeU1UQXdOekV3TVRRME0xb1hEVE15TVRBd05ERXdNVFEwTTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTHpPCldmN3YwRHRodUhvc0xTc1JHeVA1cFVjbjNHOVlLSFZIY3RPeHU5RG1tbEVQbk15WVUvSmpRekltdUFrUWRYQm0KSmtHTnpyNmxqVm9BM0VTbkJrQXBnRHZNZjZESUdzRlNFbGlydUJJWU0wc3FXVGR5bXAzQUZuRWZpdTJVcjJkegpBWG91UlJmS1VHcmtDdDNKeUhwWk9XZGFyZkhwZm1KWHMvWmczclgyNy85ZlU4dFNXZUE1amcwU1dYYU85eUFDCmRlT09nNFEzeHlscE96VHpadXZJR2NTUG9QNUM0dzBFd2pCVXdDTythOXYvOFdNNFV6cFloSzRTeEMvSjQyUGkKcUdzcDJLYkpJTlNhSnZ6SnprZzVnR25DelQ5eTV0RmFVUUxEWlFyMUVHN2FHY0xCcHdXYlBoVEt4SGp1ZDlHcwpMZVhtcDN0bWl5SEtFYUpqWDdzQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZDWlRwNEVScGtmWlIwZzFiT2tnVGVRdEtIR0RNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBQmExZUdwOXdDV2p0SlMzN3o5VAo3aDJxOVFMbzRvbWtvUSt4c0lLSXNDS1MrbjZ4K2Q5SXFQRDJJVWJYNGxNeUhnbzU1MW50bFU1emdML3R0clp0Cld3WDY4S3FIR3BhcUtFYkN2UE1IWVdVa0diL05BNDE3OXFUeWFjbytFZy9VR0JrSm9nNGkzdGZwek0wRExOc0EKa2RBMHd3ZnB3U3BkMi9ibGRnUzVkMVdMa3RLaHl0Y2NJazZaTXdoTXROdHR0YlJvR2FlMXNNWEU3YTRseDhPYQphbDUzdDB1S0hjdVFaMmdTVzlzM1FacUM2Z3ZFNnUrNEVmYWZmRUxDazVzN2pxdUhPNUVkREpaQWVZc2x2dG5OCktJMFd6SWpVSlRZa0VXdFJiOE9IcjRIZWNGdjBkVDJMdEpDTllqSjZ5MGY2U0k2R0hhd3pWZlRsN1pFekxyS2kKOENRPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - name: data-transfer-dp-frankfurt-ipv6 - - contexts: - - context: - cluster: data-transfer-dp-frankfurt-ipv6 - user: data-transfer-dp-frankfurt-ipv6 - name: data-transfer-dp-frankfurt-ipv6 - - current-context: data-transfer-dp-frankfurt-ipv6 - - users: - - name: data-transfer-dp-frankfurt-ipv6 - user: - exec: - apiVersion: client.authentication.k8s.io/v1 - command: aws - args: - - --region - - eu-central-1 - - eks - - get-token - - --cluster-name - - "data-transfer-dp-frankfurt-ipv6" - interactiveMode: IfAvailable - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: aws_preprod.yaml -dataplane_spec_s3_bucket: "double-cloud-data-transfer-preprod-specs" - -dataplane_image_k8s_image: - type: environment_variable - variable_name: DATA_PLANE_K8S_IMAGE - -notifications_api: 'https://notify.preprod.ui.internal.yadc.tech/v1/send' - -# AWS-managed prometheus -amp_region: eu-central-1 -amp_url: 'https://aps-workspaces.eu-central-1.amazonaws.com/workspaces/ws-195f8815-a36f-445e-aa51-7c9dd93e43b2/api/v1' #preprod -amp_assume_role_target: 'arn:aws:iam::840525340941:role/cp-prom-reader' - -clusters_cp_account_id: '118027436691' -cp_account_id: '840525340941' -dp_account_id: '867394682372' - -teleport: - type: enabled - telepass_url: http://telepass.telepass:80/token - proxy: doublecloud.teleport.sh:443 - labels: - Infra: preprod - Team: data-transfer - Plane: data - Type: transfer - -gcp: - byoa_common_account: 'controlplane-preprod@byoa-doublecloud.iam.gserviceaccount.com' - image_id: 'projects/mdb-dp-preprod/global/images/dc-gcp-amd64-dt-20231117092546' - project_id: 'mdb-dp-preprod' - service_account: 'common-dt-dp-sa@mdb-dp-preprod.iam.gserviceaccount.com' - creds: | - { - "type": "external_account", - "audience": "//iam.googleapis.com/projects/191556392698/locations/global/workloadIdentityPools/mdb-dt/providers/dt-aws-840525340941", - "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "environment_id": "aws1", - "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone", - "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", - "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" - }, - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/common-dt-dp-sa@mdb-dp-preprod.iam.gserviceaccount.com:generateAccessToken", - "service_account_impersonation": { - "token_lifetime_seconds": 3600 - } - } diff --git a/pkg/config/controlplane/parts/dc_prod_local.yaml b/pkg/config/controlplane/parts/dc_prod_local.yaml deleted file mode 100644 index e1bb1cf5..00000000 --- a/pkg/config/controlplane/parts/dc_prod_local.yaml +++ /dev/null @@ -1,144 +0,0 @@ -type: aws -environment: - type: aws - -dataplane_sa: - id: yc.data-transfer.agent - folder_id: yc.data-transfer.service -assume_role_target: arn:aws:iam::221736954043:role/data-transfer-mdb-cp -iam: - access_service: prod.iam.internal.double.tech:4286 - token_service: prod.iam.internal.double.tech:4282 - private_service: prod.iam.internal.double.tech:4283 -mdb_service: prod.mdb.internal.double.tech:443 -mdb_vpc_service: vpc.prod.mdb.internal.double.tech:443 -cloud_creds: - type: service_account_key - id: yc.data-transfer.agent - service_account_id: yc.data-transfer.agent - key_algorithm: RSA_4096 - private_key: - type: aws_secret_manager - secret_id: arn:aws:secretsmanager:eu-central-1:179772598917:secret:control-plane - key: sa_private_key - public_key: | - -----BEGIN PUBLIC KEY----- - MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzopUsI5QVhGP5Um+dvu6 - B5b8zIHpye4OLQNu/PkL6aG90I+WB/gb728QMJ9aLW8OCLXmAN7vqsHARceqiYyn - BwlLb5wWUve0iJoJnMbRPxiP0sgzRs/h39ugLqd54b8xnAa9I0N/wPizjqRQN1vJ - ulSArU1uqy4xTIObaRwWU4j3y/Cd3JvMbka8mTSS7BLlRadlkAWcD7on6mz9BX6O - 19K1E0SjqM1EzCVTP1aH2j2NTKI67cQabDr/QpVg5t1xG2zto/hbXFlMIv6DPvJl - R3gvlDAHtJfrIwCr9lcipsyGaVoOC2calcG2U0byh0UYN8gRRRybp9bWyoANmsO9 - DtQlfIH0Hya5kB75mAHiPUzYPKUR/YkT8RiaR5yqWFZdoSrU77GR7J+gxcxHzBGd - fMXGZIKZoMhb23Hl8GQ+Ne3/jqoXKvkzTOiLDCUKVdSd335RvbzuiJQKL7eYRtIL - JHgwVSHXmYY0Tei2ImimdrpnTmvdXaqjEj5w0F7QH9X81/HQ234pTg7K4VYKZNIM - encsvOKKWMZjHM21f6lc8qpxJtF4D1Ap+mc3xhfXb0L/68XO9rZCtLtD8mhB3g3W - 0KgoZ6J6SmZ49+suoYPQFKrJ23MuwJlIqKs44Dmu7i0aXrAmvVIqyJJuJQpBtBTp - p9rhVIMgEUbfbCUYw3P/DrMCAwEAAQ== - -----END PUBLIC KEY----- -key_id: 'arn:aws:kms:eu-central-1:179772598917:alias/data-transfer-master-key' -ami_config: - instance_profile: data-transfer-mdb-dp - region_to_ami: # Dataplane EC2 AMI per-region - ap-northeast-1: ami-070346bc4332d6617 - ap-northeast-2: ami-07b5fdb007f8f458e - ap-northeast-3: ami-033217334260bc303 - ap-south-1: ami-0737b8a3dd101d1e5 - ap-southeast-1: ami-0e0e7ddb545281179 - ap-southeast-2: ami-0a270c95e7a438e99 - ca-central-1: ami-0f2092fdc6a5826b7 - eu-central-1: ami-0302d08fed620a271 - eu-north-1: ami-09fcf05651153a2d0 - eu-west-1: ami-0d8eb723c4bc8b934 - eu-west-2: ami-092a18752a42c6f34 - eu-west-3: ami-0711782f579d0e127 - me-south-1: ami-0906566801c3e31ce - sa-east-1: ami-0f5029b9f41ed8b8f - us-east-1: ami-0667f03604b4770ae - us-east-2: ami-01ef4edfab5cdafa7 - us-west-2: ami-046aac667d1b30c7b -dataplanes: # named list of kubeconfig-s - - name: eks_data-transfer-dp-frankfurt # default data plane k8s cluster - region: 'eu-central-1' - secret: | - apiVersion: v1 - preferences: {} - kind: Config - - clusters: - - cluster: - server: https://46F0394EC1704FD4508EDFC2BEFCFC38.yl4.eu-central-1.eks.amazonaws.com - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeU1EVXdOVEV6TXpJek1Wb1hEVE15TURVd01qRXpNekl6TVZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBS1ZQCnBuZkZRN29PKzRpWTNEWkJtUUJpUjZWS2lUQm5oS3lXc2NBejBFRCtDNG16YmlqNW1pRmxZSDl3bWM2QzRZek8KMDVJVmxKQktyMFFNb0ZwMDR2RFFJcjFubDIyd2REbjBQTkxqM3I5YnNyc1dLMlV6OEU1S2RzZjdvajBTUlV3TApkMUJMQmIwaitVUFhJeGE2UytoYWFMRjYxZDhvVHQ2SFF3NFl6ZEY5S0hYZHhyb2NOb2srME5uTUx2S1d1dlczCkFyTnZGOGFNSzZvK3pnbFlYMmN5T3BCeEdESnJQQkljalBQejlSK25wbEtrQU83elZKRTJ1K2xiUEd6OEl0dzMKckFQTnVZL2FRMXV2MFhhQXlxTm9LWWNVL09UV2dHNWFiRVVuSTZnUjZyMmFIQU5wdWliaFVabHBpRHNoejMybQo5S1lOd0N6UTBzdG9jT2dSTzZjQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZJQk91MGNrUUZKdE11bXlseXJ2bzlxREZyd0dNQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBS0wyb0R2eEFGSlZVb1hvNnpEUwo1ejY0TGYzcnRwK2hTS3dVSUlENWRLRDI2UkhMUTZoUTNMYjlIdWgxVDBhUlFtMlFEMnVLTDhPMXl1ZytoYUpPClhPWXI0NVNsRERMeU5qSEdyNUU2bUtnNW9naUJrajdoOVhlMVkwUkdGLytPQXdUUnA0MEZYcnIyRURVK3M4VisKTHVDWkFSdExyUXM1bm5XcGJwNHA3U2Nxa2FYZkZ6cWFVYk4xeDRjVG1GUUNmZjVocXNueEk5K3BmdWFxcjcrTgp6NlRPazh0c1U3UVRPNDV0aFJyUVZseWpNNFIxb214U1pqTGtEcm02MUF1NkVYQ05SRVNNamdpR3Q0b2ZYMHlmClV0Y0ZWUFRGeGpFNW1aNWhQeTROVVlTQ1NXbDQvc1NsYlhCSCtxcW5nbUVNTWFSNUxReVlaOHdIUktCS08xdWsKdjE4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - name: data-transfer-dp-frankfurt-ipv6 - - contexts: - - context: - cluster: data-transfer-dp-frankfurt-ipv6 - user: data-transfer-dp-frankfurt-ipv6 - name: data-transfer-dp-frankfurt-ipv6 - - current-context: data-transfer-dp-frankfurt-ipv6 - - users: - - name: data-transfer-dp-frankfurt-ipv6 - user: - exec: - apiVersion: client.authentication.k8s.io/v1 - command: aws - args: - - --region - - eu-central-1 - - eks - - get-token - - --cluster-name - - "data-transfer-dp-frankfurt-ipv6" - interactiveMode: IfAvailable - -data_plane_config_variable_name: DATA_PLANE_CONFIG_RESOURCE -data_plane_config_variable_value: aws_prod.yaml -dataplane_spec_s3_bucket: "double-cloud-data-transfer-prod-specs" - -dataplane_image_k8s_image: - type: environment_variable - variable_name: DATA_PLANE_K8S_IMAGE - -notifications_api: 'https://notify.ui.internal.double.tech/v1/send' - -# AWS-managed prometheus -amp_region: eu-central-1 -amp_url: 'https://aps-workspaces.eu-central-1.amazonaws.com/workspaces/ws-a2ea1ab6-82a2-45a3-9bbd-5cff23d2a4e2/api/v1' #prod -amp_assume_role_target: 'arn:aws:iam::179772598917:role/cp-prom-reader' # prod role - -clusters_cp_account_id: '883433064081' -cp_account_id: '179772598917' -dp_account_id: '221736954043' - -teleport: - type: enabled - telepass_url: http://telepass.telepass:80/token - proxy: doublecloud.teleport.sh:443 - labels: - Infra: prod - Team: data-transfer - Plane: data - Type: transfer - -gcp: - byoa_common_account: 'controlplane@byoa-doublecloud.iam.gserviceaccount.com' - image_id: 'projects/mdb-dp-prod/global/images/dc-gcp-amd64-dt-20231117100928' - project_id: 'mdb-dp-prod' - service_account: 'common-dt-dp-sa@mdb-dp-prod.iam.gserviceaccount.com' - creds: | - { - "type": "external_account", - "audience": "//iam.googleapis.com/projects/113033745255/locations/global/workloadIdentityPools/mdb-dt/providers/dt-aws-179772598917", - "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/common-dt-dp-sa@mdb-dp-prod.iam.gserviceaccount.com:generateAccessToken", - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "environment_id": "aws1", - "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone", - "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", - "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" - } - } diff --git a/pkg/config/env/common.go b/pkg/config/env/common.go index 7aa84a3a..80ae5ccb 100644 --- a/pkg/config/env/common.go +++ b/pkg/config/env/common.go @@ -11,8 +11,6 @@ var ( func init() { if IsTest() { environmentProvider = NewTestEnvironmentProvider() - } else { - environmentProvider = NewProductionEnvironmentProvider() } } diff --git a/pkg/config/env/environment.go b/pkg/config/env/environment.go index 7f53337e..a2a038df 100644 --- a/pkg/config/env/environment.go +++ b/pkg/config/env/environment.go @@ -2,49 +2,7 @@ package env type Environment int -const ( - EnvironmentUnknown Environment = 0 - EnvironmentInternal Environment = 1 - EnvironmentYC Environment = 2 - EnvironmentAWS Environment = 3 - EnvironmentNebius Environment = 4 - EnvironmentKZ Environment = 5 -) - -var ( - Environments = []Environment{EnvironmentInternal, EnvironmentYC, EnvironmentAWS, EnvironmentNebius, EnvironmentKZ} -) - -func (env Environment) String() string { - switch env { - case EnvironmentInternal: - return "Internal" - case EnvironmentYC: - return "YC" - case EnvironmentAWS: - return "AWS" - case EnvironmentNebius: - return "Nebius" - case EnvironmentKZ: - return "KZ" - default: - return "Unknown" - } -} - -func GetEnvByName(name string) (Environment, bool) { - switch name { - case "Internal": - return EnvironmentInternal, true - case "YC": - return EnvironmentYC, true - case "AWS": - return EnvironmentAWS, true - case "Nebius": - return EnvironmentNebius, true - case "KZ": - return EnvironmentKZ, true - default: - return EnvironmentUnknown, false - } +type EnvironmentProvider interface { + Get() Environment + Set(env Environment) } diff --git a/pkg/config/env/environment_provider.go b/pkg/config/env/environment_provider.go deleted file mode 100644 index ef5ffae7..00000000 --- a/pkg/config/env/environment_provider.go +++ /dev/null @@ -1,64 +0,0 @@ -package env - -import ( - "github.com/doublecloud/tross/pkg/config" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" -) - -type EnvironmentProvider interface { - Get() Environment - Set(env Environment) -} - -type ProductionEnvironmentProvider struct { -} - -func NewProductionEnvironmentProvider() *ProductionEnvironmentProvider { - return &ProductionEnvironmentProvider{} -} - -func (provider *ProductionEnvironmentProvider) Get() Environment { - var environment config.Environment - if cpconfig.Common != nil { - environment = cpconfig.Common.Environment - } else if dpconfig.Common != nil { - environment = dpconfig.Common.Environment - } else { - return EnvironmentUnknown - } - - switch environment.(type) { - case *config.Internal: - return EnvironmentInternal - case *config.YC: - return EnvironmentYC - case *config.AWS: - return EnvironmentAWS - case *config.Nebius: - return EnvironmentNebius - } - return EnvironmentUnknown -} - -func (provider *ProductionEnvironmentProvider) Set(env Environment) { - panic("Change environment only available in tests") -} - -type TestEnvironmentProvider struct { - env Environment -} - -func NewTestEnvironmentProvider() *TestEnvironmentProvider { - return &TestEnvironmentProvider{ - env: EnvironmentInternal, - } -} - -func (provider *TestEnvironmentProvider) Get() Environment { - return provider.env -} - -func (provider *TestEnvironmentProvider) Set(env Environment) { - provider.env = env -} diff --git a/pkg/oplock/optimistic_lock.go b/pkg/oplock/optimistic_lock.go deleted file mode 100644 index 0dc596da..00000000 --- a/pkg/oplock/optimistic_lock.go +++ /dev/null @@ -1,194 +0,0 @@ -package oplock - -import ( - "context" - "fmt" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/yt/go/migrate" - "github.com/doublecloud/tross/yt/go/schema" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" -) - -type Lock struct { - yt yt.Client - path ypath.Path -} - -type LockRow struct { - Key string `yson:",key"` - Value map[string]interface{} `yson:"Value"` - Version uint64 `yson:"Version"` - UpdateAt time.Time `yson:"UpdateAt"` -} - -type VersionConflict struct { - CandidateVersion uint64 - ActualVersion uint64 -} - -func (vc *VersionConflict) Is(err error) bool { - _, ok := err.(*VersionConflict) - return ok -} - -func IsVersionConflict(err error) bool { - return xerrors.Is(err, new(VersionConflict)) -} - -func (vc *VersionConflict) Error() string { - return fmt.Sprintf("row was updated. Actual version: %d, Candidate version: %d", vc.ActualVersion, vc.CandidateVersion) -} - -func (l *Lock) Read(key string) (*LockRow, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - tx, err := l.yt.BeginTabletTx(ctx, nil) - if err != nil { - //nolint:descriptiveerrors - return nil, xerrors.Errorf("Cannot begin tablet transaction: %w", err) - } - r, err := l.ReadTx(ctx, tx, key) - if err != nil { - //nolint:descriptiveerrors - return nil, xerrors.Errorf("Table read failed: %w", err) - } - logger.Log.Debugf("read key %v", key) - //nolint:descriptiveerrors - return r, tx.Commit() -} - -func (l *Lock) ReadTx(ctx context.Context, tx yt.TabletTx, key string) (*LockRow, error) { - r := new(LockRow) - r.Key = key - res, err := tx.LookupRows(ctx, l.path, []interface{}{map[string]string{"Key": key}}, nil) - if err != nil { - //nolint:descriptiveerrors - return nil, err - } - for res.Next() { - if err := res.Scan(&r); err != nil { - //nolint:descriptiveerrors - return nil, err - } - } - if r.Value == nil { - r.Value = map[string]interface{}{} - } - return r, nil -} - -type countRow struct { - Count int64 -} - -func (l *Lock) Count() (int64, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - res, err := l.yt.SelectRows(ctx, fmt.Sprintf("sum(1) as Count from [%v] group by 1", l.path), nil) - if err != nil { - //nolint:descriptiveerrors - return 0, err - } - defer func() { _ = res.Close() }() - var count countRow - for res.Next() { - if err := res.Scan(&count); err != nil { - //nolint:descriptiveerrors - return 0, err - } - } - return count.Count, nil -} - -func (l *Lock) Save(row *LockRow) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - tx, err := l.yt.BeginTabletTx(ctx, nil) - if err != nil { - //nolint:descriptiveerrors - return err - } - if err := l.SaveTx(ctx, tx, row); err != nil { - //nolint:descriptiveerrors - return err - } - logger.Log.Debugf("save %v", row) - //nolint:descriptiveerrors - return tx.Commit() -} - -func (l *Lock) SaveTx(ctx context.Context, tx yt.TabletTx, row *LockRow) error { - res, err := tx.LookupRows(ctx, l.path, []interface{}{map[string]string{"Key": row.Key}}, nil) - if err != nil { - //nolint:descriptiveerrors - return err - } - - var dbRow LockRow - for res.Next() { - if err := res.Scan(&dbRow); err != nil { - //nolint:descriptiveerrors - return err - } - } - if dbRow.Version > row.Version { - //nolint:descriptiveerrors - return &VersionConflict{ActualVersion: dbRow.Version, CandidateVersion: row.Version} - } - row.Version++ - row.UpdateAt = time.Now() - if err := tx.InsertRows(ctx, l.path, []interface{}{row}, nil); err != nil { - //nolint:descriptiveerrors - return err - } - return nil -} - -func createDynTableAndMount(ctx context.Context, yc yt.Client, path ypath.YPath, schema schema.Schema, bundle string) error { - attrs := make(map[string]interface{}) - attrs["dynamic"] = true - attrs["schema"] = schema - if bundle != "" { - attrs["tablet_cell_bundle"] = bundle - } - - _, err := yc.CreateNode(ctx, path, yt.NodeTable, &yt.CreateNodeOptions{ - Recursive: true, - Attributes: attrs, - }) - - if err != nil { - //nolint:descriptiveerrors - return err - } - - //nolint:descriptiveerrors - return migrate.MountAndWait(ctx, yc, path.YPath()) -} - -func New(client yt.Client, path ypath.Path) (*Lock, error) { - if err := backoff.Retry(func() error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - exist, err := client.NodeExists(ctx, path, nil) - if err != nil { - return xerrors.Errorf("Cannot check existence of the node %s: %w", path.String(), err) - } - if !exist { - lockRow := new(LockRow) - return createDynTableAndMount(context.Background(), client, path, schema.MustInfer(lockRow), "") - } - return nil - }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5)); err != nil { - return nil, err - } - return &Lock{ - yt: client, - path: path, - }, nil -} diff --git a/pkg/oplock/optimistic_lock_test.go b/pkg/oplock/optimistic_lock_test.go deleted file mode 100644 index 4e56e493..00000000 --- a/pkg/oplock/optimistic_lock_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package oplock - -import ( - "testing" - - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yttest" - "github.com/stretchr/testify/require" -) - -func TestLock_Read(t *testing.T) { - env, cancel := yttest.NewEnv(t) - defer cancel() - defer teardown(env) - locker, err := New(env.YT, "//home/cdc/test_lock") - require.NoError(t, err) - row, err := locker.Read("foo_key") - require.NoError(t, err) - row.Value = map[string]interface{}{ - "some_val": int64(42), - } - require.NoError(t, locker.Save(row)) - updatedRow, err := locker.Read("foo_key") - require.NoError(t, err) - require.Equal(t, updatedRow.Value, map[string]interface{}{"some_val": int64(42)}) -} - -func teardown(env *yttest.Env) { - _ = env.YT.RemoveNode(env.Ctx, ypath.Path("//home/cdc/test_lock"), nil) -} - -func TestLock_Save(t *testing.T) { - env, cancel := yttest.NewEnv(t) - defer cancel() - defer teardown(env) - locker, err := New(env.YT, "//home/cdc/test_lock") - require.NoError(t, err) - row, err := locker.Read("foo_key") - require.NoError(t, err) - row2, err := locker.Read("foo_key") - require.NoError(t, err) - row.Value["some_vale"] = 42 - row2.Value["some_val"] = 43 - require.NoError(t, locker.Save(row)) - require.Error(t, locker.Save(row2)) -} - -func TestIsVersionConflictError(t *testing.T) { - abstract.CheckOpaqueErrorWrapping(t, "pointer", IsVersionConflict, func(err error) error { - return &VersionConflict{ActualVersion: 14, CandidateVersion: 88} - }) -} diff --git a/pkg/parsers/resources/abstract.go b/pkg/parsers/resources/abstract.go index af5690a8..65ad57ce 100644 --- a/pkg/parsers/resources/abstract.go +++ b/pkg/parsers/resources/abstract.go @@ -1,12 +1,5 @@ package resources -import ( - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/config/env" -) - type AbstractResources interface { GetResource(string) (string, error) @@ -18,34 +11,3 @@ type AbstractResources interface { type Resourceable interface { ResourcesObj() AbstractResources } - -func isAnyResourceNamePresent(resourceNames []string) bool { - return !(len(resourceNames) == 0 || (len(resourceNames) == 1 && resourceNames[0] == "")) -} - -func IsAllowedYTResources() bool { - return dpconfig.IsInternalCloud() && !env.IsTest() -} - -func NewResources(logger log.Logger, resourceNames []string) (AbstractResources, error) { - if isAnyResourceNamePresent(resourceNames) { - if IsAllowedYTResources() { - logger.Info("creating YtResources", log.Any("resourceNames", resourceNames)) - result, err := NewYtResources(logger, resourceNames) - if err != nil { - return nil, xerrors.Errorf("unable to create yt resource, err: %w", err) - } - return result, nil - } else { - logger.Info("creating EmbeddedResources", log.Any("resourceNames", resourceNames)) - result, err := NewEmbeddedResources(resourceNames) - if err != nil { - return nil, xerrors.Errorf("unable to create embedded resource, err: %w", err) - } - return result, nil - } - } else { - logger.Info("creating NoResources", log.Any("resourceNames", resourceNames)) - return &NoResources{}, nil - } -} diff --git a/pkg/parsers/resources/dummy.go b/pkg/parsers/resources/dummy.go new file mode 100644 index 00000000..8b774658 --- /dev/null +++ b/pkg/parsers/resources/dummy.go @@ -0,0 +1,10 @@ +//go:build !arcadia +// +build !arcadia + +package resources + +import "github.com/doublecloud/tross/internal/core/log" + +func NewResources(logger log.Logger, resourceNames []string) (AbstractResources, error) { + return &NoResources{}, nil +} diff --git a/pkg/parsers/resources/yt_resources.go b/pkg/parsers/resources/yt_resources.go deleted file mode 100644 index 4419945a..00000000 --- a/pkg/parsers/resources/yt_resources.go +++ /dev/null @@ -1,95 +0,0 @@ -package resources - -import ( - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - schemaregistry "github.com/doublecloud/tross/pkg/schemaregistry/ytschemaregistry" -) - -type YtResources struct { - schemaRegistry schemaregistry.AbstractYtSchemaRegistry - sleepCallback sleepCallback - - ytResourcesDescription []schemaregistry.YtResourcesDescription - resNameToIndex map[string]int - - logger log.Logger - - closeCh chan struct{} - outdatedCh chan struct{} -} - -type sleepCallback func() - -func (r *YtResources) GetResource(resName string) (string, error) { - if index, ok := r.resNameToIndex[resName]; ok { - return r.ytResourcesDescription[index].Content, nil - } else { - return "", xerrors.Errorf("resource %s not found", resName) - } -} - -func (r *YtResources) watcher() { - defer close(r.outdatedCh) - for { - r.sleepCallback() - - select { - case <-r.closeCh: - return - default: - } - changed, err := r.schemaRegistry.CheckIfAnyResourceChanged(r.ytResourcesDescription) - if err != nil { - r.logger.Warnf("unable to check if any resource changed, err: %s", err.Error()) - continue - } - if changed { - return - } - } -} - -func (r *YtResources) RunWatcher() { - go r.watcher() -} - -func (r *YtResources) OutdatedCh() <-chan struct{} { - return r.outdatedCh -} - -func (r *YtResources) Close() { - close(r.closeCh) -} - -func NewYtResourcesImpl(logger log.Logger, resourceNames []string, schemaRegistry schemaregistry.AbstractYtSchemaRegistry, sleepCallback sleepCallback) (*YtResources, error) { - ytResourcesDescription, err := schemaRegistry.InitYtResourcesDescription(resourceNames) - if err != nil { - return nil, xerrors.Errorf("unable to init yt resources, err: %w", err) - } - - resNameToIndex := make(map[string]int) - for i, descr := range ytResourcesDescription { - resNameToIndex[descr.Name] = i - } - - return &YtResources{ - schemaRegistry: schemaRegistry, - sleepCallback: sleepCallback, - ytResourcesDescription: ytResourcesDescription, - resNameToIndex: resNameToIndex, - logger: logger, - closeCh: make(chan struct{}), - outdatedCh: make(chan struct{}), - }, nil -} - -func NewYtResources(logger log.Logger, resourceNames []string) (*YtResources, error) { - schemaRegistry, err := schemaregistry.NewSchemaRegistry(logger) - if err != nil { - return nil, xerrors.Errorf("unable to create schemaRegistry object, err: %w", err) - } - return NewYtResourcesImpl(logger, resourceNames, schemaRegistry, func() { time.Sleep(time.Second * 60) }) -} diff --git a/pkg/parsers/resources/yt_resources_test.go b/pkg/parsers/resources/yt_resources_test.go deleted file mode 100644 index 8f1a0a58..00000000 --- a/pkg/parsers/resources/yt_resources_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package resources - -import ( - "testing" - "time" - - "github.com/doublecloud/tross/internal/logger" - schemaregistry "github.com/doublecloud/tross/pkg/schemaregistry/ytschemaregistry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" -) - -func TestWatcher(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - resourceNames := []string{"aaa", "bbb"} - ytResourcesDescr := []schemaregistry.YtResourcesDescription{ - { - Name: "aaa", - Content: "a3", - Revision: 1, - }, - { - Name: "bbb", - Content: "b3", - Revision: 1, - }, - } - - schemaRegistryMock := schemaregistry.NewMockAbstractYtSchemaRegistry(ctrl) - schemaRegistryMock.EXPECT().InitYtResourcesDescription(resourceNames).Return(ytResourcesDescr, nil) - schemaRegistryMock.EXPECT().CheckIfAnyResourceChanged(gomock.Any()).Return(false, nil) - schemaRegistryMock.EXPECT().CheckIfAnyResourceChanged(gomock.Any()).Return(true, nil) - - index := 0 - sleepCallback := func() { - index++ - } - - ytResources, err := NewYtResourcesImpl(logger.Log, resourceNames, schemaRegistryMock, sleepCallback) - require.NoError(t, err) - - ytResources.RunWatcher() - - ch := ytResources.OutdatedCh() - channelClosed := false - go func() { - <-ch - channelClosed = true - }() - - time.Sleep(time.Second) - - require.Equal(t, 2, index) - require.True(t, channelClosed) -} diff --git a/pkg/providers/yt/prepare_job_spec.go b/pkg/providers/yt/prepare_job_spec.go deleted file mode 100644 index d080b039..00000000 --- a/pkg/providers/yt/prepare_job_spec.go +++ /dev/null @@ -1,119 +0,0 @@ -//go:build !all_in_one_binary -// +build !all_in_one_binary - -package yt - -import ( - "context" - "sync" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - "github.com/doublecloud/tross/yt/go/mapreduce" - "github.com/doublecloud/tross/yt/go/mapreduce/spec" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" -) - -type clusterRevisionPair struct { - cluster string - revision string -} - -var udfFileSpecsCache = make(map[clusterRevisionPair][]spec.File) -var udfFileSpecsCacheMutex = sync.Mutex{} - -func dataplaneDir(cluster string) ypath.Path { - if cluster == "vanga" { - return "//home/transfer-manager/data-plane" - } - return "//home/data-transfer/data-plane" -} - -func DataplaneExecutablePath(cluster, revision string) ypath.Path { - return dataplaneDir(cluster).Child(revision) -} - -func getUdfFileSpecs(cluster, rev string) ([]spec.File, error) { - pair := clusterRevisionPair{ - cluster: cluster, - revision: rev, - } - udfFileSpecsCacheMutex.Lock() - defer udfFileSpecsCacheMutex.Unlock() - res, ok := udfFileSpecsCache[pair] - if ok { - return res, nil - } - res, err := backoff.RetryWithData(func() ([]spec.File, error) { - res, err := udfFileSpecs(cluster, rev) - if err != nil { - return nil, err - } - return res, nil - }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 3)) - if err != nil { - return nil, err - } - udfFileSpecsCache[pair] = res - return res, nil -} - -func udfFileSpecs(cluster, rev string) ([]spec.File, error) { - dpDir := dataplaneDir(cluster) - udfDir := dpDir.Child("udfs").Child(rev) - - ytClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, nil, &yt.Config{ - Proxy: cluster, - Token: string(cpconfig.InternalCloud.YtToken), - DisableProxyDiscovery: false, - }) - if err != nil { - return nil, xerrors.Errorf("Cannot create YT client: %w", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - var udfs []struct { - Name string `yson:",value"` - } - if err := ytClient.ListNode(ctx, udfDir, &udfs, &yt.ListNodeOptions{}); err != nil { - return nil, xerrors.Errorf("Cannot list node %s: %w", udfDir, err) - } - logger.Log.Infof("List of UDFs at %s:%s: %v", cluster, udfDir, udfs) - - fileSpecs := make([]spec.File, 0, len(udfs)) - for _, udf := range udfs { - fileSpecs = append(fileSpecs, spec.File{ - FileName: udf.Name, - Executable: true, - CypressPath: udfDir.Child(udf.Name), - }) - } - return fileSpecs, nil -} - -func PrepareDataplaneJobSpec(job *spec.Spec, cluster, revision string) ([]mapreduce.OperationOption, error) { - dpExecutablePath := DataplaneExecutablePath(cluster, revision) - job.PatchUserBinary(dpExecutablePath) - job.VisitUserScripts(func(script *spec.UserScript) { - if script.Environment == nil { - script.Environment = map[string]string{} - } - script.Environment[dataplaneVersionEnv] = revision - }) - - udfs, err := getUdfFileSpecs(cluster, revision) - if err != nil { - return nil, xerrors.Errorf("Cannot get UDF paths from YT: %w", err) - } - if len(udfs) > 0 { - job.VisitUserScripts(func(u *spec.UserScript) { - u.FilePaths = append(u.FilePaths, udfs...) - }) - } - return []mapreduce.OperationOption{mapreduce.SkipSelfUpload()}, nil -} diff --git a/pkg/schemaregistry/ytschemaregistry/parsers.go b/pkg/schemaregistry/ytschemaregistry/parsers.go deleted file mode 100644 index 50118f88..00000000 --- a/pkg/schemaregistry/ytschemaregistry/parsers.go +++ /dev/null @@ -1,147 +0,0 @@ -package ytschemaregistry - -import ( - "context" - "encoding/json" - "fmt" - "io" - "strings" - "time" - - "github.com/doublecloud/tross/internal/core/xerrors" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - "github.com/doublecloud/tross/pkg/config/env" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" -) - -const ( - ParsersConfigsPath = "//home/logfeller/configs/parsers" -) - -var parsersYtPaths = []ypath.YPath{ - ypath.Path(fmt.Sprintf("%s/parsers.auto.json", ParsersConfigsPath)), - ypath.Path(fmt.Sprintf("%s/parsers.robot.json", ParsersConfigsPath)), -} - -type CommonParserParams struct { - Precomputers []string `json:"type"` -} - -type ParserParams struct { - Type string `json:"type"` - CommonParserParams CommonParserParams `json:"common"` -} - -func GetParsers() (map[string]ParserParams, error) { - if env.IsTest() { - return map[string]ParserParams{}, nil - } - ytClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, nil, &yt.Config{ - Proxy: "hahn", - Token: string(cpconfig.InternalCloud.YtToken), - AllowRequestsFromJob: true, - DisableProxyDiscovery: false, - }) - if err != nil { - return nil, xerrors.Errorf("unable to create yt client: %w", err) - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - result := map[string]ParserParams{} - - for _, currYtPath := range parsersYtPaths { - fileContent, err := ytClient.ReadFile(ctx, currYtPath, nil) - if err != nil { - return nil, xerrors.Errorf("unable to read file %s from yt: %w", currYtPath, err) - } - - fileContentBuf := new(strings.Builder) - _, err = io.Copy(fileContentBuf, fileContent) - if err != nil { - return nil, xerrors.Errorf("unable to copy file content: %w", err) - } - - var parsers map[string]ParserParams - fileContentStr := fileContentBuf.String() - if err := json.Unmarshal([]byte(fileContentStr), &parsers); err != nil { - return nil, xerrors.Errorf("unable to unmarshal json %s, error: %w", fileContent, err) - } - - for k, v := range parsers { - result[k] = v - } - } - - return result, nil -} - -func GetParser(parserName string) (*ParserParams, error) { - parsers, err := GetParsers() - if err != nil { - return nil, err - } - if parserParams, found := parsers[parserName]; found { - return &parserParams, nil - } else { - return nil, nil - } -} - -func GetYqlFlagParserParams(parserName string) (*string, []string, error) { - parserParams, err := GetParser(parserName) - if err != nil { - return nil, nil, err - } - if parserParams == nil { - return nil, nil, nil - } - - return &parserParams.Type, parserParams.CommonParserParams.Precomputers, nil -} - -func IsYqlParser(parserType, chunkSplitterName string, precomputers []string) bool { - switch parserType { - case "TTskvWithSchemaLogParser": - case "TJsonWithSchemaLogParser": - case "TProtoLogParser": - case "TNativeProtoLogParser": - case "TBsWebvisorLogParser": - case "TYsonLogParser": - case "TNativeInnerSerializedProtoLogParser": - // do nothing - default: - return false - } - - switch chunkSplitterName { - case "line-break": - case "protoseq": - case "do-not-split": - // do nothing - default: - return false - } - - for _, precomputer := range precomputers { - switch precomputer { - case "icookie_from_separate_uuid": - case "separate_icookie": - case "separate_cookiei": - case "headers_icookie": - case "encrypted_cookies_icookie": - case "encrypted_turbo_icookie": - case "transport_topic_info": - case "extended_transport_topic_info": - case "common_access": - // do nothing - default: - return false - } - } - - return true -} diff --git a/pkg/schemaregistry/ytschemaregistry/parsers_cache.go b/pkg/schemaregistry/ytschemaregistry/parsers_cache.go deleted file mode 100644 index af86211e..00000000 --- a/pkg/schemaregistry/ytschemaregistry/parsers_cache.go +++ /dev/null @@ -1,58 +0,0 @@ -package ytschemaregistry - -import ( - "sync" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" -) - -type ParserCache struct { - lastUpdated time.Time - mutex sync.Mutex - parsers []string -} - -func (c *ParserCache) ListParsers() ([]string, error) { - c.mutex.Lock() - defer c.mutex.Unlock() - - if len(c.parsers) == 0 || c.stale() { - if err := c.update(); err != nil { - if len(c.parsers) == 0 { - return nil, xerrors.Errorf("Unable to list parsers: %w", err) - } - logger.Log.Warn("Unable to update parser cache: returning stale cached values", log.Error(err)) - } - } - - return c.parsers, nil -} - -func (c *ParserCache) stale() bool { - return time.Since(c.lastUpdated) > time.Minute -} - -func (c *ParserCache) update() error { - parserMap, err := GetParsers() - if err != nil { - return xerrors.Errorf("Unable to get parsers: %w", err) - } - c.parsers = make([]string, len(parserMap)) - i := 0 - for key := range parserMap { - c.parsers[i] = key - i++ - } - return nil -} - -func NewParserCache() *ParserCache { - return &ParserCache{ - lastUpdated: time.Time{}, - mutex: sync.Mutex{}, - parsers: []string{}, - } -} diff --git a/pkg/schemaregistry/ytschemaregistry/parsers_test.go b/pkg/schemaregistry/ytschemaregistry/parsers_test.go deleted file mode 100644 index 2a648754..00000000 --- a/pkg/schemaregistry/ytschemaregistry/parsers_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package ytschemaregistry - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestIsYqlParser(t *testing.T) { - require.Equal(t, true, IsYqlParser("TYsonLogParser", "do-not-split", []string{"extended_transport_topic_info"})) -} diff --git a/pkg/schemaregistry/ytschemaregistry/util.go b/pkg/schemaregistry/ytschemaregistry/util.go deleted file mode 100644 index 10e6914e..00000000 --- a/pkg/schemaregistry/ytschemaregistry/util.go +++ /dev/null @@ -1,63 +0,0 @@ -package ytschemaregistry - -import ( - "context" - "io" - "path/filepath" - "strings" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" -) - -func checkExtension(name string) string { - fileName := name - fileExt := filepath.Ext(name) - if fileExt == "" { - fileName += ".json" - } - return fileName -} - -func makePathToResource(ytDirPath ypath.Path, name string) ypath.Path { - return ytDirPath.Child(checkExtension(name)) -} - -func getRevision(tx yt.Tx, ytDirPath ypath.Path, name string) (uint64, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer cancel() - - currYtPath := makePathToResource(ytDirPath, name) - var revision uint64 - err := tx.GetNode(ctx, currYtPath.Attr("arcadia_revision"), &revision, nil) - if err != nil { - return 0, xerrors.Errorf("unable to get yt node 'arcadia_revision' for path: %s", currYtPath.String()) - } - return revision, nil -} - -func readResource(tx yt.Tx, ytDirPath ypath.Path, resourceName string, logger log.Logger) (string, uint64, error) { - currYtPath := makePathToResource(ytDirPath, resourceName) - logger.Info("Read file", log.Any("path", currYtPath)) - - fileContent, err := tx.ReadFile(context.Background(), currYtPath, nil) - if err != nil { - return "", 0, xerrors.Errorf("unable to read file %s from yt: %w", currYtPath, err) - } - - fileContentBuf := new(strings.Builder) - _, err = io.Copy(fileContentBuf, fileContent) - if err != nil { - return "", 0, xerrors.Errorf("unable to copy file content, err: %w", err) - } - - revision, err := getRevision(tx, ytDirPath, resourceName) - if err != nil { - return "", 0, xerrors.Errorf("unable to get arcadia_revision: %w", err) - } - - return fileContentBuf.String(), revision, nil -} diff --git a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry.go b/pkg/schemaregistry/ytschemaregistry/yt_schema_registry.go deleted file mode 100644 index df1ffa79..00000000 --- a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry.go +++ /dev/null @@ -1,16 +0,0 @@ -package ytschemaregistry - -type YtResourcesDescription struct { - Name string - Content string - Revision uint64 -} - -// how to generate mock from interface: -// export GO111MODULE=on && ya tool mockgen -source ./yt_schema_registry.go -package schemaregistry -destination ./yt_schema_registry_mock.go - -type AbstractYtSchemaRegistry interface { - InitYtResourcesDescription(resourceNames []string) ([]YtResourcesDescription, error) - CheckIfAnyResourceChanged(resources []YtResourcesDescription) (bool, error) - ListSchemas() ([]string, error) -} diff --git a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_impl.go b/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_impl.go deleted file mode 100644 index ee61af7e..00000000 --- a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_impl.go +++ /dev/null @@ -1,126 +0,0 @@ -package ytschemaregistry - -import ( - "context" - "sync" - "time" - - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - "github.com/doublecloud/tross/yt/go/ypath" - "github.com/doublecloud/tross/yt/go/yt" -) - -type SchemaRegistry struct { - logger log.Logger - ytClient yt.Client - ytDirPathParsers ypath.Path - mutex sync.Mutex - allSchemas []string - ParserCache *ParserCache -} - -func (r *SchemaRegistry) InitYtResourcesDescription(resourceNames []string) ([]YtResourcesDescription, error) { - result := make([]YtResourcesDescription, 0) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) - defer cancel() - tx, err := r.ytClient.BeginTx(ctx, nil) - if err != nil { - return nil, xerrors.Errorf("unable to create yt transaction, err: %w", err) - } - defer tx.Commit() - - for _, resourceName := range resourceNames { - resourceBody, revision, err := readResource(tx, r.ytDirPathParsers, resourceName, r.logger) - if err != nil { - return nil, xerrors.Errorf("unable to load resource '%s', err: %w", resourceName, err) - } - result = append(result, YtResourcesDescription{ - Name: resourceName, - Content: resourceBody, - Revision: revision, - }) - } - - return result, nil -} - -func (r *SchemaRegistry) CheckIfAnyResourceChanged(resources []YtResourcesDescription) (bool, error) { - tx, err := r.ytClient.BeginTx(context.Background(), nil) // context-timeout will be in getRevision() - if err != nil { - return false, xerrors.Errorf("unable to begin transaction: %s", err.Error()) - } - defer tx.Commit() - - for _, resource := range resources { - ytRevision, err := getRevision(tx, r.ytDirPathParsers, resource.Name) - if err != nil { - return false, xerrors.Errorf("unable to get revision: %s", err.Error()) - } - - if resource.Revision != ytRevision { - r.logger.Infof("resource revision is changed, resource_name:%s, start_revision:%d, now_revision:%d", resource.Name, resource.Revision, ytRevision) - return true, nil - } - } - return false, nil -} - -func (r *SchemaRegistry) ListSchemas() ([]string, error) { - r.mutex.Lock() - defer r.mutex.Unlock() - - if len(r.allSchemas) > 0 { - return r.allSchemas, nil - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - var parsers []struct { - Name string `yson:",value"` - } - size := int64(10_000) - if err := r.ytClient.ListNode(ctx, r.ytDirPathParsers, &parsers, &yt.ListNodeOptions{MaxSize: &size}); err != nil { - return nil, xerrors.Errorf("unable to list schemas: %w", err) - } - var res []string - for _, t := range parsers { - res = append(res, t.Name) - } - r.allSchemas = res - return res, nil -} - -func NewSchemaRegistry(lgr log.Logger) (*SchemaRegistry, error) { - var tkn, proxy string - if cpconfig.IsInternalCloud() { - tkn = string(cpconfig.InternalCloud.YtToken) - proxy = cpconfig.InternalCloud.YtProxy - } else if dpconfig.IsInternalCloud() { - tkn = string(dpconfig.InternalCloud.YtToken) - proxy = dpconfig.InternalCloud.YtProxy - } else { - return nil, xerrors.Errorf("schemaRegistry object should be created only in internal cloud") - } - - client, err := ytclient.NewYtClientWrapper(ytclient.HTTP, lgr, &yt.Config{ - Proxy: proxy, - Token: tkn, - AllowRequestsFromJob: true, - }) - if err != nil { - return nil, xerrors.Errorf("unable to create yt client, err: %w", err) - } - - return &SchemaRegistry{ - logger: lgr, - ytClient: client, - ytDirPathParsers: "//home/logfeller/configs/parsers", - mutex: sync.Mutex{}, - allSchemas: nil, - ParserCache: NewParserCache(), - }, nil -} diff --git a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_mock.go b/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_mock.go deleted file mode 100644 index 93686aeb..00000000 --- a/pkg/schemaregistry/ytschemaregistry/yt_schema_registry_mock.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: ./yt_schema_registry.go - -// Package schemaregistry is a generated GoMock package. -package ytschemaregistry - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockAbstractYtSchemaRegistry is a mock of AbstractYtSchemaRegistry interface. -type MockAbstractYtSchemaRegistry struct { - ctrl *gomock.Controller - recorder *MockAbstractYtSchemaRegistryMockRecorder -} - -// MockAbstractYtSchemaRegistryMockRecorder is the mock recorder for MockAbstractYtSchemaRegistry. -type MockAbstractYtSchemaRegistryMockRecorder struct { - mock *MockAbstractYtSchemaRegistry -} - -// NewMockAbstractYtSchemaRegistry creates a new mock instance. -func NewMockAbstractYtSchemaRegistry(ctrl *gomock.Controller) *MockAbstractYtSchemaRegistry { - mock := &MockAbstractYtSchemaRegistry{ctrl: ctrl} - mock.recorder = &MockAbstractYtSchemaRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAbstractYtSchemaRegistry) EXPECT() *MockAbstractYtSchemaRegistryMockRecorder { - return m.recorder -} - -// CheckIfAnyResourceChanged mocks base method. -func (m *MockAbstractYtSchemaRegistry) CheckIfAnyResourceChanged(resources []YtResourcesDescription) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckIfAnyResourceChanged", resources) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckIfAnyResourceChanged indicates an expected call of CheckIfAnyResourceChanged. -func (mr *MockAbstractYtSchemaRegistryMockRecorder) CheckIfAnyResourceChanged(resources interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckIfAnyResourceChanged", reflect.TypeOf((*MockAbstractYtSchemaRegistry)(nil).CheckIfAnyResourceChanged), resources) -} - -// InitYtResourcesDescription mocks base method. -func (m *MockAbstractYtSchemaRegistry) InitYtResourcesDescription(resourceNames []string) ([]YtResourcesDescription, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InitYtResourcesDescription", resourceNames) - ret0, _ := ret[0].([]YtResourcesDescription) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// InitYtResourcesDescription indicates an expected call of InitYtResourcesDescription. -func (mr *MockAbstractYtSchemaRegistryMockRecorder) InitYtResourcesDescription(resourceNames interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitYtResourcesDescription", reflect.TypeOf((*MockAbstractYtSchemaRegistry)(nil).InitYtResourcesDescription), resourceNames) -} - -// ListSchemas mocks base method. -func (m *MockAbstractYtSchemaRegistry) ListSchemas() ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListSchemas") - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListSchemas indicates an expected call of ListSchemas. -func (mr *MockAbstractYtSchemaRegistryMockRecorder) ListSchemas() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemas", reflect.TypeOf((*MockAbstractYtSchemaRegistry)(nil).ListSchemas)) -} diff --git a/pkg/worker/tasks/oss_job.go b/pkg/worker/tasks/oss_job.go new file mode 100644 index 00000000..2bdda3a3 --- /dev/null +++ b/pkg/worker/tasks/oss_job.go @@ -0,0 +1,46 @@ +//go:build !arcadia +// +build !arcadia + +package tasks + +import ( + "context" + "time" + + "github.com/doublecloud/tross/internal/logger" + "github.com/doublecloud/tross/pkg/abstract/coordinator" + server "github.com/doublecloud/tross/pkg/abstract/model" + "github.com/doublecloud/tross/pkg/errors" + "github.com/doublecloud/tross/pkg/errors/categories" +) + +func StartJob(ctx context.Context, cp coordinator.Coordinator, transfer server.Transfer, task *server.TransferOperation) error { + if !transfer.IsMain() { + return nil + } + transfer.Status = server.Running + if transfer.SnapshotOnly() { + transfer.Status = server.Completed + } + if err := cp.SetStatus(transfer.ID, transfer.Status); err != nil { + return errors.CategorizedErrorf(categories.Internal, "Cannot transit transfer into the %s state: %w", string(transfer.Status), err) + } + if transfer.SnapshotOnly() { + return nil + } + + logger.Log.Info("Wait for transfer status change to apply") + time.Sleep(10 * time.Second) + logger.Log.Info("Transfer status change is considered completed") + time.Sleep(time.Second) + return nil +} + +func StopJob(cp coordinator.Coordinator, transfer server.Transfer) error { + if transfer.SnapshotOnly() { + return nil + } + logger.Log.Infof("Wait to change apply") + time.Sleep(10 * time.Second) + return nil +} diff --git a/pkg/worker/tasks/start_job.go b/pkg/worker/tasks/start_job.go deleted file mode 100644 index 69a256b5..00000000 --- a/pkg/worker/tasks/start_job.go +++ /dev/null @@ -1,216 +0,0 @@ -package tasks - -import ( - "context" - "fmt" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/cloud/bitbucket/public-api/yandex/cloud/compute/v1/instancegroup" - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/metrics/solomon" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/errors" - "github.com/doublecloud/tross/pkg/errors/categories" - "github.com/doublecloud/tross/pkg/oplock" - "github.com/doublecloud/tross/pkg/providers" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - ycsdk "github.com/doublecloud/tross/pkg/yc/sdk" - "github.com/doublecloud/tross/yt/go/guid" - "github.com/doublecloud/tross/yt/go/yt" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func StartJob(ctx context.Context, cp coordinator.Coordinator, transfer server.Transfer, task *server.TransferOperation) error { - if !transfer.IsMain() { - return nil - } - transfer.Status = server.Running - if transfer.SnapshotOnly() { - transfer.Status = server.Completed - } - if err := cp.SetStatus(transfer.ID, transfer.Status); err != nil { - return errors.CategorizedErrorf(categories.Internal, "Cannot transit transfer into the %s state: %w", string(transfer.Status), err) - } - if transfer.SnapshotOnly() { - return nil - } - - var startJobErr error = nil - switch r := transfer.Runtime.(type) { - case *abstract.LocalRuntime, *abstract.CrossDcYtRuntime, *abstract.DedicatedRuntime, *abstract.K8SRuntime: - // do nothing - case *abstract.YtRuntime: - startJobErr = startJobInYTRuntime(ctx, r, transfer) - case *abstract.NirvactorRuntime: - startJobErr = startJobInNirvactorRuntime(ctx, cp, transfer, task) - case *abstract.YcRuntime: - startJobErr = startJobInYCRuntime(ctx, transfer) - case nil: - logger.Log.Warn("StartJob called for a nil runtime") - // this is possible in tests. Finish immediately - return nil - default: - startJobErr = abstract.NewFatalError(xerrors.Errorf("unknown runtime of type %T", r)) - } - if startJobErr != nil { - return errors.CategorizedErrorf(categories.Internal, "failed to start job: %w", startJobErr) - } - - logger.Log.Info("Wait for transfer status change to apply") - time.Sleep(10 * time.Second) - logger.Log.Info("Transfer status change is considered completed") - time.Sleep(time.Second) - return nil -} - -func startJobInYTRuntime(ctx context.Context, r *abstract.YtRuntime, transfer server.Transfer) error { - var ytProxy, ytToken string - if dpconfig.IsDataPlane() { - ytProxy = dpconfig.InternalCloud.YtProxy - ytToken = string(dpconfig.InternalCloud.YtToken) - } else { - ytProxy = cpconfig.InternalCloud.YtProxy - ytToken = string(cpconfig.InternalCloud.YtToken) - } - lockClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, logger.Log, &yt.Config{ - Proxy: ytProxy, - Token: ytToken, - AllowRequestsFromJob: true, - }) - if err != nil { - return xerrors.Errorf("Cannot create YT HTTP Lock client: %w", err) - } - operations, err := oplock.New(lockClient, "//home/transfer-manager/replications") - if err != nil { - return xerrors.Errorf("Operation lock error: %w", err) - } - var opID yt.OperationID - if err := backoff.Retry(func() error { - op, err := operations.Read(transfer.ID) - if err != nil { - return xerrors.Errorf("Cannot resolve YT operation by transfer ID (%s): %w", transfer.ID, err) - } - id, ok := op.Value["operation_id"].(string) - if !ok { - return ErrNoActiveOperation - } - parsedID, err := guid.ParseString(id) - if err != nil { - return xerrors.Errorf("Cannot parse operation ID (%s): %w", id, err) - } - opID = yt.OperationID(parsedID) - return nil - }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5)); err != nil { - if xerrors.Is(err, ErrNoActiveOperation) { - return nil - } - return xerrors.Errorf("failed to obtain or read YT lock for transfer %s: %w", transfer.ID, err) - } - jobClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, logger.Log, &yt.Config{ - Proxy: string(r.Cluster), - Token: ytToken, - AllowRequestsFromJob: true, - }) - if err != nil { - return xerrors.Errorf("Cannot create YT HTTP Job client: %w", err) - } - logger.Log.Info("Waiting until operation starts", log.String("operation_id", opID.String())) - for { - op, err := jobClient.GetOperation(ctx, opID, nil) - if err != nil { - return xerrors.Errorf("Cannot get operation by ID %s: %w", opID.String(), err) - } - if op.State.IsFinished() { - logger.Log.Warn("Started operation is finished", log.String("operation_id", opID.String()), log.Any("state", op.State)) - return nil - } - if op.State == yt.StateRunning { - logger.Log.Info("Operation is running", log.String("operation_id", opID.String())) - return nil - } - logger.Log.Info("Operation is not running yet; waiting 1 second", log.String("operation_id", opID.String()), log.String("operation_status", string(op.State))) - time.Sleep(time.Second) - } -} - -func startJobInNirvactorRuntime(ctx context.Context, cp coordinator.Coordinator, transfer server.Transfer, task *server.TransferOperation) error { - transfer.Status = server.Creating - err := cp.SetStatus(transfer.ID, transfer.Status) - if err != nil { - logger.Log.Error("unable to update transfer", log.Error(err)) - } - - // just in case, redeploy - activator, ok := providers.Source[providers.Activator](logger.Log, solomon.NewRegistry(solomon.NewRegistryOpts()), cp, &transfer) - if ok { - if err := activator.Activate(ctx, task, nil, providers.NopActivateCallback); err != nil { - return xerrors.Errorf("Cannot execute %s activate hook: %w", transfer.SrcType(), err) - } - } - transfer.Status = server.Running - if err := cp.SetStatus(transfer.ID, transfer.Status); err != nil { - logger.Log.Error("unable to update transfer", log.Error(err)) - } - - return nil -} - -func startJobInYCRuntime(ctx context.Context, transfer server.Transfer) error { - logger.Log.Info("wait for instance group to be inited") - sdk, err := ycsdk.Instance() - if err != nil { - logger.Log.Warn("unable to init sdk", log.Error(err)) - return xerrors.Errorf("unable to init SDK: %w", err) - } - ctxLimited, cancel := context.WithTimeout(ctx, time.Minute*3) - defer cancel() - for { - igListFilter := fmt.Sprintf("name='%v'", transfer.ID) - l, err := sdk.InstanceGroup().InstanceGroup().List(ctxLimited, &instancegroup.ListInstanceGroupsRequest{ - Filter: igListFilter, - FolderId: serviceFolderID(), - }) - if st, ok := status.FromError(err); ok && st.Code() == codes.DeadlineExceeded { - return xerrors.Errorf("unable to ensure ig inited: %w", err) - } - if err == context.DeadlineExceeded { - return xerrors.Errorf("unable to ensure ig inited: %w", err) - } - if err != nil { - logger.Log.Warn("unable to get instance group", log.Error(err)) - time.Sleep(time.Second) - continue - } - if len(l.InstanceGroups) == 0 { - logger.Log.Infof("no instance group found for filter: %v, wait for 10 second more", igListFilter) - time.Sleep(time.Second * 10) - continue - } - if len(l.InstanceGroups) > 1 { - return xerrors.Errorf("found more than one replication instance: %v", l.InstanceGroups) - } - ig := l.InstanceGroups[0] - logger.Log.Infof("inited IG: %v, %v running for %v", ig.Id, ig.Name, time.Since(ig.CreatedAt.AsTime())) - return nil - } -} - -func serviceFolderID() string { - if dpconfig.IsDataPlane() { - return dpconfig.ExternalCloud.ServiceFolderID - } else { - return cpconfig.ExternalCloud.ServiceFolderID - } -} - -func startJobInK8SRuntime(transfer server.Transfer) error { - return nil -} diff --git a/pkg/worker/tasks/stop_job.go b/pkg/worker/tasks/stop_job.go deleted file mode 100644 index 5a48ecab..00000000 --- a/pkg/worker/tasks/stop_job.go +++ /dev/null @@ -1,149 +0,0 @@ -package tasks - -import ( - "context" - "fmt" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/doublecloud/tross/cloud/bitbucket/public-api/yandex/cloud/compute/v1/instancegroup" - "github.com/doublecloud/tross/internal/core/log" - "github.com/doublecloud/tross/internal/core/xerrors" - "github.com/doublecloud/tross/internal/logger" - "github.com/doublecloud/tross/pkg/abstract" - "github.com/doublecloud/tross/pkg/abstract/coordinator" - server "github.com/doublecloud/tross/pkg/abstract/model" - cpconfig "github.com/doublecloud/tross/pkg/config/controlplane" - dpconfig "github.com/doublecloud/tross/pkg/config/dataplane" - "github.com/doublecloud/tross/pkg/oplock" - ytclient "github.com/doublecloud/tross/pkg/providers/yt/client" - ycsdk "github.com/doublecloud/tross/pkg/yc/sdk" - "github.com/doublecloud/tross/yt/go/guid" - "github.com/doublecloud/tross/yt/go/yt" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var ErrNoActiveOperation = xerrors.NewSentinel("TM: missed operation id") - -func StopJob(cp coordinator.Coordinator, transfer server.Transfer) error { - if transfer.SnapshotOnly() { - return nil - } - if r, ok := transfer.Runtime.(*abstract.YtRuntime); ok { - var ytProxy, ytToken string - if dpconfig.IsDataPlane() { - ytProxy = dpconfig.InternalCloud.YtProxy - ytToken = string(dpconfig.InternalCloud.YtToken) - } else { - ytProxy = cpconfig.InternalCloud.YtProxy - ytToken = string(cpconfig.InternalCloud.YtToken) - } - err := cp.SetStatus(transfer.ID, server.Stopping) - if err != nil { - return xerrors.Errorf("unable to set controlplane status: %w", err) - } - - lockClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, logger.Log, &yt.Config{ - Proxy: ytProxy, - Token: ytToken, - AllowRequestsFromJob: true, - }) - if err != nil { - return xerrors.Errorf("unable to create new yt http client: %w", err) - } - operations, err := oplock.New(lockClient, "//home/transfer-manager/replications") - if err != nil { - return xerrors.Errorf("unable to create new oplock: %w", err) - } - var opID yt.OperationID - if err := backoff.Retry(func() error { - op, err := operations.Read(transfer.ID) - if err != nil { - return xerrors.Errorf("unable to read transfer ID: %w", err) - } - id, ok := op.Value["operation_id"].(string) - if !ok { - return ErrNoActiveOperation - } - parsedID, err := guid.ParseString(id) - if err != nil { - return xerrors.Errorf("unable to parse ID string: %w", err) - } - opID = yt.OperationID(parsedID) - return nil - }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 5)); err != nil { - if err == ErrNoActiveOperation { - logger.Log.Infof("Operation is already stopped") - return nil - } - return err - } - jobClient, err := ytclient.NewYtClientWrapper(ytclient.HTTP, logger.Log, &yt.Config{ - Proxy: string(r.Cluster), - Token: ytToken, - AllowRequestsFromJob: true, - }) - if err != nil { - return xerrors.Errorf("unable to create new job client: %w", err) - } - ctx := context.Background() - for { - op, err := jobClient.GetOperation(ctx, opID, nil) - if err != nil { - time.Sleep(time.Second) - continue - } - if op.State.IsFinished() { - break - } - time.Sleep(time.Second) - } - if err := cp.SetStatus(transfer.ID, server.Stop); err != nil { - return xerrors.Errorf("unable to set controlplane status: %w", err) - } - } else if _, ok := transfer.Runtime.(*abstract.YcRuntime); ok { - logger.Log.Info("wait for instance group to be deleted") - sdk, err := ycsdk.Instance() - if err != nil { - logger.Log.Warn("unable to init sdk", log.Error(err)) - return xerrors.Errorf("unable to init SDK: %w", err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*3) - defer cancel() - igListFilter := fmt.Sprintf("name='%v'", transfer.ID) - l, err := sdk.InstanceGroup().InstanceGroup().List(ctx, &instancegroup.ListInstanceGroupsRequest{ - Filter: igListFilter, - FolderId: serviceFolderID(), - }) - if err != nil { - logger.Log.Warn("unable to get instance group", log.Error(err)) - return xerrors.Errorf("unable to list instance groups: %w", err) - } - if len(l.InstanceGroups) == 0 { - logger.Log.Infof("no instance group found for filter: %v", igListFilter) - return nil - } - if len(l.InstanceGroups) > 1 { - return xerrors.Errorf("found more than one replication instance: %v", l.InstanceGroups) - } - ig := l.InstanceGroups[0] - for { - _, err := sdk.InstanceGroup().InstanceGroup().Get(ctx, &instancegroup.GetInstanceGroupRequest{InstanceGroupId: ig.Id}) - if st, ok := status.FromError(err); ok && (st.Code() == codes.PermissionDenied || st.Code() == codes.NotFound) { - logger.Log.Infof("instance: %v deleted, was running for %v", ig.Id, time.Since(ig.CreatedAt.AsTime())) - return nil - } - if err != nil { - logger.Log.Warn("unable to get instance", log.Error(err)) - return xerrors.Errorf("unable to get ig: %w", err) - } - time.Sleep(time.Second * 10) - logger.Log.Infof("wait for instance: %v to be deleted", ig.Id) - } - } else { - logger.Log.Infof("Wait to change apply") - time.Sleep(10 * time.Second) - } - return nil -}