From 608ba43d1b4e60d76661a6b7546d14c63132d47c Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Thu, 22 Aug 2024 15:46:57 +0200 Subject: [PATCH] test(ydbcp): add docker-compose for ydbcp --- docker-compose.yaml | 87 +++++++++++++++++++ dockerfile | 20 +++++ go.mod | 2 +- go.sum | 2 + init_db/create_tables.sh | 6 ++ internal/config/config.go | 1 + internal/connectors/client/connector.go | 36 ++++---- .../db/yql/schema/create_tables.yql | 4 - .../connectors/db/yql/schema/drop_tables.yql | 3 + .../server/services/backup/backupservice.go | 5 +- internal/types/backup.go | 2 +- local_config.yaml | 25 ++++++ 12 files changed, 168 insertions(+), 25 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 dockerfile create mode 100755 init_db/create_tables.sh create mode 100644 internal/connectors/db/yql/schema/drop_tables.yql create mode 100644 local_config.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..bcea4237 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,87 @@ +services: + ydb: + image: cr.ai.nebius.cloud/crnca8q7ti1i7vpqs28l/ydb-local:24.1.16-stream-nb-1.1-1 + platform: linux/amd64 + hostname: local-ydb + #volumes: + # - ./ydbd/ydb_certs:/ydb_certs + # - ./ydbd/ydb_data:/ydb_data + environment: + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + - YDB_ANONYMOUS_CREDENTIALS=true + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_FEATURE_FLAGS=enable_uuid_as_primary_key,enable_implicit_query_parameter_types + ports: + - "8765:8765" + #- "2135:2135" + #- "2136:2136" + networks: + - ydbcp-net + + setup_ydb: + image: cr.ai.nebius.cloud/crnca8q7ti1i7vpqs28l/ydb-local:24.1.16-stream-nb-1.1-1 + platform: linux/amd64 + volumes: + - ./init_db:/init_db + - ./internal/connectors/db/yql/schema:/init_db/schema + depends_on: + ydb: + condition: service_healthy + restart: "no" + command: bash -c "chmod +x ./init_db/create_tables.sh && ./init_db/create_tables.sh" + networks: + - ydbcp-net + + s3: + image: quay.io/minio/minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ydbcp + MINIO_ROOT_PASSWORD: password + ENABLE_VIRTUAL_STYLE: 0 + ports: + #- "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 10s + timeout: 20s + retries: 3 + networks: + - ydbcp-net + + setup_s3: + image: amazon/aws-cli + environment: + - AWS_ACCESS_KEY_ID=ydbcp + - AWS_SECRET_ACCESS_KEY=password + - AWS_DEFAULT_REGION=us-east-1 + - AWS_ENDPOINT_URL=http://ydbcp-s3-1.ydbcp-net:9000 + depends_on: + s3: + condition: service_healthy + command: s3api create-bucket --bucket test-bucket + networks: + - ydbcp-net + + ydbcp: + image: ydbcp:local + depends_on: + setup_ydb: + condition: service_completed_successfully + setup_s3: + condition: service_completed_successfully + build: + context: . + dockerfile: ./dockerfile + ports: + - "50051:50051" + networks: + - ydbcp-net + +networks: + ydbcp-net: + name: ydbcp-net + external: true diff --git a/dockerfile b/dockerfile new file mode 100644 index 00000000..483de4e7 --- /dev/null +++ b/dockerfile @@ -0,0 +1,20 @@ +# Use the official Golang image as the base image +FROM golang:1.22-alpine as builder + +# Set the Current Working Directory inside the container +WORKDIR ./ydbcp + +# Copy source code into the container +COPY ./ ./ + +# Download all dependencies +RUN go mod download + +# Install grpcurl +#RUN go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest + +# Build the Go app +RUN go build -o . ./cmd/ydbcp/main.go + +# Command to run the executable +CMD ["./main", "--config=local_config.yaml"] \ No newline at end of file diff --git a/go.mod b/go.mod index 87a12f16..703c4571 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/jonboulle/clockwork v0.3.0 github.com/stretchr/testify v1.8.1 - github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 + github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8 github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2 go.uber.org/automaxprocs v1.5.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 32d0be7b..c72c2384 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 h1:nL8XwD6fSst7xFUirkaWJmE7kM0CdWRYgu6+YQer1d4= github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8 h1:ZWxYw6L51aNAMLbTpC/VbXP0rcnvsCAJqx7EI/CjWmc= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2 h1:thrbvktqKA6LFZTnZrGuQi8LQVel1J2dDfoQFsgvcYs= github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2/go.mod h1:QMmT1fMKZnpid73USXLJawh+32bKySSE2WtEnBUIKd8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= diff --git a/init_db/create_tables.sh b/init_db/create_tables.sh new file mode 100755 index 00000000..f2e316bb --- /dev/null +++ b/init_db/create_tables.sh @@ -0,0 +1,6 @@ +# create ydbcp tables +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local scripting yql -f init_db/schema/create_tables.yql + +# create and fill user table kv_test +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local workload kv init +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local workload kv run upsert --rows 100 \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 137dd9d7..ef7048f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,7 @@ type S3Config struct { PathPrefix string `yaml:"path_prefix"` AccessKeyIDPath string `yaml:"access_key_id_path"` SecretAccessKeyPath string `yaml:"secret_access_key_path"` + S3ForcePathStyle bool `yaml:"s3_force_path_style"` } type YDBConnectionConfig struct { diff --git a/internal/connectors/client/connector.go b/internal/connectors/client/connector.go index cabab61e..cf6c5e26 100644 --- a/internal/connectors/client/connector.go +++ b/internal/connectors/client/connector.go @@ -213,7 +213,7 @@ func prepareItemsForExport( destinationPrefix := path.Join( s3Settings.DestinationPrefix, clientDb.Scheme().Database(), - time.Now().Format(types.BackupTimestampFormat)+"_"+s3Settings.BackupID, + time.Now().Format(types.BackupTimestampFormat), strings.TrimPrefix(source, clientDb.Scheme().Database()+"/"), ) @@ -255,14 +255,15 @@ func (d *ClientYdbConnector) ExportToS3( CancelAfter: durationpb.New(time.Second), }, Settings: &Ydb_Export.ExportToS3Settings{ - Endpoint: s3Settings.Endpoint, - Bucket: s3Settings.Bucket, - Region: s3Settings.Region, - AccessKey: s3Settings.AccessKey, - SecretKey: s3Settings.SecretKey, - Description: s3Settings.Description, - NumberOfRetries: s3Settings.NumberOfRetries, - Items: items, + Endpoint: s3Settings.Endpoint, + Bucket: s3Settings.Bucket, + Region: s3Settings.Region, + AccessKey: s3Settings.AccessKey, + SecretKey: s3Settings.SecretKey, + Description: s3Settings.Description, + NumberOfRetries: s3Settings.NumberOfRetries, + Items: items, + DisableVirtualAddressing: s3Settings.S3ForcePathStyle, }, }, ) @@ -364,14 +365,15 @@ func (d *ClientYdbConnector) ImportFromS3(ctx context.Context, clientDb *ydb.Dri CancelAfter: durationpb.New(time.Second), }, Settings: &Ydb_Import.ImportFromS3Settings{ - Endpoint: s3Settings.Endpoint, - Bucket: s3Settings.Bucket, - Region: s3Settings.Region, - AccessKey: s3Settings.AccessKey, - SecretKey: s3Settings.SecretKey, - Description: s3Settings.Description, - NumberOfRetries: s3Settings.NumberOfRetries, - Items: items, + Endpoint: s3Settings.Endpoint, + Bucket: s3Settings.Bucket, + Region: s3Settings.Region, + AccessKey: s3Settings.AccessKey, + SecretKey: s3Settings.SecretKey, + Description: s3Settings.Description, + NumberOfRetries: s3Settings.NumberOfRetries, + Items: items, + DisableVirtualAddressing: s3Settings.S3ForcePathStyle, }, }, ) diff --git a/internal/connectors/db/yql/schema/create_tables.yql b/internal/connectors/db/yql/schema/create_tables.yql index de3b45ab..165ed129 100644 --- a/internal/connectors/db/yql/schema/create_tables.yql +++ b/internal/connectors/db/yql/schema/create_tables.yql @@ -1,7 +1,3 @@ -DROP TABLE Backups; -DROP TABLE OperationTypes; -DROP TABLE Operations; - CREATE TABLE Backups ( id String NOT NULL, container_id String NOT NULL, diff --git a/internal/connectors/db/yql/schema/drop_tables.yql b/internal/connectors/db/yql/schema/drop_tables.yql new file mode 100644 index 00000000..74619965 --- /dev/null +++ b/internal/connectors/db/yql/schema/drop_tables.yql @@ -0,0 +1,3 @@ +DROP TABLE Backups; +DROP TABLE OperationTypes; +DROP TABLE Operations; \ No newline at end of file diff --git a/internal/server/services/backup/backupservice.go b/internal/server/services/backup/backupservice.go index 3bc3fcac..3bd82358 100644 --- a/internal/server/services/backup/backupservice.go +++ b/internal/server/services/backup/backupservice.go @@ -151,7 +151,7 @@ func (s *BackupService) MakeBackup(ctx context.Context, req *pb.MakeBackupReques SourcePaths: req.GetSourcePaths(), SourcePathToExclude: req.GetSourcePathsToExclude(), DestinationPrefix: s.s3.PathPrefix, - BackupID: types.GenerateObjectID(), // TODO: do we need backup id? + S3ForcePathStyle: s.s3.S3ForcePathStyle, } clientOperationID, err := s.clientConn.ExportToS3(ctx, client, s3Settings) @@ -250,6 +250,7 @@ func (s *BackupService) MakeRestore(ctx context.Context, req *pb.MakeRestoreRequ s3Settings := types.ImportSettings{ Endpoint: s.s3.Endpoint, + Region: s.s3.Region, Bucket: s.s3.Bucket, AccessKey: accessKey, SecretKey: secretKey, @@ -257,7 +258,7 @@ func (s *BackupService) MakeRestore(ctx context.Context, req *pb.MakeRestoreRequ NumberOfRetries: 10, // TODO: get value from configuration BackupID: req.GetBackupId(), SourcePaths: req.GetSourcePaths(), - S3ForcePathStyle: true, + S3ForcePathStyle: s.s3.S3ForcePathStyle, DestinationPrefix: req.GetDestinationPrefix(), } diff --git a/internal/types/backup.go b/internal/types/backup.go index 281920b0..6a425f3a 100644 --- a/internal/types/backup.go +++ b/internal/types/backup.go @@ -339,7 +339,7 @@ type ExportSettings struct { SourcePaths []string SourcePathToExclude []string DestinationPrefix string - BackupID string + S3ForcePathStyle bool } type ImportSettings struct { diff --git a/local_config.yaml b/local_config.yaml new file mode 100644 index 00000000..cf8af782 --- /dev/null +++ b/local_config.yaml @@ -0,0 +1,25 @@ +operation_ttl_seconds: 86400 # 24 hours + +db_connection: + connection_string: "grpcs://ydbcp-ydb-1.ydbcp-net:2135/local" + insecure: true + discovery: false + +client_connection: + insecure: true + discovery: false + allowed_endpoint_domains: + - ydbcp-ydb-1.ydbcp-net + allow_insecure_endpoint: false + +s3: + endpoint: "http://ydbcp-s3-1.ydbcp-net:9000" + region: "us-east-1" + bucket: "test-bucket" + path_prefix: "test-prefix" + access_key_id_path: "access_key" + secret_access_key_path: "secret_key" + s3_force_path_style: true + +grpc_server: + bind_port: 50051