From 53a257236e34b8496af9081ad09d18e49159094d Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Thu, 22 Aug 2024 15:46:57 +0200 Subject: [PATCH] test(ydbcp): add docker-compose for ydbcp --- cmd/ydbcp/config.yaml | 15 ++-- docker-compose.yaml | 85 +++++++++++++++++++ dockerfile | 20 +++++ go.mod | 2 +- go.sum | 2 + init_db.sh | 0 init_db/create_tables.sh | 6 ++ internal/config/config.go | 1 + internal/connectors/client/connector.go | 36 ++++---- .../db/yql/schema/create_tables.yql | 4 - .../connectors/db/yql/schema/drop_tables.yql | 3 + .../server/services/backup/backupservice.go | 5 +- internal/types/backup.go | 2 +- test.sh | 4 +- 14 files changed, 151 insertions(+), 34 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 dockerfile create mode 100644 init_db.sh create mode 100755 init_db/create_tables.sh create mode 100644 internal/connectors/db/yql/schema/drop_tables.yql diff --git a/cmd/ydbcp/config.yaml b/cmd/ydbcp/config.yaml index 2217a9ec..718b55b7 100644 --- a/cmd/ydbcp/config.yaml +++ b/cmd/ydbcp/config.yaml @@ -1,7 +1,7 @@ operation_ttl_seconds: 86400 # 24 hours db_connection: - connection_string: "grpcs://localhost:2135/domain/database" + connection_string: "grpcs://ydbcp-ydb-1.ydbcp-net:2135/local" insecure: true discovery: false @@ -14,12 +14,13 @@ client_connection: allow_insecure_endpoint: false s3: - endpoint: s3.endpoint.com - region: s3-region - bucket: s3-bucket - path_prefix: cluster-domain - access_key_id_path: path-to-s3-key - secret_access_key_path: path-to-s3-sec + endpoint: "http://ydbcp-s3-1.ydbcp-net:9000" + region: "us-east-1" + bucket: "test-bucket" + path_prefix: "test-prefix" + access_key_id_path: "cmd/ydbcp/access_key" + secret_access_key_path: "cmd/ydbcp/secret_key" + s3_force_path_style: true auth: plugin_path: path/to/plugin.so diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..c91b1c41 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,85 @@ +services: + ydb: + image: cr.ai.nebius.cloud/crnca8q7ti1i7vpqs28l/ydb-local:24.1.16-stream-nb-1.1-1 + platform: linux/amd64 + hostname: local-ydb + #volumes: + # - ./ydbd/ydb_certs:/ydb_certs + # - ./ydbd/ydb_data:/ydb_data + environment: + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + - YDB_ANONYMOUS_CREDENTIALS=true + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_FEATURE_FLAGS=enable_uuid_as_primary_key,enable_implicit_query_parameter_types + ports: + - "8765:8765" + #- "2135:2135" + #- "2136:2136" + networks: + - ydbcp-net + + setup_ydb: + image: cr.ai.nebius.cloud/crnca8q7ti1i7vpqs28l/ydb-local:24.1.16-stream-nb-1.1-1 + platform: linux/amd64 + volumes: + - ./init_db:/init_db + - ./internal/connectors/db/yql/schema:/init_db/schema + depends_on: + ydb: + condition: service_healthy + restart: "no" + command: bash -c "chmod +x ./init_db/create_tables.sh && ./init_db/create_tables.sh" + networks: + - ydbcp-net + + s3: + image: quay.io/minio/minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ydbcp + MINIO_ROOT_PASSWORD: password + ENABLE_VIRTUAL_STYLE: 0 + ports: + #- "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 10s + timeout: 20s + retries: 3 + networks: + - ydbcp-net + + setup_s3: + image: amazon/aws-cli + environment: + - AWS_ACCESS_KEY_ID=ydbcp + - AWS_SECRET_ACCESS_KEY=password + - AWS_DEFAULT_REGION=us-east-1 + - AWS_ENDPOINT_URL=http://ydbcp-s3-1.ydbcp-net:9000 + depends_on: + s3: + condition: service_healthy + command: s3api create-bucket --bucket test-bucket + networks: + - ydbcp-net + + ydbcp: + image: ydbcp:local + depends_on: + setup_ydb: + condition: service_completed_successfully + setup_s3: + condition: service_completed_successfully + build: + context: . + dockerfile: ./dockerfile + networks: + - ydbcp-net + +networks: + ydbcp-net: + name: ydbcp-net + external: true diff --git a/dockerfile b/dockerfile new file mode 100644 index 00000000..2e40506e --- /dev/null +++ b/dockerfile @@ -0,0 +1,20 @@ +# Use the official Golang image as the base image +FROM golang:1.22 as builder + +# Set the Current Working Directory inside the container +WORKDIR ./ydbcp + +# Copy source code into the container +COPY ./ ./ + +# Download all dependencies +RUN go mod download + +# Install grpcurl +RUN go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest + +# Build the Go app +RUN go build -o . ./cmd/ydbcp/main.go + +# Command to run the executable +CMD ["./main"] \ No newline at end of file diff --git a/go.mod b/go.mod index 87a12f16..703c4571 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/jonboulle/clockwork v0.3.0 github.com/stretchr/testify v1.8.1 - github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 + github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8 github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2 go.uber.org/automaxprocs v1.5.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 32d0be7b..c72c2384 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 h1:nL8XwD6fSst7xFUirkaWJmE7kM0CdWRYgu6+YQer1d4= github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8 h1:ZWxYw6L51aNAMLbTpC/VbXP0rcnvsCAJqx7EI/CjWmc= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240821162910-6cb364b2ccc8/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2 h1:thrbvktqKA6LFZTnZrGuQi8LQVel1J2dDfoQFsgvcYs= github.com/ydb-platform/ydb-go-sdk/v3 v3.75.2/go.mod h1:QMmT1fMKZnpid73USXLJawh+32bKySSE2WtEnBUIKd8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= diff --git a/init_db.sh b/init_db.sh new file mode 100644 index 00000000..e69de29b diff --git a/init_db/create_tables.sh b/init_db/create_tables.sh new file mode 100755 index 00000000..f2e316bb --- /dev/null +++ b/init_db/create_tables.sh @@ -0,0 +1,6 @@ +# create ydbcp tables +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local scripting yql -f init_db/schema/create_tables.yql + +# create and fill user table kv_test +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local workload kv init +./ydb -e grpc://ydbcp-ydb-1.ydbcp-net:2136 -d /local workload kv run upsert --rows 100 \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 137dd9d7..ef7048f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,7 @@ type S3Config struct { PathPrefix string `yaml:"path_prefix"` AccessKeyIDPath string `yaml:"access_key_id_path"` SecretAccessKeyPath string `yaml:"secret_access_key_path"` + S3ForcePathStyle bool `yaml:"s3_force_path_style"` } type YDBConnectionConfig struct { diff --git a/internal/connectors/client/connector.go b/internal/connectors/client/connector.go index cabab61e..cf6c5e26 100644 --- a/internal/connectors/client/connector.go +++ b/internal/connectors/client/connector.go @@ -213,7 +213,7 @@ func prepareItemsForExport( destinationPrefix := path.Join( s3Settings.DestinationPrefix, clientDb.Scheme().Database(), - time.Now().Format(types.BackupTimestampFormat)+"_"+s3Settings.BackupID, + time.Now().Format(types.BackupTimestampFormat), strings.TrimPrefix(source, clientDb.Scheme().Database()+"/"), ) @@ -255,14 +255,15 @@ func (d *ClientYdbConnector) ExportToS3( CancelAfter: durationpb.New(time.Second), }, Settings: &Ydb_Export.ExportToS3Settings{ - Endpoint: s3Settings.Endpoint, - Bucket: s3Settings.Bucket, - Region: s3Settings.Region, - AccessKey: s3Settings.AccessKey, - SecretKey: s3Settings.SecretKey, - Description: s3Settings.Description, - NumberOfRetries: s3Settings.NumberOfRetries, - Items: items, + Endpoint: s3Settings.Endpoint, + Bucket: s3Settings.Bucket, + Region: s3Settings.Region, + AccessKey: s3Settings.AccessKey, + SecretKey: s3Settings.SecretKey, + Description: s3Settings.Description, + NumberOfRetries: s3Settings.NumberOfRetries, + Items: items, + DisableVirtualAddressing: s3Settings.S3ForcePathStyle, }, }, ) @@ -364,14 +365,15 @@ func (d *ClientYdbConnector) ImportFromS3(ctx context.Context, clientDb *ydb.Dri CancelAfter: durationpb.New(time.Second), }, Settings: &Ydb_Import.ImportFromS3Settings{ - Endpoint: s3Settings.Endpoint, - Bucket: s3Settings.Bucket, - Region: s3Settings.Region, - AccessKey: s3Settings.AccessKey, - SecretKey: s3Settings.SecretKey, - Description: s3Settings.Description, - NumberOfRetries: s3Settings.NumberOfRetries, - Items: items, + Endpoint: s3Settings.Endpoint, + Bucket: s3Settings.Bucket, + Region: s3Settings.Region, + AccessKey: s3Settings.AccessKey, + SecretKey: s3Settings.SecretKey, + Description: s3Settings.Description, + NumberOfRetries: s3Settings.NumberOfRetries, + Items: items, + DisableVirtualAddressing: s3Settings.S3ForcePathStyle, }, }, ) diff --git a/internal/connectors/db/yql/schema/create_tables.yql b/internal/connectors/db/yql/schema/create_tables.yql index de3b45ab..165ed129 100644 --- a/internal/connectors/db/yql/schema/create_tables.yql +++ b/internal/connectors/db/yql/schema/create_tables.yql @@ -1,7 +1,3 @@ -DROP TABLE Backups; -DROP TABLE OperationTypes; -DROP TABLE Operations; - CREATE TABLE Backups ( id String NOT NULL, container_id String NOT NULL, diff --git a/internal/connectors/db/yql/schema/drop_tables.yql b/internal/connectors/db/yql/schema/drop_tables.yql new file mode 100644 index 00000000..74619965 --- /dev/null +++ b/internal/connectors/db/yql/schema/drop_tables.yql @@ -0,0 +1,3 @@ +DROP TABLE Backups; +DROP TABLE OperationTypes; +DROP TABLE Operations; \ No newline at end of file diff --git a/internal/server/services/backup/backupservice.go b/internal/server/services/backup/backupservice.go index 3bc3fcac..3bd82358 100644 --- a/internal/server/services/backup/backupservice.go +++ b/internal/server/services/backup/backupservice.go @@ -151,7 +151,7 @@ func (s *BackupService) MakeBackup(ctx context.Context, req *pb.MakeBackupReques SourcePaths: req.GetSourcePaths(), SourcePathToExclude: req.GetSourcePathsToExclude(), DestinationPrefix: s.s3.PathPrefix, - BackupID: types.GenerateObjectID(), // TODO: do we need backup id? + S3ForcePathStyle: s.s3.S3ForcePathStyle, } clientOperationID, err := s.clientConn.ExportToS3(ctx, client, s3Settings) @@ -250,6 +250,7 @@ func (s *BackupService) MakeRestore(ctx context.Context, req *pb.MakeRestoreRequ s3Settings := types.ImportSettings{ Endpoint: s.s3.Endpoint, + Region: s.s3.Region, Bucket: s.s3.Bucket, AccessKey: accessKey, SecretKey: secretKey, @@ -257,7 +258,7 @@ func (s *BackupService) MakeRestore(ctx context.Context, req *pb.MakeRestoreRequ NumberOfRetries: 10, // TODO: get value from configuration BackupID: req.GetBackupId(), SourcePaths: req.GetSourcePaths(), - S3ForcePathStyle: true, + S3ForcePathStyle: s.s3.S3ForcePathStyle, DestinationPrefix: req.GetDestinationPrefix(), } diff --git a/internal/types/backup.go b/internal/types/backup.go index 281920b0..6a425f3a 100644 --- a/internal/types/backup.go +++ b/internal/types/backup.go @@ -339,7 +339,7 @@ type ExportSettings struct { SourcePaths []string SourcePathToExclude []string DestinationPrefix string - BackupID string + S3ForcePathStyle bool } type ImportSettings struct { diff --git a/test.sh b/test.sh index ef371161..5bc7120f 100755 --- a/test.sh +++ b/test.sh @@ -29,11 +29,11 @@ if [[ "ListBackups" == "$1" ]]; then doneflag=1 fi if [[ "ListOperations" == "$1" ]]; then - $GRPCURL "${ARGS[@]}" -d '{"databaseNameMask": "%", "containerId": "'"$CONTAINER_ID"'"}' localhost:50051 ydbcp.v1alpha1.OperationService.ListOperations + grpcurl -plaintext -d '{"databaseNameMask": "/local", "containerId": ""}' localhost:50051 ydbcp.v1alpha1.OperationService.ListOperations doneflag=1 fi if [[ "MakeBackup" == "$1" ]]; then - $GRPCURL "${ARGS[@]}" -d '{"database_name": "/testing-global/ydbc", "database_endpoint": "grpcs://localhost:2135", "source_paths": ["/testing-global/ydbc"], "containerId": "'"$CONTAINER_ID"'"}' localhost:50051 ydbcp.v1alpha1.BackupService.MakeBackup + grpcurl -plaintext -d '{"database_name": "/local", "database_endpoint": "grpcs://ydbcp-ydb-1.ydbcp-net:2135", "source_paths": ["/local/kv_test"]}' localhost:50051 ydbcp.v1alpha1.BackupService.MakeBackup doneflag=1 fi if [[ 0 == $doneflag ]]; then