local-sst-writer: test only logical content and test many keys (#57203)

ref #57105

DEPS.bzl

@@ -4580,13 +4580,13 @@ def go_deps():
         name = "com_github_lance6716_pebble",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/lance6716/pebble",
-        sha256 = "fad807913eaf5ff14651929475dda5e5c6f6838dbaeef11b7eddefe771f01f1e",
-        strip_prefix = "github.com/lance6716/[email protected]20241104073946-6f55c09bd183",
+        sha256 = "8fe7ce7009c4d2b0ae28f4c3d62f38b6256ee20e47944490b58cb1d33a155725",
+        strip_prefix = "github.com/lance6716/[email protected]20241108073934-da961314c63f",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241104073946-6f55c09bd183.zip",
-            "http://ats.apps.svc/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241104073946-6f55c09bd183.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241104073946-6f55c09bd183.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241104073946-6f55c09bd183.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241108073934-da961314c63f.zip",
+            "http://ats.apps.svc/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241108073934-da961314c63f.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241108073934-da961314c63f.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/lance6716/pebble/com_github_lance6716_pebble-v0.0.0-20241108073934-da961314c63f.zip",
         ],
     )
     go_repository(

go.mod

@@ -73,7 +73,7 @@ require (
 	github.com/kisielk/errcheck v1.7.0
 	github.com/klauspost/compress v1.17.9
 	github.com/ks3sdklib/aws-sdk-go v1.2.9
-	github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183
+	github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f
 	github.com/lestrrat-go/jwx/v2 v2.0.21
 	github.com/mgechev/revive v1.4.0
 	github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7

go.sum

@@ -558,8 +558,8 @@ github.com/ks3sdklib/aws-sdk-go v1.2.9 h1:Eg0fM56r4Gjp9PiK1Bg9agJUxCAWCk236qq9DI
 github.com/ks3sdklib/aws-sdk-go v1.2.9/go.mod h1:xBNbOrxSnd36AQpZ8o99mGGu+blblUd9rI0MKGmeufo=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183 h1:CrFmpCAT5PGMgmQadTa2lXZrjuvpknONB1/pyxiyDsM=
-github.com/lance6716/pebble v0.0.0-20241104073946-6f55c09bd183/go.mod h1:ZxnWA3Ab0ufDIyppyzL16j6HFNpdXeiU/1cE4Wlv/lQ=
+github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f h1:KyBFXtlZDJTrwuD3wBHGk+TtdOpCB17AEm0X4eqhRdg=
+github.com/lance6716/pebble v0.0.0-20241108073934-da961314c63f/go.mod h1:ZxnWA3Ab0ufDIyppyzL16j6HFNpdXeiU/1cE4Wlv/lQ=
 github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
 github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=

lightning/tests/config/pd-no-tls.toml

@@ -0,0 +1,2 @@
+[replication]
+enable-placement-rules = true

lightning/tests/config/tikv-no-tls.toml

@@ -0,0 +1,23 @@
+# config of tikv
+[storage]
+reserve-space = "1KB"
+data-dir = "/tmp/lightning_test/tikv1/"
+
+[coprocessor]
+region-max-keys = 100
+region-split-keys = 60
+
+[rocksdb]
+max-open-files = 4096
+[raftdb]
+max-open-files = 4096
+
+[raftstore]
+# true (default value) for high reliability, this can prevent data loss when power failure.
+sync-log = false
+capacity = "10GB"
+# Speed up TiKV region heartbeat
+pd-heartbeat-tick-interval = "1s"
+
+[cdc]
+hibernate-regions-compatible=false

lightning/tests/run_local_sst_test.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# currently the script is WIP and not used in the CI.
+
+set -eu
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+export UTILS_DIR="$CUR/../../tests/_utils"
+export PATH="$PATH:$CUR/../../bin:$UTILS_DIR"
+export TEST_DIR=/tmp/lightning_test
+export COV_DIR="/tmp/group_cover"
+mkdir -p $COV_DIR || true
+export TIKV_CONFIG="$CUR/config/tikv-no-tls.toml"
+export PD_CONFIG="$CUR/config/pd-no-tls.toml"
+export TESTS_ROOT="$CUR"
+source $UTILS_DIR/run_services
+
+export TIKV_COUNT=1
+export PD_HTTP_PROTO="http"
+
+# Create COV_DIR if not exists
+if [ -d "$COV_DIR" ]; then
+  mkdir -p $COV_DIR
+fi
+
+# Reset TEST_DIR
+rm -rf $TEST_DIR && mkdir -p $TEST_DIR
+
+trap stop_services EXIT
+start_services $@ --no-tiflash --no-tidb
+
+rm /tmp/*.sst || true
+
+# change to project root
+cd $CUR/../..
+go test ./pkg/lightning/tikv -tikv-write-test -test.v -test.run TestIntegrationTest
+cp /tmp/lightning_test/tikv1/import/*_write_*.sst /tmp/tikv-write-cf.sst
+
+for prefix in go tikv; do
+  bin/tikv-ctl sst_dump --file=/tmp/$prefix-write-cf.sst --command=scan --output_hex --show_properties \
+    > /tmp/$prefix-write-cf-scan.txt
+  awk "
+    /from \[\] to \[\]/ { f1 = 1; next }
+    /Table Properties:/ { f1 = 0; f2 = 1; next }
+    f1 { print > \"/tmp/$prefix-write-cf-data.txt\" }
+    f2 { print > \"/tmp/$prefix-write-cf-properties.txt\" }
+  " /tmp/$prefix-write-cf-scan.txt
+  # filter some properties that are not deterministic by logical content
+  grep -v -F -e "data block size" -e "index block size" -e "entries for filter" -e "(estimated) table size" \
+    -e "DB identity" -e "DB session identity" -e "DB host id" -e "original file number" -e "unique ID" \
+    /tmp/$prefix-write-cf-properties.txt > /tmp/$prefix-write-cf-properties.txt.filtered
+done
+
+diff /tmp/tikv-write-cf-data.txt /tmp/go-write-cf-data.txt
+diff /tmp/tikv-write-cf-properties.txt.filtered /tmp/go-write-cf-properties.txt.filtered
+
+# clean tikv-ctl temporary files
+rm -rf ctl-engine-info-log || true

pkg/lightning/tikv/BUILD.bazel

@@ -47,16 +47,16 @@ go_test(
         "local_sst_writer_test.go",
         "tikv_test.go",
     ],
+    data = glob(["sst-examples/**"]),
     embed = [":tikv"],
     flaky = True,
     shard_count = 7,
     deps = [
-        "@com_github_cockroachdb_pebble//:pebble",
-        "@com_github_cockroachdb_pebble//sstable",
-        "@com_github_cockroachdb_pebble//vfs",
         "@com_github_coreos_go_semver//semver",
         "@com_github_google_uuid//:uuid",
+        "@com_github_lance6716_pebble//:pebble",
         "@com_github_lance6716_pebble//sstable",
+        "@com_github_lance6716_pebble//vfs",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_kvproto//pkg/import_sstpb",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",

pkg/lightning/tikv/local_sst_writer.go

@@ -38,15 +38,15 @@ type writeCFWriter struct {
 func newWriteCFWriter(
 	sstPath string,
 	ts uint64,
-	identity *rockssst.Identity,
 ) (*writeCFWriter, error) {
 	f, err := vfs.Default.Create(sstPath)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
 	writable := objstorageprovider.NewFileWritable(f)
 	writer := rockssst.NewWriter(writable, rockssst.WriterOptions{
-		// TODO(lance6716): should read TiKV config to know compression algorithm.
+		// TODO(lance6716): should read TiKV config to know these values.
+		BlockSize:   32 * 1024,
 		Compression: rocks.ZstdCompression,
 		// TODO(lance6716): should check the behaviour is the exactly same.
 		FilterPolicy: rocksbloom.FilterPolicy(10),
@@ -63,7 +63,7 @@ func newWriteCFWriter(
 				return mockCollector{name: "BlobFileSizeCollector"}
 			},
 		},
-	}, identity)
+	})
 	return &writeCFWriter{sstWriter: writer, ts: ts}, nil
 }