From 1d55ad6a1d04600c90c891da84b936901a07961b Mon Sep 17 00:00:00 2001 From: Patrick Weizhi Xu Date: Thu, 2 Jan 2025 13:56:54 +0800 Subject: [PATCH] enhance: [2.5] add param for tuning max VARCHAR length and restore limit to 65535 (#38883) issue: #38882 pr: https://github.com/milvus-io/milvus/pull/38884 Signed-off-by: Patrick Weizhi Xu (cherry picked from commit a64a7377ea8258efe36c74bd1cf9db556726ba2d) --- internal/proxy/task.go | 5 +++-- internal/proxy/util.go | 6 +++--- pkg/util/paramtable/component_param.go | 9 +++++++++ tests/go_client/common/consts.go | 2 +- tests/go_client/testcases/collection_test.go | 4 ++-- tests/python_client/testcases/test_collection.py | 6 +++--- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/internal/proxy/task.go b/internal/proxy/task.go index 8cbfacf9c2d1f..786e2171eb1f4 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -1256,8 +1256,9 @@ func (t *alterCollectionFieldTask) PreExecute(ctx context.Context) error { return merr.WrapErrParameterInvalid("%s should be an integer, but got %T", prop.Key, prop.Value) } - if value > defaultMaxVarCharLength { - return merr.WrapErrParameterInvalid("%s exceeds the maximum allowed value 1048576", prop.Value) + defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64() + if int64(value) > defaultMaxVarCharLength { + return merr.WrapErrParameterInvalidMsg("%s exceeds the maximum allowed value %s", prop.Value, strconv.FormatInt(defaultMaxVarCharLength, 10)) } } } diff --git a/internal/proxy/util.go b/internal/proxy/util.go index bf7ae45b09ee8..1a5bb46265118 100644 --- a/internal/proxy/util.go +++ b/internal/proxy/util.go @@ -63,8 +63,6 @@ const ( // enableMultipleVectorFields indicates whether to enable multiple vector fields. enableMultipleVectorFields = true - defaultMaxVarCharLength = 1048576 - defaultMaxArrayCapacity = 4096 defaultMaxSearchRequest = 1024 @@ -363,8 +361,10 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema) if err != nil { return err } + + defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64() if maxLengthPerRow > defaultMaxVarCharLength || maxLengthPerRow <= 0 { - return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, 1048576]") + return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, %d]", defaultMaxVarCharLength) } exist = true } diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 68d139b659a23..1263a02054856 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -1305,6 +1305,7 @@ type proxyConfig struct { SkipAutoIDCheck ParamItem `refreshable:"true"` SkipPartitionKeyCheck ParamItem `refreshable:"true"` EnablePublicPrivilege ParamItem `refreshable:"false"` + MaxVarCharLength ParamItem `refreshable:"false"` AccessLog AccessLogConfig @@ -1717,6 +1718,14 @@ please adjust in embedded Milvus: false`, } p.EnablePublicPrivilege.Init(base.mgr) + p.MaxVarCharLength = ParamItem{ + Key: "proxy.maxVarCharLength", + Version: "2.4.19", // hotfix + DefaultValue: strconv.Itoa(65535), // 64K + Doc: "maximum number of characters for a varchar field; this value is overridden by the value in a pre-existing schema if applicable", + } + p.MaxVarCharLength.Init(base.mgr) + p.GracefulStopTimeout = ParamItem{ Key: "proxy.gracefulStopTimeout", Version: "2.3.7", diff --git a/tests/go_client/common/consts.go b/tests/go_client/common/consts.go index 5162b0917dcf5..91a6df94fb219 100644 --- a/tests/go_client/common/consts.go +++ b/tests/go_client/common/consts.go @@ -57,7 +57,7 @@ const ( DefaultRgName = "__default_resource_group" DefaultDb = "default" MaxDim = 32768 - MaxLength = int64(1048576) + MaxLength = int64(65535) MaxCollectionNameLen = 255 DefaultRgCapacity = 1000000 RetentionDuration = 40 // common.retentionDuration diff --git a/tests/go_client/testcases/collection_test.go b/tests/go_client/testcases/collection_test.go index 248206c56e269..e5e009bb2b7a9 100644 --- a/tests/go_client/testcases/collection_test.go +++ b/tests/go_client/testcases/collection_test.go @@ -836,7 +836,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) { for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} { arrayVarcharField.WithMaxLength(invalidLength) err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema)) - common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]") + common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]") } } @@ -858,7 +858,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) { for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} { varcharField.WithMaxLength(invalidLength) err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema)) - common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]") + common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]") } } diff --git a/tests/python_client/testcases/test_collection.py b/tests/python_client/testcases/test_collection.py index 5cf191ee55f74..7e5aeb18474c4 100644 --- a/tests/python_client/testcases/test_collection.py +++ b/tests/python_client/testcases/test_collection.py @@ -3875,10 +3875,10 @@ def test_collection_string_field_with_exceed_max_len(self): c_name = cf.gen_unique_str(prefix) int_field = cf.gen_int64_field(is_primary=True) vec_field = cf.gen_float_vec_field() - max_length = 1048576 + 1 + max_length = 65535 + 1 string_field = cf.gen_string_field(max_length=max_length) schema = cf.gen_collection_schema([int_field, string_field, vec_field]) - error = {ct.err_code: 1048576, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 1048576]"} + error = {ct.err_code: 65535, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 65535]"} self.collection_wrap.init_collection(name=c_name, schema=schema, check_task=CheckTasks.err_res, check_items=error) @@ -4117,7 +4117,7 @@ def test_collection_string_array_max_length_invalid(self, max_length): self.init_collection_wrap(schema=array_schema, check_task=CheckTasks.err_res, check_items={ct.err_code: 65535, ct.err_msg: "the maximum length specified for a VarChar " - "should be in (0, 1048576]"}) + "should be in (0, 65535]"}) @pytest.mark.tags(CaseLabel.L2) def test_collection_array_field_all_datatype(self):