enhance: [2.5] add param for tuning max VARCHAR length and restore li…

…mit to 65535 (#38883) issue: #38882 pr: #38884 Signed-off-by: Patrick Weizhi Xu <[email protected]> (cherry picked from commit a64a737)
milvus-io · Jan 2, 2025 · 1d55ad6 · 1d55ad6
1 parent a2c4cd5
commit 1d55ad6
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 11 deletions.
diff --git a/internal/proxy/task.go b/internal/proxy/task.go
@@ -1256,8 +1256,9 @@ func (t *alterCollectionFieldTask) PreExecute(ctx context.Context) error {
 				return merr.WrapErrParameterInvalid("%s should be an integer, but got %T", prop.Key, prop.Value)
 			}
 
-			if value > defaultMaxVarCharLength {
-				return merr.WrapErrParameterInvalid("%s exceeds the maximum allowed value 1048576", prop.Value)
+			defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
+			if int64(value) > defaultMaxVarCharLength {
+				return merr.WrapErrParameterInvalidMsg("%s exceeds the maximum allowed value %s", prop.Value, strconv.FormatInt(defaultMaxVarCharLength, 10))
 			}
 		}
 	}

diff --git a/internal/proxy/util.go b/internal/proxy/util.go
@@ -63,8 +63,6 @@ const (
 	// enableMultipleVectorFields indicates whether to enable multiple vector fields.
 	enableMultipleVectorFields = true
 
-	defaultMaxVarCharLength = 1048576
-
 	defaultMaxArrayCapacity = 4096
 
 	defaultMaxSearchRequest = 1024
@@ -363,8 +361,10 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema)
 		if err != nil {
 			return err
 		}
+
+		defaultMaxVarCharLength := Params.ProxyCfg.MaxVarCharLength.GetAsInt64()
 		if maxLengthPerRow > defaultMaxVarCharLength || maxLengthPerRow <= 0 {
-			return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, 1048576]")
+			return merr.WrapErrParameterInvalidMsg("the maximum length specified for a VarChar should be in (0, %d]", defaultMaxVarCharLength)
 		}
 		exist = true
 	}

diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go
@@ -1305,6 +1305,7 @@ type proxyConfig struct {
 	SkipAutoIDCheck              ParamItem `refreshable:"true"`
 	SkipPartitionKeyCheck        ParamItem `refreshable:"true"`
 	EnablePublicPrivilege        ParamItem `refreshable:"false"`
+	MaxVarCharLength             ParamItem `refreshable:"false"`
 
 	AccessLog AccessLogConfig
 
@@ -1717,6 +1718,14 @@ please adjust in embedded Milvus: false`,
 	}
 	p.EnablePublicPrivilege.Init(base.mgr)
 
+	p.MaxVarCharLength = ParamItem{
+		Key:          "proxy.maxVarCharLength",
+		Version:      "2.4.19",            // hotfix
+		DefaultValue: strconv.Itoa(65535), // 64K
+		Doc:          "maximum number of characters for a varchar field; this value is overridden by the value in a pre-existing schema if applicable",
+	}
+	p.MaxVarCharLength.Init(base.mgr)
+
 	p.GracefulStopTimeout = ParamItem{
 		Key:          "proxy.gracefulStopTimeout",
 		Version:      "2.3.7",

diff --git a/tests/go_client/common/consts.go b/tests/go_client/common/consts.go
@@ -57,7 +57,7 @@ const (
 	DefaultRgName           = "__default_resource_group"
 	DefaultDb               = "default"
 	MaxDim                  = 32768
-	MaxLength               = int64(1048576)
+	MaxLength               = int64(65535)
 	MaxCollectionNameLen    = 255
 	DefaultRgCapacity       = 1000000
 	RetentionDuration       = 40   // common.retentionDuration

diff --git a/tests/go_client/testcases/collection_test.go b/tests/go_client/testcases/collection_test.go
@@ -836,7 +836,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
 	for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
 		arrayVarcharField.WithMaxLength(invalidLength)
 		err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
-		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
+		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
 	}
 }
 
@@ -858,7 +858,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) {
 	for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
 		varcharField.WithMaxLength(invalidLength)
 		err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
-		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 1048576]")
+		common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
 	}
 }
 

diff --git a/tests/python_client/testcases/test_collection.py b/tests/python_client/testcases/test_collection.py
@@ -3875,10 +3875,10 @@ def test_collection_string_field_with_exceed_max_len(self):
         c_name = cf.gen_unique_str(prefix)
         int_field = cf.gen_int64_field(is_primary=True)
         vec_field = cf.gen_float_vec_field()
-        max_length = 1048576 + 1
+        max_length = 65535 + 1
         string_field = cf.gen_string_field(max_length=max_length)
         schema = cf.gen_collection_schema([int_field, string_field, vec_field])
-        error = {ct.err_code: 1048576, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 1048576]"}
+        error = {ct.err_code: 65535, ct.err_msg: "the maximum length specified for a VarChar should be in (0, 65535]"}
         self.collection_wrap.init_collection(name=c_name, schema=schema,
                                              check_task=CheckTasks.err_res, check_items=error)
 
@@ -4117,7 +4117,7 @@ def test_collection_string_array_max_length_invalid(self, max_length):
         self.init_collection_wrap(schema=array_schema, check_task=CheckTasks.err_res,
                                   check_items={ct.err_code: 65535,
                                                ct.err_msg: "the maximum length specified for a VarChar "
-                                                           "should be in (0, 1048576]"})
+                                                           "should be in (0, 65535]"})
 
     @pytest.mark.tags(CaseLabel.L2)
     def test_collection_array_field_all_datatype(self):