From acae3f044257d81599b166fd39efa093ea916bf4 Mon Sep 17 00:00:00 2001 From: Alex Aizman Date: Thu, 31 Oct 2024 20:34:56 -0400 Subject: [PATCH] set-primary with force (part one) Signed-off-by: Alex Aizman --- ais/metasync.go | 2 +- ais/test/multiproxy_test.go | 15 +++------------ api/apc/actmsg.go | 13 +++++++------ api/cluster.go | 13 ++++++++++--- cmd/cli/cli/cluster_hdlr.go | 23 ++++++++++++++++++++--- cmd/cli/cli/const.go | 4 ---- cmd/cli/cli/object_hdlr.go | 3 +++ cmd/cli/go.mod | 2 +- cmd/cli/go.sum | 35 +++++++++++++++++++++++++++++++++-- 9 files changed, 78 insertions(+), 32 deletions(-) diff --git a/ais/metasync.go b/ais/metasync.go index 4e21806325..b013d35652 100644 --- a/ais/metasync.go +++ b/ais/metasync.go @@ -287,7 +287,7 @@ drain: } } y.workCh <- revsReq{} - nlog.Infof("%s: becoming non-primary", y.p) + nlog.Infoln(y.p.String(), "becoming non-primary") } // main method; see top of the file; returns number of "sync" failures diff --git a/ais/test/multiproxy_test.go b/ais/test/multiproxy_test.go index 2098d2c0bb..c554193520 100644 --- a/ais/test/multiproxy_test.go +++ b/ais/test/multiproxy_test.go @@ -10,7 +10,6 @@ import ( "fmt" "math" "net/http" - "net/url" "path/filepath" "reflect" "sync" @@ -1007,7 +1006,7 @@ func setPrimaryTo(t *testing.T, proxyURL string, smap *meta.Smap, directURL, toI baseParams := tools.BaseAPIParams(directURL) tlog.Logf("Setting primary from %s to %s\n", smap.Primary.ID(), toID) - err := api.SetPrimaryProxy(baseParams, toID, false /*force*/) + err := api.SetPrimary(baseParams, toID, "" /*toURL*/, false /*force*/) tassert.CheckFatal(t, err) newSmap, err = tools.WaitForNewSmap(proxyURL, smap.Version) @@ -1243,17 +1242,9 @@ func networkFailurePrimary(t *testing.T) { } // Forcefully set new primary for the original one + baseParams := tools.BaseAPIParams(oldPrimaryURL) - baseParams.Method = http.MethodPut - reqParams := &api.ReqParams{ - BaseParams: baseParams, - Path: apc.URLPathDaeProxy.Join(newPrimaryID), - Query: url.Values{ - apc.QparamForce: {"true"}, - apc.QparamPrimaryCandidate: {newPrimaryURL}, - }, - } - err = reqParams.DoRequest() + err = api.SetPrimary(baseParams, newPrimaryID, newPrimaryURL /*toURL*/, true /*force*/) tassert.CheckFatal(t, err) smap, err = tools.WaitForClusterState( diff --git a/api/apc/actmsg.go b/api/apc/actmsg.go index 979b0d2e1e..e0672f64e2 100644 --- a/api/apc/actmsg.go +++ b/api/apc/actmsg.go @@ -103,12 +103,13 @@ const ( // internal use const ( - ActAddRemoteBck = "add-remote-bck" // add to BMD existing remote bucket, usually on the fly - ActRmNodeUnsafe = "rm-unsafe" // primary => the node to be removed - ActStartGFN = "start-gfn" // get-from-neighbor - ActStopGFN = "stop-gfn" // off - ActCleanupMarkers = "cleanup-markers" // part of the target joining sequence - ActSelfRemove = "self-initiated-removal" // e.g., when losing last mountpath + ActAddRemoteBck = "add-remote-bck" // add to BMD existing remote bucket, usually on the fly + ActRmNodeUnsafe = "rm-unsafe" // primary => the node to be removed + ActStartGFN = "start-gfn" // get-from-neighbor + ActStopGFN = "stop-gfn" // off + ActCleanupMarkers = "cleanup-markers" // part of the target joining sequence + ActSelfRemove = "self-initiated-removal" // e.g., when losing last mountpath + ActSetPrimaryForce = "set-primary-force" // BEWARE! advanced usage only ) const ( diff --git a/api/cluster.go b/api/cluster.go index 63aea2bcbd..735540a80d 100644 --- a/api/cluster.go +++ b/api/cluster.go @@ -202,13 +202,20 @@ func JoinCluster(bp BaseParams, nodeInfo *meta.Snode) (rebID, sid string, err er // SetPrimaryProxy given a daemonID sets that corresponding proxy as the // primary proxy of the cluster. -func SetPrimaryProxy(bp BaseParams, newPrimaryID string, force bool) error { +func SetPrimary(bp BaseParams, newPrimaryID, newPrimaryURL string, force bool) error { bp.Method = http.MethodPut reqParams := AllocRp() reqParams.BaseParams = bp reqParams.Path = apc.URLPathCluProxy.Join(newPrimaryID) - if force { - reqParams.Query = url.Values{apc.QparamForce: []string{"true"}} + if force || newPrimaryURL != "" { + q := make(url.Values, 2) + if force { + q.Set(apc.QparamForce, "true") + } + if newPrimaryURL != "" { + q.Set(apc.QparamPrimaryCandidate, newPrimaryURL) + } + reqParams.Query = q } err := reqParams.DoRequest() FreeRp(reqParams) diff --git a/cmd/cli/cli/cluster_hdlr.go b/cmd/cli/cli/cluster_hdlr.go index 1b55f68288..64bb98b4e9 100644 --- a/cmd/cli/cli/cluster_hdlr.go +++ b/cmd/cli/cli/cluster_hdlr.go @@ -137,7 +137,7 @@ var ( { Name: cmdPrimary, Usage: "select a new primary proxy/gateway", - ArgsUsage: nodeIDArgument, + ArgsUsage: nodeIDArgument + " [URL]", Flags: clusterCmdsFlags[cmdPrimary], Action: setPrimaryHandler, BashComplete: suggestProxies, @@ -492,7 +492,24 @@ func setPrimaryHandler(c *cli.Context) error { if c.NArg() == 0 { return missingArgumentsError(c, c.Command.ArgsUsage) } - node, sname, err := getNode(c, c.Args().Get(0)) + var ( + toID = c.Args().Get(0) + toURL = c.Args().Get(1) + force = flagIsSet(c, forceFlag) + ) + if force { + err := api.SetPrimary(apiBP, toID, toURL, force) + if err == nil { + var s string + if toURL != "" { + s = " (at " + toURL + ")" + } + actionDone(c, fmt.Sprintf("%s%s is now a new primary", toID, s)) + } + return err + } + + node, sname, err := getNode(c, toID) if err != nil { return err } @@ -509,7 +526,7 @@ func setPrimaryHandler(c *cli.Context) error { return fmt.Errorf("%s is non-electable", sname) } - err = api.SetPrimaryProxy(apiBP, node.ID(), flagIsSet(c, forceFlag)) + err = api.SetPrimary(apiBP, toID, toURL, force) if err == nil { actionDone(c, sname+" is now a new primary") } diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go index b407bda26b..478226cdc3 100644 --- a/cmd/cli/cli/const.go +++ b/cmd/cli/cli/const.go @@ -287,10 +287,6 @@ const ( renameObjectArgument = objectArgument + " NEW_OBJECT_NAME" - setCustomArgument = objectArgument + " " + jsonKeyValueArgument + " | " + keyValuePairsArgument + ", e.g.:\n" + - indent1 + - "mykey1=value1 mykey2=value2 OR '{\"mykey1\":\"value1\", \"mykey2\":\"value2\"}'" - // nodes nodeIDArgument = "NODE_ID" optionalNodeIDArgument = "[NODE_ID]" diff --git a/cmd/cli/cli/object_hdlr.go b/cmd/cli/cli/object_hdlr.go index 3d27f5a656..3557849d4f 100644 --- a/cmd/cli/cli/object_hdlr.go +++ b/cmd/cli/cli/object_hdlr.go @@ -70,6 +70,9 @@ const concatUsage = "append a file, a directory, or multiple files and/or direct indent1 + "as a new " + objectArgument + " if doesn't exists, and to an existing " + objectArgument + " otherwise, e.g.:\n" + indent1 + "$ ais object concat docs ais://nnn/all-docs ### concatenate all files from docs/ directory." +const setCustomArgument = objectArgument + " " + jsonKeyValueArgument + " | " + keyValuePairsArgument + ", e.g.:\n" + + indent1 + "mykey1=value1 mykey2=value2 OR (same) '{\"mykey1\":\"value1\", \"mykey2\":\"value2\"}'" + var ( objectCmdsFlags = map[string][]cli.Flag{ commandRemove: append( diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index 52621adf9c..986a22c832 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli go 1.23.2 require ( - github.com/NVIDIA/aistore v1.3.26-0.20241030184258-5f8152698f76 + github.com/NVIDIA/aistore v1.3.26-0.20241031134731-416fbbe723fa github.com/fatih/color v1.17.0 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo/v2 v2.20.2 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index 10c9a4c6d9..2b6d118204 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/NVIDIA/aistore v1.3.26-0.20241030184258-5f8152698f76 h1:LxjNbiwXn4X4W3spJPsT0TjvffFHPzFtZ/ja6dNRiUQ= -github.com/NVIDIA/aistore v1.3.26-0.20241030184258-5f8152698f76/go.mod h1:Q6J3YIeiL4A6oWga3qCJ8+XI1CUvdde7Gua/HfueGlQ= +github.com/NVIDIA/aistore v1.3.26-0.20241031134731-416fbbe723fa h1:9e30zpzZ5uybijndwwuN0ZKPCQaBDyb+0xshj0oVxds= +github.com/NVIDIA/aistore v1.3.26-0.20241031134731-416fbbe723fa/go.mod h1:IGdDyXEbwtj194tZukn6ptpn8ldL2pApqfOSIyTQyw4= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= @@ -13,6 +13,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= @@ -28,6 +30,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.4.0 h1:rCSCih1FnSWJEel/eub9wclBSqpF2F/PuvxUWGWnbO8= github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -35,6 +39,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -61,6 +67,8 @@ github.com/google/pprof v0.0.0-20241009165004-a3522334989c h1:NDovD0SMpBYXlE1zJm github.com/google/pprof v0.0.0-20241009165004-a3522334989c/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -187,6 +195,22 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy03iVTXP6ffeN2iXrxfGsZGCjVx0/4KlizjyBwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -238,6 +262,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 h1:nFS3IivktIU5Mk6KQa+v6RKkHUpdQpphqGNLxqNnbEk= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=