From 999362add70c73cd791fb1a2716254706b3e3600 Mon Sep 17 00:00:00 2001 From: Iwan BK Date: Tue, 13 Aug 2024 10:56:48 +0700 Subject: [PATCH 1/5] storage: concurrent disk image write. Write it concurrently to speed it up from the previous sequential write. --- pkg/storage/disk.go | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go index f9083717c..d087dd65a 100644 --- a/pkg/storage/disk.go +++ b/pkg/storage/disk.go @@ -14,6 +14,7 @@ import ( log "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" + "golang.org/x/sync/errgroup" ) const ( @@ -145,10 +146,35 @@ func (s *Module) DiskWrite(name string, image string) error { return fmt.Errorf("image size is bigger than disk") } - _, err = io.Copy(file, source) - if err != nil { + // writing the image concurrently to speedup the previous sequential write. + // the sequential write is slow because the data source is from the remote server. + var ( + // use errgroup because there is no point in continuing if one of the goroutines failed + group = new(errgroup.Group) + concurrentNum int = 5 + imgSize int64 = imgStat.Size() + chunkSize = imgSize / int64(concurrentNum) + ) + + log.Info().Int("concurrentNum", concurrentNum).Msg("writing image concurrently") + for i := 0; i < concurrentNum; i++ { + index := i + group.Go(func() error { + start := chunkSize * int64(index) + len := chunkSize + if index == concurrentNum-1 { //last chunk + len = imgSize - start + } + wr := io.NewOffsetWriter(file, start) + rd := io.NewSectionReader(source, start, len) + _, err = io.Copy(wr, rd) + return err + }) + } + if err := group.Wait(); err != nil { return errors.Wrap(err, "failed to write disk image") } + log.Info().Msg("writing image finished") return nil } From d35f3c592ee0ba155c3710b690fd7c14a904d4c8 Mon Sep 17 00:00:00 2001 From: Iwan BK Date: Tue, 13 Aug 2024 16:08:38 +0700 Subject: [PATCH 2/5] wording change --- pkg/storage/disk.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go index d087dd65a..cad8e61c4 100644 --- a/pkg/storage/disk.go +++ b/pkg/storage/disk.go @@ -150,19 +150,19 @@ func (s *Module) DiskWrite(name string, image string) error { // the sequential write is slow because the data source is from the remote server. var ( // use errgroup because there is no point in continuing if one of the goroutines failed - group = new(errgroup.Group) - concurrentNum int = 5 - imgSize int64 = imgStat.Size() - chunkSize = imgSize / int64(concurrentNum) + group = new(errgroup.Group) + numWorkers int = 5 + imgSize int64 = imgStat.Size() + chunkSize = imgSize / int64(numWorkers) ) - log.Info().Int("concurrentNum", concurrentNum).Msg("writing image concurrently") - for i := 0; i < concurrentNum; i++ { + log.Info().Int("numWorkers", numWorkers).Msg("writing image concurrently") + for i := 0; i < numWorkers; i++ { index := i group.Go(func() error { start := chunkSize * int64(index) len := chunkSize - if index == concurrentNum-1 { //last chunk + if index == numWorkers-1 { //last chunk len = imgSize - start } wr := io.NewOffsetWriter(file, start) From caf83ac67e274e6ea04e640701eb7506fd8c500d Mon Sep 17 00:00:00 2001 From: Iwan BK Date: Fri, 16 Aug 2024 09:53:38 +0700 Subject: [PATCH 3/5] don't do concurrent copy on HDD only nodes --- pkg/storage/disk.go | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go index cad8e61c4..a2a78cd17 100644 --- a/pkg/storage/disk.go +++ b/pkg/storage/disk.go @@ -14,6 +14,7 @@ import ( log "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" "golang.org/x/sync/errgroup" ) @@ -146,6 +147,23 @@ func (s *Module) DiskWrite(name string, image string) error { return fmt.Errorf("image size is bigger than disk") } + // do sequential copy on HDD only node + // otherwise do concurrent copy + if n, _ := s.Total(zos.SSDDevice); n == uint64(0) { + _, err = io.Copy(file, source) + + } else { + err = s.concurrentCopy(file, source, imgStat) + } + + if err != nil { + return errors.Wrap(err, "failed to write disk image") + } + + return nil +} + +func (s *Module) concurrentCopy(dst io.WriterAt, src io.ReaderAt, imgStat os.FileInfo) error { // writing the image concurrently to speedup the previous sequential write. // the sequential write is slow because the data source is from the remote server. var ( @@ -165,17 +183,16 @@ func (s *Module) DiskWrite(name string, image string) error { if index == numWorkers-1 { //last chunk len = imgSize - start } - wr := io.NewOffsetWriter(file, start) - rd := io.NewSectionReader(source, start, len) - _, err = io.Copy(wr, rd) + wr := io.NewOffsetWriter(dst, start) + rd := io.NewSectionReader(src, start, len) + _, err := io.Copy(wr, rd) return err }) } if err := group.Wait(); err != nil { - return errors.Wrap(err, "failed to write disk image") + return err } - log.Info().Msg("writing image finished") - + log.Info().Msg("writing image concurrently finished") return nil } From bfca35942e4bad7bbb692af7078d2a2c80651048 Mon Sep 17 00:00:00 2001 From: Iwan BK Date: Fri, 16 Aug 2024 10:58:57 +0700 Subject: [PATCH 4/5] remove separate gocyclo and staticcheck that already included in golangci-lint --- pkg/Makefile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/Makefile b/pkg/Makefile index 2af25b9a0..a55d047b2 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -5,13 +5,11 @@ all: build getdeps: @echo "Installing golint" && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.0 - @echo "Installing gocyclo" && go install github.com/fzipp/gocyclo/cmd/gocyclo@latest @echo "Installing deadcode" && go install github.com/remyoudompheng/go-misc/deadcode@latest @echo "Installing misspell" && go install github.com/client9/misspell/cmd/misspell@latest @echo "Installing ineffassign" && go install github.com/gordonklaus/ineffassign@latest - @echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest -verifiers: vet fmt lint cyclo spelling static #deadcode +verifiers: vet fmt lint spelling#deadcode vet: @echo "Running $@" @@ -29,10 +27,6 @@ ineffassign: @echo "Running $@" @${GOPATH}/bin/ineffassign . -cyclo: - @echo "Running $@" - @${GOPATH}/bin/gocyclo -over 100 . - deadcode: @echo "Running $@" @${GOPATH}/bin/deadcode -test $(shell go list ./...) || true @@ -40,9 +34,6 @@ deadcode: spelling: @${GOPATH}/bin/misspell -i "monitord,forumla,etherent" -error `find .` -static: - @${GOPATH}/bin/staticcheck -- ./... - check: test test: verifiers build go test -vet=off -v $(shell go list ./... | grep -Ev "stubs|network" ) From ffc59d8acc7c81e4758dec6fb49c6f9a31d5eb7f Mon Sep 17 00:00:00 2001 From: Iwan BK Date: Fri, 16 Aug 2024 11:03:50 +0700 Subject: [PATCH 5/5] push .golangci config --- pkg/.golangci.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pkg/.golangci.yaml diff --git a/pkg/.golangci.yaml b/pkg/.golangci.yaml new file mode 100644 index 000000000..dc19bb965 --- /dev/null +++ b/pkg/.golangci.yaml @@ -0,0 +1,9 @@ +linters-settings: + gocyclo: + # Minimal code complexity to report. + # Default: 30 (but we recommend 10-20) + min-complexity: 100 + staticcheck: + # SAxxxx checks in https://staticcheck.io/docs/configuration/options/#checks + # Default: ["*"] + checks: ["all"]