storage: concurrent disk image write.

pkg/.golangci.yaml

@@ -0,0 +1,9 @@
+linters-settings:
+  gocyclo:
+    # Minimal code complexity to report.
+    # Default: 30 (but we recommend 10-20)
+    min-complexity: 100
+  staticcheck:
+    # SAxxxx checks in https://staticcheck.io/docs/configuration/options/#checks
+    # Default: ["*"]
+    checks: ["all"]

pkg/Makefile

@@ -5,13 +5,11 @@ all: build
 
 getdeps:
 	@echo "Installing golint" && go install github.com/golangci/golangci-lint/cmd/[email protected]
-	@echo "Installing gocyclo" && go install github.com/fzipp/gocyclo/cmd/gocyclo@latest
 	@echo "Installing deadcode" && go install github.com/remyoudompheng/go-misc/deadcode@latest
 	@echo "Installing misspell" && go install github.com/client9/misspell/cmd/misspell@latest
 	@echo "Installing ineffassign" && go install github.com/gordonklaus/ineffassign@latest
-	@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
 
-verifiers: vet fmt lint cyclo spelling static #deadcode
+verifiers: vet fmt lint spelling#deadcode
 
 vet:
 	@echo "Running $@"
@@ -29,20 +27,13 @@ ineffassign:
 	@echo "Running $@"
 	@${GOPATH}/bin/ineffassign .
 
-cyclo:
-	@echo "Running $@"
-	@${GOPATH}/bin/gocyclo -over 100 .
-
 deadcode:
 	@echo "Running $@"
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
 	@${GOPATH}/bin/misspell -i "monitord,forumla,etherent" -error `find .`
 
-static:
-	@${GOPATH}/bin/staticcheck -- ./...
-
 check: test
 test: verifiers build
 	go test -vet=off -v $(shell go list ./... | grep -Ev "stubs|network" )

pkg/storage/disk.go

@@ -14,6 +14,8 @@ import (
 	log "github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
+	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -145,14 +147,55 @@ func (s *Module) DiskWrite(name string, image string) error {
 		return fmt.Errorf("image size is bigger than disk")
 	}
 
-	_, err = io.Copy(file, source)
+	// do sequential copy on HDD only node
+	// otherwise do concurrent copy
+	if n, _ := s.Total(zos.SSDDevice); n == uint64(0) {
+		_, err = io.Copy(file, source)
+
+	} else {
+		err = s.concurrentCopy(file, source, imgStat)
+	}
+
 	if err != nil {
 		return errors.Wrap(err, "failed to write disk image")
 	}
 
 	return nil
 }
 
+func (s *Module) concurrentCopy(dst io.WriterAt, src io.ReaderAt, imgStat os.FileInfo) error {
+	// writing the image concurrently to speedup the previous sequential write.
+	// the sequential write is slow because the data source is from the remote server.
+	var (
+		// use errgroup because there is no point in continuing if one of the goroutines failed
+		group            = new(errgroup.Group)
+		numWorkers int   = 5
+		imgSize    int64 = imgStat.Size()
+		chunkSize        = imgSize / int64(numWorkers)
+	)
+
+	log.Info().Int("numWorkers", numWorkers).Msg("writing image concurrently")
+	for i := 0; i < numWorkers; i++ {
+		index := i
+		group.Go(func() error {
+			start := chunkSize * int64(index)
+			len := chunkSize
+			if index == numWorkers-1 { //last chunk
+				len = imgSize - start
+			}
+			wr := io.NewOffsetWriter(dst, start)
+			rd := io.NewSectionReader(src, start, len)
+			_, err := io.Copy(wr, rd)
+			return err
+		})
+	}
+	if err := group.Wait(); err != nil {
+		return err
+	}
+	log.Info().Msg("writing image concurrently finished")
+	return nil
+}
+
 // DiskCreate with given size, return path to virtual disk (size in MB)
 func (s *Module) DiskCreate(name string, size gridtypes.Unit) (disk pkg.VDisk, err error) {
 	path, err := s.findDisk(name)