Switch to retriable HTTP

client/_http/node.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"path/filepath"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 )
@@ -85,7 +86,7 @@ func (n *NodeClient) Deploy(dl *gridtypes.Deployment, update bool) error {
 		return errors.Wrap(err, "failed to sign request")
 	}
 
-	response, err := http.DefaultClient.Do(request)
+	response, err := retryablehttp.DefaultClient.Do(request)
 	if err != nil {
 		return err
 	}
@@ -110,7 +111,7 @@ func (n *NodeClient) Get(twin, deployment uint32) (dl gridtypes.Deployment, err
 		return dl, errors.Wrap(err, "failed to sign request")
 	}
 
-	response, err := http.DefaultClient.Do(request)
+	response, err := retryablehttp.DefaultClient.Do(request)
 	if err != nil {
 		return dl, err
 	}
@@ -135,7 +136,7 @@ func (n *NodeClient) Delete(twin, deployment uint32) (err error) {
 		return errors.Wrap(err, "failed to sign request")
 	}
 
-	response, err := http.DefaultClient.Do(request)
+	response, err := retryablehttp.DefaultClient.Do(request)
 	if err != nil {
 		return err
 	}
@@ -155,7 +156,7 @@ func (n *NodeClient) Counters() (total gridtypes.Capacity, used gridtypes.Capaci
 		return total, used, errors.Wrap(err, "failed to build request")
 	}
 
-	response, err := http.DefaultClient.Do(request)
+	response, err := retryablehttp.DefaultClient.Do(request)
 	if err != nil {
 		return total, used, err
 	}

cmds/identityd/ssh.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/environment"
@@ -72,7 +73,7 @@ func manageSSHKeys() error {
 
 	for _, user := range authorizedUsers {
 		fetchKey := func() error {
-			res, err := http.Get(fmt.Sprintf("https://github.com/%s.keys", user))
+			res, err := retryablehttp.Get(fmt.Sprintf("https://github.com/%s.keys", user))
 
 			if err != nil {
 				return fmt.Errorf("failed to fetch user keys: %+w", err)

go.mod

@@ -62,6 +62,8 @@ require (
 	gotest.tools v2.2.0+incompatible
 )
 
+require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+
 require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/Microsoft/hcsshim v0.8.25 // indirect
@@ -94,6 +96,7 @@ require (
 	github.com/hanwen/go-fuse/v2 v2.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/holiman/uint256 v1.2.3 // indirect
 	github.com/jarcoal/httpmock v1.3.1

go.sum

@@ -287,8 +287,12 @@ github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1r
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=

pkg/environment/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 )
 
@@ -51,11 +52,11 @@ func GetConfig() (base Config, err error) {
 
 // GetConfig returns extend config for specific run mode
 func GetConfigForMode(mode RunMode) (Config, error) {
-	httpClient := &http.Client{
-		Timeout: defaultHttpTimeout,
-	}
+	httpClient := retryablehttp.NewClient()
+	httpClient.HTTPClient.Timeout = defaultHttpTimeout
+	httpClient.RetryMax = 5
 
-	return getConfig(mode, baseExtendedURL, httpClient)
+	return getConfig(mode, baseExtendedURL, httpClient.StandardClient())
 }
 
 func uniqueStr(slice []string) []string {

pkg/flist/flist.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
@@ -126,7 +127,7 @@ type flistModule struct {
 	commander commander
 	system    system
 
-	httpClient *http.Client
+	httpClient *retryablehttp.Client
 }
 
 func newFlister(root string, storage volumeAllocator, commander commander, system system) *flistModule {
@@ -155,6 +156,9 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
 		}
 	}
 
+	httpClient := retryablehttp.NewClient()
+	httpClient.HTTPClient.Timeout = defaultHubCallTimeout
+	httpClient.RetryMax = 5
 	return &flistModule{
 		root:       root,
 		flist:      filepath.Join(root, "flist"),
@@ -168,9 +172,7 @@ func newFlister(root string, storage volumeAllocator, commander commander, syste
 		commander: commander,
 		system:    system,
 
-		httpClient: &http.Client{
-			Timeout: defaultHubCallTimeout,
-		},
+		httpClient: httpClient,
 	}
 }
 
@@ -731,14 +733,14 @@ func (f *flistModule) downloadInNamespace(name, u string) (resp *http.Response,
 			return errors.Wrap(err, "failed to start tcp connection")
 		}
 
-		cl := http.Client{
-			Transport: &http.Transport{
-				DisableKeepAlives: true,
-				DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-					return con, nil
-				},
+		cl := retryablehttp.NewClient()
+		cl.HTTPClient.Transport = &http.Transport{
+			DisableKeepAlives: true,
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return con, nil
 			},
 		}
+		cl.RetryMax = 5
 
 		resp, err = cl.Get(u)
 		return err

pkg/gateway/metrics.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
@@ -139,14 +140,14 @@ func metrics(rawUrl string) (map[string]*metric, error) {
 
 	defer con.Close()
 
-	cl := http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return con, nil
-			},
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Transport = &http.Transport{
+		DisableKeepAlives: true,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return con, nil
 		},
 	}
+	cl.RetryMax = 5
 
 	response, err := cl.Get(rawUrl)
 

pkg/geoip/geoip.go

@@ -3,9 +3,9 @@ package geoip
 import (
 	"encoding/json"
 	"errors"
-	"net/http"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/rs/zerolog/log"
 )
 
@@ -22,9 +22,7 @@ type Location struct {
 var (
 	geoipURLs = []string{"https://geoip.grid.tf/", "https://02.geoip.grid.tf/", "https://03.geoip.grid.tf/"}
 
-	defaultHTTPClient = &http.Client{
-		Timeout: 10 * time.Second,
-	}
+	defaultHTTPClient = retryablehttp.NewClient()
 )
 
 // Fetch retrieves the location of the system calling this function
@@ -51,6 +49,8 @@ func getLocation(geoIPService string) (Location, error) {
 		City:      "Unknown",
 	}
 
+	defaultHTTPClient.HTTPClient.Timeout = 10 * time.Second
+	defaultHTTPClient.RetryMax = 5
 	resp, err := defaultHTTPClient.Get(geoIPService)
 	if err != nil {
 		return l, err

pkg/perf/publicip/publicip_task.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/rs/zerolog/log"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zos/pkg/environment"
@@ -271,14 +272,15 @@ func getRealPublicIP() (net.IP, error) {
 
 	defer con.Close()
 
-	cl := http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return con, nil
-			},
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Transport = &http.Transport{
+		DisableKeepAlives: true,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return con, nil
 		},
 	}
+	cl.RetryMax = 5
+
 	response, err := cl.Get("https://api.ipify.org/")
 	if err != nil {
 		return nil, errors.Join(err, errPublicIPLookup)

pkg/provision/engine.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/joncrlsn/dque"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
@@ -91,6 +92,8 @@ const (
 	opPause
 	// opResume resumes a deployment
 	opResume
+	// servers default timeout
+	defaultHttpTimeout = 10 * time.Second
 )
 
 // engineJob is a persisted job instance that is
@@ -1210,11 +1213,11 @@ func isTwinVerified(twinID uint32) (verified bool, err error) {
 	q.Set("twin_id", fmt.Sprint(twinID))
 	request.URL.RawQuery = q.Encode()
 
-	cl := &http.Client{
-		Timeout: 10 * time.Second,
-	}
+	cl := retryablehttp.NewClient()
+	cl.HTTPClient.Timeout = defaultHttpTimeout
+	cl.RetryMax = 5
 
-	response, err := cl.Do(request)
+	response, err := cl.StandardClient().Do(request)
 	if err != nil {
 		return
 	}

pkg/upgrade/hub/hub.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/0-fs/meta"
@@ -65,15 +66,16 @@ func MatchType(typ FListType) FListFilter {
 
 // HubClient API for f-list
 type HubClient struct {
-	httpClient *http.Client
+	httpClient *retryablehttp.Client
 }
 
 // NewHubClient create new hub client with the passed option for the http client
 func NewHubClient(timeout time.Duration) *HubClient {
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryMax = 5
+	httpClient.HTTPClient.Timeout = timeout
 	return &HubClient{
-		httpClient: &http.Client{
-			Timeout: timeout,
-		},
+		httpClient: httpClient,
 	}
 }
 
@@ -295,9 +297,10 @@ func (b *Regular) Files(repo string) ([]FileInfo, error) {
 	}
 
 	u.Path = filepath.Join("api", "flist", repo, b.Name)
-	cl := &http.Client{
-		Timeout: defaultHubCallTimeout,
-	}
+
+	cl := retryablehttp.NewClient()
+	cl.RetryMax = 5
+	cl.HTTPClient.Timeout = defaultHubCallTimeout
 
 	response, err := cl.Get(u.String())
 	if err != nil {