Add offline files resolve support (#155)

* Add offline files resolve support

* Fetch supported formats before the build

.github/workflows/debricked.yml

@@ -23,5 +23,9 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+      - name: Pull Supported Formats
+        run: |
+          cd cmd/debricked
+          go generate -v -x
       - run: |
           go run cmd/debricked/main.go scan -t ${{ secrets.DEBRICKED_TOKEN }} -e "pkg/**" -e "test/**" -e "**/testdata/**"

.github/workflows/test.yml

@@ -35,6 +35,11 @@ jobs:
           restore-keys: |
             ${{ matrix.os }}-go
 
+      - name: Pull Supported Formats
+        run: |
+          cd cmd/debricked
+          go generate -v -x
+
       - name: Build
         run: go build -v ./...
 
@@ -85,6 +90,11 @@ jobs:
           restore-keys: |
             ${{ matrix.os }}-go
 
+      - name: Pull Supported Formats
+        run: |
+          cd cmd/debricked
+          go generate -v -x
+
       - name: Build
         run: go build -v ./...
 
@@ -140,6 +150,10 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: '1.20'
+      - name: Pull Supported Formats
+        run: |
+          cd cmd/debricked
+          go generate -v -x
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:

.gitignore

@@ -12,6 +12,7 @@ dist/
 test/resolve/testdata/pip/requirements.txt.venv/
 test/resolve/testdata/pip/requirements.txt.pip.debricked.lock
 internal/cmd/scan/testdata/npm/yarn.lock
+internal/file/embedded/supported_formats.json
 internal/resolution/pm/gradle/.gradle-init-script.debricked.groovy
 internal/callgraph/language/java11/testdata/mvnproj/target
 test/resolve/testdata/npm/yarn.lock

build/docker/alpine.Dockerfile

@@ -5,6 +5,8 @@ RUN apk update \
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 COPY . .
+RUN mkdir -p internal/file/embedded && \
+    wget -O internal/file/embedded/supported_formats.json https://debricked.com/api/1.0/open/files/supported-formats
 RUN go build -o debricked ./cmd/debricked
 ENTRYPOINT ["debricked"]
 

build/docker/debian.Dockerfile

@@ -3,6 +3,8 @@ WORKDIR /cli
 RUN apt -y update && apt -y upgrade && apt -y install git && \
     apt -y clean && rm -rf /var/lib/apt/lists/*
 COPY go.mod go.sum ./
+RUN mkdir -p internal/file/embedded && \
+    wget -O internal/file/embedded/supported_formats.json https://debricked.com/api/1.0/open/files/supported-formats
 RUN go mod download && go mod verify
 COPY . .
 RUN go build -o debricked ./cmd/debricked

cmd/debricked/main.go

@@ -7,6 +7,8 @@ import (
 	"github.com/debricked/cli/internal/wire"
 )
 
+//go:generate sh ../../scripts/fetch_supported_formats.sh
+
 var version string // Set at compile time
 
 func main() {

internal/client/deb_client.go

@@ -7,20 +7,24 @@ import (
 )
 
 const DefaultDebrickedUri = "https://debricked.com"
+const DefaultTimeout = 15
 
 type IDebClient interface {
 	// Post makes a POST request to one of Debricked's API endpoints
 	Post(uri string, contentType string, body *bytes.Buffer, timeout int) (*http.Response, error)
 	// Get makes a GET request to one of Debricked's API endpoints
 	Get(uri string, format string) (*http.Response, error)
 	SetAccessToken(accessToken *string)
+	ConfigureClientSettings(retry bool, timeout int)
 }
 
 type DebClient struct {
 	host        *string
 	httpClient  IClient
 	accessToken *string
 	jwtToken    string
+	retry       bool
+	timeout     int
 }
 
 func NewDebClient(accessToken *string, httpClient IClient) *DebClient {
@@ -34,6 +38,8 @@ func NewDebClient(accessToken *string, httpClient IClient) *DebClient {
 		httpClient:  httpClient,
 		accessToken: initAccessToken(accessToken),
 		jwtToken:    "",
+		retry:       true,
+		timeout:     DefaultTimeout,
 	}
 }
 
@@ -46,13 +52,18 @@ func (debClient *DebClient) Post(uri string, contentType string, body *bytes.Buf
 }
 
 func (debClient *DebClient) Get(uri string, format string) (*http.Response, error) {
-	return get(uri, debClient, true, format)
+	return get(uri, debClient, debClient.retry, format)
 }
 
 func (debClient *DebClient) SetAccessToken(accessToken *string) {
 	debClient.accessToken = initAccessToken(accessToken)
 }
 
+func (debClient *DebClient) ConfigureClientSettings(retry bool, timeout int) {
+	debClient.retry = retry
+	debClient.timeout = timeout
+}
+
 func initAccessToken(accessToken *string) *string {
 	if accessToken == nil {
 		accessToken = new(string)

internal/client/request.go

@@ -15,6 +15,7 @@ import (
 )
 
 var NoResErr = errors.New("failed to get response. Check out the Debricked status page: https://status.debricked.com/")
+var SupportedFormatsFallbackError = errors.New("get supported formats from the server. Using cached data instead")
 
 func get(uri string, debClient *DebClient, retry bool, format string) (*http.Response, error) {
 	request, err := newRequest("GET", *debClient.host+uri, debClient.jwtToken, format, nil)
@@ -35,6 +36,14 @@ func post(uri string, debClient *DebClient, contentType string, body *bytes.Buff
 		return nil, err
 	}
 	request.Header.Add("Content-Type", contentType)
+
+	if debClient.timeout > 0 {
+		timeoutDuration := time.Duration(debClient.timeout) * time.Second
+		ctx, cancel := context.WithTimeout(request.Context(), timeoutDuration)
+		defer cancel()
+		request = request.WithContext(ctx)
+	}
+
 	res, err := debClient.httpClient.Do(request)
 	if err != nil {
 		return nil, err

internal/client/testdata/deb_client_mock.go

@@ -45,16 +45,16 @@ func (mock *DebClientMock) Get(uri string, format string) (*http.Response, error
 func (mock *DebClientMock) Post(uri string, format string, body *bytes.Buffer, timeout int) (*http.Response, error) {
 	response, err := mock.popResponse(mock.RemoveQueryParamsFromUri(uri))
 
-	if response != nil {
+	if response != nil || !mock.serviceUp {
 		return response, err
 	}
 
 	return mock.realDebClient.Post(uri, format, body, timeout)
 }
 
-func (mock *DebClientMock) SetAccessToken(_ *string) {
+func (mock *DebClientMock) SetAccessToken(_ *string) {}
 
-}
+func (mock *DebClientMock) ConfigureClientSettings(retry bool, timeout int) {}
 
 type MockResponse struct {
 	StatusCode   int

internal/cmd/files/files_test.go

@@ -4,11 +4,12 @@ import (
 	"testing"
 
 	"github.com/debricked/cli/internal/file"
+	"github.com/debricked/cli/internal/io"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewFilesCmd(t *testing.T) {
-	finder, _ := file.NewFinder(nil)
+	finder, _ := file.NewFinder(nil, io.FileSystem{})
 	cmd := NewFilesCmd(finder)
 	commands := cmd.Commands()
 	nbrOfCommands := 1

internal/file/finder.go

@@ -1,32 +1,43 @@
 package file
 
 import (
+	"embed"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
 
 	"github.com/debricked/cli/internal/client"
+	ioFs "github.com/debricked/cli/internal/io"
+	"github.com/fatih/color"
 )
 
+//go:embed embedded/supported_formats.json
+var supportedFormats embed.FS
+
+const SupportedFormatsFallbackFilePath = "embedded/supported_formats.json"
+const SupportedFormatsUri = "/api/1.0/open/files/supported-formats"
+
 type IFinder interface {
 	GetGroups(rootPath string, exclusions []string, lockfileOnly bool, strictness int) (Groups, error)
 	GetSupportedFormats() ([]*CompiledFormat, error)
 }
 
 type Finder struct {
-	debClient client.IDebClient
+	debClient  client.IDebClient
+	filesystem ioFs.IFileSystem
 }
 
-func NewFinder(c client.IDebClient) (*Finder, error) {
+func NewFinder(c client.IDebClient, fs ioFs.IFileSystem) (*Finder, error) {
 	if c == nil {
 		return nil, errors.New("client is nil")
 	}
 
-	return &Finder{c}, nil
+	return &Finder{c, fs}, nil
 }
 
 // GetGroups return all file groups in specified path recursively.
@@ -68,15 +79,11 @@ func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOn
 
 // GetSupportedFormats returns all supported dependency file formats
 func (finder *Finder) GetSupportedFormats() ([]*CompiledFormat, error) {
-	res, err := finder.debClient.Get("/api/1.0/open/files/supported-formats", "application/json")
+	body, err := finder.GetSupportedFormatsJson()
 	if err != nil {
 		return nil, err
 	}
-	if res.StatusCode != http.StatusOK {
-		return nil, errors.New("failed to fetch supported formats")
-	}
-	defer res.Body.Close()
-	body, _ := io.ReadAll(res.Body)
+
 	var formats []*Format
 	err = json.Unmarshal(body, &formats)
 	if err != nil {
@@ -95,3 +102,35 @@ func (finder *Finder) GetSupportedFormats() ([]*CompiledFormat, error) {
 
 	return compiledDependencyFileFormats, nil
 }
+
+func (finder *Finder) GetSupportedFormatsJson() ([]byte, error) {
+	finder.debClient.ConfigureClientSettings(false, 5)
+	defer finder.debClient.ConfigureClientSettings(true, 15)
+
+	res, err := finder.debClient.Get(SupportedFormatsUri, "application/json")
+
+	if err != nil || res.StatusCode != http.StatusOK {
+		fmt.Printf("%s Unable to get supported formats from the server. Using cached data instead.\n", color.YellowString("⚠️"))
+
+		return finder.GetSupportedFormatsFallbackJson()
+	}
+
+	defer res.Body.Close()
+
+	return io.ReadAll(res.Body)
+}
+
+func (finder *Finder) GetSupportedFormatsFallbackJson() ([]byte, error) {
+	jsonFile, err := finder.filesystem.FsOpenEmbed(supportedFormats, SupportedFormatsFallbackFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer finder.filesystem.FsCloseFile(jsonFile)
+
+	jsonData, err := finder.filesystem.FsReadAll(jsonFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return jsonData, nil
+}

internal/file/finder_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	"github.com/debricked/cli/internal/client/testdata"
+	ioFs "github.com/debricked/cli/internal/io"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -55,21 +56,23 @@ func (mock *debClientMock) Get(_ string, _ string) (*http.Response, error) {
 
 func (mock *debClientMock) SetAccessToken(_ *string) {}
 
+func (mock *debClientMock) ConfigureClientSettings(retry bool, timeout int) {}
+
 var finder *Finder
 
 func setUp(auth bool) {
-	finder, _ = NewFinder(&debClientMock{})
+	finder, _ = NewFinder(&debClientMock{}, ioFs.FileSystem{})
 	authorized = auth
 }
 
 func TestNewFinder(t *testing.T) {
-	finder, err := NewFinder(nil)
+	finder, err := NewFinder(nil, ioFs.FileSystem{})
 
 	assert.NotNil(t, err)
 	assert.Nil(t, finder)
 	assert.ErrorContains(t, err, "client is nil")
 
-	finder, err = NewFinder(testdata.NewDebClientMock())
+	finder, err = NewFinder(testdata.NewDebClientMock(), ioFs.FileSystem{})
 	assert.Nil(t, err)
 	assert.NotNil(t, finder)
 }
@@ -85,12 +88,15 @@ func TestGetSupportedFormats(t *testing.T) {
 	}
 }
 
-func TestGetSupportedFormatsFailed(t *testing.T) {
+func TestGetSupportedFormatsEndpointUnaccessible(t *testing.T) {
 	setUp(false)
 	formats, err := finder.GetSupportedFormats()
-	assert.Error(t, err)
-	assert.ErrorContains(t, err, "failed to fetch supported formats")
-	assert.Empty(t, formats)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(formats), 1)
+	for _, format := range formats {
+		hasContent := format.ManifestFileRegex != nil || len(format.LockFileRegexes) > 0
+		assert.True(t, hasContent, "failed to assert that format had content")
+	}
 }
 
 func TestGetGroups(t *testing.T) {