diff --git a/debian/parser.go b/debian/parser.go index 66f32bfeb..e9f6f0952 100644 --- a/debian/parser.go +++ b/debian/parser.go @@ -70,17 +70,6 @@ func (u *updater) Parse(ctx context.Context, r io.ReadCloser) ([]*claircore.Vuln }, } vs = append(vs, &v) - - for _, bin := range u.sm.Get(d.VersionCodeName, src) { - // Shallow copy. - vuln := v - vuln.Package = &claircore.Package{ - Name: bin, - Kind: claircore.BINARY, - } - - vs = append(vs, &vuln) - } } } } diff --git a/debian/sourcemapper.go b/debian/sourcemapper.go deleted file mode 100644 index 539443192..000000000 --- a/debian/sourcemapper.go +++ /dev/null @@ -1,168 +0,0 @@ -package debian - -import ( - "bufio" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/textproto" - "net/url" - "path" - "strings" - "sync" - - "github.com/quay/zlog" - "golang.org/x/sync/errgroup" -) - -var sourceRepos = [3]string{"main", "contrib", "non-free"} - -// NewSourcesMap returns a SourcesMap but does not perform any -// inserts into the map. That needs to be done explicitly by calling -// the Update method. -func newSourcesMap(client *http.Client, srcs []sourceURL) *sourcesMap { - return &sourcesMap{ - urls: srcs, - sourceMap: make(map[string]map[string]map[string]struct{}), - etagMap: make(map[string]string), - client: client, - } -} - -type sourceURL struct { - distro string - url *url.URL -} - -// sourcesMap wraps a map that defines the relationship between a source -// package and its associated binaries. It offers an Update method -// to populate and update the map. It is Release-centric. -// -// It should have the same lifespan as the Updater to save allocations -// and take advantage of the entity tag that Debian sends back. -type sourcesMap struct { - urls []sourceURL - mu, etagMu sync.RWMutex - // sourceMap maps distribution -> source package -> binary packages - sourceMap map[string]map[string]map[string]struct{} - etagMap map[string]string - client *http.Client -} - -// Get returns all the binaries associated with a source package -// identified by a string. Empty slice is returned if the source -// doesn't exist in the map. -func (m *sourcesMap) Get(distro, source string) []string { - m.mu.RLock() - defer m.mu.RUnlock() - bins := []string{} - for bin := range m.sourceMap[distro][source] { - bins = append(bins, bin) - } - return bins -} - -// Update pulls the Sources.gz files for the different repos and saves -// the resulting source to binary relationships. -func (m *sourcesMap) Update(ctx context.Context) error { - g, ctx := errgroup.WithContext(ctx) - for _, source := range m.urls { - // Required as source is overwritten upon each iteration, - // which may cause a race condition when used below in g.Go. - src := source - for _, r := range sourceRepos { - u, err := source.url.Parse(path.Join(r, `source`, `Sources.gz`)) - g.Go(func() error { - if err != nil { - return fmt.Errorf("unable to construct URL: %w", err) - } - if err := m.fetchSources(ctx, src.distro, u.String()); err != nil { - return fmt.Errorf("unable to fetch sources: %w", err) - } - return nil - }) - } - } - return g.Wait() -} - -func (m *sourcesMap) fetchSources(ctx context.Context, distro, url string) error { - ctx = zlog.ContextWithValues(ctx, - "component", "debian/sourcemapper.fetchSources", - "url", url) - zlog.Debug(ctx).Msg("attempting fetch of Sources file") - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - m.etagMu.RLock() - etag := m.etagMap[url] - m.etagMu.RUnlock() - req.Header.Set("If-None-Match", etag) - - resp, err := m.client.Do(req) - if err != nil { - return err - } - - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - if etag == "" || etag != resp.Header.Get("etag") { - break - } - fallthrough - case http.StatusNotModified: - zlog.Debug(ctx).Msg("already processed the latest version of the file") - return nil - default: - return fmt.Errorf("received status code %d querying mapping url %s", resp.StatusCode, url) - } - m.etagMu.Lock() - m.etagMap[url] = resp.Header.Get("etag") - m.etagMu.Unlock() - - var reader io.ReadCloser - switch resp.Header.Get("Content-Type") { - case "application/gzip", "application/x-gzip": - reader, err = gzip.NewReader(resp.Body) - if err != nil { - return err - } - defer reader.Close() - default: - return fmt.Errorf("received bad content-type %s querying mapping url %s", resp.Header.Get("Content-Type"), url) - } - - tp := textproto.NewReader(bufio.NewReader(reader)) - hdr, err := tp.ReadMIMEHeader() - for ; err == nil && len(hdr) > 0; hdr, err = tp.ReadMIMEHeader() { - source := hdr.Get("Package") - if source == "linux" { - continue - } - binaries := hdr.Get("Binary") - m.mu.Lock() - if m.sourceMap[distro] == nil { - m.sourceMap[distro] = make(map[string]map[string]struct{}) - } - if m.sourceMap[distro][source] == nil { - m.sourceMap[distro][source] = make(map[string]struct{}) - } - for _, bin := range strings.Split(binaries, ", ") { - m.sourceMap[distro][source][bin] = struct{}{} - } - m.mu.Unlock() - } - switch { - case errors.Is(err, io.EOF): - default: - return fmt.Errorf("could not read Sources file %s: %w", url, err) - } - - return nil -} diff --git a/debian/sourcemapper_test.go b/debian/sourcemapper_test.go deleted file mode 100644 index 20efea79f..000000000 --- a/debian/sourcemapper_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package debian - -import ( - "bytes" - "context" - "io" - "net/http" - "net/url" - "os" - "testing" - - "github.com/quay/zlog" -) - -type TestClientFunc func(req *http.Request) *http.Response - -func (f TestClientFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req), nil -} - -func newTestClient() (*http.Client, error) { - f, err := os.Open("testdata/Bullseye-Sources.gz") - if err != nil { - return nil, err - } - b, err := io.ReadAll(f) - if err != nil { - return nil, err - } - f.Close() - - return &http.Client{ - Transport: TestClientFunc( - func(req *http.Request) *http.Response { - w := &http.Response{ - StatusCode: http.StatusOK, - Header: make(http.Header), - Body: io.NopCloser(bytes.NewReader(b)), - } - w.Header.Set("Content-Type", "application/gzip") - return w - }, - ), - }, nil -} - -func TestCreateSourcesMap(t *testing.T) { - ctx := zlog.Test(context.Background(), t) - client, err := newTestClient() - if err != nil { - t.Fatalf("got the error %v", err) - } - u, err := url.Parse("http://[::1]/") - if err != nil { - t.Fatal(err) - } - mapper := newSourcesMap(client, []sourceURL{{distro: "bullseye", url: u}}) - - err = mapper.Update(ctx) - if err != nil { - t.Fatalf("unexpected error %v", err) - } - opensshBinaries := mapper.Get("bullseye", "aalib") - if len(opensshBinaries) != 3 { - t.Fatalf("expected 3 binaries related to aalib found %d - %v", len(opensshBinaries), opensshBinaries) - } - - tarBinaries := mapper.Get("bullseye", "389-ds-base") - if len(tarBinaries) != 6 { - t.Fatalf("expected 6 binaries related to 389-ds-base found %d - %v", len(tarBinaries), tarBinaries) - } -} diff --git a/debian/testdata/Bullseye-Sources.gz b/debian/testdata/Bullseye-Sources.gz deleted file mode 100644 index c054c7ce2..000000000 Binary files a/debian/testdata/Bullseye-Sources.gz and /dev/null differ diff --git a/debian/updater.go b/debian/updater.go index 21b1de5ff..7522c3515 100644 --- a/debian/updater.go +++ b/debian/updater.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "io" "net/http" @@ -14,7 +15,6 @@ import ( "strconv" "strings" - "github.com/quay/claircore" "github.com/quay/zlog" "github.com/quay/claircore/libvuln/driver" @@ -125,8 +125,7 @@ var ( func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { s := driver.NewUpdaterSet() - ds, err := f.findReleases(ctx, f.mirror) - if err != nil { + if err := f.findReleases(ctx, f.mirror); err != nil { return s, fmt.Errorf("debian: examining remote: %w", err) } @@ -134,17 +133,6 @@ func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { u := &updater{ jsonURL: f.json.String(), } - for _, d := range ds { - src, err := f.mirror.Parse(path.Join("dists", d.VersionCodeName) + "/") - if err != nil { - return s, fmt.Errorf("debian: unable to construct source URL: %w", err) - } - - u.dists = append(u.dists, sourceURL{ - distro: d.VersionCodeName, - url: src, - }) - } if err := s.Add(u); err != nil { return s, fmt.Errorf("debian: unable to add updater: %w", err) @@ -154,32 +142,31 @@ func (f *Factory) UpdaterSet(ctx context.Context) (driver.UpdaterSet, error) { } // FindReleases is split out as a method to make it easier to examine the mirror and the archive. -func (f *Factory) findReleases(ctx context.Context, u *url.URL) ([]*claircore.Distribution, error) { +func (f *Factory) findReleases(ctx context.Context, u *url.URL) error { dir, err := u.Parse("dists/") if err != nil { - return nil, fmt.Errorf("debian: unable to construct URL: %w", err) + return fmt.Errorf("debian: unable to construct URL: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, dir.String(), nil) if err != nil { - return nil, fmt.Errorf("debian: unable to construct request: %w", err) + return fmt.Errorf("debian: unable to construct request: %w", err) } res, err := f.c.Do(req) if err != nil { - return nil, fmt.Errorf("debian: unable to do request: %w", err) + return fmt.Errorf("debian: unable to do request: %w", err) } defer res.Body.Close() switch res.StatusCode { case http.StatusOK: default: - return nil, fmt.Errorf("debian: unexpected status fetching %q: %s", dir.String(), res.Status) + return fmt.Errorf("debian: unexpected status fetching %q: %s", dir.String(), res.Status) } var buf bytes.Buffer if _, err := buf.ReadFrom(res.Body); err != nil { - return nil, fmt.Errorf("debian: unable to read dists listing: %w", err) + return fmt.Errorf("debian: unable to read dists listing: %w", err) } ms := linkRegexp.FindAllStringSubmatch(buf.String(), -1) - var todos []*claircore.Distribution Listing: for _, m := range ms { dist := m[1] @@ -257,20 +244,18 @@ Listing: continue } - todos = append(todos, mkDist(dist, int(ver))) + mkDist(dist, int(ver)) } - return todos, nil + return nil } // Updater implements [driver.updater]. type updater struct { // jsonURL is the URL from which to fetch JSON vulnerability data jsonURL string - dists []sourceURL - c *http.Client - sm *sourcesMap + c *http.Client } // UpdaterConfig is the configuration for the updater. @@ -278,9 +263,9 @@ type UpdaterConfig struct { // Deprecated: Use JSONURL instead. OVALURL string `json:"url" yaml:"url"` JSONURL string `json:"json_url" yaml:"json_url"` - // Deprecated: Use DistsURLs instead. - DistsURL string `json:"dists_url" yaml:"dists_url"` - DistsURLs []sourceURL `json:"dists_urls" yaml:"dists_urls"` + // Deprecated: DistURL and DistsURLs are unused. + DistsURL string `json:"dists_url" yaml:"dists_url"` + DistsURLs []json.RawMessage `json:"dists_urls" yaml:"dists_urls"` } // Name implements [driver.Updater]. @@ -307,21 +292,6 @@ func (u *updater) Configure(ctx context.Context, f driver.ConfigUnmarshaler, c * zlog.Info(ctx). Msg("configured JSON database URL") } - if len(cfg.DistsURLs) != 0 { - u.dists = cfg.DistsURLs - zlog.Info(ctx). - Msg("configured dists URLs") - } - - var srcs []sourceURL - for _, dist := range u.dists { - src, err := url.Parse(dist.url.String()) - if err != nil { - return fmt.Errorf("debian: unable to parse dist URL: %w", err) - } - srcs = append(srcs, sourceURL{distro: dist.distro, url: src}) - } - u.sm = newSourcesMap(u.c, srcs) return nil } @@ -385,11 +355,6 @@ func (u *updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io } zlog.Info(ctx).Msg("fetched latest json database successfully") - if err := u.sm.Update(ctx); err != nil { - return nil, "", fmt.Errorf("could not update source to binary map: %w", err) - } - zlog.Info(ctx).Msg("updated the debian source to binary map successfully") success = true - return f, driver.Fingerprint(fp), err }