rpm: consult dnf database for repository information

rhel/coalescer.go

@@ -2,6 +2,7 @@ package rhel
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/indexer"
@@ -116,10 +117,15 @@ func (*Coalescer) Coalesce(ctx context.Context, artifacts []*indexer.LayerArtifa
 					PackageDB:      pkg.PackageDB,
 					IntroducedIn:   layerArtifacts.Hash,
 					DistributionID: distID,
-					RepositoryIDs:  make([]string, len(layerArtifacts.Repos)),
 				}
-				for i := range layerArtifacts.Repos {
-					environment.RepositoryIDs[i] = layerArtifacts.Repos[i].ID
+				v, _ := url.ParseQuery(pkg.RepositoryHint) // Ignore error
+				if id := v.Get("repoid"); id != "" {
+					environment.RepositoryIDs = v["repoid"]
+				} else {
+					environment.RepositoryIDs = make([]string, len(layerArtifacts.Repos))
+					for i := range layerArtifacts.Repos {
+						environment.RepositoryIDs[i] = layerArtifacts.Repos[i].ID
+					}
 				}
 				db.packages[pkg.ID] = pkg
 				db.environments[pkg.ID] = environment

rpm/dnf.go

@@ -0,0 +1,153 @@
+package rpm
+
+import (
+	"context"
+	"database/sql"
+	_ "embed" // embed query
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+
+	"github.com/quay/zlog"
+	_ "modernc.org/sqlite" // register the sqlite driver
+)
+
+// BUG(hank) The dnf mapping is less useful than it could be because there's no
+// reliable way to turn the RepoID that it reports into something with meaning
+// outside of the Red Hat build system's builder's context. See [CLAIRDEV-45]
+// for more information.
+//
+// [CLAIRDEV-45]: https://issues.redhat.com/browse/CLAIRDEV-45
+
+// RepoMap reports the latest NEVRA → RepoID mapping, as extracted from the dnf
+// or dnf5 database.
+func repoMap(ctx context.Context, sys fs.FS) (map[string]string, error) {
+	var toOpen string
+	var isdnf5 bool
+Look:
+	for i, p := range []string{
+		`usr/lib/sysimage/libdnf5/transaction_history.sqlite`, // dnf5 location
+		`var/lib/dnf/history.sqlite`,                          // dnf3/4 location
+	} {
+		switch _, err := fs.Stat(sys, p); {
+		case errors.Is(err, nil):
+			toOpen = p
+			isdnf5 = i == 0
+			break Look
+		case errors.Is(err, fs.ErrNotExist): // OK
+		default:
+			return nil, fmt.Errorf("rpm: unexpected error opening dnf history: %w", err)
+		}
+	}
+	if toOpen == "" {
+		// Nothing found.
+		return nil, nil
+	}
+
+	zlog.Debug(ctx).
+		Str("path", toOpen).
+		Bool("is-5", isdnf5).
+		Msg("found dnf history database")
+	r, err := sys.Open(toOpen)
+	switch {
+	case errors.Is(err, nil):
+	case errors.Is(err, fs.ErrNotExist):
+		return nil, nil
+	default:
+		return nil, fmt.Errorf("rpm: unexpected error opening dnf history: %w", err)
+	}
+	defer func() {
+		if err := r.Close(); err != nil {
+			zlog.Warn(ctx).Err(err).Msg("unable to close tarfs sqlite db")
+		}
+	}()
+
+	db, cleanup, err := openHistoryDatabase(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	defer cleanup()
+	defer db.Close()
+
+	rows, err := db.QueryContext(ctx, queryFinalState, removedEnum(isdnf5))
+	if err != nil {
+		return nil, fmt.Errorf("rpm: error querying dnf database: %w", err)
+	}
+	defer func() {
+		if err := rows.Close(); err != nil {
+			zlog.Warn(ctx).Err(err).Msg("error closing rows object")
+		}
+	}()
+
+	ret := make(map[string]string)
+	var k, v string
+	for rows.Next() {
+		if err := rows.Scan(&k, &v); err != nil {
+			return nil, fmt.Errorf("rpm: error reading dnf database: %w", err)
+		}
+		ret[k] = v
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("rpm: error reading dnf database: %w", err)
+	}
+
+	return ret, nil
+}
+
+// RemovedEnum reports the enum for a "removed" action for the indicated
+// database version.
+func removedEnum(is5 bool) int {
+	// Defined here:
+	// https://github.com/rpm-software-management/dnf5/blob/13886935418e28482de7b675169482b85303845d/include/libdnf/transaction/transaction_item_action.hpp#L35
+	if is5 {
+		return 5
+	}
+	// Defined here:
+	// https://github.com/rpm-software-management/libdnf/blob/93759bc5cac262906e52b6a173d7b157914ec29e/libdnf/transaction/Types.hpp#L45
+	return 8
+}
+
+// OpenHistoryDatabase contains all the logic for opening the provided [fs.File]
+// as a [sql.DB].
+func openHistoryDatabase(ctx context.Context, r fs.File) (*sql.DB, func(), error) {
+	// Currently needs to be linked into the filesystem.
+	// See also: quay/claircore#720
+	f, err := os.CreateTemp(os.TempDir(), `dnf.sqlite.*`)
+	if err != nil {
+		return nil, noop, fmt.Errorf("rpm: error reading sqlite db: %w", err)
+	}
+	cleanup := func() {
+		if err := os.Remove(f.Name()); err != nil {
+			zlog.Error(ctx).Err(err).Msg("unable to unlink sqlite db")
+		}
+		if err := f.Close(); err != nil {
+			zlog.Warn(ctx).Err(err).Msg("unable to close sqlite db")
+		}
+	}
+	zlog.Debug(ctx).Str("file", f.Name()).Msg("copying sqlite db out of tar")
+	if _, err := io.Copy(f, r); err != nil {
+		cleanup()
+		return nil, noop, fmt.Errorf("rpm: error reading sqlite db: %w", err)
+	}
+	if err := f.Sync(); err != nil {
+		cleanup()
+		return nil, noop, fmt.Errorf("rpm: error reading sqlite db: %w", err)
+	}
+
+	db, err := sql.Open("sqlite", f.Name())
+	if err != nil {
+		cleanup()
+		return nil, noop, fmt.Errorf("rpm: error reading sqlite db: %w", err)
+	}
+	return db, cleanup, nil
+}
+
+func noop() {}
+
+// QueryFinalState returns (nerva, repoid) rows and takes a single argument, the
+// "removed" enum to disregard.
+//
+//go:embed dnf_finalstate.sql
+var queryFinalState string

rpm/dnf_finalstate.sql

@@ -0,0 +1,27 @@
+SELECT
+	name ||'-'||
+	CASE
+		WHEN epoch = 0 THEN ''
+		ELSE epoch || ':'
+	END ||
+	version ||'-'||
+	release ||'.'||
+	arch
+		AS nerva,
+	repoid
+FROM
+	trans_item
+	JOIN (
+		SELECT
+			max(id) AS uniq
+		FROM
+			trans_item
+		WHERE
+			action <> ?
+		GROUP BY
+			item_id
+		) ON (uniq = trans_item.id)
+	JOIN
+		repo ON (repo.id = repo_id)
+	JOIN
+		rpm USING (item_id);

rpm/native_db.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"path"
 	"regexp"
 	"runtime/trace"
@@ -79,7 +80,7 @@ func packagesFromDB(ctx context.Context, pkgdb string, db nativeDB) ([]*claircor
 		}
 		p.Module = modStream
 		p.Version = constructEVR(&b, &info)
-		p.RepositoryHint = constructHint(&b, &info)
+		p.RepositoryHint = constructHint(&info)
 
 		if s, ok := src[info.SourceNEVR]; ok {
 			p.Source = s
@@ -263,14 +264,12 @@ func constructEVR(b *strings.Builder, info *Info) string {
 	return b.String()
 }
 
-func constructHint(b *strings.Builder, info *Info) string {
-	b.Reset()
+func constructHint(info *Info) string {
+	v := url.Values{}
 	if info.Digest != "" {
-		b.WriteString("hash:")
 		switch info.DigestAlgo {
 		case 8:
-			b.WriteString("sha256:")
-			b.WriteString(info.Digest)
+			v.Add("hash", fmt.Sprintf("sha256:%s", info.Digest))
 		}
 	}
 	if len(info.Signature) != 0 {
@@ -282,20 +281,11 @@ func constructHint(b *strings.Builder, info *Info) string {
 				if p.SigType != 0 {
 					continue
 				}
-				if b.Len() != 0 {
-					b.WriteByte('|')
-				}
-				fmt.Fprintf(b, "key:%016x", p.IssuerKeyId)
+				v.Add("key", fmt.Sprintf("%016x", p.IssuerKeyId))
 			case *packet.Signature:
-				if p.SigType != 0 || p.IssuerKeyId == nil {
-					continue
-				}
-				if b.Len() != 0 {
-					b.WriteByte('|')
-				}
-				fmt.Fprintf(b, "key:%016x", *p.IssuerKeyId)
+				v.Add("key", fmt.Sprintf("%016x", p.IssuerKeyId))
 			}
 		}
 	}
-	return b.String()
+	return v.Encode()
 }

rpm/packagescanner.go

@@ -3,9 +3,11 @@ package rpm
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
+	"net/url"
 	"os"
 	"path"
 	"runtime/trace"
@@ -30,21 +32,21 @@ var (
 	_ indexer.PackageScanner   = (*Scanner)(nil)
 )
 
-// Scanner implements the scanner.PackageScanner interface.
+// Scanner implements the [indexer.PackageScanner] interface.
 //
 // This looks for directories that look like rpm databases and examines the
 // files it finds there.
 //
 // The zero value is ready to use.
 type Scanner struct{}
 
-// Name implements scanner.VersionedScanner.
+// Name implements [indexer.VersionedScanner].
 func (*Scanner) Name() string { return pkgName }
 
-// Version implements scanner.VersionedScanner.
+// Version implements [indexer.VersionedScanner].
 func (*Scanner) Version() string { return pkgVersion }
 
-// Kind implements scanner.VersionedScanner.
+// Kind implements [indexer.VersionedScanner].
 func (*Scanner) Kind() string { return pkgKind }
 
 // Scan attempts to find rpm databases within the layer and enumerate the
@@ -174,6 +176,38 @@ func (ps *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*clairco
 		}
 		pkgs = append(pkgs, ps...)
 	}
+	rm, err := repoMap(ctx, sys)
+	switch {
+	case errors.Is(err, nil) && len(rm) != 0: // OK
+		zlog.Debug(ctx).Msg("using dnf-discovered repoids")
+		for _, pkg := range pkgs {
+			nerva := fmt.Sprintf("%s-%s.%s", pkg.Name, pkg.Version, pkg.Arch)
+			repoid, ok := rm[nerva]
+			if !ok {
+				// Packages not installed via dnf, which may happen during
+				// bootstrapping, aren't present in the dnf history database.
+				// This means the process shouldn't bail if the package is
+				// missing.
+				continue
+			}
+			v, err := url.ParseQuery(pkg.RepositoryHint)
+			if err != nil { // Shouldn't happen:
+				zlog.Warn(ctx).
+					AnErr("url.ParseQuery", err).
+					Msg("malformed RepositoryHint")
+				continue
+			}
+			v.Add("reposrc", "dnf")
+			v.Add("repoid", repoid)
+			pkg.RepositoryHint = v.Encode()
+		}
+	case errors.Is(err, nil) && len(rm) == 0: // nothing found
+		zlog.Debug(ctx).Msg("no dnf-discovered repoids available")
+	default: // some error
+		zlog.Warn(ctx).
+			AnErr("repoMap", err).
+			Msg("unable to open dnf history database")
+	}
 
 	return pkgs, nil
 }