feat(cmd): add arch

.github/workflows/fetch.yml

@@ -245,3 +245,63 @@ jobs:
       - name: fetch redis
         if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
         run: ./gost fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" microsoft
+
+  fetch-arch:
+    name: fetch-arch
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_DATABASE: test
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+      - name: build
+        id: build
+        run: make build
+      - name: fetch sqlite3
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./gost fetch --dbtype sqlite3 arch
+      - name: fetch mysql
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./gost fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" arch
+      - name: fetch postgres
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./gost fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" arch
+      - name: fetch redis
+        if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
+        run: ./gost fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" arch

cmd/arch.go

@@ -0,0 +1,73 @@
+package cmd
+
+import (
+	"time"
+
+	"github.com/inconshreveable/log15"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/vulsio/gost/db"
+	"github.com/vulsio/gost/fetcher"
+	"github.com/vulsio/gost/models"
+	"github.com/vulsio/gost/util"
+	"golang.org/x/xerrors"
+)
+
+var archCmd = &cobra.Command{
+	Use:   "arch",
+	Short: "Fetch the CVE information from Arch Linux",
+	Long:  `Fetch the CVE information from Arch Linux`,
+	RunE:  fetchArch,
+}
+
+func init() {
+	fetchCmd.AddCommand(archCmd)
+}
+
+func fetchArch(_ *cobra.Command, _ []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
+	log15.Info("Initialize Database")
+	driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
+	if err != nil {
+		if xerrors.Is(err, db.ErrDBLocked) {
+			return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err)
+		}
+		return xerrors.Errorf("Failed to open DB. err: %w", err)
+	}
+
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err)
+	}
+	if fetchMeta.OutDated() {
+		return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+	}
+	// If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion.
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err)
+	}
+
+	log15.Info("Fetched all CVEs from Arch Linux")
+	advJSONs, err := fetcher.FetchArch()
+	if err != nil {
+		return xerrors.Errorf("Failed to fetch Arch. err: %w", err)
+	}
+	advs := models.ConvertArch(advJSONs)
+
+	log15.Info("Fetched", "Advisories", len(advs))
+
+	log15.Info("Insert Arch Linux CVEs into DB", "db", driver.Name())
+	if err := driver.InsertArch(advs); err != nil {
+		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	fetchMeta.LastFetchedAt = time.Now()
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
+	}
+
+	return nil
+}

db/arch.go

@@ -0,0 +1,143 @@
+package db
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/cheggaaa/pb/v3"
+	"github.com/spf13/viper"
+	"github.com/vulsio/gost/models"
+	"golang.org/x/xerrors"
+	"gorm.io/gorm"
+)
+
+// GetArch :
+func (r *RDBDriver) GetArch(advID string) (*models.ArchADV, error) {
+	var a models.ArchADV
+	if err := r.conn.
+		Preload("Packages").
+		Preload("Issues").
+		Preload("Advisories").
+		Where(&models.ArchADV{Name: advID}).
+		First(&a).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		return nil, xerrors.Errorf("Failed to find first record by %s. err: %w", advID, err)
+	}
+	return &a, nil
+}
+
+// GetArchMulti :
+func (r *RDBDriver) GetArchMulti(advIDs []string) (map[string]models.ArchADV, error) {
+	m := make(map[string]models.ArchADV)
+	for _, id := range advIDs {
+		a, err := r.GetArch(id)
+		if err != nil {
+			return nil, xerrors.Errorf("Failed to get Arch. err: %w", err)
+		}
+		if a != nil {
+			m[id] = *a
+		}
+	}
+	return m, nil
+}
+
+// InsertArch :
+func (r *RDBDriver) InsertArch(advs []models.ArchADV) error {
+	if err := r.deleteAndInsertArch(advs); err != nil {
+		return xerrors.Errorf("Failed to insert Arch Advisory data. err: %w", err)
+	}
+	return nil
+}
+
+func (r *RDBDriver) deleteAndInsertArch(advs []models.ArchADV) (err error) {
+	bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer {
+		if viper.GetBool("log-json") {
+			return io.Discard
+		}
+		return os.Stderr
+	}())
+	tx := r.conn.Begin()
+
+	defer func() {
+		if err != nil {
+			tx.Rollback()
+			return
+		}
+		tx.Commit()
+	}()
+
+	// Delete all old records
+	for _, table := range []interface{}{models.ArchAdvisory{}, models.ArchIssue{}, models.ArchPackage{}, models.ArchADV{}} {
+		if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(table).Error; err != nil {
+			return xerrors.Errorf("Failed to delete old records. err: %w", err)
+		}
+	}
+
+	batchSize := viper.GetInt("batch-size")
+	if batchSize < 1 {
+		return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly")
+	}
+
+	for idx := range chunkSlice(len(advs), batchSize) {
+		if err = tx.Create(advs[idx.From:idx.To]).Error; err != nil {
+			return xerrors.Errorf("Failed to insert. err: %w", err)
+		}
+		bar.Add(idx.To - idx.From)
+	}
+	bar.Finish()
+
+	return nil
+}
+
+// GetUnfixedAdvsArch :
+func (r *RDBDriver) GetUnfixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) {
+	return r.getAdvsArchWithFixStatus(pkgName, "Vulnerable")
+}
+
+// GetFixedAdvsArch :
+func (r *RDBDriver) GetFixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) {
+	return r.getAdvsArchWithFixStatus(pkgName, "Fixed")
+}
+
+func (r *RDBDriver) getAdvsArchWithFixStatus(pkgName, fixStatus string) (map[string]models.ArchADV, error) {
+	var as []models.ArchADV
+	if err := r.conn.
+		Joins("JOIN arch_packages ON arch_packages.arch_adv_id = arch_advs.id AND arch_packages.name = ?", pkgName).
+		Preload("Packages").
+		Preload("Issues").
+		Preload("Advisories").
+		Where(&models.ArchADV{Status: fixStatus}).
+		Find(&as).Error; err != nil {
+		return nil, xerrors.Errorf("Failed to find advisory by pkgname: %s, fix status: %s. err: %w", pkgName, fixStatus, err)
+	}
+
+	m := make(map[string]models.ArchADV)
+	for _, a := range as {
+		m[a.Name] = a
+	}
+	return m, nil
+}
+
+// GetAdvisoriesArch gets AdvisoryID: []CVE IDs
+func (r *RDBDriver) GetAdvisoriesArch() (map[string][]string, error) {
+	m := make(map[string][]string)
+	var as []models.ArchADV
+	// the maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0.
+	// https://www.sqlite.org/limits.html Maximum Number Of Host Parameters In A Single SQL Statement
+	if err := r.conn.Preload("Issues").FindInBatches(&as, 999, func(_ *gorm.DB, _ int) error {
+		for _, a := range as {
+			for _, i := range a.Issues {
+				m[a.Name] = append(m[a.Name], i.Issue)
+			}
+		}
+		return nil
+	}).Error; err != nil {
+		return nil, xerrors.Errorf("Failed to find Arch. err: %w", err)
+	}
+
+	return m, nil
+}

db/db.go

@@ -40,11 +40,17 @@ type DB interface {
 	GetRelatedProducts(string, []string) ([]string, error)
 	GetFilteredCvesMicrosoft([]string, []string) (map[string]models.MicrosoftCVE, error)
 	GetAdvisoriesMicrosoft() (map[string][]string, error)
+	GetArch(string) (*models.ArchADV, error)
+	GetArchMulti([]string) (map[string]models.ArchADV, error)
+	GetFixedAdvsArch(string) (map[string]models.ArchADV, error)
+	GetUnfixedAdvsArch(string) (map[string]models.ArchADV, error)
+	GetAdvisoriesArch() (map[string][]string, error)
 
 	InsertRedhat([]models.RedhatCVE) error
 	InsertDebian([]models.DebianCVE) error
 	InsertUbuntu([]models.UbuntuCVE) error
 	InsertMicrosoft([]models.MicrosoftCVE, []models.MicrosoftKBRelation) error
+	InsertArch([]models.ArchADV) error
 }
 
 // Option :

db/rdb.go

@@ -159,6 +159,11 @@ func (r *RDBDriver) MigrateDB() error {
 		&models.MicrosoftKB{},
 		&models.MicrosoftKBRelation{},
 		&models.MicrosoftSupersededBy{},
+
+		&models.ArchADV{},
+		&models.ArchPackage{},
+		&models.ArchIssue{},
+		&models.ArchAdvisory{},
 	); err != nil {
 		switch r.name {
 		case dialectSqlite3: