From b6ea60c58db7ce391ed2aee17f9d8dd72f6f0dd7 Mon Sep 17 00:00:00 2001 From: MaineK00n Date: Fri, 30 Aug 2024 14:43:01 +0900 Subject: [PATCH] feat(cmd): add arch --- .github/workflows/fetch.yml | 60 ++++++++ cmd/arch.go | 73 ++++++++++ db/arch.go | 143 ++++++++++++++++++ db/db.go | 6 + db/rdb.go | 5 + db/redis.go | 280 +++++++++++++++++++++++++++++++----- fetcher/arch.go | 26 ++++ models/arch.go | 89 ++++++++++++ server/server.go | 76 ++++++++++ 9 files changed, 722 insertions(+), 36 deletions(-) create mode 100644 cmd/arch.go create mode 100644 db/arch.go create mode 100644 fetcher/arch.go create mode 100644 models/arch.go diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml index a6c66c9..e8d96a4 100644 --- a/.github/workflows/fetch.yml +++ b/.github/workflows/fetch.yml @@ -245,3 +245,63 @@ jobs: - name: fetch redis if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} run: ./gost fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" microsoft + + fetch-arch: + name: fetch-arch + runs-on: ubuntu-latest + services: + mysql: + image: mysql + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: test + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + postgres: + image: postgres + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: password + POSTGRES_DB: test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + - name: build + id: build + run: make build + - name: fetch sqlite3 + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./gost fetch --dbtype sqlite3 arch + - name: fetch mysql + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./gost fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" arch + - name: fetch postgres + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./gost fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" arch + - name: fetch redis + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./gost fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" arch diff --git a/cmd/arch.go b/cmd/arch.go new file mode 100644 index 0000000..5ec3bd8 --- /dev/null +++ b/cmd/arch.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "time" + + "github.com/inconshreveable/log15" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/vulsio/gost/db" + "github.com/vulsio/gost/fetcher" + "github.com/vulsio/gost/models" + "github.com/vulsio/gost/util" + "golang.org/x/xerrors" +) + +var archCmd = &cobra.Command{ + Use: "arch", + Short: "Fetch the CVE information from Arch Linux", + Long: `Fetch the CVE information from Arch Linux`, + RunE: fetchArch, +} + +func init() { + fetchCmd.AddCommand(archCmd) +} + +func fetchArch(_ *cobra.Command, _ []string) (err error) { + if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { + return xerrors.Errorf("Failed to SetLogger. err: %w", err) + } + + log15.Info("Initialize Database") + driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) + if err != nil { + if xerrors.Is(err, db.ErrDBLocked) { + return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) + } + return xerrors.Errorf("Failed to open DB. err: %w", err) + } + + fetchMeta, err := driver.GetFetchMeta() + if err != nil { + return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) + } + if fetchMeta.OutDated() { + return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) + } + // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) + } + + log15.Info("Fetched all CVEs from Arch Linux") + advJSONs, err := fetcher.FetchArch() + if err != nil { + return xerrors.Errorf("Failed to fetch Arch. err: %w", err) + } + advs := models.ConvertArch(advJSONs) + + log15.Info("Fetched", "Advisories", len(advs)) + + log15.Info("Insert Arch Linux CVEs into DB", "db", driver.Name()) + if err := driver.InsertArch(advs); err != nil { + return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + fetchMeta.LastFetchedAt = time.Now() + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + return nil +} diff --git a/db/arch.go b/db/arch.go new file mode 100644 index 0000000..983b7b9 --- /dev/null +++ b/db/arch.go @@ -0,0 +1,143 @@ +package db + +import ( + "errors" + "fmt" + "io" + "os" + + "github.com/cheggaaa/pb/v3" + "github.com/spf13/viper" + "github.com/vulsio/gost/models" + "golang.org/x/xerrors" + "gorm.io/gorm" +) + +// GetArch : +func (r *RDBDriver) GetArch(advID string) (*models.ArchADV, error) { + var a models.ArchADV + if err := r.conn. + Preload("Packages"). + Preload("Issues"). + Preload("Advisories"). + Where(&models.ArchADV{Name: advID}). + First(&a).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, xerrors.Errorf("Failed to find first record by %s. err: %w", advID, err) + } + return &a, nil +} + +// GetArchMulti : +func (r *RDBDriver) GetArchMulti(advIDs []string) (map[string]models.ArchADV, error) { + m := make(map[string]models.ArchADV) + for _, id := range advIDs { + a, err := r.GetArch(id) + if err != nil { + return nil, xerrors.Errorf("Failed to get Arch. err: %w", err) + } + if a != nil { + m[id] = *a + } + } + return m, nil +} + +// InsertArch : +func (r *RDBDriver) InsertArch(advs []models.ArchADV) error { + if err := r.deleteAndInsertArch(advs); err != nil { + return xerrors.Errorf("Failed to insert Arch Advisory data. err: %w", err) + } + return nil +} + +func (r *RDBDriver) deleteAndInsertArch(advs []models.ArchADV) (err error) { + bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer { + if viper.GetBool("log-json") { + return io.Discard + } + return os.Stderr + }()) + tx := r.conn.Begin() + + defer func() { + if err != nil { + tx.Rollback() + return + } + tx.Commit() + }() + + // Delete all old records + for _, table := range []interface{}{models.ArchAdvisory{}, models.ArchIssue{}, models.ArchPackage{}, models.ArchADV{}} { + if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(table).Error; err != nil { + return xerrors.Errorf("Failed to delete old records. err: %w", err) + } + } + + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + + for idx := range chunkSlice(len(advs), batchSize) { + if err = tx.Create(advs[idx.From:idx.To]).Error; err != nil { + return xerrors.Errorf("Failed to insert. err: %w", err) + } + bar.Add(idx.To - idx.From) + } + bar.Finish() + + return nil +} + +// GetUnfixedAdvsArch : +func (r *RDBDriver) GetUnfixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { + return r.getAdvsArchWithFixStatus(pkgName, "Vulnerable") +} + +// GetFixedAdvsArch : +func (r *RDBDriver) GetFixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { + return r.getAdvsArchWithFixStatus(pkgName, "Fixed") +} + +func (r *RDBDriver) getAdvsArchWithFixStatus(pkgName, fixStatus string) (map[string]models.ArchADV, error) { + var as []models.ArchADV + if err := r.conn. + Joins("JOIN arch_packages ON arch_packages.arch_adv_id = arch_advs.id AND arch_packages.name = ?", pkgName). + Preload("Packages"). + Preload("Issues"). + Preload("Advisories"). + Where(&models.ArchADV{Status: fixStatus}). + Find(&as).Error; err != nil { + return nil, xerrors.Errorf("Failed to find advisory by pkgname: %s, fix status: %s. err: %w", pkgName, fixStatus, err) + } + + m := make(map[string]models.ArchADV) + for _, a := range as { + m[a.Name] = a + } + return m, nil +} + +// GetAdvisoriesArch gets AdvisoryID: []CVE IDs +func (r *RDBDriver) GetAdvisoriesArch() (map[string][]string, error) { + m := make(map[string][]string) + var as []models.ArchADV + // the maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0. + // https://www.sqlite.org/limits.html Maximum Number Of Host Parameters In A Single SQL Statement + if err := r.conn.Preload("Issues").FindInBatches(&as, 999, func(_ *gorm.DB, _ int) error { + for _, a := range as { + for _, i := range a.Issues { + m[a.Name] = append(m[a.Name], i.Issue) + } + } + return nil + }).Error; err != nil { + return nil, xerrors.Errorf("Failed to find Arch. err: %w", err) + } + + return m, nil +} diff --git a/db/db.go b/db/db.go index ffa54c8..cbc683c 100644 --- a/db/db.go +++ b/db/db.go @@ -40,11 +40,17 @@ type DB interface { GetRelatedProducts(string, []string) ([]string, error) GetFilteredCvesMicrosoft([]string, []string) (map[string]models.MicrosoftCVE, error) GetAdvisoriesMicrosoft() (map[string][]string, error) + GetArch(string) (*models.ArchADV, error) + GetArchMulti([]string) (map[string]models.ArchADV, error) + GetFixedAdvsArch(string) (map[string]models.ArchADV, error) + GetUnfixedAdvsArch(string) (map[string]models.ArchADV, error) + GetAdvisoriesArch() (map[string][]string, error) InsertRedhat([]models.RedhatCVE) error InsertDebian([]models.DebianCVE) error InsertUbuntu([]models.UbuntuCVE) error InsertMicrosoft([]models.MicrosoftCVE, []models.MicrosoftKBRelation) error + InsertArch([]models.ArchADV) error } // Option : diff --git a/db/rdb.go b/db/rdb.go index ab820be..a6f7fc3 100644 --- a/db/rdb.go +++ b/db/rdb.go @@ -159,6 +159,11 @@ func (r *RDBDriver) MigrateDB() error { &models.MicrosoftKB{}, &models.MicrosoftKBRelation{}, &models.MicrosoftSupersededBy{}, + + &models.ArchADV{}, + &models.ArchPackage{}, + &models.ArchIssue{}, + &models.ArchAdvisory{}, ); err != nil { switch r.name { case dialectSqlite3: diff --git a/db/redis.go b/db/redis.go index 14ed4eb..6a205d1 100644 --- a/db/redis.go +++ b/db/redis.go @@ -43,49 +43,57 @@ import ( │ 5 │ GOST#MS#PKG#P#$KBID │ $PRODUCT │ (Microsoft) GET []PRODUCT NAME BY KBID │ ├───┼──────────────────────────┼──────────────┼─────────────────────────────────────────────┤ │ 6 │ GOST#MS#PKG#K#$KBID │ $KBID │ (Microsoft) GET SUPERSEDEDBY []KBID BY KBID │ + ├───┼──────────────────────────┼──────────────┼─────────────────────────────────────────────┤ + │ 7 │ GOST#ARCH#PKG#K#$PKGNAME │ $ADVID │ (Arch) GET RELATED []ADVID BY PKGNAME │ └───┴──────────────────────────┴──────────────┴─────────────────────────────────────────────┘ - Hash - ┌───┬────────────────┬───────────────┬───────────┬────────────────────────────────────────────────┐ - │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ - └───┴────────────────┴───────────────┴───────────┴────────────────────────────────────────────────┘ - ┌───┬────────────────┬───────────────┬───────────┬────────────────────────────────────────────────┐ - │ 1 │ GOST#RH#CVE │ $CVEID │ $CVEJSON │ (RedHat) TO GET CVEJSON BY CVEID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 2 │ GOST#RH#ADV │ $AdvisoryID │ []CVEID │ (RedHat) TO GET CVEIDs BY ADVISORYID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 3 │ GOST#DEB#CVE │ $CVEID │ $CVEJSON │ (Debian) TO GET CVEJSON BY CVEID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 4 │ GOST#UBU#CVE │ $CVEID │ $CVEJSON │ (Ubuntu) TO GET CVEJSON BY CVEID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 5 │ GOST#UBU#ADV │ $AdvisoryID │ []CVEID │ (Ubuntu) TO GET CVEIDs BY ADVISORYID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 6 │ GOST#MS#CVE │ $CVEID │ $CVEJSON │ (Microsoft) TO GET CVEJSON BY CVEID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 7 │ GOST#MS#ADV │ $AdvisoryID │ []CVEID │ (Microsoft) TO GET CVEIDs BY ADVISORYID │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 8 │ GOST#DEP │ RH/DEB/UBU/MS │ JSON │ TO DELETE OUTDATED AND UNNEEDED KEY AND MEMBER │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 9 │ GOST#FETCHMETA │ Revision │ string │ GET Gost Binary Revision │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 10│ GOST#FETCHMETA │ SchemaVersion │ uint │ GET Gost Schema Version │ - ├───┼────────────────┼───────────────┼───────────┼────────────────────────────────────────────────┤ - │ 11│ GOST#FETCHMETA │ LastFetchedAt │ time.Time │ GET Gost Last Fetched Time │ - └───┴────────────────┴───────────────┴───────────┴────────────────────────────────────────────────┘ + ┌───┬────────────────┬────────────────────┬───────────┬────────────────────────────────────────────────┐ + │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ + └───┴────────────────┴────────────────────┴───────────┴────────────────────────────────────────────────┘ + ┌───┬────────────────┬────────────────────┬───────────┬────────────────────────────────────────────────┐ + │ 1 │ GOST#RH#CVE │ $CVEID │ $CVEJSON │ (RedHat) TO GET CVEJSON BY CVEID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 2 │ GOST#RH#ADV │ $AdvisoryID │ []CVEID │ (RedHat) TO GET CVEIDs BY ADVISORYID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 3 │ GOST#DEB#CVE │ $CVEID │ $CVEJSON │ (Debian) TO GET CVEJSON BY CVEID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 4 │ GOST#UBU#CVE │ $CVEID │ $CVEJSON │ (Ubuntu) TO GET CVEJSON BY CVEID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 5 │ GOST#UBU#ADV │ $AdvisoryID │ []CVEID │ (Ubuntu) TO GET CVEIDs BY ADVISORYID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 6 │ GOST#MS#CVE │ $CVEID │ $CVEJSON │ (Microsoft) TO GET CVEJSON BY CVEID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 7 │ GOST#MS#ADV │ $AdvisoryID │ []CVEID │ (Microsoft) TO GET CVEIDs BY ADVISORYID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 8 │ GOST#ARCH#DATA │ $AdvisoryID │ $ADVJSON │ (Arch) TO GET ADVJSON BY ADVISORYID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 9 │ GOST#ARCH#ADV │ $AdvisoryID │ []CVEID │ (Arch) TO GET CVEIDs BY ADVISORYID │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 10│ GOST#DEP │ RH/DEB/UBU/MS/ARCH │ JSON │ TO DELETE OUTDATED AND UNNEEDED KEY AND MEMBER │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 11│ GOST#FETCHMETA │ Revision │ string │ GET Gost Binary Revision │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 12│ GOST#FETCHMETA │ SchemaVersion │ uint │ GET Gost Schema Version │ + ├───┼────────────────┼────────────────────┼───────────┼────────────────────────────────────────────────┤ + │ 13│ GOST#FETCHMETA │ LastFetchedAt │ time.Time │ GET Gost Last Fetched Time │ + └───┴────────────────┴────────────────────┴───────────┴────────────────────────────────────────────────┘ **/ const ( - dialectRedis = "redis" - cveKeyFormat = "GOST#%s#CVE" - pkgKeyFormat = "GOST#%s#PKG#%s" - advKeyFormat = "GOST#%s#ADV" - redhatName = "RH" - debianName = "DEB" - ubuntuName = "UBU" - microsoftName = "MS" - depKey = "GOST#DEP" - fetchMetaKey = "GOST#FETCHMETA" + dialectRedis = "redis" + cveKeyFormat = "GOST#%s#CVE" + pkgKeyFormat = "GOST#%s#PKG#%s" + archDataKeyFormat = "GOST#ARCH#DATA" + advKeyFormat = "GOST#%s#ADV" + redhatName = "RH" + debianName = "DEB" + ubuntuName = "UBU" + microsoftName = "MS" + archName = "ARCH" + depKey = "GOST#DEP" + fetchMetaKey = "GOST#FETCHMETA" ) // RedisDriver is Driver for Redis @@ -751,6 +759,90 @@ func (r *RedisDriver) GetFilteredCvesMicrosoft(products []string, kbs []string) return detected, nil } +// GetArch : +func (r *RedisDriver) GetArch(advID string) (*models.ArchADV, error) { + s, err := r.conn.HGet(context.TODO(), archDataKeyFormat, advID).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return nil, nil + } + return nil, xerrors.Errorf("Failed to HGet. err: %w", err) + } + + var a models.ArchADV + if err := json.Unmarshal([]byte(s), &a); err != nil { + return nil, xerrors.Errorf("Failed to Unmarshal json. err: %w", err) + } + return &a, nil +} + +// GetArchMulti : +func (r *RedisDriver) GetArchMulti(advIDs []string) (map[string]models.ArchADV, error) { + if len(advIDs) == 0 { + return map[string]models.ArchADV{}, nil + } + + ss, err := r.conn.HMGet(context.TODO(), archDataKeyFormat, advIDs...).Result() + if err != nil { + return nil, xerrors.Errorf("Failed to HMGet. err: %w", err) + } + + m := make(map[string]models.ArchADV) + for _, s := range ss { + if s == nil { + continue + } + + var a models.ArchADV + if err := json.Unmarshal([]byte(s.(string)), &a); err != nil { + return nil, xerrors.Errorf("Failed to Unmarshal json. err: %w", err) + } + m[a.Name] = a + } + return m, nil +} + +// GetUnfixedAdvsArch : +func (r *RedisDriver) GetUnfixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { + return r.getAdvsArchWithFixStatus(pkgName, "Vulnerable") +} + +// GetFixedAdvsArch : +func (r *RedisDriver) GetFixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { + return r.getAdvsArchWithFixStatus(pkgName, "Fixed") +} + +func (r *RedisDriver) getAdvsArchWithFixStatus(pkgName, fixStatus string) (map[string]models.ArchADV, error) { + ctx := context.TODO() + advIDs, err := r.conn.SMembers(ctx, fmt.Sprintf(pkgKeyFormat, archName, pkgName)).Result() + if err != nil { + return nil, xerrors.Errorf("Failed to SMembers. err: %w", err) + } + + m, err := r.GetArchMulti(advIDs) + if err != nil { + return nil, xerrors.Errorf("Failed to GetArchMulti. err: %w", err) + } + + m2 := make(map[string]models.ArchADV) + for advID, adv := range m { + if adv.Status != fixStatus { + continue + } + + var pkgs []models.ArchPackage + for _, p := range adv.Packages { + if p.Name == pkgName { + pkgs = append(pkgs, p) + } + } + adv.Packages = pkgs + + m2[advID] = adv + } + return m2, nil +} + // GetAdvisoriesRedHat gets AdvisoryID: []CVE IDs func (r *RedisDriver) GetAdvisoriesRedHat() (map[string][]string, error) { return r.getAdvisories(fmt.Sprintf(advKeyFormat, redhatName)) @@ -766,6 +858,11 @@ func (r *RedisDriver) GetAdvisoriesMicrosoft() (map[string][]string, error) { return r.getAdvisories(fmt.Sprintf(advKeyFormat, microsoftName)) } +// GetAdvisoriesArch gets AdvisoryID: []CVE IDs +func (r *RedisDriver) GetAdvisoriesArch() (map[string][]string, error) { + return r.getAdvisories(fmt.Sprintf(advKeyFormat, archName)) +} + func (r *RedisDriver) getAdvisories(key string) (map[string][]string, error) { v, err := r.conn.HGetAll(context.Background(), key).Result() if err != nil { @@ -1334,3 +1431,114 @@ func (r *RedisDriver) InsertMicrosoft(cves []models.MicrosoftCVE, relations []mo return nil } + +// InsertArch : +func (r *RedisDriver) InsertArch(advs []models.ArchADV) error { + ctx := context.TODO() + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return xerrors.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + + // newDeps, oldDeps: {"ADVID": {"PKGNAME": {}}, "advisories": {"ADVID": {}}} + newDeps := map[string]map[string]struct{}{"advisories": {}} + oldDepsStr, err := r.conn.HGet(ctx, depKey, archName).Result() + if err != nil { + if !errors.Is(err, redis.Nil) { + return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) + } + oldDepsStr = "{}" + } + var oldDeps map[string]map[string]struct{} + if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { + return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) + } + + log15.Info("Insert Advisories", "advs", len(advs)) + bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer { + if viper.GetBool("log-json") { + return io.Discard + } + return os.Stderr + }()) + for idx := range chunkSlice(len(advs), batchSize) { + pipe := r.conn.Pipeline() + for _, adv := range advs[idx.From:idx.To] { + j, err := json.Marshal(adv) + if err != nil { + return xerrors.Errorf("Failed to marshal json. err: %w", err) + } + + _ = pipe.HSet(ctx, archDataKeyFormat, adv.Name, string(j)) + if _, ok := newDeps[adv.Name]; !ok { + newDeps[adv.Name] = make(map[string]struct{}) + } + + for _, pkg := range adv.Packages { + _ = pipe.SAdd(ctx, fmt.Sprintf(pkgKeyFormat, archName, pkg.Name), adv.Name) + newDeps[adv.Name][pkg.Name] = struct{}{} + if _, ok := oldDeps[adv.Name]; ok { + delete(oldDeps[adv.Name], pkg.Name) + } + } + if _, ok := oldDeps[adv.Name]; ok { + if len(oldDeps[adv.Name]) == 0 { + delete(oldDeps, adv.Name) + } + } + + j, err = json.Marshal(func() []string { + is := make([]string, 0, len(adv.Issues)) + for _, i := range adv.Issues { + is = append(is, i.Issue) + } + return is + }()) + if err != nil { + return xerrors.Errorf("Failed to marshal json. err: %w", err) + } + _ = pipe.HSet(ctx, fmt.Sprintf(advKeyFormat, archName), adv.Name, string(j)) + newDeps["advisories"][adv.Name] = struct{}{} + if _, ok := oldDeps["advisories"]; ok { + delete(oldDeps["advisories"], adv.Name) + } + if _, ok := oldDeps["advisories"]; ok { + if len(oldDeps["advisories"]) == 0 { + delete(oldDeps, "advisories") + } + } + } + if _, err := pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + bar.Add(idx.To - idx.From) + } + bar.Finish() + + pipe := r.conn.Pipeline() + for k, v := range oldDeps { + switch k { + case "advisories": + for advID := range v { + _ = pipe.HDel(ctx, fmt.Sprintf(advKeyFormat, archName), advID) + } + default: + for pkgName := range v { + _ = pipe.SRem(ctx, fmt.Sprintf(pkgKeyFormat, archName, pkgName), k) + } + if _, ok := newDeps[k]; !ok { + _ = pipe.HDel(ctx, archDataKeyFormat, k) + } + } + } + newDepsJSON, err := json.Marshal(newDeps) + if err != nil { + return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) + } + _ = pipe.HSet(ctx, depKey, archName, string(newDepsJSON)) + if _, err = pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + + return nil +} diff --git a/fetcher/arch.go b/fetcher/arch.go new file mode 100644 index 0000000..89fa92c --- /dev/null +++ b/fetcher/arch.go @@ -0,0 +1,26 @@ +package fetcher + +import ( + "encoding/json" + + "github.com/vulsio/gost/models" + "github.com/vulsio/gost/util" + "golang.org/x/xerrors" +) + +const archAdvURL = "https://security.archlinux.org/json" + +// FetchArch fetch Advisory JSONs +func FetchArch() ([]models.ArchADVJSON, error) { + bs, err := util.FetchURL(archAdvURL) + if err != nil { + return nil, xerrors.Errorf("Failed to fetch Security Advisory from Arch Linux. err: %w", err) + } + + var advs []models.ArchADVJSON + if err := json.Unmarshal(bs, &advs); err != nil { + return nil, xerrors.Errorf("Failed to unmarshal Arch Linux Security Advisory JSON. err: %w", err) + } + + return advs, nil +} diff --git a/models/arch.go b/models/arch.go new file mode 100644 index 0000000..0483e63 --- /dev/null +++ b/models/arch.go @@ -0,0 +1,89 @@ +package models + +// ArchADVJSON : +type ArchADVJSON struct { + Advisories []string `json:"advisories"` + Affected string `json:"affected"` + Fixed *string `json:"fixed"` + Issues []string `json:"issues"` + Name string `json:"name"` + Packages []string `json:"packages"` + Severity string `json:"severity"` + Status string `json:"status"` + Ticket *string `json:"ticket"` + Type string `json:"type"` +} + +// ArchADV : +type ArchADV struct { + ID int64 `json:"-"` + Name string `json:"name" gorm:"type:varchar(255)"` + Packages []ArchPackage `json:"packages"` + Status string `json:"status" gorm:"type:varchar(255)"` + Severity string `json:"severity" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` + Affected string `json:"affected" gorm:"type:varchar(255)"` + Fixed *string `json:"fixed" gorm:"type:varchar(255)"` + Ticket *string `json:"ticket" gorm:"type:varchar(255)"` + Issues []ArchIssue `json:"issues"` + Advisories []ArchAdvisory `json:"advisories"` +} + +// ArchPackage : +type ArchPackage struct { + ID int64 `json:"-"` + ArchADVID int64 `json:"-"` + Name string `json:"name" gorm:"type:varchar(255);index:idx_arch_packages_name"` +} + +// ArchIssue : +type ArchIssue struct { + ID int64 `json:"-"` + ArchADVID int64 `json:"-"` + Issue string `json:"issue" gorm:"type:varchar(255);index:idx_arch_issues_issue"` +} + +// ArchAdvisory : +type ArchAdvisory struct { + ID int64 `json:"-"` + ArchADVID int64 `json:"-"` + Advisory string `json:"advisory" gorm:"type:varchar(255)"` +} + +// ConvertArch : +func ConvertArch(advJSONs []ArchADVJSON) []ArchADV { + advs := make([]ArchADV, 0, len(advJSONs)) + for _, aj := range advJSONs { + advs = append(advs, ArchADV{ + Name: aj.Name, + Packages: func() []ArchPackage { + ps := make([]ArchPackage, 0, len(aj.Packages)) + for _, p := range aj.Packages { + ps = append(ps, ArchPackage{Name: p}) + } + return ps + }(), + Status: aj.Status, + Severity: aj.Severity, + Type: aj.Type, + Affected: aj.Affected, + Fixed: aj.Fixed, + Ticket: aj.Ticket, + Issues: func() []ArchIssue { + is := make([]ArchIssue, 0, len(aj.Issues)) + for _, i := range aj.Issues { + is = append(is, ArchIssue{Issue: i}) + } + return is + }(), + Advisories: func() []ArchAdvisory { + as := make([]ArchAdvisory, 0, len(aj.Advisories)) + for _, a := range aj.Advisories { + as = append(as, ArchAdvisory{Advisory: a}) + } + return as + }(), + }) + } + return advs +} diff --git a/server/server.go b/server/server.go index aa57aad..baef6dc 100644 --- a/server/server.go +++ b/server/server.go @@ -42,10 +42,12 @@ func Start(logToFile bool, logDir string, driver db.DB) error { e.GET("/debian/cves/:id", getDebianCve(driver)) e.GET("/ubuntu/cves/:id", getUbuntuCve(driver)) e.GET("/microsoft/cves/:id", getMicrosoftCve(driver)) + e.GET("/arch/advs/:id", getArchAdv(driver)) e.POST("/redhat/multi-cves", getRedhatMultiCve(driver)) e.POST("/debian/multi-cves", getDebianMultiCve(driver)) e.POST("/ubuntu/multi-cves", getUbuntuMultiCve(driver)) e.POST("/microsoft/multi-cves", getMicrosoftMultiCve(driver)) + e.POST("/arch/multi-advs", getArchMultiAdv(driver)) e.GET("/redhat/:release/pkgs/:name/unfixed-cves", getUnfixedCvesRedhat(driver)) e.GET("/debian/:release/pkgs/:name/unfixed-cves", getUnfixedCvesDebian(driver)) e.GET("/debian/:release/pkgs/:name/fixed-cves", getFixedCvesDebian(driver)) @@ -54,9 +56,12 @@ func Start(logToFile bool, logDir string, driver db.DB) error { e.GET("/redhat/advisories", getRedhatAdvisories(driver)) e.GET("/ubuntu/advisories", getUbuntuAdvisories(driver)) e.GET("/microsoft/advisories", getMicrosoftAdvisories(driver)) + e.GET("/arch/advisories", getArchAdvisories(driver)) e.POST("/microsoft/kbs", getExpandKB(driver)) e.POST("/microsoft/products", getRelatedProducts(driver)) e.POST("/microsoft/filtered-cves", getFilteredCvesMicrosoft(driver)) + e.GET("/arch/pkgs/:name/unfixed-advs", getUnfixedAdvsArch(driver)) + e.GET("/arch/pkgs/:name/fixed-advs", getFixedAdvsArch(driver)) bindURL := fmt.Sprintf("%s:%s", viper.GetString("bind"), viper.GetString("port")) log15.Info("Listening", "URL", bindURL) @@ -123,6 +128,19 @@ func getMicrosoftCve(driver db.DB) echo.HandlerFunc { } } +// Handler +func getArchAdv(driver db.DB) echo.HandlerFunc { + return func(c echo.Context) error { + cveid := c.Param("id") + cveDetail, err := driver.GetArch(cveid) + if err != nil { + log15.Error("Failed to get Arch by Advisory ID.", "err", err) + return err + } + return c.JSON(http.StatusOK, &cveDetail) + } +} + type cveIDs struct { CveIDs []string `json:"cveIDs"` } @@ -191,6 +209,26 @@ func getMicrosoftMultiCve(driver db.DB) echo.HandlerFunc { } } +// Handler +func getArchMultiAdv(driver db.DB) echo.HandlerFunc { + type advIDs struct { + AdvIDs []string `json:"advIDs"` + } + + return func(c echo.Context) error { + var advIDs advIDs + if err := c.Bind(&advIDs); err != nil { + return err + } + cveDetails, err := driver.GetArchMulti(advIDs.AdvIDs) + if err != nil { + log15.Error("Failed to get Arch by Advisory IDs.", "err", err) + return err + } + return c.JSON(http.StatusOK, &cveDetails) + } +} + // Handler func getUnfixedCvesRedhat(driver db.DB) echo.HandlerFunc { return func(c echo.Context) error { @@ -356,3 +394,41 @@ func getMicrosoftAdvisories(driver db.DB) echo.HandlerFunc { return c.JSON(http.StatusOK, &m) } } + +// Handler +func getArchAdvisories(driver db.DB) echo.HandlerFunc { + return func(c echo.Context) error { + m, err := driver.GetAdvisoriesArch() + if err != nil { + log15.Error("Failed to get Arch Advisories.", "err", err) + return err + } + return c.JSON(http.StatusOK, &m) + } +} + +// Handler +func getUnfixedAdvsArch(driver db.DB) echo.HandlerFunc { + return func(c echo.Context) error { + pkgName := c.Param("name") + cveDetail, err := driver.GetUnfixedAdvsArch(pkgName) + if err != nil { + log15.Error("Failed to get Unfixed Advisories in Arch", "err", err) + return err + } + return c.JSON(http.StatusOK, &cveDetail) + } +} + +// Handler +func getFixedAdvsArch(driver db.DB) echo.HandlerFunc { + return func(c echo.Context) error { + pkgName := c.Param("name") + cveDetail, err := driver.GetFixedAdvsArch(pkgName) + if err != nil { + log15.Error("Failed to get Fixed Advisories in Arch", "err", err) + return err + } + return c.JSON(http.StatusOK, &cveDetail) + } +}