breaking-change(redis): deprecated expire option (#88)

* breaking-change(redis): deprecated expire option * chore: expand var in func args * refactor: move ConvertToModels from db to models package * chore: use pb/v3
vulsio · Oct 4, 2021 · 69dd7f6 · 69dd7f6
1 parent 9776906
commit 69dd7f6
Show file tree

Hide file tree

Showing 21 changed files with 698 additions and 840 deletions.
diff --git a/cmd/debian.go b/cmd/debian.go
@@ -49,10 +49,11 @@ func fetchDebian(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	log15.Info("Fetched all CVEs from Debian")
-	cves, err := fetcher.RetrieveDebianCveDetails()
+	cveJSONs, err := fetcher.RetrieveDebianCveDetails()
 	if err != nil {
 		return err
 	}
+	cves := models.ConvertDebian(cveJSONs)
 
 	log15.Info("Fetched", "CVEs", len(cves))
 

diff --git a/cmd/fetch.go b/cmd/fetch.go
@@ -23,7 +23,4 @@ func init() {
 
 	fetchCmd.PersistentFlags().Int("batch-size", 15, "The number of batch size to insert.")
 	_ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size"))
-
-	fetchCmd.PersistentFlags().Uint("expire", 0, "timeout to set for Redis keys in seconds. If set to 0, the key is persistent.")
-	_ = viper.BindPFlag("expire", fetchCmd.PersistentFlags().Lookup("expire"))
 }
diff --git a/cmd/microsoft.go b/cmd/microsoft.go
@@ -59,18 +59,18 @@ func fetchMicrosoft(cmd *cobra.Command, args []string) (err error) {
 	if len(apiKey) == 0 {
 		return errors.New("apikey is required")
 	}
-	cves, err := fetcher.RetrieveMicrosoftCveDetails(apiKey)
+	cveXMLs, err := fetcher.RetrieveMicrosoftCveDetails(apiKey)
 	if err != nil {
 		return err
 	}
-
-	xls, err := fetcher.RetrieveMicrosoftBulletinSearch()
+	cveXls, err := fetcher.RetrieveMicrosoftBulletinSearch()
 	if err != nil {
 		return err
 	}
+	cves, product := models.ConvertMicrosoft(cveXMLs, cveXls)
 
 	log15.Info("Insert Microsoft CVEs into DB", "db", driver.Name())
-	if err := driver.InsertMicrosoft(cves, xls); err != nil {
+	if err := driver.InsertMicrosoft(cves, product); err != nil {
 		return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err)
 	}
 

diff --git a/cmd/notify.go b/cmd/notify.go
@@ -60,7 +60,7 @@ func notifyRedhat(conf config.Config) error {
 		return err
 	}
 
-	cves, err := db.ConvertRedhat(cveJSONs)
+	cves, err := models.ConvertRedhat(cveJSONs)
 	if err != nil {
 		return nil
 
@@ -89,11 +89,11 @@ func notifyRedhat(conf config.Config) error {
 		if err != nil {
 			return err
 		}
-		db.ClearIDRedhat(c)
+		notifier.ClearIDRedhat(c)
 
 		cve.Cvss3.Cvss3BaseScore = "10 (This is dummy)"
 		cve.ThreatSeverity = "High (This is dummy)"
-		body := util.DiffRedhat(c, &cve, conf.Redhat[cve.Name])
+		body := notifier.DiffRedhat(c, &cve, conf.Redhat[cve.Name])
 		if body != "" {
 			subject := fmt.Sprintf("%s Update %s", conf.EMail.SubjectPrefix, cve.Name)
 			body = fmt.Sprintf("%s\nhttps://access.redhat.com/security/cve/%s\n========================================================\n",

diff --git a/cmd/redhat.go b/cmd/redhat.go
@@ -28,9 +28,13 @@ func fetchRedHat(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
 	}
 
-	cves, err := fetcher.FetchRedHatVulnList()
+	cveJSONs, err := fetcher.FetchRedHatVulnList()
 	if err != nil {
-		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
+		return xerrors.Errorf("Failed to initialize vulnerability DB . err: %w", err)
+	}
+	cves, err := models.ConvertRedhat(cveJSONs)
+	if err != nil {
+		return xerrors.Errorf("Failed to convert RedhatCVE. err: %w", err)
 	}
 
 	log15.Info("Initialize Database")

diff --git a/cmd/redhatapi.go b/cmd/redhatapi.go
@@ -79,10 +79,14 @@ func fetchRedHatAPI(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	log15.Info(fmt.Sprintf("Fetched %d CVEs", len(entries)))
-	cves, err := fetcher.RetrieveRedhatCveDetails(resourceURLs)
+	cveJSONs, err := fetcher.RetrieveRedhatCveDetails(resourceURLs)
 	if err != nil {
 		return xerrors.Errorf("Failed to fetch the CVE details. err: %w", err)
 	}
+	cves, err := models.ConvertRedhat(cveJSONs)
+	if err != nil {
+		return xerrors.Errorf("Failed to convert RedhatCVE. err: %w", err)
+	}
 
 	log15.Info("Insert RedHat into DB", "db", driver.Name())
 	if err := driver.InsertRedhat(cves); err != nil {

diff --git a/cmd/ubuntu.go b/cmd/ubuntu.go
@@ -28,10 +28,11 @@ func fetchUbuntu(cmd *cobra.Command, args []string) (err error) {
 		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
 	}
 
-	cves, err := fetcher.FetchUbuntuVulnList()
+	cveJSONs, err := fetcher.FetchUbuntuVulnList()
 	if err != nil {
-		return xerrors.Errorf("error in vulnerability DB initialize: %w", err)
+		return xerrors.Errorf("Failed to initialize vulnerability DB . err: %w", err)
 	}
+	cves := models.ConvertUbuntu(cveJSONs)
 
 	log15.Info("Initialize Database")
 	driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"))

diff --git a/db/db.go b/db/db.go
@@ -35,10 +35,10 @@ type DB interface {
 	GetUnfixedCvesUbuntu(string, string) (map[string]models.UbuntuCVE, error)
 	GetFixedCvesUbuntu(string, string) (map[string]models.UbuntuCVE, error)
 
-	InsertRedhat([]models.RedhatCVEJSON) error
-	InsertDebian(models.DebianJSON) error
-	InsertUbuntu([]models.UbuntuCVEJSON) error
-	InsertMicrosoft([]models.MicrosoftXML, []models.MicrosoftBulletinSearch) error
+	InsertRedhat([]models.RedhatCVE) error
+	InsertDebian([]models.DebianCVE) error
+	InsertUbuntu([]models.UbuntuCVE) error
+	InsertMicrosoft([]models.MicrosoftCVE, []models.MicrosoftProduct) error
 }
 
 // NewDB returns db driver

diff --git a/db/debian.go b/db/debian.go
@@ -4,10 +4,10 @@ import (
 	"errors"
 	"fmt"
 
+	pb "github.com/cheggaaa/pb/v3"
 	"github.com/spf13/viper"
 	"github.com/vulsio/gost/models"
 	"golang.org/x/xerrors"
-	pb "gopkg.in/cheggaaa/pb.v1"
 	"gorm.io/gorm"
 )
 
@@ -52,8 +52,7 @@ func (r *RDBDriver) GetDebianMulti(cveIDs []string) (map[string]models.DebianCVE
 }
 
 // InsertDebian :
-func (r *RDBDriver) InsertDebian(cveJSON models.DebianJSON) (err error) {
-	cves := ConvertDebian(cveJSON)
+func (r *RDBDriver) InsertDebian(cves []models.DebianCVE) (err error) {
 	if err = r.deleteAndInsertDebian(cves); err != nil {
 		return xerrors.Errorf("Failed to insert Debian CVE data. err: %w", err)
 	}
@@ -98,47 +97,6 @@ func (r *RDBDriver) deleteAndInsertDebian(cves []models.DebianCVE) (err error) {
 	return nil
 }
 
-// ConvertDebian :
-func ConvertDebian(cveJSONs models.DebianJSON) (cves []models.DebianCVE) {
-	uniqCve := map[string]models.DebianCVE{}
-	for pkgName, cveMap := range cveJSONs {
-		for cveID, cve := range cveMap {
-			var releases []models.DebianRelease
-			for release, releaseInfo := range cve.Releases {
-				r := models.DebianRelease{
-					ProductName:  release,
-					Status:       releaseInfo.Status,
-					FixedVersion: releaseInfo.FixedVersion,
-					Urgency:      releaseInfo.Urgency,
-					Version:      releaseInfo.Repositories[release],
-				}
-				releases = append(releases, r)
-			}
-
-			pkg := models.DebianPackage{
-				PackageName: pkgName,
-				Release:     releases,
-			}
-
-			pkgs := []models.DebianPackage{pkg}
-			if oldCve, ok := uniqCve[cveID]; ok {
-				pkgs = append(pkgs, oldCve.Package...)
-			}
-
-			uniqCve[cveID] = models.DebianCVE{
-				CveID:       cveID,
-				Scope:       cve.Scope,
-				Description: cve.Description,
-				Package:     pkgs,
-			}
-		}
-	}
-	for _, c := range uniqCve {
-		cves = append(cves, c)
-	}
-	return cves
-}
-
 var debVerCodename = map[string]string{
 	"8":  "jessie",
 	"9":  "stretch",