diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml index 4e51b02a..322baaa4 100644 --- a/.github/workflows/fetch.yml +++ b/.github/workflows/fetch.yml @@ -229,6 +229,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + env: + Version: 5 6 7 8 9 steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 @@ -241,16 +243,16 @@ jobs: run: make build - name: fetch sqlite3 if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype sqlite3 oracle + run: ./goval-dictionary fetch --dbtype sqlite3 oracle $Version - name: fetch mysql if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" oracle + run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" oracle $Version - name: fetch postgres if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" oracle + run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" oracle $Version - name: fetch redis if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" oracle + run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" oracle $Version fetch-amazon: name: fetch-amazon @@ -289,6 +291,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + env: + Version: 1 2 2022 2023 steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 @@ -301,16 +305,16 @@ jobs: run: make build - name: fetch sqlite3 if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype sqlite3 amazon + run: ./goval-dictionary fetch --dbtype sqlite3 amazon $Version - name: fetch mysql if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" amazon + run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" amazon $Version - name: fetch postgres if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" amazon + run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" amazon $Version - name: fetch redis if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" amazon + run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" amazon $Version fetch-fedora: name: fetch-fedora diff --git a/GNUmakefile b/GNUmakefile index 88d6e16b..bebdf5f6 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -103,8 +103,8 @@ fetch-rdb: integration/goval-dict.old fetch debian --dbpath=$(PWD)/integration/oval.old.sqlite3 7 8 9 10 11 integration/goval-dict.old fetch ubuntu --dbpath=$(PWD)/integration/oval.old.sqlite3 14 16 18 19 20 21 22 integration/goval-dict.old fetch redhat --dbpath=$(PWD)/integration/oval.old.sqlite3 5 6 7 8 9 - integration/goval-dict.old fetch oracle --dbpath=$(PWD)/integration/oval.old.sqlite3 - integration/goval-dict.old fetch amazon --dbpath=$(PWD)/integration/oval.old.sqlite3 + integration/goval-dict.old fetch oracle --dbpath=$(PWD)/integration/oval.old.sqlite3 5 6 7 8 9 + integration/goval-dict.old fetch amazon --dbpath=$(PWD)/integration/oval.old.sqlite3 1 2 2022 2023 integration/goval-dict.old fetch alpine --dbpath=$(PWD)/integration/oval.old.sqlite3 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 integration/goval-dict.old fetch suse --dbpath=$(PWD)/integration/oval.old.sqlite3 --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed integration/goval-dict.old fetch suse --dbpath=$(PWD)/integration/oval.old.sqlite3 --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3 @@ -115,8 +115,8 @@ fetch-rdb: integration/goval-dict.new fetch debian --dbpath=$(PWD)/integration/oval.new.sqlite3 7 8 9 10 11 integration/goval-dict.new fetch ubuntu --dbpath=$(PWD)/integration/oval.new.sqlite3 14 16 18 19 20 21 22 integration/goval-dict.new fetch redhat --dbpath=$(PWD)/integration/oval.new.sqlite3 5 6 7 8 9 - integration/goval-dict.new fetch oracle --dbpath=$(PWD)/integration/oval.new.sqlite3 - integration/goval-dict.new fetch amazon --dbpath=$(PWD)/integration/oval.new.sqlite3 + integration/goval-dict.new fetch oracle --dbpath=$(PWD)/integration/oval.new.sqlite3 5 6 7 8 9 + integration/goval-dict.new fetch amazon --dbpath=$(PWD)/integration/oval.new.sqlite3 1 2 2022 2023 integration/goval-dict.new fetch alpine --dbpath=$(PWD)/integration/oval.new.sqlite3 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 integration/goval-dict.new fetch suse --dbpath=$(PWD)/integration/oval.new.sqlite3 --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed integration/goval-dict.new fetch suse --dbpath=$(PWD)/integration/oval.new.sqlite3 --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3 @@ -131,8 +131,8 @@ fetch-redis: integration/goval-dict.old fetch debian --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 7 8 9 10 11 integration/goval-dict.old fetch ubuntu --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 14 16 18 19 20 21 22 integration/goval-dict.old fetch redhat --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 5 6 7 8 9 - integration/goval-dict.old fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6379/0" - integration/goval-dict.old fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6379/0" + integration/goval-dict.old fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 5 6 7 8 9 + integration/goval-dict.old fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 1 2 2022 2023 integration/goval-dict.old fetch alpine --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 integration/goval-dict.old fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed integration/goval-dict.old fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3 @@ -143,8 +143,8 @@ fetch-redis: integration/goval-dict.new fetch debian --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 7 8 9 10 11 integration/goval-dict.new fetch ubuntu --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 14 16 18 19 20 21 22 integration/goval-dict.new fetch redhat --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 5 6 7 8 9 - integration/goval-dict.new fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6380/0" - integration/goval-dict.new fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6380/0" + integration/goval-dict.new fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 5 6 7 8 9 + integration/goval-dict.new fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 1 2 2022 2023 integration/goval-dict.new fetch alpine --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 integration/goval-dict.new fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed integration/goval-dict.new fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3 diff --git a/README.md b/README.md index 02ec8f7c..c974b293 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ $ goval-dictionary fetch suse --suse-type suse-enterprise-desktop 10 11 12 15 - [Oracle Linux](https://linux.oracle.com/security/oval/) ```bash - $ goval-dictionary fetch oracle + $ goval-dictionary fetch oracle 5 6 7 8 9 ``` ### Usage: Fetch alpine-secdb as OVAL data type @@ -173,7 +173,7 @@ See [here](https://secdb.alpinelinux.org/) for a list of supported alpines. Amazon ALAS provideis Vulnerability data as `no-OVAL-format`, but it is supported by goval-dictionary to make Amazon ALAS easier to handle from Vuls. ```bash - $ goval-dictionary fetch amazon + $ goval-dictionary fetch amazon 1 2 2022 2023 ``` #### Usage: Fetch Security Updates from Fedora diff --git a/commands/fetch-alpine.go b/commands/fetch-alpine.go index a122954d..ae846515 100644 --- a/commands/fetch-alpine.go +++ b/commands/fetch-alpine.go @@ -16,15 +16,18 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/alpine" + "github.com/vulsio/goval-dictionary/util" ) // fetchAlpineCmd is Subcommand for fetch Alpine secdb // https://secdb.alpinelinux.org/ var fetchAlpineCmd = &cobra.Command{ - Use: "alpine", - Short: "Fetch Vulnerability dictionary from Alpine secdb", - Long: `Fetch Vulnerability dictionary from Alpine secdb`, - RunE: fetchAlpine, + Use: "alpine [version]", + Short: "Fetch Vulnerability dictionary from Alpine secdb", + Long: `Fetch Vulnerability dictionary from Alpine secdb`, + Args: cobra.MinimumNArgs(1), + RunE: fetchAlpine, + Example: "$ goval-dictionary fetch alpine 3.16 3.17", } func init() { @@ -36,20 +39,6 @@ func fetchAlpine(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch alpine command. err: specify versions to fetch") - } - - // Distinct - v := map[string]bool{} - vers := []string{} - for _, arg := range args { - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) if err != nil { if locked { @@ -70,7 +59,7 @@ func fetchAlpine(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - results, err := fetcher.FetchFiles(vers) + results, err := fetcher.FetchFiles(util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } diff --git a/commands/fetch-amazon.go b/commands/fetch-amazon.go index 06af41bf..64e01d92 100644 --- a/commands/fetch-amazon.go +++ b/commands/fetch-amazon.go @@ -1,7 +1,6 @@ package commands import ( - "fmt" "time" "github.com/inconshreveable/log15" @@ -15,22 +14,25 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/amazon" + "github.com/vulsio/goval-dictionary/util" ) // fetchAmazonCmd is Subcommand for fetch Amazon ALAS RSS // https://alas.aws.amazon.com/alas.rss var fetchAmazonCmd = &cobra.Command{ - Use: "amazon", - Short: "Fetch Vulnerability dictionary from Amazon ALAS", - Long: `Fetch Vulnerability dictionary from Amazon ALAS`, - RunE: fetchAmazon, + Use: "amazon [version]", + Short: "Fetch Vulnerability dictionary from Amazon ALAS", + Long: `Fetch Vulnerability dictionary from Amazon ALAS`, + Args: cobra.MinimumNArgs(1), + RunE: fetchAmazon, + Example: "$ goval-dictionary fetch amazon 1 2 2022 2023", } func init() { fetchCmd.AddCommand(fetchAmazonCmd) } -func fetchAmazon(_ *cobra.Command, _ []string) (err error) { +func fetchAmazon(_ *cobra.Command, args []string) (err error) { if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } @@ -61,64 +63,22 @@ func fetchAmazon(_ *cobra.Command, _ []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - uinfo, err := fetcher.FetchUpdateInfoAmazonLinux1() + m, err := fetcher.FetchFiles(util.Unique(args)) if err != nil { - return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux1. err: %w", err) - } - root := models.Root{ - Family: c.Amazon, - OSVersion: "1", - Definitions: amazon.ConvertToModel(uinfo), - Timestamp: time.Now(), - } - log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux1. Inserting to DB", len(root.Definitions))) - if err := execute(driver, &root); err != nil { - return xerrors.Errorf("Failed to Insert Amazon1. err: %w", err) - } - - uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2() - if err != nil { - return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2. err: %w", err) - } - root = models.Root{ - Family: c.Amazon, - OSVersion: "2", - Definitions: amazon.ConvertToModel(uinfo), - Timestamp: time.Now(), - } - log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2. Inserting to DB", len(root.Definitions))) - if err := execute(driver, &root); err != nil { - return xerrors.Errorf("Failed to Insert Amazon2. err: %w", err) - } - - uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2022() - if err != nil { - return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2022. err: %w", err) - } - root = models.Root{ - Family: c.Amazon, - OSVersion: "2022", - Definitions: amazon.ConvertToModel(uinfo), - Timestamp: time.Now(), - } - log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2022. Inserting to DB", len(root.Definitions))) - if err := execute(driver, &root); err != nil { - return xerrors.Errorf("Failed to Insert Amazon2022. err: %w", err) - } + return xerrors.Errorf("Failed to fetch files. err: %w", err) + } + for ver, us := range m { + root := models.Root{ + Family: c.Amazon, + OSVersion: ver, + Definitions: amazon.ConvertToModel(us), + Timestamp: time.Now(), + } - uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2023() - if err != nil { - return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2023. err: %w", err) - } - root = models.Root{ - Family: c.Amazon, - OSVersion: "2023", - Definitions: amazon.ConvertToModel(uinfo), - Timestamp: time.Now(), - } - log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2023. Inserting to DB", len(root.Definitions))) - if err := execute(driver, &root); err != nil { - return xerrors.Errorf("Failed to Insert Amazon2023. err: %w", err) + if err := driver.InsertOval(&root); err != nil { + return xerrors.Errorf("Failed to insert OVAL. err: %w", err) + } + log15.Info("Finish", "Updated", len(root.Definitions)) } fetchMeta.LastFetchedAt = time.Now() @@ -128,12 +88,3 @@ func fetchAmazon(_ *cobra.Command, _ []string) (err error) { return nil } - -func execute(driver db.DB, root *models.Root) error { - if err := driver.InsertOval(root); err != nil { - return xerrors.Errorf("Failed to insert OVAL. err: %w", err) - } - log15.Info("Finish", "Updated", len(root.Definitions)) - - return nil -} diff --git a/commands/fetch-debian.go b/commands/fetch-debian.go index 6e54edaf..460e07d4 100644 --- a/commands/fetch-debian.go +++ b/commands/fetch-debian.go @@ -18,14 +18,17 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/debian" + "github.com/vulsio/goval-dictionary/util" ) // fetchDebianCmd is Subcommand for fetch Debian OVAL var fetchDebianCmd = &cobra.Command{ - Use: "debian", - Short: "Fetch Vulnerability dictionary from Debian", - Long: `Fetch Vulnerability dictionary from Debian`, - RunE: fetchDebian, + Use: "debian [version]", + Short: "Fetch Vulnerability dictionary from Debian", + Long: `Fetch Vulnerability dictionary from Debian`, + Args: cobra.MinimumNArgs(1), + RunE: fetchDebian, + Example: "$ goval-dictionary fetch debian 10 11", } func init() { @@ -37,10 +40,6 @@ func fetchDebian(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch debian command. err: specify versions to fetch") - } - driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) if err != nil { if locked { @@ -64,17 +63,7 @@ func fetchDebian(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - // Distinct - vers := []string{} - v := map[string]bool{} - for _, arg := range args { - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - - results, err := fetcher.FetchFiles(vers) + results, err := fetcher.FetchFiles(util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } diff --git a/commands/fetch-fedora.go b/commands/fetch-fedora.go index 7104ae65..1dafc6dd 100644 --- a/commands/fetch-fedora.go +++ b/commands/fetch-fedora.go @@ -2,7 +2,6 @@ package commands import ( "fmt" - "strconv" "time" "github.com/inconshreveable/log15" @@ -16,14 +15,17 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/fedora" + "github.com/vulsio/goval-dictionary/util" ) // fetchFedoraCmd is Subcommand for fetch Fedora OVAL var fetchFedoraCmd = &cobra.Command{ - Use: "fedora", - Short: "Fetch Vulnerability dictionary from Fedora", - Long: `Fetch Vulnerability dictionary from Fedora`, - RunE: fetchFedora, + Use: "fedora [version]", + Short: "Fetch Vulnerability dictionary from Fedora", + Long: `Fetch Vulnerability dictionary from Fedora`, + Args: cobra.MinimumNArgs(1), + RunE: fetchFedora, + Example: "$ goval-dictionary fetch fedora 37", } func init() { @@ -35,10 +37,6 @@ func fetchFedora(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch fedora command. err: specify versions to fetch") - } - driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) if err != nil { if locked { @@ -65,23 +63,7 @@ func fetchFedora(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - // Distinct - vers := []string{} - v := map[string]bool{} - for _, arg := range args { - ver, err := strconv.Atoi(arg) - // Fedora versions prior to version 32 have no update information - // https://dl.fedoraproject.org/pub/fedora/linux/updates/ - if err != nil || ver < 32 { - return xerrors.Errorf("Specify version to fetch (from 32 to latest Fedora version). arg: %s", arg) - } - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - - uinfos, err := fetcher.FetchUpdateInfosFedora(vers) + uinfos, err := fetcher.FetchUpdateInfosFedora(util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } @@ -94,9 +76,10 @@ func fetchFedora(_ *cobra.Command, args []string) (err error) { Timestamp: time.Now(), } log15.Info(fmt.Sprintf("%d CVEs for Fedora %s. Inserting to DB", len(root.Definitions), k)) - if err := execute(driver, &root); err != nil { - return xerrors.Errorf("Failed to Insert Fedora %s. err: %w", k, err) + if err := driver.InsertOval(&root); err != nil { + return xerrors.Errorf("Failed to insert OVAL. err: %w", err) } + log15.Info("Finish", "Updated", len(root.Definitions)) } return nil } diff --git a/commands/fetch-oracle.go b/commands/fetch-oracle.go index 40fa85c9..691b4fc0 100644 --- a/commands/fetch-oracle.go +++ b/commands/fetch-oracle.go @@ -8,6 +8,7 @@ import ( "github.com/inconshreveable/log15" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/exp/slices" "golang.org/x/xerrors" c "github.com/vulsio/goval-dictionary/config" @@ -20,17 +21,19 @@ import ( // fetchOracleCmd is Subcommand for fetch Oracle OVAL var fetchOracleCmd = &cobra.Command{ - Use: "oracle", - Short: "Fetch Vulnerability dictionary from Oracle", - Long: `Fetch Vulnerability dictionary from Oracle`, - RunE: fetchOracle, + Use: "oracle [version]", + Short: "Fetch Vulnerability dictionary from Oracle", + Long: `Fetch Vulnerability dictionary from Oracle`, + Args: cobra.MinimumNArgs(1), + RunE: fetchOracle, + Example: "$ goval-dictionary fetch oracle 8 9", } func init() { fetchCmd.AddCommand(fetchOracleCmd) } -func fetchOracle(_ *cobra.Command, _ []string) (err error) { +func fetchOracle(_ *cobra.Command, args []string) (err error) { if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } @@ -76,7 +79,9 @@ func fetchOracle(_ *cobra.Command, _ []string) (err error) { } for osVer, defs := range oracle.ConvertToModel(&ovalroot) { - osVerDefs[osVer] = append(osVerDefs[osVer], defs...) + if slices.Contains(args, osVer) { + osVerDefs[osVer] = append(osVerDefs[osVer], defs...) + } } } diff --git a/commands/fetch-redhat.go b/commands/fetch-redhat.go index a11f0617..aabfaa83 100644 --- a/commands/fetch-redhat.go +++ b/commands/fetch-redhat.go @@ -2,7 +2,6 @@ package commands import ( "encoding/xml" - "strconv" "strings" "time" @@ -17,14 +16,17 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/redhat" + "github.com/vulsio/goval-dictionary/util" ) // fetchRedHatCmd is Subcommand for fetch RedHat OVAL var fetchRedHatCmd = &cobra.Command{ - Use: "redhat", - Short: "Fetch Vulnerability dictionary from RedHat", - Long: `Fetch Vulnerability dictionary from RedHat`, - RunE: fetchRedHat, + Use: "redhat [version]", + Short: "Fetch Vulnerability dictionary from RedHat", + Long: `Fetch Vulnerability dictionary from RedHat`, + Args: cobra.MinimumNArgs(1), + RunE: fetchRedHat, + Example: "$ goval-dictionary fetch redhat 8 9", } func init() { @@ -36,10 +38,6 @@ func fetchRedHat(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch redhat command. err: specify versions to fetch") - } - driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) if err != nil { if locked { @@ -60,21 +58,7 @@ func fetchRedHat(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - // Distinct - vers := []string{} - v := map[string]bool{} - for _, arg := range args { - ver, err := strconv.Atoi(arg) - if err != nil || ver < 5 { - return xerrors.Errorf("Specify version to fetch (from 5 to latest RHEL version). arg: %s", arg) - } - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - - results, err := fetcher.FetchFiles(vers) + results, err := fetcher.FetchFiles(util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } diff --git a/commands/fetch-suse.go b/commands/fetch-suse.go index 3e262b7b..fbc0552e 100644 --- a/commands/fetch-suse.go +++ b/commands/fetch-suse.go @@ -16,20 +16,19 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/suse" + "github.com/vulsio/goval-dictionary/util" ) // fetchSUSECmd is Subcommand for fetch SUSE OVAL var fetchSUSECmd = &cobra.Command{ - Use: "suse", + Use: "suse [version]", Short: "Fetch Vulnerability dictionary from SUSE", - Long: `Fetch Vulnerability dictionary from SUSE - -$ goval-dictionary fetch suse --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed -$ goval-dictionary fetch suse --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3 -$ goval-dictionary fetch suse --suse-type suse-enterprise-server 9 10 11 12 15 -$ goval-dictionary fetch suse --suse-type suse-enterprise-desktop 10 11 12 15 -`, - RunE: fetchSUSE, + Long: `Fetch Vulnerability dictionary from SUSE`, + RunE: fetchSUSE, + Example: `$ goval-dictionary fetch suse --suse-type opensuse 13.2 tumbleweed +$ goval-dictionary fetch suse --suse-type opensuse-leap 15.2 15.3 +$ goval-dictionary fetch suse --suse-type suse-enterprise-server 12 15 +$ goval-dictionary fetch suse --suse-type suse-enterprise-desktop 12 15`, } func init() { @@ -44,20 +43,6 @@ func fetchSUSE(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch suse command. err: specify versions to fetch. Oval files are here: http://ftp.suse.com/pub/projects/security/oval/") - } - - // Distinct - v := map[string]bool{} - vers := []string{} - for _, arg := range args { - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - var suseType string switch viper.GetString("suse-type") { case "opensuse": @@ -92,7 +77,7 @@ func fetchSUSE(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - results, err := fetcher.FetchFiles(suseType, vers) + results, err := fetcher.FetchFiles(suseType, util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } diff --git a/commands/fetch-ubuntu.go b/commands/fetch-ubuntu.go index 9c0de9dc..c84e7d5c 100644 --- a/commands/fetch-ubuntu.go +++ b/commands/fetch-ubuntu.go @@ -16,14 +16,17 @@ import ( "github.com/vulsio/goval-dictionary/log" "github.com/vulsio/goval-dictionary/models" "github.com/vulsio/goval-dictionary/models/ubuntu" + "github.com/vulsio/goval-dictionary/util" ) // fetchUbuntuCmd is Subcommand for fetch Ubuntu OVAL var fetchUbuntuCmd = &cobra.Command{ - Use: "ubuntu", - Short: "Fetch Vulnerability dictionary from Ubuntu", - Long: `Fetch Vulnerability dictionary from Ubuntu`, - RunE: fetchUbuntu, + Use: "ubuntu [version]", + Short: "Fetch Vulnerability dictionary from Ubuntu", + Long: `Fetch Vulnerability dictionary from Ubuntu`, + Args: cobra.MinimumNArgs(1), + RunE: fetchUbuntu, + Example: "$ goval-dictionary fetch ubuntu 20 22", } func init() { @@ -35,10 +38,6 @@ func fetchUbuntu(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to SetLogger. err: %w", err) } - if len(args) == 0 { - return xerrors.New("Failed to fetch ubuntu command. err: specify versions to fetch") - } - driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) if err != nil { if locked { @@ -59,17 +58,7 @@ func fetchUbuntu(_ *cobra.Command, args []string) (err error) { return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) } - // Distinct - v := map[string]bool{} - vers := []string{} - for _, arg := range args { - v[arg] = true - } - for k := range v { - vers = append(vers, k) - } - - results, err := fetcher.FetchFiles(vers) + results, err := fetcher.FetchFiles(util.Unique(args)) if err != nil { return xerrors.Errorf("Failed to fetch files. err: %w", err) } diff --git a/commands/version.go b/commands/version.go index 61952062..143b52e9 100644 --- a/commands/version.go +++ b/commands/version.go @@ -16,7 +16,7 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Show version", Long: `Show version`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { fmt.Printf("goval-dictionary %s %s\n", config.Version, config.Revision) }, } diff --git a/fetcher/amazon/amazon.go b/fetcher/amazon/amazon.go index f90fb399..74287025 100644 --- a/fetcher/amazon/amazon.go +++ b/fetcher/amazon/amazon.go @@ -10,10 +10,8 @@ import ( "fmt" "net/url" "path" - "sort" "github.com/inconshreveable/log15" - "golang.org/x/net/html/charset" "golang.org/x/xerrors" "github.com/vulsio/goval-dictionary/fetcher/util" @@ -21,127 +19,71 @@ import ( ) // updateinfo for x86_64 also contains information for aarch64 -const ( - al1MirrorListURI = "http://repo.us-west-2.amazonaws.com/2018.03/updates/x86_64/mirror.list" - al2CoreMirrorListURI = "https://cdn.amazonlinux.com/2/core/latest/x86_64/mirror.list" - al2ExtraCatalogURI = "http://amazonlinux.default.amazonaws.com/2/extras-catalog.json" - al2ExtraMirrorListURIFormat = "https://cdn.amazonlinux.com/2/extras/%s/latest/x86_64/mirror.list" - al2022ReleasemdURI = "https://cdn.amazonlinux.com/al2022/core/releasemd.xml" - al2023ReleasemdURI = "https://cdn.amazonlinux.com/al2023/core/releasemd.xml" -) - -var errNoUpdateInfo = xerrors.New("No updateinfo field in the repomd") - -// FetchUpdateInfoAmazonLinux1 fetches a list of Amazon Linux1 updateinfo -func FetchUpdateInfoAmazonLinux1() (*models.Updates, error) { - return fetchUpdateInfoAmazonLinux(al1MirrorListURI) -} - -// FetchUpdateInfoAmazonLinux2 fetches a list of Amazon Linux2 updateinfo -func FetchUpdateInfoAmazonLinux2() (*models.Updates, error) { - updates, err := fetchUpdateInfoAmazonLinux(al2CoreMirrorListURI) - if err != nil { - return nil, xerrors.Errorf("Failed to fetch Amazon Linux 2 core updateinfo. err: %w", err) - } - for i := range updates.UpdateList { - updates.UpdateList[i].Repository = "amzn2-core" - } - - rs, err := util.FetchFeedFiles([]util.FetchRequest{{URL: al2ExtraCatalogURI, MIMEType: util.MIMETypeJSON}}) - if err != nil || len(rs) != 1 { - return nil, xerrors.Errorf("Failed to fetch extras-catalog.json for Amazon Linux 2. url: %s, err: %w", al2ExtraCatalogURI, err) - } - - var catalog extrasCatalog - if err := json.Unmarshal(rs[0].Body, &catalog); err != nil { - return nil, xerrors.Errorf("Failed to unmarshal extras-catalog.json for Amazon Linux 2. err: %w", err) - } - - for _, t := range catalog.Topics { - us, err := fetchUpdateInfoAmazonLinux(fmt.Sprintf(al2ExtraMirrorListURIFormat, t.N)) - if err != nil { - if errors.Is(err, errNoUpdateInfo) { - continue - } - return nil, xerrors.Errorf("Failed to fetch Amazon Linux 2 %s updateinfo. err: %w", t.N, err) - } - for _, u := range us.UpdateList { - u.Repository = fmt.Sprintf("amzn2extra-%s", t.N) - updates.UpdateList = append(updates.UpdateList, u) - } - } - return updates, nil +type mirror struct { + core string + extra string } -// FetchUpdateInfoAmazonLinux2022 fetches a list of Amazon Linux2022 updateinfo -func FetchUpdateInfoAmazonLinux2022() (*models.Updates, error) { - uri, err := getAmazonLinux2022MirrorListURI() - if err != nil { - return nil, err - } - return fetchUpdateInfoAmazonLinux(uri) +var mirrors = map[string]mirror{ + "1": {core: "http://repo.us-west-2.amazonaws.com/2018.03/updates/x86_64/mirror.list"}, + "2": { + core: "https://cdn.amazonlinux.com/2/core/latest/x86_64/mirror.list", + extra: "http://amazonlinux.default.amazonaws.com/2/extras-catalog.json", + }, + "2022": {core: "https://cdn.amazonlinux.com/al2022/core/mirrors/latest/x86_64/mirror.list"}, + "2023": {core: "https://cdn.amazonlinux.com/al2023/core/mirrors/latest/x86_64/mirror.list"}, } -func getAmazonLinux2022MirrorListURI() (uri string, err error) { - results, err := util.FetchFeedFiles([]util.FetchRequest{{URL: al2022ReleasemdURI, MIMEType: util.MIMETypeXML}}) - if err != nil || len(results) != 1 { - return "", xerrors.Errorf("Failed to fetch releasemd.xml for AL2022. url: %s, err: %w", al2022ReleasemdURI, err) - } - - var root root - // Since the XML charset encoding is defined as `utf8` instead of `utf-8`, the following error will occur if it do not set decoder.CharsetReader. - // `Failed to fetch updateinfo for Amazon Linux2022. err: xml: encoding "utf8" declared but Decoder.CharsetReader is nil` - decoder := xml.NewDecoder(bytes.NewReader(results[0].Body)) - decoder.CharsetReader = charset.NewReaderLabel - if err := decoder.Decode(&root); err != nil { - return "", xerrors.Errorf("Failed to decode releasemd.xml for AL2022. err: %w", err) - } +var errNoUpdateInfo = xerrors.New("No updateinfo field in the repomd") - versions := []string{} - for _, release := range root.Releases.Release { - versions = append(versions, release.Version) - } - if len(versions) == 0 { - return "", xerrors.Errorf("Failed to get the latest version of al2022. url: %s", al2022ReleasemdURI) - } - sort.Sort(sort.Reverse(sort.StringSlice(versions))) - return fmt.Sprintf("https://cdn.amazonlinux.com/al2022/core/mirrors/%s/x86_64/mirror.list", versions[0]), nil -} +// FetchFiles fetch from Amazon ALAS +func FetchFiles(versions []string) (map[string]*models.Updates, error) { + m := map[string]*models.Updates{} + for _, v := range versions { + switch v { + case "1", "2022", "2023": + us, err := fetchUpdateInfoAmazonLinux(mirrors[v].core) + if err != nil { + return nil, xerrors.Errorf("Failed to fetch Amazon Linux %s UpdateInfo. err: %w", v, err) + } + m[v] = us + case "2": + updates, err := fetchUpdateInfoAmazonLinux(mirrors[v].core) + if err != nil { + return nil, xerrors.Errorf("Failed to fetch Amazon Linux %s UpdateInfo. err: %w", v, err) + } -// FetchUpdateInfoAmazonLinux2023 fetches a list of Amazon Linux2023 updateinfo -func FetchUpdateInfoAmazonLinux2023() (*models.Updates, error) { - uri, err := getAmazonLinux2023MirrorListURI() - if err != nil { - return nil, err - } - return fetchUpdateInfoAmazonLinux(uri) -} + rs, err := util.FetchFeedFiles([]util.FetchRequest{{URL: mirrors[v].extra, MIMEType: util.MIMETypeJSON}}) + if err != nil || len(rs) != 1 { + return nil, xerrors.Errorf("Failed to fetch extras-catalog.json for Amazon Linux 2. url: %s, err: %w", mirrors[v].extra, err) + } -func getAmazonLinux2023MirrorListURI() (uri string, err error) { - results, err := util.FetchFeedFiles([]util.FetchRequest{{URL: al2023ReleasemdURI, MIMEType: util.MIMETypeXML}}) - if err != nil || len(results) != 1 { - return "", xerrors.Errorf("Failed to fetch releasemd.xml for AL2023. url: %s, err: %w", al2023ReleasemdURI, err) - } + var catalog extrasCatalog + if err := json.Unmarshal(rs[0].Body, &catalog); err != nil { + return nil, xerrors.Errorf("Failed to unmarshal extras-catalog.json for Amazon Linux 2. err: %w", err) + } - var root root - // Since the XML charset encoding is defined as `utf8` instead of `utf-8`, the following error will occur if it do not set decoder.CharsetReader. - // `Failed to fetch updateinfo for Amazon Linux2023. err: xml: encoding "utf8" declared but Decoder.CharsetReader is nil` - decoder := xml.NewDecoder(bytes.NewReader(results[0].Body)) - decoder.CharsetReader = charset.NewReaderLabel - if err := decoder.Decode(&root); err != nil { - return "", xerrors.Errorf("Failed to decode releasemd.xml for AL2023. err: %w", err) - } + for _, t := range catalog.Topics { + us, err := fetchUpdateInfoAmazonLinux(fmt.Sprintf("https://cdn.amazonlinux.com/2/extras/%s/latest/x86_64/mirror.list", t.N)) + if err != nil { + if errors.Is(err, errNoUpdateInfo) { + continue + } + return nil, xerrors.Errorf("Failed to fetch Amazon Linux 2 %s updateinfo. err: %w", t.N, err) + } + for _, u := range us.UpdateList { + u.Repository = fmt.Sprintf("amzn2extra-%s", t.N) + updates.UpdateList = append(updates.UpdateList, u) + } + } - versions := []string{} - for _, release := range root.Releases.Release { - versions = append(versions, release.Version) - } - if len(versions) == 0 { - return "", xerrors.Errorf("Failed to get the latest version of al2023. url: %s", al2023ReleasemdURI) + m[v] = updates + default: + log15.Warn("Skip unknown amazon.", "version", v) + } } - sort.Sort(sort.Reverse(sort.StringSlice(versions))) - return fmt.Sprintf("https://cdn.amazonlinux.com/al2023/core/mirrors/%s/x86_64/mirror.list", versions[0]), nil + return m, nil } func fetchUpdateInfoAmazonLinux(mirrorListURL string) (uinfo *models.Updates, err error) { diff --git a/fetcher/amazon/types.go b/fetcher/amazon/types.go index 7c320fcb..a2b94cee 100644 --- a/fetcher/amazon/types.go +++ b/fetcher/amazon/types.go @@ -1,23 +1,5 @@ package amazon -import "encoding/xml" - -// root is a struct of releasemd.xml for AL2022 -// curl https://al2022-repos-us-west-2-9761ab97.s3.dualstack.us-west-2.amazonaws.com/core/releasemd.xml -type root struct { - XMLName xml.Name `xml:"root"` - Releases struct { - Release []struct { - Version string `xml:"version,attr"` - Update []struct { - Name string `xml:"name"` - VersionString string `xml:"version_string"` - ReleaseNotes string `xml:"release_notes"` - } `xml:"update"` - } `xml:"release"` - } `xml:"releases"` -} - // extrasCatalog is a struct of extras-catalog.json for Amazon Linux 2 Extra Repository type extrasCatalog struct { Topics []struct { diff --git a/fetcher/debian/debian.go b/fetcher/debian/debian.go index e73c5d58..cdf1b5ad 100644 --- a/fetcher/debian/debian.go +++ b/fetcher/debian/debian.go @@ -46,7 +46,7 @@ func debianName(major string) string { } } -// FetchFiles fetch OVAL from RedHat +// FetchFiles fetch OVAL from Debian func FetchFiles(versions []string) ([]util.FetchResult, error) { reqs := newFetchRequests(versions) if len(reqs) == 0 { diff --git a/fetcher/fedora/fedora.go b/fetcher/fedora/fedora.go index 01e68731..ed5d6d8c 100644 --- a/fetcher/fedora/fedora.go +++ b/fetcher/fedora/fedora.go @@ -78,10 +78,20 @@ func FetchUpdateInfosFedora(versions []string) (map[string]*models.Updates, erro func newFedoraFetchRequests(target []string, arch string) (reqs []util.FetchRequest, moduleReqs []util.FetchRequest) { for _, v := range target { var updateURL, moduleURL string - if n, _ := strconv.Atoi(v); n < 36 { + n, err := strconv.Atoi(v) + if err != nil { + log15.Warn("Skip unknown fedora.", "version", v) + continue + } + + switch { + case n < 32: + log15.Warn("Skip fedora because no vulnerability information provided.", "version", v) + continue + case n < 36: updateURL = archiveUpdateURL moduleURL = archiveModuleURL - } else { + default: updateURL = pubUpdateURL moduleURL = pubModuleURL } diff --git a/fetcher/redhat/redhat.go b/fetcher/redhat/redhat.go index 1a9fb5c0..0f33a2b0 100644 --- a/fetcher/redhat/redhat.go +++ b/fetcher/redhat/redhat.go @@ -2,18 +2,29 @@ package redhat import ( "fmt" + "strconv" + "github.com/inconshreveable/log15" "golang.org/x/xerrors" "github.com/vulsio/goval-dictionary/fetcher/util" ) func newFetchRequests(target []string) (reqs []util.FetchRequest) { - const t = "https://www.redhat.com/security/data/oval/com.redhat.rhsa-RHEL%s.xml.bz2" for _, v := range target { + n, err := strconv.Atoi(v) + if err != nil { + log15.Warn("Skip unknown redhat.", "version", v) + continue + } + + if n < 5 { + log15.Warn("Skip redhat because no vulnerability information provided.", "version", v) + continue + } reqs = append(reqs, util.FetchRequest{ Target: v, - URL: fmt.Sprintf(t, v), + URL: fmt.Sprintf("https://www.redhat.com/security/data/oval/com.redhat.rhsa-RHEL%s.xml.bz2", v), MIMEType: util.MIMETypeBzip2, Concurrently: false, }) diff --git a/fetcher/suse/suse.go b/fetcher/suse/suse.go index 66e727c5..a8b407dc 100644 --- a/fetcher/suse/suse.go +++ b/fetcher/suse/suse.go @@ -25,7 +25,7 @@ func newFetchRequests(suseType string, target []string) (reqs []util.FetchReques return } -// FetchFiles fetch OVAL from RedHat +// FetchFiles fetch OVAL from SUSE func FetchFiles(suseType string, versions []string) ([]util.FetchResult, error) { reqs := newFetchRequests(suseType, versions) if len(reqs) == 0 { diff --git a/go.mod b/go.mod index 7c7cc666..80ec0a16 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/cheggaaa/pb/v3 v3.0.8 github.com/go-redis/redis/v8 v8.11.3 - github.com/google/go-cmp v0.5.7 + github.com/google/go-cmp v0.5.8 github.com/hashicorp/go-version v1.4.0 github.com/htcat/htcat v1.0.2 github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac @@ -16,6 +16,7 @@ require ( github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 github.com/ulikunitz/xz v0.5.10 + golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 golang.org/x/net v0.0.0-20211206223403-eba003a116a9 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/yaml.v2 v2.4.0 @@ -63,7 +64,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect + golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index 2272aa2f..afb94f5b 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -591,6 +591,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -742,8 +744,9 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/util/util.go b/util/util.go new file mode 100644 index 00000000..78f9d4f1 --- /dev/null +++ b/util/util.go @@ -0,0 +1,12 @@ +package util + +import "golang.org/x/exp/maps" + +// Unique return unique elements +func Unique[T comparable](s []T) []T { + m := map[T]struct{}{} + for _, v := range s { + m[v] = struct{}{} + } + return maps.Keys(m) +}