Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(command): specify version in amazon or oracle #289

Merged
merged 2 commits into from
Apr 17, 2023
Merged

refactor(command): specify version in amazon or oracle #289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a1b6d3
refactor(command): specify version in amazon or oracle
MaineK00n Mar 16, 2023
c15e6c8
style: fix typo
MaineK00n Mar 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions .github/workflows/fetch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,8 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
Version: 5 6 7 8 9
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
Expand All @@ -241,16 +243,16 @@ jobs:
run: make build
- name: fetch sqlite3
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype sqlite3 oracle
run: ./goval-dictionary fetch --dbtype sqlite3 oracle $Version
- name: fetch mysql
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" oracle
run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" oracle $Version
- name: fetch postgres
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" oracle
run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" oracle $Version
- name: fetch redis
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" oracle
run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" oracle $Version

fetch-amazon:
name: fetch-amazon
Expand Down Expand Up @@ -289,6 +291,8 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
Version: 1 2 2022 2023
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
Expand All @@ -301,16 +305,16 @@ jobs:
run: make build
- name: fetch sqlite3
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype sqlite3 amazon
run: ./goval-dictionary fetch --dbtype sqlite3 amazon $Version
- name: fetch mysql
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" amazon
run: ./goval-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" amazon $Version
- name: fetch postgres
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" amazon
run: ./goval-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" amazon $Version
- name: fetch redis
if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}}
run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" amazon
run: ./goval-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" amazon $Version

fetch-fedora:
name: fetch-fedora
Expand Down
16 changes: 8 additions & 8 deletions GNUmakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ fetch-rdb:
integration/goval-dict.old fetch debian --dbpath=$(PWD)/integration/oval.old.sqlite3 7 8 9 10 11
integration/goval-dict.old fetch ubuntu --dbpath=$(PWD)/integration/oval.old.sqlite3 14 16 18 19 20 21 22
integration/goval-dict.old fetch redhat --dbpath=$(PWD)/integration/oval.old.sqlite3 5 6 7 8 9
integration/goval-dict.old fetch oracle --dbpath=$(PWD)/integration/oval.old.sqlite3
integration/goval-dict.old fetch amazon --dbpath=$(PWD)/integration/oval.old.sqlite3
integration/goval-dict.old fetch oracle --dbpath=$(PWD)/integration/oval.old.sqlite3 5 6 7 8 9
integration/goval-dict.old fetch amazon --dbpath=$(PWD)/integration/oval.old.sqlite3 1 2 2022 2023
integration/goval-dict.old fetch alpine --dbpath=$(PWD)/integration/oval.old.sqlite3 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16
integration/goval-dict.old fetch suse --dbpath=$(PWD)/integration/oval.old.sqlite3 --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed
integration/goval-dict.old fetch suse --dbpath=$(PWD)/integration/oval.old.sqlite3 --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3
Expand All @@ -115,8 +115,8 @@ fetch-rdb:
integration/goval-dict.new fetch debian --dbpath=$(PWD)/integration/oval.new.sqlite3 7 8 9 10 11
integration/goval-dict.new fetch ubuntu --dbpath=$(PWD)/integration/oval.new.sqlite3 14 16 18 19 20 21 22
integration/goval-dict.new fetch redhat --dbpath=$(PWD)/integration/oval.new.sqlite3 5 6 7 8 9
integration/goval-dict.new fetch oracle --dbpath=$(PWD)/integration/oval.new.sqlite3
integration/goval-dict.new fetch amazon --dbpath=$(PWD)/integration/oval.new.sqlite3
integration/goval-dict.new fetch oracle --dbpath=$(PWD)/integration/oval.new.sqlite3 5 6 7 8 9
integration/goval-dict.new fetch amazon --dbpath=$(PWD)/integration/oval.new.sqlite3 1 2 2022 2023
integration/goval-dict.new fetch alpine --dbpath=$(PWD)/integration/oval.new.sqlite3 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16
integration/goval-dict.new fetch suse --dbpath=$(PWD)/integration/oval.new.sqlite3 --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed
integration/goval-dict.new fetch suse --dbpath=$(PWD)/integration/oval.new.sqlite3 --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3
Expand All @@ -131,8 +131,8 @@ fetch-redis:
integration/goval-dict.old fetch debian --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 7 8 9 10 11
integration/goval-dict.old fetch ubuntu --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 14 16 18 19 20 21 22
integration/goval-dict.old fetch redhat --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 5 6 7 8 9
integration/goval-dict.old fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/goval-dict.old fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
integration/goval-dict.old fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 5 6 7 8 9
integration/goval-dict.old fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 1 2 2022 2023
integration/goval-dict.old fetch alpine --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16
integration/goval-dict.old fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed
integration/goval-dict.old fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3
Expand All @@ -143,8 +143,8 @@ fetch-redis:
integration/goval-dict.new fetch debian --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 7 8 9 10 11
integration/goval-dict.new fetch ubuntu --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 14 16 18 19 20 21 22
integration/goval-dict.new fetch redhat --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 5 6 7 8 9
integration/goval-dict.new fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/goval-dict.new fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
integration/goval-dict.new fetch oracle --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 5 6 7 8 9
integration/goval-dict.new fetch amazon --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 1 2 2022 2023
integration/goval-dict.new fetch alpine --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16
integration/goval-dict.new fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --suse-type opensuse 10.2 10.3 11.0 11.1 11.2 11.3 11.4 12.1 12.2 12.3 13.1 13.2 tumbleweed
integration/goval-dict.new fetch suse --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --suse-type opensuse-leap 42.1 42.2 42.3 15.0 15.1 15.2 15.3
Expand Down
4 changes: 2 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ $ goval-dictionary fetch suse --suse-type suse-enterprise-desktop 10 11 12 15
- [Oracle Linux](https://linux.oracle.com/security/oval/)

```bash
$ goval-dictionary fetch oracle
$ goval-dictionary fetch oracle 5 6 7 8 9
```

### Usage: Fetch alpine-secdb as OVAL data type
Expand All @@ -173,7 +173,7 @@ See [here](https://secdb.alpinelinux.org/) for a list of supported alpines.
Amazon ALAS provideis Vulnerability data as `no-OVAL-format`, but it is supported by goval-dictionary to make Amazon ALAS easier to handle from Vuls.

```bash
$ goval-dictionary fetch amazon
$ goval-dictionary fetch amazon 1 2 2022 2023
```

#### Usage: Fetch Security Updates from Fedora
Expand Down
27 changes: 8 additions & 19 deletions commands/fetch-alpine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@ import (
"github.com/vulsio/goval-dictionary/log"
"github.com/vulsio/goval-dictionary/models"
"github.com/vulsio/goval-dictionary/models/alpine"
"github.com/vulsio/goval-dictionary/util"
)

// fetchAlpineCmd is Subcommand for fetch Alpine secdb
// https://secdb.alpinelinux.org/
var fetchAlpineCmd = &cobra.Command{
Use: "alpine",
Short: "Fetch Vulnerability dictionary from Alpine secdb",
Long: `Fetch Vulnerability dictionary from Alpine secdb`,
RunE: fetchAlpine,
Use: "alpine [version]",
Short: "Fetch Vulnerability dictionary from Alpine secdb",
Long: `Fetch Vulnerability dictionary from Alpine secdb`,
Args: cobra.MinimumNArgs(1),
RunE: fetchAlpine,
Example: "$ goval-dictionary fetch alpine 3.16 3.17",
}

func init() {
Expand All @@ -36,20 +39,6 @@ func fetchAlpine(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

if len(args) == 0 {
return xerrors.New("Failed to fetch alpine command. err: specify versions to fetch")
}

// Distinct
v := map[string]bool{}
vers := []string{}
for _, arg := range args {
v[arg] = true
}
for k := range v {
vers = append(vers, k)
}

driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if locked {
Expand All @@ -70,7 +59,7 @@ func fetchAlpine(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err)
}

results, err := fetcher.FetchFiles(vers)
results, err := fetcher.FetchFiles(util.Unique(args))
if err != nil {
return xerrors.Errorf("Failed to fetch files. err: %w", err)
}
Expand Down
93 changes: 22 additions & 71 deletions commands/fetch-amazon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package commands

import (
"fmt"
"time"

"github.com/inconshreveable/log15"
Expand All @@ -15,22 +14,25 @@ import (
"github.com/vulsio/goval-dictionary/log"
"github.com/vulsio/goval-dictionary/models"
"github.com/vulsio/goval-dictionary/models/amazon"
"github.com/vulsio/goval-dictionary/util"
)

// fetchAmazonCmd is Subcommand for fetch Amazon ALAS RSS
// https://alas.aws.amazon.com/alas.rss
var fetchAmazonCmd = &cobra.Command{
Use: "amazon",
Short: "Fetch Vulnerability dictionary from Amazon ALAS",
Long: `Fetch Vulnerability dictionary from Amazon ALAS`,
RunE: fetchAmazon,
Use: "amazon [version]",
Short: "Fetch Vulnerability dictionary from Amazon ALAS",
Long: `Fetch Vulnerability dictionary from Amazon ALAS`,
Args: cobra.MinimumNArgs(1),
RunE: fetchAmazon,
Example: "$ goval-dictionary fetch amazon 1 2 2022 2023",
}

func init() {
fetchCmd.AddCommand(fetchAmazonCmd)
}

func fetchAmazon(_ *cobra.Command, _ []string) (err error) {
func fetchAmazon(_ *cobra.Command, args []string) (err error) {
if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}
Expand Down Expand Up @@ -61,64 +63,22 @@ func fetchAmazon(_ *cobra.Command, _ []string) (err error) {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err)
}

uinfo, err := fetcher.FetchUpdateInfoAmazonLinux1()
m, err := fetcher.FetchFiles(util.Unique(args))
if err != nil {
return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux1. err: %w", err)
}
root := models.Root{
Family: c.Amazon,
OSVersion: "1",
Definitions: amazon.ConvertToModel(uinfo),
Timestamp: time.Now(),
}
log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux1. Inserting to DB", len(root.Definitions)))
if err := execute(driver, &root); err != nil {
return xerrors.Errorf("Failed to Insert Amazon1. err: %w", err)
}

uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2()
if err != nil {
return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2. err: %w", err)
}
root = models.Root{
Family: c.Amazon,
OSVersion: "2",
Definitions: amazon.ConvertToModel(uinfo),
Timestamp: time.Now(),
}
log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2. Inserting to DB", len(root.Definitions)))
if err := execute(driver, &root); err != nil {
return xerrors.Errorf("Failed to Insert Amazon2. err: %w", err)
}

uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2022()
if err != nil {
return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2022. err: %w", err)
}
root = models.Root{
Family: c.Amazon,
OSVersion: "2022",
Definitions: amazon.ConvertToModel(uinfo),
Timestamp: time.Now(),
}
log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2022. Inserting to DB", len(root.Definitions)))
if err := execute(driver, &root); err != nil {
return xerrors.Errorf("Failed to Insert Amazon2022. err: %w", err)
}
return xerrors.Errorf("Failed to fetch files. err: %w", err)
}
for ver, us := range m {
root := models.Root{
Family: c.Amazon,
OSVersion: ver,
Definitions: amazon.ConvertToModel(us),
Timestamp: time.Now(),
}

uinfo, err = fetcher.FetchUpdateInfoAmazonLinux2023()
if err != nil {
return xerrors.Errorf("Failed to fetch updateinfo for Amazon Linux2023. err: %w", err)
}
root = models.Root{
Family: c.Amazon,
OSVersion: "2023",
Definitions: amazon.ConvertToModel(uinfo),
Timestamp: time.Now(),
}
log15.Info(fmt.Sprintf("%d CVEs for Amazon Linux2023. Inserting to DB", len(root.Definitions)))
if err := execute(driver, &root); err != nil {
return xerrors.Errorf("Failed to Insert Amazon2023. err: %w", err)
if err := driver.InsertOval(&root); err != nil {
return xerrors.Errorf("Failed to insert OVAL. err: %w", err)
}
log15.Info("Finish", "Updated", len(root.Definitions))
}

fetchMeta.LastFetchedAt = time.Now()
Expand All @@ -128,12 +88,3 @@ func fetchAmazon(_ *cobra.Command, _ []string) (err error) {

return nil
}

func execute(driver db.DB, root *models.Root) error {
if err := driver.InsertOval(root); err != nil {
return xerrors.Errorf("Failed to insert OVAL. err: %w", err)
}
log15.Info("Finish", "Updated", len(root.Definitions))

return nil
}
27 changes: 8 additions & 19 deletions commands/fetch-debian.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@ import (
"github.com/vulsio/goval-dictionary/log"
"github.com/vulsio/goval-dictionary/models"
"github.com/vulsio/goval-dictionary/models/debian"
"github.com/vulsio/goval-dictionary/util"
)

// fetchDebianCmd is Subcommand for fetch Debian OVAL
var fetchDebianCmd = &cobra.Command{
Use: "debian",
Short: "Fetch Vulnerability dictionary from Debian",
Long: `Fetch Vulnerability dictionary from Debian`,
RunE: fetchDebian,
Use: "debian [version]",
Short: "Fetch Vulnerability dictionary from Debian",
Long: `Fetch Vulnerability dictionary from Debian`,
Args: cobra.MinimumNArgs(1),
RunE: fetchDebian,
Example: "$ goval-dictionary fetch debian 10 11",
}

func init() {
Expand All @@ -37,10 +40,6 @@ func fetchDebian(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to SetLogger. err: %w", err)
}

if len(args) == 0 {
return xerrors.New("Failed to fetch debian command. err: specify versions to fetch")
}

driver, locked, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{})
if err != nil {
if locked {
Expand All @@ -64,17 +63,7 @@ func fetchDebian(_ *cobra.Command, args []string) (err error) {
return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err)
}

// Distinct
vers := []string{}
v := map[string]bool{}
for _, arg := range args {
v[arg] = true
}
for k := range v {
vers = append(vers, k)
}

results, err := fetcher.FetchFiles(vers)
results, err := fetcher.FetchFiles(util.Unique(args))
if err != nil {
return xerrors.Errorf("Failed to fetch files. err: %w", err)
}
Expand Down
Loading