Skip to content

Commit

Permalink
feat: aprl #246
Browse files Browse the repository at this point in the history
  • Loading branch information
cmendible committed Jun 17, 2024
1 parent 07bc220 commit fdfeb78
Show file tree
Hide file tree
Showing 57 changed files with 824 additions and 130 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "internal/aprl"]
path = internal/aprl
url = https://github.com/Azure/Azure-Proactive-Resiliency-Library-v2.git
14 changes: 11 additions & 3 deletions cmd/azqr/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package azqr
import (
"fmt"
"sort"
"strings"

"github.com/Azure/azqr/internal"
"github.com/Azure/azqr/internal/scanners"
Expand All @@ -23,9 +24,10 @@ var rulesCmd = &cobra.Command{
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
serviceScanners := internal.GetScanners()
aprl := internal.GetAprlRecommendations()

fmt.Println("# | Id | Category | Impact | Recommendation | More Info")
fmt.Println("---|---|---|---|---|---")
fmt.Println("# | Resource Type | Id | Category | Impact | Recommendation | Learn")
fmt.Println("---|---|---|---|---|---|---")

i := 0
for _, scanner := range serviceScanners {
Expand All @@ -45,7 +47,13 @@ var rulesCmd = &cobra.Command{
for _, k := range keys {
rule := rules[k]
i++
fmt.Printf("%s | %s | %s | %s | %s | [Learn](%s)", fmt.Sprint(i), rule.Id, rule.Category, rule.Impact, rule.Recommendation, rule.Url)
fmt.Printf("%s | %s | %s | %s | %s | %s | [Learn](%s)", fmt.Sprint(i), rule.Id, scanner.ResourceType(), rule.Category, rule.Impact, rule.Recommendation, rule.Url)
fmt.Println()
}

for _, r := range aprl[strings.ToLower(scanner.ResourceType())] {
i++
fmt.Printf("%s | %s | %s | %s | %s | %s | [Learn](%s)", fmt.Sprint(i), r.RecommendationID, scanner.ResourceType(), r.Category, r.Impact, r.Recommendation, r.LearnMoreLink[0].Url)
fmt.Println()
}
}
Expand Down
5 changes: 4 additions & 1 deletion cmd/azqr/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ func init() {
scanCmd.PersistentFlags().BoolP("defender", "d", true, "Scan Defender Status")
scanCmd.PersistentFlags().BoolP("advisor", "a", true, "Scan Azure Advisor Recommendations")
scanCmd.PersistentFlags().BoolP("costs", "c", false, "Scan Azure Costs")
scanCmd.PersistentFlags().BoolP("excel", "x", false, "Create excel report")
scanCmd.PersistentFlags().BoolP("excel", "x", true, "Create excel report")
scanCmd.PersistentFlags().StringP("output-name", "o", "", "Output file name without extension")
scanCmd.PersistentFlags().BoolP("mask", "m", true, "Mask the subscription id in the report")
scanCmd.PersistentFlags().BoolP("azure-cli-credential", "f", false, "Force the use of Azure CLI Credential")
scanCmd.PersistentFlags().BoolP("debug", "", false, "Set log level to debug")
scanCmd.PersistentFlags().StringP("exclusions", "e", "", "Exclusions file (YAML format)")
scanCmd.PersistentFlags().BoolP("azqr", "", true, "Scan Azure Quick Review Recommendations (default)")

rootCmd.AddCommand(scanCmd)
}
Expand Down Expand Up @@ -49,6 +50,7 @@ func scan(cmd *cobra.Command, serviceScanners []scanners.IAzureScanner) {
debug, _ := cmd.Flags().GetBool("debug")
forceAzureCliCredential, _ := cmd.Flags().GetBool("azure-cli-credential")
exclusionFile, _ := cmd.Flags().GetString("exclusions")
azqr, _ := cmd.Flags().GetBool("azqr")

params := internal.ScanParams{
SubscriptionID: subscriptionID,
Expand All @@ -63,6 +65,7 @@ func scan(cmd *cobra.Command, serviceScanners []scanners.IAzureScanner) {
ServiceScanners: serviceScanners,
ForceAzureCliCredential: forceAzureCliCredential,
ExclusionsFile: exclusionFile,
UseAzqrRecommendations: azqr,
}

internal.Scan(&params)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/synapse/armsynapse v0.8.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/webpubsub/armwebpubsub v1.2.0
github.com/google/uuid v1.6.0
github.com/rs/zerolog v1.32.0
github.com/spf13/cobra v1.8.0
github.com/xuri/excelize/v2 v2.8.1
Expand All @@ -57,7 +58,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand Down
1 change: 1 addition & 0 deletions internal/aprl
Submodule aprl added at 42e88e
87 changes: 87 additions & 0 deletions internal/aprl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package internal

import (
"embed"
"io/fs"
"strings"

"github.com/Azure/azqr/internal/scanners"
"gopkg.in/yaml.v3"
)

//go:embed aprl/azure-resources/**/**/*.yaml
//go:embed aprl/azure-resources/**/**/kql/*.kql
//go:embed aprl/azure-specialized-workloads/**/*.yaml
//go:embed aprl/azure-specialized-workloads/**/kql/*.kql
var embededFiles embed.FS

// GetAprlRecommendations
func GetAprlRecommendations() map[string]map[string]scanners.AzureAprlRecommendation {
r := map[string]map[string]scanners.AzureAprlRecommendation{}

fsys, err := fs.Sub(embededFiles, "aprl/azure-resources")
if err != nil {
return nil
}

q := map[string]string{}
err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() && strings.HasSuffix(path, ".kql") {
content, err := fs.ReadFile(fsys, path)
if err != nil {
return err
}

fileName := strings.TrimSuffix(d.Name(), ".kql")
q[fileName] = string(content)
}
return nil
})
if err != nil {
return nil
}

err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() && strings.HasSuffix(path, ".yaml") {
content, err := fs.ReadFile(fsys, path)
if err != nil {
return err
}

var recommendations []scanners.AzureAprlRecommendation
err = yaml.Unmarshal(content, &recommendations)
if err != nil {
return err
}

for _, recommendation := range recommendations {
t := strings.ToLower(recommendation.ResourceType)
if _, ok := r[t]; !ok {
r[t] = map[string]scanners.AzureAprlRecommendation{}
}

if i, ok := q[recommendation.RecommendationID]; ok {
recommendation.GraphQuery = i
}

r[t][recommendation.RecommendationID] = recommendation
}

}
return nil
})
if err != nil {
return nil
}

return r
}
32 changes: 30 additions & 2 deletions internal/graph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package graph

import (
"context"
"time"

"github.com/Azure/azqr/internal/to"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
Expand Down Expand Up @@ -55,15 +56,42 @@ func (q *GraphQuery) Query(ctx context.Context, query string, subscriptionIDs []
for ok := true; ok; ok = skipToken != nil {
request.Options.SkipToken = skipToken
// Run the query and get the results
results, err := q.client.Resources(ctx, request, nil)
results, err := q.retry(ctx, 3, 10*time.Second, request)
if err == nil {
result.Count = *results.TotalRecords
result.Data = append(result.Data, results.Data.([]interface{})...)
skipToken = results.SkipToken
} else {
log.Fatal().Err(err).Msg("Failed to run Resource Graph query")
log.Fatal().Err(err).Msgf("Failed to run Resource Graph query: %s", query)
return nil
}
}
return &result
}

func (q *GraphQuery) retry(ctx context.Context, attempts int, sleep time.Duration, request arg.QueryRequest) (arg.ClientResourcesResponse, error) {
var err error
for i := 0; ; i++ {
res, err := q.client.Resources(ctx, request, nil)
if err == nil {
return res, nil
}

// if shouldSkipError(err) {
// return []scanners.AzureServiceResult{}, nil
// }

errAsString := err.Error()

if i >= (attempts - 1) {
log.Info().Msgf("Retry limit reached. Error: %s", errAsString)
break
}

log.Debug().Msgf("Retrying after error: %s", errAsString)

time.Sleep(sleep)
sleep *= 2
}
return arg.ClientResourcesResponse{}, err
}
6 changes: 6 additions & 0 deletions internal/renderers/csv/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ func CreateCsvReport(data *renderers.ReportData) {
records := data.ServicesTable()
writeData(records, data.OutputFileName, "services")

records = data.RecommendationsTable()
writeData(records, data.OutputFileName, "recommendations")

records = data.ImpactedTable()
writeData(records, data.OutputFileName, "impacted")

records = data.DefenderTable()
writeData(records, data.OutputFileName, "defender")

Expand Down
3 changes: 2 additions & 1 deletion internal/renderers/excel/excel.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (
_ "image/png"
"unicode/utf8"

"github.com/Azure/azqr/internal/renderers"
"github.com/Azure/azqr/internal/embeded"
"github.com/Azure/azqr/internal/renderers"
"github.com/rs/zerolog/log"
"github.com/xuri/excelize/v2"
)
Expand All @@ -25,6 +25,7 @@ func CreateExcelReport(data *renderers.ReportData) {
}()

renderRecommendations(f, data)
renderImpactedResources(f, data)
renderServices(f, data)
renderDefender(f, data)
renderAdvisor(f, data)
Expand Down
46 changes: 46 additions & 0 deletions internal/renderers/excel/impacted.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package excel

import (
_ "image/png"

"github.com/Azure/azqr/internal/renderers"
"github.com/rs/zerolog/log"
"github.com/xuri/excelize/v2"
)

func renderImpactedResources(f *excelize.File, data *renderers.ReportData) {
sheetName := "Impacted Resources"
if len(data.AprlData) > 0 {
_, err := f.NewSheet(sheetName)
if err != nil {
log.Fatal().Err(err).Msg("Failed to create APRL sheet")
}

records := data.ImpactedTable()
headers := records[0]
records = records[1:]

createFirstRow(f, sheetName, headers)

currentRow := 4
for _, row := range records {
currentRow += 1
cell, err := excelize.CoordinatesToCellName(1, currentRow)
if err != nil {
log.Fatal().Err(err).Msg("Failed to get cell")
}
err = f.SetSheetRow(sheetName, cell, &row)
if err != nil {
log.Fatal().Err(err).Msg("Failed to set row")
}
setHyperLink(f, sheetName, 12, currentRow)
}

configureSheet(f, sheetName, headers, currentRow)
} else {
log.Info().Msgf("Skipping %s. No data to render", sheetName)
}
}
43 changes: 13 additions & 30 deletions internal/renderers/excel/recommendations.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,52 +12,35 @@ import (
)

func renderRecommendations(f *excelize.File, data *renderers.ReportData) {
if len(data.MainData) > 0 {
err := f.SetSheetName("Sheet1", "Recommendations")
sheetName := "Recommendations"
if len(data.Recomendations) > 0 {
err := f.SetSheetName("Sheet1", sheetName)
if err != nil {
log.Fatal().Err(err).Msg("Failed to create Recommendations sheet")
log.Fatal().Err(err).Msgf("Failed to create %s sheet", sheetName)
}

renderedRules := map[string]bool{}

headers := []string{"Category", "Impact", "Recommendation", "Learn", "RId"}
rows := [][]string{}
for _, result := range data.MainData {
for _, rr := range result.Rules {
_, exists := renderedRules[rr.Id]
if !exists && rr.NotCompliant {
rulesToRender := map[string]string{
"Category": string(rr.Category),
"Impact": string(rr.Impact),
"Recommendation": rr.Recommendation,
"Learn": rr.Learn,
"RId": rr.Id,
}
renderedRules[rr.Id] = true
rows = append(rows, mapToRow(headers, rulesToRender)...)
}
}
}
records := data.RecommendationsTable()
headers := records[0]
records = records[1:]

createFirstRow(f, "Recommendations", headers)
createFirstRow(f, sheetName, headers)

currentRow := 4
for _, row := range rows {
for _, row := range records {
currentRow += 1
cell, err := excelize.CoordinatesToCellName(1, currentRow)
if err != nil {
log.Fatal().Err(err).Msg("Failed to get cell")
}
err = f.SetSheetRow("Recommendations", cell, &row)
err = f.SetSheetRow(sheetName, cell, &row)
if err != nil {
log.Fatal().Err(err).Msg("Failed to set row")
}

setHyperLink(f, "Recommendations", 6, currentRow)
// setHyperLink(f, sheetName, 12, currentRow)
}

configureSheet(f, "Recommendations", headers, currentRow)
configureSheet(f, sheetName, headers, currentRow)
} else {
log.Info().Msg("Skipping Recommendations. No data to render")
log.Info().Msgf("Skipping %s. No data to render", sheetName)
}
}
2 changes: 1 addition & 1 deletion internal/renderers/excel/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

func renderServices(f *excelize.File, data *renderers.ReportData) {
if len(data.MainData) > 0 {
if len(data.AzqrData) > 0 {
_, err := f.NewSheet("Services")
if err != nil {
log.Fatal().Err(err).Msg("Failed to create Services sheet")
Expand Down
Loading

0 comments on commit fdfeb78

Please sign in to comment.