AST-16676: added query mapping option

cmd/args.go

@@ -45,6 +45,10 @@ func GetArgs(cmd *cobra.Command, productName string) internal.Args {
 	if err != nil {
 		panic(err)
 	}
+	args.QueryMappingFile, err = cmd.Flags().GetString(queryMapping)
+	if err != nil {
+		panic(err)
+	}
 
 	args.OutputPath, err = os.Getwd()
 	if err != nil {

cmd/root.go

@@ -23,6 +23,7 @@ const (
 	verboseArg             = "verbose"
 	projectsIds            = "project-id"
 	teamName               = "project-team"
+	queryMapping           = "query-mapping"
 
 	projectsActiveSinceDefaultValue = 180
 )
@@ -105,6 +106,7 @@ func init() {
 	rootCmd.Flags().StringP(userArg, "", "", "SAST username")
 	rootCmd.Flags().StringP(passArg, "", "", "SAST password")
 	rootCmd.Flags().StringP(urlArg, "", "", "SAST url")
+	rootCmd.Flags().StringP(queryMapping, "", "", "Path to file query mapping IDs from AST for triage")
 	rootCmd.Flags().StringP(teamName, "", "", "Team name filter")
 	rootCmd.Flags().StringP(projectsIds, "", "", "Project ID filter")
 	rootCmd.Flags().StringSliceP(exportArg, "", export.GetOptions(), "SAST export options")

internal/app/astquery/provider.go

@@ -1,9 +1,11 @@
 package astquery
 
 import (
+	"encoding/json"
 	"encoding/xml"
 	"fmt"
 	"hash/fnv"
+	"os"
 	"strconv"
 
 	"github.com/checkmarxDev/ast-sast-export/internal/app/interfaces"
@@ -15,17 +17,47 @@ const (
 	notCustomPackageType = "Cx"
 )
 
-type Provider struct {
-	queryProvider interfaces.QueriesRepo
-}
+type (
+	QueryMap struct {
+		AstID  string `json:"astId"`
+		SastID string `json:"sastId"`
+	}
+
+	MapSource struct {
+		Mappings []QueryMap `json:"mappings"`
+	}
+
+	Provider struct {
+		queryProvider interfaces.QueriesRepo
+		mapping       []QueryMap
+	}
+)
+
+func NewProvider(queryProvider interfaces.QueriesRepo, queryMappingPath string) (*Provider, error) {
+	var mapping []QueryMap
+	if queryMappingPath != "" {
+		var mapSource MapSource
+		data, err := os.ReadFile(queryMappingPath)
+		if err != nil {
+			return nil, err
+		}
+		if jsonErr := json.Unmarshal(data, &mapSource); jsonErr != nil {
+			return nil, jsonErr
+		}
+		mapping = mapSource.Mappings
+	}
 
-func NewProvider(queryProvider interfaces.QueriesRepo) (*Provider, error) {
 	return &Provider{
 		queryProvider: queryProvider,
+		mapping:       mapping,
 	}, nil
 }
 
-func (e *Provider) GetQueryID(language, name, group string) (string, error) {
+func (e *Provider) GetQueryID(language, name, group, sastQueryID string) (string, error) {
+	mappedAstID := e.getMappedID(sastQueryID)
+	if mappedAstID != "" {
+		return mappedAstID, nil
+	}
 	sourcePath := fmt.Sprintf("queries/%s/%s/%s/%s.cs", language, group, name, name)
 	queryID, queryIDErr := hash(sourcePath)
 	if queryIDErr != nil {
@@ -57,6 +89,15 @@ func (e *Provider) GetCustomQueriesList() (*soap.GetQueryCollectionResponse, err
 	return &output, nil
 }
 
+func (e *Provider) getMappedID(sastID string) string {
+	for _, queryMap := range e.mapping {
+		if queryMap.SastID == sastID {
+			return queryMap.AstID
+		}
+	}
+	return ""
+}
+
 func hash(s string) (uint64, error) {
 	h := fnv.New64()
 	_, err := h.Write([]byte(s))

internal/app/astquery/provider_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 type QueryIDTest struct {
-	Language, Group, Name, Expected string
+	Language, Group, Name, SastID, Expected string
 }
 
 type CustomQueriesListTest struct {
@@ -25,17 +25,17 @@ func TestProvider_GetQueryID(t *testing.T) {
 	queryProvider := mock_interfaces_queries.NewMockQueriesRepo(ctrl)
 
 	queryIDTests := []QueryIDTest{
-		{"Kotlin", "Kotlin_High_Risk", "Code_Injection", "15158446363146771540"},
-		{"CSharp", "General", "Find_SQL_Injection_Evasion_Attack", "8984835614866342550"},
-		{"Go", "General", "Find_Command_Injection_Sanitize", "9498204717545098527"},
+		{"Kotlin", "Kotlin_High_Risk", "Code_Injection", "1", "15158446363146771540"},
+		{"CSharp", "General", "Find_SQL_Injection_Evasion_Attack", "2", "8984835614866342550"},
+		{"Go", "General", "Find_Command_Injection_Sanitize", "3", "9498204717545098527"},
 	}
 	for _, test := range queryIDTests {
 		testName := fmt.Sprintf("%s %s %s", test.Language, test.Group, test.Name)
 		t.Run(testName, func(t *testing.T) {
-			repo, repoErr := NewProvider(queryProvider)
+			repo, repoErr := NewProvider(queryProvider, "")
 			assert.NoError(t, repoErr)
 
-			result, err := repo.GetQueryID(test.Language, test.Name, test.Group)
+			result, err := repo.GetQueryID(test.Language, test.Name, test.Group, test.SastID)
 			assert.NoError(t, err)
 			assert.Equal(t, test.Expected, result)
 		})
@@ -46,7 +46,7 @@ func TestProvider_GetCustomQueries(t *testing.T) {
 	var queriesObj, customQueriesObj soap.GetQueryCollectionResponse
 	ctrl := gomock.NewController(t)
 	queryProvider := mock_interfaces_queries.NewMockQueriesRepo(ctrl)
-	repo, repoErr := NewProvider(queryProvider)
+	repo, repoErr := NewProvider(queryProvider, "")
 	assert.NoError(t, repoErr)
 
 	t.Run("Successful getting custom queries", func(t *testing.T) {

internal/app/export/export.go

@@ -57,6 +57,7 @@ const (
 type Exporter interface {
 	AddFile(fileName string, data []byte) error
 	AddFileWithDataSource(fileName string, dataSource func() ([]byte, error)) error
+	CopyFile(destName, sourceName string) error
 	CreateExportPackage(prefix, outputPath string) (string, error)
 	Clean() error
 	GetTmpDir() string
@@ -99,6 +100,33 @@ func (e *Export) AddFileWithDataSource(fileName string, dataSource func() ([]byt
 	return e.AddFile(fileName, content)
 }
 
+func (e *Export) CopyFile(destName, sourceName string) error {
+	sourceFileStat, err := os.Stat(sourceName)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", sourceName)
+	}
+
+	source, err := os.Open(sourceName)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	filePath := path.Join(e.tmpDir, destName)
+	destination, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, filePerm)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	e.fileList = append(e.fileList, destName)
+	_, err = io.Copy(destination, source)
+	return err
+}
+
 // CreateExportPackage compresses and encrypts all files added so far
 func (e *Export) CreateExportPackage(prefix, outputPath string) (string, error) {
 	// create zip

internal/app/interfaces/ast_query.go

@@ -3,6 +3,6 @@ package interfaces
 import "github.com/checkmarxDev/ast-sast-export/internal/integration/soap"
 
 type ASTQueryProvider interface {
-	GetQueryID(language, name, group string) (string, error)
+	GetQueryID(language, name, group, sastQueryID string) (string, error)
 	GetCustomQueriesList() (*soap.GetQueryCollectionResponse, error)
 }

internal/app/interfaces/ast_query_id.go

@@ -1,7 +1,7 @@
 package interfaces
 
 type ASTQueryIDProvider interface {
-	GetQueryID(language, name, group string) (string, error)
+	GetQueryID(language, name, group, sastQueryID string) (string, error)
 }
 
 type ASTQuery struct {

internal/app/metadata/metadata.go

@@ -43,7 +43,7 @@ func (e *MetadataFactory) GetMetadataRecord(scanID string, queries []*Query) (*R
 
 	for queryIdx, query := range queries {
 		output.Queries = append(output.Queries, &RecordQuery{QueryID: query.QueryID})
-		astQueryID, astQueryIDErr := e.astQueryIDProvider.GetQueryID(query.Language, query.Name, query.Group)
+		astQueryID, astQueryIDErr := e.astQueryIDProvider.GetQueryID(query.Language, query.Name, query.Group, query.QueryID)
 		if astQueryIDErr != nil {
 			return nil, errors.Wrapf(
 				astQueryIDErr,

internal/app/metadata/metadata_test.go

@@ -74,7 +74,7 @@ func TestMetadataFactory_GetMetadataForQueryAndResult(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	tmpDir := t.TempDir()
 	astQueryIDProviderMock := mock_app_ast_query_id.NewMockASTQueryIDProvider(ctrl)
-	astQueryIDProviderMock.EXPECT().GetQueryID(metaQuery.Language, metaQuery.Name, metaQuery.Group).Return(astQueryID, nil)
+	astQueryIDProviderMock.EXPECT().GetQueryID(metaQuery.Language, metaQuery.Name, metaQuery.Group, metaQuery.QueryID).Return(astQueryID, nil)
 	similarityIDProviderMock := mock_integration_similarity.NewMockSimilarityIDProvider(ctrl)
 	similarityIDProviderMock.EXPECT().Calculate(
 		gomock.Any(), metaResult1.FirstNode.Name, metaResult1.FirstNode.Line, metaResult1.FirstNode.Column, metaResult1Data.MethodLines[0],

internal/models.go

@@ -12,6 +12,7 @@ type Args struct {
 	DBConnectionString,
 	ProjectsIds,
 	TeamName string
+	QueryMappingFile string
 }
 
 type ReportJob struct {

internal/process.go

@@ -44,6 +44,8 @@ const (
 	scanReportCreateAttempts = 10
 	scanReportCreateMinSleep = 1 * time.Second
 	scanReportCreateMaxSleep = 5 * time.Minute
+
+	destQueryMappingFile = "query_mapping.json"
 )
 
 type ReportConsumeOutput struct {
@@ -58,6 +60,7 @@ func RunExport(args *Args) error {
 	log.Debug().
 		Str("url", args.URL).
 		Str("export", fmt.Sprintf("%v", args.Export)).
+		Str("queryMapping", args.QueryMappingFile).
 		Int("projectsActiveSince", args.ProjectsActiveSince).
 		Bool("debug", args.Debug).
 		Int("consumers", consumerCount).
@@ -109,7 +112,7 @@ func RunExport(args *Args) error {
 	queriesRepo := queries.NewRepo(soapClient)
 	presetRepo := presetrepo.NewRepo(soapClient)
 
-	astQueryProvider, astQueryProviderErr := astquery.NewProvider(queriesRepo)
+	astQueryProvider, astQueryProviderErr := astquery.NewProvider(queriesRepo, args.QueryMappingFile)
 	if astQueryProviderErr != nil {
 		return errors.Wrap(astQueryProviderErr, "could not create AST query provider")
 	}
@@ -134,6 +137,11 @@ func RunExport(args *Args) error {
 
 	metadataSource := metadata.NewMetadataFactory(astQueryProvider, similarityIDCalculator, sourceRepo, methodLineRepo, metadataTempDir)
 
+	copyErr := copyQueryMappingFile(args.QueryMappingFile, &exportValues)
+	if copyErr != nil {
+		return errors.Wrap(copyErr, "could not copy query mapping file")
+	}
+
 	fetchErr := fetchSelectedData(client, &exportValues, args, scanReportCreateAttempts, scanReportCreateMinSleep,
 		scanReportCreateMaxSleep, metadataSource, astQueryProvider, presetProvider)
 	if fetchErr != nil {
@@ -675,3 +683,11 @@ func getRetryHttpClient() *retryablehttp.Client {
 		},
 	}
 }
+
+func copyQueryMappingFile(queryFileMapping string, exporter export2.Exporter) error {
+	if queryFileMapping == "" {
+		log.Info().Msg("not set query mapping file param")
+		return nil
+	}
+	return exporter.CopyFile(destQueryMappingFile, queryFileMapping)
+}