From d4cbb87ecdf6471b26d1b09fa7a0f67794b76945 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Wed, 27 Nov 2024 10:55:58 +0800 Subject: [PATCH] support host static html resources of test report Signed-off-by: Patrick Zhao --- .../agent/step/archive/step_tar_archive.go | 18 +- .../zadig-agent/internal/agent/step/step.go | 7 +- pkg/config/config.go | 4 + .../aslan/core/clean_cache_files.go | 101 ++++++++ .../common/repository/models/workflow_v4.go | 11 + pkg/microservice/aslan/core/service.go | 2 + .../aslan/core/stat/handler/openapi.go | 6 +- .../aslan/core/system/handler/capacity.go | 4 + .../service/workflow/job/job_testing.go | 61 ++--- .../core/workflow/testing/handler/router.go | 5 +- .../workflow/testing/handler/test_task.go | 77 +++--- .../core/workflow/testing/handler/testing.go | 31 +-- .../workflow/testing/service/test_task.go | 1 - .../core/workflow/testing/service/testing.go | 223 ++++++++++-------- .../core/service/step/step_tar_archive.go | 7 +- .../user/core/service/permission/authn.go | 13 +- pkg/setting/consts.go | 6 +- pkg/tool/tar/tar.go | 77 ++++++ pkg/types/step/step_tar_archive.go | 25 +- 19 files changed, 451 insertions(+), 228 deletions(-) create mode 100644 pkg/microservice/aslan/core/clean_cache_files.go create mode 100644 pkg/tool/tar/tar.go diff --git a/pkg/cli/zadig-agent/internal/agent/step/archive/step_tar_archive.go b/pkg/cli/zadig-agent/internal/agent/step/archive/step_tar_archive.go index 82efb8394e..3c17a731fb 100644 --- a/pkg/cli/zadig-agent/internal/agent/step/archive/step_tar_archive.go +++ b/pkg/cli/zadig-agent/internal/agent/step/archive/step_tar_archive.go @@ -31,7 +31,6 @@ import ( "github.com/koderover/zadig/v2/pkg/cli/zadig-agent/internal/common/types" "github.com/koderover/zadig/v2/pkg/tool/s3" "github.com/koderover/zadig/v2/pkg/types/step" - "github.com/koderover/zadig/v2/pkg/util/fs" ) type TarArchiveStep struct { @@ -78,18 +77,18 @@ func (s *TarArchiveStep) Run(ctx context.Context) error { cmdAndArtifactFullPaths := make([]string, 0) cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, "-czf") cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, tarName) + if s.spec.ChangeTarDir { + cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, "--exclude", tarName, "-C", s.spec.TarDir) + } + for _, artifactPath := range s.spec.ResultDirs { if len(artifactPath) == 0 { continue } artifactPath = helper.ReplaceEnvWithValue(artifactPath, envMap) - artifactPath = strings.TrimPrefix(artifactPath, "/") - - artifactPath := filepath.Join(s.workspace, artifactPath) - isDir, err := fs.IsDir(artifactPath) - if err != nil || !isDir { - s.logger.Errorf("artifactPath is not exist %s or is not dir, err: %s", artifactPath, err) - continue + if !s.spec.AbsResultDir { + artifactPath = strings.TrimPrefix(artifactPath, "/") + artifactPath = filepath.Join(s.workspace, artifactPath) } cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, artifactPath) } @@ -111,8 +110,11 @@ func (s *TarArchiveStep) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to close %s err: %s", tarName, err) } + cmd := exec.Command("tar", cmdAndArtifactFullPaths...) cmd.Stderr = os.Stderr + + log.Debugf("tar cmd: %s", cmd.String()) if err = cmd.Run(); err != nil { if s.spec.IgnoreErr { s.logger.Errorf("failed to compress %s, cmd: %s, err: %s", tarName, cmd.String(), err) diff --git a/pkg/cli/zadig-agent/internal/agent/step/step.go b/pkg/cli/zadig-agent/internal/agent/step/step.go index e28d8b20a9..d24c0fc44e 100644 --- a/pkg/cli/zadig-agent/internal/agent/step/step.go +++ b/pkg/cli/zadig-agent/internal/agent/step/step.go @@ -121,10 +121,9 @@ func RunStep(ctx context.Context, jobCtx *jobctl.JobContext, step *commonmodels. case "debug_after": return nil default: - //err := fmt.Errorf("step type: %s does not match any known type", step.StepType) - //log.Error(err) - //return err - logger.Infof(fmt.Sprintf("step type: %s does not match any known type", step.StepType)) + err := fmt.Errorf("step type: %s does not match any known type", step.StepType) + log.Error(err) + return err } if err := stepInstance.Run(ctx); err != nil { return err diff --git a/pkg/config/config.go b/pkg/config/config.go index c30d725c02..ae920ca55b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -219,6 +219,10 @@ func LocalChartTemplatePath(name string) string { return LocalTemplatePath(name, setting.ChartTemplatesPath) } +func LocalHtmlReportPath(projectName, workflowName string, taskID int64) string { + return filepath.Join(DataPath(), "project", projectName, "workflow", workflowName, "task", fmt.Sprintf("%d", taskID), "html-report") + "/" +} + func MongoURI() string { return viper.GetString(setting.ENVMongoDBConnectionString) } diff --git a/pkg/microservice/aslan/core/clean_cache_files.go b/pkg/microservice/aslan/core/clean_cache_files.go new file mode 100644 index 0000000000..1c47b1c57c --- /dev/null +++ b/pkg/microservice/aslan/core/clean_cache_files.go @@ -0,0 +1,101 @@ +/* +Copyright 2024 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "fmt" + "os" + "path/filepath" + "time" + + commonconfig "github.com/koderover/zadig/v2/pkg/config" + "github.com/koderover/zadig/v2/pkg/tool/log" +) + +func cleanCacheFiles() { + projectPath := filepath.Join(commonconfig.DataPath(), "project") + err := cleanProjectFiles(projectPath) + if err != nil { + log.Errorf("[cleanCacheFiles] failed to clean project cache files: %v", err) + } +} + +func cleanProjectFiles(path string) error { + return traverseDir(path, cleanWorkflowFiles) +} + +func cleanWorkflowFiles(path string, info os.FileInfo) error { + workflowPath := filepath.Join(path, "workflow") + return traverseDir(workflowPath, cleanTaskFiles) +} + +func cleanTaskFiles(path string, info os.FileInfo) error { + taskPath := filepath.Join(path, "task") + return traverseDir(taskPath, cleanHtmlReportFiles) +} + +func cleanHtmlReportFiles(path string, info os.FileInfo) error { + htmlReportPath := filepath.Join(path, "html-report") + htmlReportInfo, err := os.Stat(htmlReportPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to stat htmlReportPath: %s, err: %v", htmlReportPath, err) + } + + if time.Since(htmlReportInfo.ModTime()) > 7*24*time.Hour { + err = os.RemoveAll(htmlReportPath) + if err != nil { + return fmt.Errorf("failed to remove htmlReportPath: %s, err: %v", htmlReportPath, err) + } + } + return nil +} + +func traverseDir(path string, fn func(string, os.FileInfo) error) error { + pathInfo, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to stat path: %s, err: %v", path, err) + } + + if !pathInfo.IsDir() { + return fmt.Errorf("path is not a directory: %s", path) + } + + files, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("failed to read directory: %s, err: %v", path, err) + } + + for _, file := range files { + info, err := file.Info() + if err != nil { + return fmt.Errorf("failed to get file info: %s, err: %v", file.Name(), err) + } + if info.IsDir() { + err = fn(filepath.Join(path, file.Name()), info) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go index b103f8aa0f..e3158a3b21 100644 --- a/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go +++ b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go @@ -93,6 +93,17 @@ func (w *WorkflowV4) CalculateHash() [md5.Size]byte { return md5.Sum(jsonBytes) } +type ParameterSettingType string + +const ( + StringType ParameterSettingType = "string" + ChoiceType ParameterSettingType = "choice" + ImageType ParameterSettingType = "image" + Script ParameterSettingType = "script" + // Deprecated + ExternalType ParameterSettingType = "external" +) + type WorkflowStage struct { Name string `bson:"name" yaml:"name" json:"name"` Parallel bool `bson:"parallel" yaml:"parallel" json:"parallel"` diff --git a/pkg/microservice/aslan/core/service.go b/pkg/microservice/aslan/core/service.go index acc291ad26..ffc2cb9290 100644 --- a/pkg/microservice/aslan/core/service.go +++ b/pkg/microservice/aslan/core/service.go @@ -177,6 +177,8 @@ func initCron() { log.Infof("[CRONJOB] gitlab token updated....") }) + Scheduler.Every(1).Days().At("04:00").Do(cleanCacheFiles) + Scheduler.StartAsync() } diff --git a/pkg/microservice/aslan/core/stat/handler/openapi.go b/pkg/microservice/aslan/core/stat/handler/openapi.go index a3b252218c..1bb87f2e36 100644 --- a/pkg/microservice/aslan/core/stat/handler/openapi.go +++ b/pkg/microservice/aslan/core/stat/handler/openapi.go @@ -78,10 +78,6 @@ type getRollbackStatDetail struct { } func (req *getRollbackStatDetail) Validate() error { - if req.ProjectKey == "" { - return e.ErrInvalidParam.AddDesc("projectKey is empty") - } - if req.StartTime == 0 || req.EndTime == 0 { return e.ErrInvalidParam.AddDesc("starTime and endTime is empty") } @@ -109,7 +105,7 @@ func (req *getRollbackStatDetail) Validate() error { // @Tags OpenAPI // @Accept json // @Produce json -// @Param projectKey query string true "项目标识" +// @Param projectKey query string false "项目标识" // @Param envName query string false "环境名称" // @Param serviceName query string false "服务名称" // @Param startTime query int true "开始时间，格式为时间戳" diff --git a/pkg/microservice/aslan/core/system/handler/capacity.go b/pkg/microservice/aslan/core/system/handler/capacity.go index f950975fa7..b8731085af 100644 --- a/pkg/microservice/aslan/core/system/handler/capacity.go +++ b/pkg/microservice/aslan/core/system/handler/capacity.go @@ -17,6 +17,7 @@ limitations under the License. package handler import ( + "encoding/json" "fmt" "github.com/gin-gonic/gin" @@ -59,6 +60,9 @@ func UpdateStrategy(c *gin.Context) { return } + bs, _ := json.Marshal(args) + internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "任务配置", "", string(bs), ctx.Logger) + if err := service.UpdateSysCapStrategy(args); err != nil { ctx.RespErr = err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go index fe37877b1f..ad0fd391e4 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go @@ -565,51 +565,36 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul StepType: config.StepDebugAfter, } jobTaskSpec.Steps = append(jobTaskSpec.Steps, debugAfterStep) - // init archive html step - if len(testingInfo.TestReportPath) > 0 { - ext := filepath.Ext(testingInfo.TestReportPath) - if ext != ".html" { - return jobTask, fmt.Errorf("test report path: %s is not a html file", testingInfo.TestReportPath) - } - outputPath := strings.TrimSuffix(testingInfo.TestReportPath, ext) + "-archive" + ext - archiveHtmlStep := &commonmodels.StepTask{ - Name: config.TestJobHTMLReportArchiveStepName, - JobName: jobTask.Name, - StepType: config.StepArchiveHtml, - Onfailure: true, - Spec: step.StepArchiveHtmlSpec{ - HtmlPath: testingInfo.TestReportPath, - OutputPath: outputPath, - }, - } - jobTaskSpec.Steps = append(jobTaskSpec.Steps, archiveHtmlStep) + tarDestDir := "/tmp" + if testingInfo.ScriptType == types.ScriptTypeBatchFile { + tarDestDir = "%TMP%" + } else if testingInfo.ScriptType == types.ScriptTypePowerShell { + tarDestDir = "%TMP%" + } - uploads := []*step.Upload{ - { - FilePath: outputPath, - DestinationPath: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "html"), - }, - } - archiveStep := &commonmodels.StepTask{ + // init archive html step + if len(testingInfo.TestReportPath) > 0 { + testReportDir := filepath.Dir(testingInfo.TestReportPath) + testReportName := filepath.Base(testingInfo.TestReportPath) + tarArchiveStep := &commonmodels.StepTask{ Name: config.TestJobHTMLReportStepName, JobName: jobTask.Name, - StepType: config.StepArchive, + StepType: config.StepTarArchive, Onfailure: true, - Spec: step.StepArchiveSpec{ - UploadDetail: uploads, - S3: modelS3toS3(defaultS3), + Spec: &step.StepTarArchiveSpec{ + FileName: setting.HtmlReportArchivedFileName, + AbsResultDir: true, + ResultDirs: []string{testReportName}, + ChangeTarDir: true, + TarDir: "$WORKSPACE/" + testReportDir, + DestDir: tarDestDir, + S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "html-report"), }, } - jobTaskSpec.Steps = append(jobTaskSpec.Steps, archiveStep) + jobTaskSpec.Steps = append(jobTaskSpec.Steps, tarArchiveStep) } - destDir := "/tmp" - if testingInfo.ScriptType == types.ScriptTypeBatchFile { - destDir = "%TMP%" - } else if testingInfo.ScriptType == types.ScriptTypePowerShell { - destDir = "%TMP%" - } // init test result storage step if len(testingInfo.ArtifactPaths) > 0 { tarArchiveStep := &commonmodels.StepTask{ @@ -621,7 +606,7 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul ResultDirs: testingInfo.ArtifactPaths, S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "test-result"), FileName: setting.ArtifactResultOut, - DestDir: destDir, + DestDir: tarDestDir, }, } if len(testingInfo.ArtifactPaths) > 1 || testingInfo.ArtifactPaths[0] != "" { @@ -644,7 +629,7 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "junit"), TestName: testing.Name, TestProject: testing.ProjectName, - DestDir: destDir, + DestDir: tarDestDir, FileName: "merged.xml", ServiceName: serviceName, ServiceModule: serviceModule, diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/router.go b/pkg/microservice/aslan/core/workflow/testing/handler/router.go index 758416e45f..a5cf41c1cb 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/router.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/router.go @@ -26,8 +26,8 @@ func (*Router) Inject(router *gin.RouterGroup) { // 查看html测试报告不做鉴权 testReport := router.Group("report") { - testReport.GET("", GetHTMLTestReport) - testReport.GET("workflowv4/:workflowName/id/:id/job/:jobName", GetWorkflowV4HTMLTestReport) + testReport.GET("/html/testing/:projectName/:testingName/:taskID/*path", GetTestTaskHtmlReportInfo) + testReport.GET("/html/workflowv4/:projectName/:workflowName/:taskID/*path", GetWorkflowV4HTMLTestReport) } // sse apis @@ -103,7 +103,6 @@ func (*Router) Inject(router *gin.RouterGroup) { testTask.DELETE("", CancelTestTaskV3) testTask.GET("/detail", GetTestTaskInfo) testTask.GET("/report", GetTestTaskJUnitReportInfo) - testTask.GET("/html_report", GetTestTaskHtmlReportInfo) testTask.POST("/restart", RestartTestTaskV2) testTask.GET("/artifact", GetTestingTaskArtifact) // TODO: below is the deprecated apis, remove after 2.2.0 diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go index 21251ee741..92c0046e73 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/test_task.go @@ -22,6 +22,8 @@ import ( "encoding/json" "fmt" "io" + "os" + "path/filepath" "strconv" "time" @@ -29,10 +31,10 @@ import ( "github.com/gin-gonic/gin/binding" "k8s.io/apimachinery/pkg/util/wait" - commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/notify" + commonutil "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/util" workflowservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/testing/service" "github.com/koderover/zadig/v2/pkg/setting" @@ -263,47 +265,64 @@ func GetTestTaskJUnitReportInfo(c *gin.Context) { } func GetTestTaskHtmlReportInfo(c *gin.Context) { - ctx, err := internalhandler.NewContextWithAuthorization(c) - defer func() { internalhandler.JSONResponse(c, ctx) }() + _ = c.Param("projectName") + testName := c.Param("testingName") + taskIDStr := c.Param("taskID") + filepath := c.Param("path") + taskID, err := strconv.ParseInt(taskIDStr, 10, 64) if err != nil { - ctx.RespErr = fmt.Errorf("authorization Info Generation failed: err %s", err) - ctx.UnAuthorized = true + c.JSON(500, gin.H{"err": fmt.Sprintf("failed to parse taskID %s, err :%s", taskIDStr, err)}) return } - projectKey := c.Query("projectName") - testName := c.Query("testName") - taskIDStr := c.Query("taskID") - - // authorization check - if !ctx.Resources.IsSystemAdmin { - if _, ok := ctx.Resources.ProjectAuthInfo[projectKey]; !ok { - ctx.UnAuthorized = true - return - } - - if !ctx.Resources.ProjectAuthInfo[projectKey].IsProjectAdmin && - !ctx.Resources.ProjectAuthInfo[projectKey].Test.View { - ctx.UnAuthorized = true - return - } + htmlReportDir, err := service.GetTestTaskHTMLTestReport(testName, taskID, ginzap.WithContext(c).Sugar()) + if err != nil { + c.JSON(500, gin.H{"err": fmt.Sprintf("failed to get testing task html report, err :%s", err)}) + return } - taskID, err := strconv.ParseInt(taskIDStr, 10, 64) + filepath, err = findDefaultHtmlReportFilePath(htmlReportDir, filepath) if err != nil { - ctx.RespErr = e.ErrInvalidParam.AddDesc(fmt.Sprintf("taskID args err :%s", err)) + c.JSON(500, gin.H{"err": err.Error()}) return } - content, err := service.GetTestTaskHTMLTestReport(testName, taskID, ginzap.WithContext(c).Sugar()) + log.Debugf("html report file path: %s", filepath) + c.FileFromFS(filepath, gin.Dir(htmlReportDir, false)) +} + +func findDefaultHtmlReportFilePath(htmlReportDir string, htmlFilepath string) (string, error) { + // user specified path, just return + if htmlFilepath != "/" { + return htmlFilepath, nil + } + + // user not specified, find default html file path for user + files, err := os.ReadDir(htmlReportDir) if err != nil { - c.JSON(500, gin.H{"err": err}) - return + return "", fmt.Errorf("failed to read dir %s, err :%s", htmlReportDir, err) + } + + fileName := "" + for _, file := range files { + if !file.IsDir() { + if file.Name() == "index.html" { + fileName = file.Name() + break + } + if fileName == "" && filepath.Ext(file.Name()) == ".html" { + fileName = file.Name() + } + } } - c.Header("content-type", "text/html") - c.String(200, content) + if fileName != "" { + htmlFilepath = "/" + fileName + return htmlFilepath, nil + } else { + return "", fmt.Errorf("no html file found in %s", htmlReportDir) + } } func RestartTestTaskV2(c *gin.Context) { @@ -450,4 +469,4 @@ func GetTestingTaskSSE(c *gin.Context) { ctx.Logger.Error(err) } }, ctx.Logger) -} \ No newline at end of file +} diff --git a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go index 2edbd32285..b2284d7e60 100644 --- a/pkg/microservice/aslan/core/workflow/testing/handler/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/handler/testing.go @@ -276,35 +276,24 @@ func DeleteTestModule(c *gin.Context) { ctx.RespErr = commonservice.DeleteTestModule(name, projectKey, ctx.RequestID, ctx.Logger) } -func GetHTMLTestReport(c *gin.Context) { - content, err := service.GetHTMLTestReport( - c.Query("pipelineName"), - c.Query("pipelineType"), - c.Query("taskID"), - c.Query("testName"), - ginzap.WithContext(c).Sugar(), - ) +func GetWorkflowV4HTMLTestReport(c *gin.Context) { + filepath := c.Param("path") + taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) if err != nil { - c.JSON(500, gin.H{"err": err}) + c.JSON(500, gin.H{"err": fmt.Sprintf("invalid taskID %s", c.Param("id"))}) return } - - c.Header("content-type", "text/html") - c.String(200, content) -} - -func GetWorkflowV4HTMLTestReport(c *gin.Context) { - taskID, err := strconv.ParseInt(c.Param("id"), 10, 64) + htmlReportDir, err := service.GetWorkflowV4HTMLTestReport(c.Param("workflowName"), c.Param("jobName"), taskID, ginzap.WithContext(c).Sugar()) if err != nil { - c.JSON(500, gin.H{"err": err}) + c.JSON(500, gin.H{"err": fmt.Sprintf("get workflow html test report failed, err: %v", err)}) return } - content, err := service.GetWorkflowV4HTMLTestReport(c.Param("workflowName"), c.Param("jobName"), taskID, ginzap.WithContext(c).Sugar()) + + filepath, err = findDefaultHtmlReportFilePath(htmlReportDir, filepath) if err != nil { - c.JSON(500, gin.H{"err": err}) + c.JSON(500, gin.H{"err": err.Error()}) return } - c.Header("content-type", "text/html") - c.String(200, content) + c.FileFromFS(filepath, gin.Dir(htmlReportDir, false)) } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/test_task.go b/pkg/microservice/aslan/core/workflow/testing/service/test_task.go index bdfe83ed76..d4dd9dace0 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/test_task.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/test_task.go @@ -219,7 +219,6 @@ func ListTestTask(testName, projectKey string, pageNum, pageSize int, log *zap.S Skips: testResult.SkipCaseNum, Errors: testResult.ErrorCaseNum, Time: testResult.TestTime, - TestCases: testResult.TestCases, Name: testResult.TestName, } } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/testing.go b/pkg/microservice/aslan/core/workflow/testing/service/testing.go index c1308da67a..7af4aa8951 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/testing.go @@ -23,13 +23,13 @@ import ( "path" "path/filepath" "strconv" - "strings" "time" "github.com/koderover/zadig/v2/pkg/tool/log" "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" + pkgconfig "github.com/koderover/zadig/v2/pkg/config" "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models/msg_queue" @@ -43,9 +43,9 @@ import ( "github.com/koderover/zadig/v2/pkg/setting" e "github.com/koderover/zadig/v2/pkg/tool/errors" s3tool "github.com/koderover/zadig/v2/pkg/tool/s3" + tartool "github.com/koderover/zadig/v2/pkg/tool/tar" "github.com/koderover/zadig/v2/pkg/types" "github.com/koderover/zadig/v2/pkg/types/step" - "github.com/koderover/zadig/v2/pkg/util" ) func CreateTesting(username string, testing *commonmodels.Testing, log *zap.SugaredLogger) error { @@ -387,67 +387,6 @@ func ListCronjob(name, jobType string) ([]*commonmodels.Cronjob, error) { }) } -func GetHTMLTestReport(pipelineName, pipelineType, taskIDStr, testName string, log *zap.SugaredLogger) (string, error) { - if err := validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName, log); err != nil { - return "", e.ErrGetTestReport.AddErr(err) - } - - taskID, err := strconv.ParseInt(taskIDStr, 10, 64) - if err != nil { - log.Errorf("invalid taskID: %s, err: %s", taskIDStr, err) - return "", e.ErrGetTestReport.AddDesc("invalid taskID") - } - - task, err := commonrepo.NewTaskColl().Find(taskID, pipelineName, config.PipelineType(pipelineType)) - if err != nil { - log.Errorf("find task failed, pipelineName: %s, type: %s, taskID: %s, err: %s", pipelineName, config.PipelineType(pipelineType), taskIDStr, err) - return "", e.ErrGetTestReport.AddErr(err) - } - - store, err := s3.UnmarshalNewS3StorageFromEncrypted(task.StorageURI) - if err != nil { - log.Errorf("parse storageURI failed, err: %s", err) - return "", e.ErrGetTestReport.AddErr(err) - } - - if store.Subfolder != "" { - store.Subfolder = fmt.Sprintf("%s/%s/%d/%s", store.Subfolder, pipelineName, taskID, "test") - } else { - store.Subfolder = fmt.Sprintf("%s/%d/%s", pipelineName, taskID, "test") - } - - fileName := fmt.Sprintf("%s-%s-%s-%s-%s-html", pipelineType, pipelineName, taskIDStr, config.TaskTestingV2, testName) - fileName = strings.Replace(strings.ToLower(fileName), "_", "-", -1) - - tmpFilename, err := util.GenerateTmpFile() - if err != nil { - log.Errorf("generate temp file error: %s", err) - return "", e.ErrGetTestReport.AddErr(err) - } - defer func() { - _ = os.Remove(tmpFilename) - }() - client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Region, store.Insecure, store.Provider) - if err != nil { - log.Errorf("download html test report error: %s", err) - return "", e.ErrGetTestReport.AddErr(err) - } - objectKey := store.GetObjectPath(fileName) - err = client.Download(store.Bucket, objectKey, tmpFilename) - if err != nil { - log.Errorf("download html test report error: %s", err) - return "", e.ErrGetTestReport.AddErr(err) - } - - content, err := os.ReadFile(tmpFilename) - if err != nil { - log.Errorf("parse test report file error: %s", err) - return "", e.ErrGetTestReport.AddErr(err) - } - - return string(content), nil -} - func GetWorkflowV4HTMLTestReport(workflowName, jobName string, taskID int64, log *zap.SugaredLogger) (string, error) { workflowTask, err := mongodb.NewworkflowTaskv4Coll().Find(workflowName, taskID) if err != nil { @@ -469,7 +408,7 @@ func GetWorkflowV4HTMLTestReport(workflowName, jobName string, taskID int64, log return "", fmt.Errorf("cannot find job task, workflow name: %s, task id: %d, job name: %s", workflowName, taskID, jobName) } - return downloadHtmlReportFromJobTask(jobTask, workflowName, taskID, log) + return downloadHtmlReportFromJobTask(jobTask, workflowTask.ProjectName, workflowName, taskID, log) } func GetTestTaskHTMLTestReport(testName string, taskID int64, log *zap.SugaredLogger) (string, error) { @@ -491,13 +430,15 @@ func GetTestTaskHTMLTestReport(testName string, taskID int64, log *zap.SugaredLo return "", fmt.Errorf("cannot find job task for test task, workflow name: %s, task id: %d", workflowName, taskID) } - return downloadHtmlReportFromJobTask(jobTask, workflowName, taskID, log) + return downloadHtmlReportFromJobTask(jobTask, workflowTask.ProjectName, workflowName, taskID, log) } -func downloadHtmlReportFromJobTask(jobTask *commonmodels.JobTask, workflowName string, taskID int64, log *zap.SugaredLogger) (string, error) { +func downloadHtmlReportFromJobTask(jobTask *commonmodels.JobTask, projectName, workflowName string, taskID int64, log *zap.SugaredLogger) (string, error) { jobSpec := &commonmodels.JobTaskFreestyleSpec{} if err := commonmodels.IToi(jobTask.Spec, jobSpec); err != nil { - return "", fmt.Errorf("unmashal job spec error: %v", err) + err := fmt.Errorf("unmashal job spec error: %v", err) + log.Error(err) + return "", err } var stepTask *commonmodels.StepTask @@ -505,57 +446,147 @@ func downloadHtmlReportFromJobTask(jobTask *commonmodels.JobTask, workflowName s if step.Name != config.TestJobHTMLReportStepName { continue } - if step.StepType != config.StepArchive { - return "", fmt.Errorf("step: %s was not a html report step", step.Name) + if step.StepType != config.StepArchive && step.StepType != config.StepTarArchive { + err := fmt.Errorf("step: %s was not a html report step", step.Name) + log.Error(err) + return "", err } stepTask = step } if stepTask == nil { - return "", fmt.Errorf("cannot find step task for test task, workflow name: %s, task id: %d", workflowName, taskID) - } - stepSpec := &step.StepArchiveSpec{} - if err := commonmodels.IToi(stepTask.Spec, stepSpec); err != nil { - return "", fmt.Errorf("unmashal step spec error: %v", err) - } - filePath := "" - for _, artifact := range stepSpec.UploadDetail { - fileName := path.Base(artifact.FilePath) - filePath = filepath.Join(artifact.DestinationPath, fileName) + err := fmt.Errorf("cannot find step task for test task, workflow name: %s, task id: %d", workflowName, taskID) + log.Error(err) + return "", err } - store, err := s3.FindDefaultS3() - if err != nil { - log.Errorf("parse storageURI failed, err: %s", err) + htmlReportPath := pkgconfig.LocalHtmlReportPath(projectName, workflowName, taskID) + + // check if exist first + _, err := os.Stat(htmlReportPath) + if err != nil && !os.IsNotExist(err) { + // err + err = fmt.Errorf("failed to check html report file, path: %s, err: %s", htmlReportPath, err) + log.Error(err) return "", e.ErrGetTestReport.AddErr(err) + } else if err == nil { + // exist + return htmlReportPath, nil } - tmpFilename, err := util.GenerateTmpFile() + // not exist + err = os.MkdirAll(htmlReportPath, os.ModePerm) if err != nil { - log.Errorf("generate temp file error: %s", err) + err = fmt.Errorf("failed to create html report path, path: %s, err: %s", htmlReportPath, err) + log.Error(err) return "", e.ErrGetTestReport.AddErr(err) } - defer func() { - _ = os.Remove(tmpFilename) - }() - client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Region, store.Insecure, store.Provider) + + // download it from s3 + store, err := s3.FindDefaultS3() if err != nil { - log.Errorf("download html test report error: %s", err) + err = fmt.Errorf("failed to find default s3, err: %s", err) + log.Error(err) return "", e.ErrGetTestReport.AddErr(err) } - objectKey := store.GetObjectPath(filePath) - err = client.Download(store.Bucket, objectKey, tmpFilename) + + client, err := s3tool.NewClient(store.Endpoint, store.Ak, store.Sk, store.Region, store.Insecure, store.Provider) if err != nil { log.Errorf("download html test report error: %s", err) return "", e.ErrGetTestReport.AddErr(err) } - content, err := os.ReadFile(tmpFilename) - if err != nil { - log.Errorf("parse test report file error: %s", err) - return "", e.ErrGetTestReport.AddErr(err) + if stepTask.StepType == config.StepArchive { + stepSpec := &step.StepArchiveSpec{} + if err := commonmodels.IToi(stepTask.Spec, stepSpec); err != nil { + return "", fmt.Errorf("unmashal step spec error: %v", err) + } + + fpath := "" + fname := "" + for _, artifact := range stepSpec.UploadDetail { + fname = path.Base(artifact.FilePath) + fpath = filepath.Join(artifact.DestinationPath, fname) + } + + objectKey := store.GetObjectPath(fpath) + downloadDest := filepath.Join(htmlReportPath, fname) + err = client.Download(store.Bucket, objectKey, downloadDest) + if err != nil { + err = fmt.Errorf("download html test report error: %s", err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + + return htmlReportPath, nil + } else if stepTask.StepType == config.StepTarArchive { + stepSpec := &step.StepTarArchiveSpec{} + if err := commonmodels.IToi(stepTask.Spec, stepSpec); err != nil { + return "", e.ErrGetTestReport.AddErr(fmt.Errorf("unmashal step spec error: %v", err)) + } + + downloadDest := filepath.Join(htmlReportPath, setting.HtmlReportArchivedFileName) + objectKey := filepath.Join(stepSpec.S3DestDir, stepSpec.FileName) + err = client.Download(store.Bucket, objectKey, downloadDest) + if err != nil { + err = fmt.Errorf("download html test report error: %s", err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + + err = tartool.Untar(downloadDest, htmlReportPath, true) + if err != nil { + err = fmt.Errorf("Untar %s err: %v", downloadDest, err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + + if len(stepSpec.ResultDirs) == 0 { + return "", e.ErrGetTestReport.AddErr(fmt.Errorf("not found html report step in job task")) + } + + unTarFilePath := filepath.Join(htmlReportPath, stepSpec.ResultDirs[0]) + unTarFileInfo, err := os.Stat(unTarFilePath) + if err != nil { + err = fmt.Errorf("failed to stat untar files %s, err: %v", unTarFilePath, err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + + if unTarFileInfo.IsDir() { + untarFiles, err := os.ReadDir(unTarFilePath) + if err != nil { + err = fmt.Errorf("failed to read files in extracted directory, path: %s, err: %s", htmlReportPath, err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + + // Batch move files + for _, file := range untarFiles { + oldPath := filepath.Join(unTarFilePath, file.Name()) + newPath := filepath.Join(htmlReportPath, file.Name()) + err := os.Rename(oldPath, newPath) + if err != nil { + err = fmt.Errorf("failed to move file from %s to %s, err: %s", oldPath, newPath, err) + log.Error(err) + return "", e.ErrGetTestReport.AddErr(err) + } + } + + err = os.Remove(unTarFilePath) + if err != nil { + log.Errorf("remove extracted directory %s err: %v", downloadDest, err) + } + } + + err = os.Remove(downloadDest) + if err != nil { + log.Errorf("remove download file %s err: %v", downloadDest, err) + } + + return htmlReportPath, nil } - return string(content), nil + return "", e.ErrGetTestReport.AddErr(fmt.Errorf("not found html report step in job task")) } func validateTestReportParam(pipelineName, pipelineType, taskIDStr, testName string, log *zap.SugaredLogger) error { diff --git a/pkg/microservice/jobexecutor/core/service/step/step_tar_archive.go b/pkg/microservice/jobexecutor/core/service/step/step_tar_archive.go index 2d4e1b6f35..7031a32051 100644 --- a/pkg/microservice/jobexecutor/core/service/step/step_tar_archive.go +++ b/pkg/microservice/jobexecutor/core/service/step/step_tar_archive.go @@ -32,7 +32,6 @@ import ( "github.com/koderover/zadig/v2/pkg/tool/log" "github.com/koderover/zadig/v2/pkg/tool/s3" "github.com/koderover/zadig/v2/pkg/types/step" - "github.com/koderover/zadig/v2/pkg/util/fs" ) type TarArchiveStep struct { @@ -90,11 +89,6 @@ func (s *TarArchiveStep) Run(ctx context.Context) error { artifactPath = strings.TrimPrefix(artifactPath, "/") artifactPath = filepath.Join(s.workspace, artifactPath) } - isDir, err := fs.IsDir(artifactPath) - if err != nil || !isDir { - log.Errorf("artifactPath is not exist %s or is not dir, err: %s", artifactPath, err) - continue - } cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, artifactPath) } @@ -138,6 +132,7 @@ func (s *TarArchiveStep) Run(ctx context.Context) error { } }() + log.Debugf("tar cmd: %s", cmd.String()) if err = cmd.Run(); err != nil { if s.spec.IgnoreErr { log.Errorf("failed to compress %s, cmd: %s, err: %s", tarName, cmd.String(), err) diff --git a/pkg/microservice/user/core/service/permission/authn.go b/pkg/microservice/user/core/service/permission/authn.go index d303a07e37..fdeb40f46e 100644 --- a/pkg/microservice/user/core/service/permission/authn.go +++ b/pkg/microservice/user/core/service/permission/authn.go @@ -39,7 +39,8 @@ const ( serviceDeployableURLRegExp = `^\/api\/aslan\/service\/services\/[\w-]+\/environments\/deployable$` generalWebhookURLRegExp = `^\/api\/aslan\/workflow\/v4\/generalhook\/[\w-]+\/[^/]+\/webhook$` codeHostAuthURLRegExp = `^\/api\/v1\/codehosts\/\w+\/auth$` - testReportURLRegExp = `^\/api\/aslan\/testing\/report\/workflowv4\/[\w-]+\/id\/\w+\/job\/[^/]+$` + // workflowTestTaskReportURLRegExp = `^\/api\/aslan\/testing\/report\/workflowv4\/[\w-]+\/id\/\w+\/job\/[^/]+$` + // testingTaskReportURLRegExp = `^\/api\/aslan\/testing\/testtask\/[\w-]+\/\w+\/[^/]+$` ) func IsPublicURL(reqPath, method string) bool { @@ -202,7 +203,10 @@ func IsPublicURL(reqPath, method string) bool { return true } - if realPath == "/api/aslan/testing/report" && method == http.MethodGet { + if strings.HasPrefix(realPath, "/api/aslan/testing/report/html/workflowv4") && method == http.MethodGet { + return true + } + if strings.HasPrefix(realPath, "/api/aslan/testing/report/html/testing") && method == http.MethodGet { return true } @@ -257,11 +261,6 @@ func IsPublicURL(reqPath, method string) bool { return true } - match, _ = regexp.MatchString(testReportURLRegExp, realPath) - if match && method == http.MethodGet { - return true - } - return false } diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index b11cc04e98..b2fac07ad6 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -747,7 +747,11 @@ const ( // Note: **Restricted because of product design since v1.9.0**. const AttachedClusterNamespace = "koderover-agent" -const ArtifactResultOut = "artifactResultOut.tar.gz" +// testing constants +const ( + ArtifactResultOut = "artifactResultOut.tar.gz" + HtmlReportArchivedFileName = "htmlReportArchived.tar.gz" +) const ( DefaultReleaseNaming = "$Service$" diff --git a/pkg/tool/tar/tar.go b/pkg/tool/tar/tar.go new file mode 100644 index 0000000000..ba1cbe824a --- /dev/null +++ b/pkg/tool/tar/tar.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" +) + +func Untar(tarFile, destDir string, gzipCompressed bool) error { + file, err := os.Open(tarFile) + if err != nil { + return err + } + defer file.Close() + + var tarReader *tar.Reader + if gzipCompressed { + gzipReader, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzipReader.Close() + tarReader = tar.NewReader(gzipReader) + } else { + tarReader = tar.NewReader(file) + } + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + target := filepath.Join(destDir, header.Name) + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + case tar.TypeReg: + outFile, err := os.Create(target) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tarReader); err != nil { + outFile.Close() + return err + } + outFile.Close() + default: + fmt.Printf("Unable to untar type: %c in file %s", header.Typeflag, header.Name) + } + } + return nil +} diff --git a/pkg/types/step/step_tar_archive.go b/pkg/types/step/step_tar_archive.go index 8ea26ab212..5f382ffc57 100644 --- a/pkg/types/step/step_tar_archive.go +++ b/pkg/types/step/step_tar_archive.go @@ -17,13 +17,20 @@ limitations under the License. package step type StepTarArchiveSpec struct { - ResultDirs []string `bson:"result_dirs" json:"result_dirs" yaml:"result_dirs"` - AbsResultDir bool `bson:"abs_result_dir" json:"abs_result_dir" yaml:"abs_result_dir"` - DestDir string `bson:"dest_dir" json:"dest_dir" yaml:"dest_dir"` - S3DestDir string `bson:"s3_dest_dir" json:"s3_dest_dir" yaml:"s3_dest_dir"` - TarDir string `bson:"tar_dir" json:"tar_dir" yaml:"tar_dir"` - ChangeTarDir bool `bson:"change_tar_dir" json:"change_tar_dir" yaml:"change_tar_dir"` - FileName string `bson:"file_name" json:"file_name" yaml:"file_name"` - IgnoreErr bool `bson:"ignore_err" json:"ignore_err" yaml:"ignore_err"` - S3Storage *S3 `bson:"s3_storage" json:"s3_storage" yaml:"s3_storage"` + // Source file/dir path + ResultDirs []string `bson:"result_dirs" json:"result_dirs" yaml:"result_dirs"` + // Is absolute path in result dir + AbsResultDir bool `bson:"abs_result_dir" json:"abs_result_dir" yaml:"abs_result_dir"` + // Tar dest dir + DestDir string `bson:"dest_dir" json:"dest_dir" yaml:"dest_dir"` + // S3 dest dir + S3DestDir string `bson:"s3_dest_dir" json:"s3_dest_dir" yaml:"s3_dest_dir"` + // Change tar dir, equal to "-C $TarDir" in tar command + TarDir string `bson:"tar_dir" json:"tar_dir" yaml:"tar_dir"` + // Enable change tar dir, equal to "-C" in tar command + ChangeTarDir bool `bson:"change_tar_dir" json:"change_tar_dir" yaml:"change_tar_dir"` + // File name + FileName string `bson:"file_name" json:"file_name" yaml:"file_name"` + IgnoreErr bool `bson:"ignore_err" json:"ignore_err" yaml:"ignore_err"` + S3Storage *S3 `bson:"s3_storage" json:"s3_storage" yaml:"s3_storage"` }