Skip to content

Commit

Permalink
support host static html resources of test report
Browse files Browse the repository at this point in the history
Signed-off-by: Patrick Zhao <zhaoyu@koderover.com>
  • Loading branch information
PetrusZ committed Nov 29, 2024
1 parent 700f5a9 commit d4cbb87
Show file tree
Hide file tree
Showing 19 changed files with 451 additions and 228 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
"github.com/koderover/zadig/v2/pkg/cli/zadig-agent/internal/common/types"
"github.com/koderover/zadig/v2/pkg/tool/s3"
"github.com/koderover/zadig/v2/pkg/types/step"
"github.com/koderover/zadig/v2/pkg/util/fs"
)

type TarArchiveStep struct {
Expand Down Expand Up @@ -78,18 +77,18 @@ func (s *TarArchiveStep) Run(ctx context.Context) error {
cmdAndArtifactFullPaths := make([]string, 0)
cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, "-czf")
cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, tarName)
if s.spec.ChangeTarDir {
cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, "--exclude", tarName, "-C", s.spec.TarDir)
}

for _, artifactPath := range s.spec.ResultDirs {
if len(artifactPath) == 0 {
continue
}
artifactPath = helper.ReplaceEnvWithValue(artifactPath, envMap)
artifactPath = strings.TrimPrefix(artifactPath, "/")

artifactPath := filepath.Join(s.workspace, artifactPath)
isDir, err := fs.IsDir(artifactPath)
if err != nil || !isDir {
s.logger.Errorf("artifactPath is not exist %s or is not dir, err: %s", artifactPath, err)
continue
if !s.spec.AbsResultDir {
artifactPath = strings.TrimPrefix(artifactPath, "/")
artifactPath = filepath.Join(s.workspace, artifactPath)
}
cmdAndArtifactFullPaths = append(cmdAndArtifactFullPaths, artifactPath)
}
Expand All @@ -111,8 +110,11 @@ func (s *TarArchiveStep) Run(ctx context.Context) error {
if err != nil {
return fmt.Errorf("failed to close %s err: %s", tarName, err)
}

cmd := exec.Command("tar", cmdAndArtifactFullPaths...)
cmd.Stderr = os.Stderr

log.Debugf("tar cmd: %s", cmd.String())
if err = cmd.Run(); err != nil {
if s.spec.IgnoreErr {
s.logger.Errorf("failed to compress %s, cmd: %s, err: %s", tarName, cmd.String(), err)
Expand Down
7 changes: 3 additions & 4 deletions pkg/cli/zadig-agent/internal/agent/step/step.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,9 @@ func RunStep(ctx context.Context, jobCtx *jobctl.JobContext, step *commonmodels.
case "debug_after":
return nil
default:
//err := fmt.Errorf("step type: %s does not match any known type", step.StepType)
//log.Error(err)
//return err
logger.Infof(fmt.Sprintf("step type: %s does not match any known type", step.StepType))
err := fmt.Errorf("step type: %s does not match any known type", step.StepType)
log.Error(err)
return err
}
if err := stepInstance.Run(ctx); err != nil {
return err
Expand Down
4 changes: 4 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,10 @@ func LocalChartTemplatePath(name string) string {
return LocalTemplatePath(name, setting.ChartTemplatesPath)
}

func LocalHtmlReportPath(projectName, workflowName string, taskID int64) string {
return filepath.Join(DataPath(), "project", projectName, "workflow", workflowName, "task", fmt.Sprintf("%d", taskID), "html-report") + "/"
}

func MongoURI() string {
return viper.GetString(setting.ENVMongoDBConnectionString)
}
Expand Down
101 changes: 101 additions & 0 deletions pkg/microservice/aslan/core/clean_cache_files.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2024 The KodeRover Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
"fmt"
"os"
"path/filepath"
"time"

commonconfig "github.com/koderover/zadig/v2/pkg/config"
"github.com/koderover/zadig/v2/pkg/tool/log"
)

func cleanCacheFiles() {
projectPath := filepath.Join(commonconfig.DataPath(), "project")
err := cleanProjectFiles(projectPath)
if err != nil {
log.Errorf("[cleanCacheFiles] failed to clean project cache files: %v", err)
}
}

func cleanProjectFiles(path string) error {
return traverseDir(path, cleanWorkflowFiles)
}

func cleanWorkflowFiles(path string, info os.FileInfo) error {
workflowPath := filepath.Join(path, "workflow")
return traverseDir(workflowPath, cleanTaskFiles)
}

func cleanTaskFiles(path string, info os.FileInfo) error {
taskPath := filepath.Join(path, "task")
return traverseDir(taskPath, cleanHtmlReportFiles)
}

func cleanHtmlReportFiles(path string, info os.FileInfo) error {
htmlReportPath := filepath.Join(path, "html-report")
htmlReportInfo, err := os.Stat(htmlReportPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to stat htmlReportPath: %s, err: %v", htmlReportPath, err)
}

if time.Since(htmlReportInfo.ModTime()) > 7*24*time.Hour {
err = os.RemoveAll(htmlReportPath)
if err != nil {
return fmt.Errorf("failed to remove htmlReportPath: %s, err: %v", htmlReportPath, err)
}
}
return nil
}

func traverseDir(path string, fn func(string, os.FileInfo) error) error {
pathInfo, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to stat path: %s, err: %v", path, err)
}

if !pathInfo.IsDir() {
return fmt.Errorf("path is not a directory: %s", path)
}

files, err := os.ReadDir(path)
if err != nil {
return fmt.Errorf("failed to read directory: %s, err: %v", path, err)
}

for _, file := range files {
info, err := file.Info()
if err != nil {
return fmt.Errorf("failed to get file info: %s, err: %v", file.Name(), err)
}
if info.IsDir() {
err = fn(filepath.Join(path, file.Name()), info)
if err != nil {
return err
}
}
}
return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,17 @@ func (w *WorkflowV4) CalculateHash() [md5.Size]byte {
return md5.Sum(jsonBytes)
}

type ParameterSettingType string

const (
StringType ParameterSettingType = "string"
ChoiceType ParameterSettingType = "choice"
ImageType ParameterSettingType = "image"
Script ParameterSettingType = "script"
// Deprecated
ExternalType ParameterSettingType = "external"
)

type WorkflowStage struct {
Name string `bson:"name" yaml:"name" json:"name"`
Parallel bool `bson:"parallel" yaml:"parallel" json:"parallel"`
Expand Down
2 changes: 2 additions & 0 deletions pkg/microservice/aslan/core/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,8 @@ func initCron() {
log.Infof("[CRONJOB] gitlab token updated....")
})

Scheduler.Every(1).Days().At("04:00").Do(cleanCacheFiles)

Scheduler.StartAsync()
}

Expand Down
6 changes: 1 addition & 5 deletions pkg/microservice/aslan/core/stat/handler/openapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ type getRollbackStatDetail struct {
}

func (req *getRollbackStatDetail) Validate() error {
if req.ProjectKey == "" {
return e.ErrInvalidParam.AddDesc("projectKey is empty")
}

if req.StartTime == 0 || req.EndTime == 0 {
return e.ErrInvalidParam.AddDesc("starTime and endTime is empty")
}
Expand Down Expand Up @@ -109,7 +105,7 @@ func (req *getRollbackStatDetail) Validate() error {
// @Tags OpenAPI
// @Accept json
// @Produce json
// @Param projectKey query string true "项目标识"
// @Param projectKey query string false "项目标识"
// @Param envName query string false "环境名称"
// @Param serviceName query string false "服务名称"
// @Param startTime query int true "开始时间,格式为时间戳"
Expand Down
4 changes: 4 additions & 0 deletions pkg/microservice/aslan/core/system/handler/capacity.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package handler

import (
"encoding/json"
"fmt"

"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -59,6 +60,9 @@ func UpdateStrategy(c *gin.Context) {
return
}

bs, _ := json.Marshal(args)
internalhandler.InsertOperationLog(c, ctx.UserName, "", "更新", "任务配置", "", string(bs), ctx.Logger)

if err := service.UpdateSysCapStrategy(args); err != nil {
ctx.RespErr = err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -565,51 +565,36 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul
StepType: config.StepDebugAfter,
}
jobTaskSpec.Steps = append(jobTaskSpec.Steps, debugAfterStep)
// init archive html step
if len(testingInfo.TestReportPath) > 0 {
ext := filepath.Ext(testingInfo.TestReportPath)
if ext != ".html" {
return jobTask, fmt.Errorf("test report path: %s is not a html file", testingInfo.TestReportPath)
}
outputPath := strings.TrimSuffix(testingInfo.TestReportPath, ext) + "-archive" + ext

archiveHtmlStep := &commonmodels.StepTask{
Name: config.TestJobHTMLReportArchiveStepName,
JobName: jobTask.Name,
StepType: config.StepArchiveHtml,
Onfailure: true,
Spec: step.StepArchiveHtmlSpec{
HtmlPath: testingInfo.TestReportPath,
OutputPath: outputPath,
},
}
jobTaskSpec.Steps = append(jobTaskSpec.Steps, archiveHtmlStep)
tarDestDir := "/tmp"
if testingInfo.ScriptType == types.ScriptTypeBatchFile {
tarDestDir = "%TMP%"
} else if testingInfo.ScriptType == types.ScriptTypePowerShell {
tarDestDir = "%TMP%"
}

uploads := []*step.Upload{
{
FilePath: outputPath,
DestinationPath: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "html"),
},
}
archiveStep := &commonmodels.StepTask{
// init archive html step
if len(testingInfo.TestReportPath) > 0 {
testReportDir := filepath.Dir(testingInfo.TestReportPath)
testReportName := filepath.Base(testingInfo.TestReportPath)
tarArchiveStep := &commonmodels.StepTask{
Name: config.TestJobHTMLReportStepName,
JobName: jobTask.Name,
StepType: config.StepArchive,
StepType: config.StepTarArchive,
Onfailure: true,
Spec: step.StepArchiveSpec{
UploadDetail: uploads,
S3: modelS3toS3(defaultS3),
Spec: &step.StepTarArchiveSpec{
FileName: setting.HtmlReportArchivedFileName,
AbsResultDir: true,
ResultDirs: []string{testReportName},
ChangeTarDir: true,
TarDir: "$WORKSPACE/" + testReportDir,
DestDir: tarDestDir,
S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "html-report"),
},
}
jobTaskSpec.Steps = append(jobTaskSpec.Steps, archiveStep)
jobTaskSpec.Steps = append(jobTaskSpec.Steps, tarArchiveStep)
}

destDir := "/tmp"
if testingInfo.ScriptType == types.ScriptTypeBatchFile {
destDir = "%TMP%"
} else if testingInfo.ScriptType == types.ScriptTypePowerShell {
destDir = "%TMP%"
}
// init test result storage step
if len(testingInfo.ArtifactPaths) > 0 {
tarArchiveStep := &commonmodels.StepTask{
Expand All @@ -621,7 +606,7 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul
ResultDirs: testingInfo.ArtifactPaths,
S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "test-result"),
FileName: setting.ArtifactResultOut,
DestDir: destDir,
DestDir: tarDestDir,
},
}
if len(testingInfo.ArtifactPaths) > 1 || testingInfo.ArtifactPaths[0] != "" {
Expand All @@ -644,7 +629,7 @@ func (j *TestingJob) toJobtask(jobSubTaskID int, testing *commonmodels.TestModul
S3DestDir: path.Join(j.workflow.Name, fmt.Sprint(taskID), jobTask.Name, "junit"),
TestName: testing.Name,
TestProject: testing.ProjectName,
DestDir: destDir,
DestDir: tarDestDir,
FileName: "merged.xml",
ServiceName: serviceName,
ServiceModule: serviceModule,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ func (*Router) Inject(router *gin.RouterGroup) {
// 查看html测试报告不做鉴权
testReport := router.Group("report")
{
testReport.GET("", GetHTMLTestReport)
testReport.GET("workflowv4/:workflowName/id/:id/job/:jobName", GetWorkflowV4HTMLTestReport)
testReport.GET("/html/testing/:projectName/:testingName/:taskID/*path", GetTestTaskHtmlReportInfo)
testReport.GET("/html/workflowv4/:projectName/:workflowName/:taskID/*path", GetWorkflowV4HTMLTestReport)
}

// sse apis
Expand Down Expand Up @@ -103,7 +103,6 @@ func (*Router) Inject(router *gin.RouterGroup) {
testTask.DELETE("", CancelTestTaskV3)
testTask.GET("/detail", GetTestTaskInfo)
testTask.GET("/report", GetTestTaskJUnitReportInfo)
testTask.GET("/html_report", GetTestTaskHtmlReportInfo)
testTask.POST("/restart", RestartTestTaskV2)
testTask.GET("/artifact", GetTestingTaskArtifact)
// TODO: below is the deprecated apis, remove after 2.2.0
Expand Down
Loading

0 comments on commit d4cbb87

Please sign in to comment.