diff --git a/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/token/TemporaryTokenCreateRequest.kt b/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/token/TemporaryTokenCreateRequest.kt index d6fdad51d4..75ecd29abb 100644 --- a/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/token/TemporaryTokenCreateRequest.kt +++ b/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/pojo/token/TemporaryTokenCreateRequest.kt @@ -52,5 +52,7 @@ data class TemporaryTokenCreateRequest( @ApiModelProperty("允许访问次数,为空表示无限制") val permits: Int? = null, @ApiModelProperty("token类型") - val type: TokenType + val type: TokenType, + @ApiModelProperty("创建人") + val createdBy: String? = null, ) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/TemporaryTokenServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/TemporaryTokenServiceImpl.kt index 47175ce8cf..3a43acc5f3 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/TemporaryTokenServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/impl/TemporaryTokenServiceImpl.kt @@ -66,9 +66,9 @@ class TemporaryTokenServiceImpl( token = generateToken(), permits = permits, type = type, - createdBy = SecurityUtils.getUserId(), + createdBy = createdBy ?: SecurityUtils.getUserId(), createdDate = LocalDateTime.now(), - lastModifiedBy = SecurityUtils.getUserId(), + lastModifiedBy = createdBy ?: SecurityUtils.getUserId(), lastModifiedDate = LocalDateTime.now() ) temporaryTokenRepository.save(temporaryToken) diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt index d3e35b666e..47c557ca64 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt @@ -86,6 +86,7 @@ const val ANALYSIS_EXECUTOR_SERVICE_NAME = "\${service.prefix:}analysis-executor const val HELM_SERVICE_NAME = "\${service.prefix:}helm\${service.suffix:}" const val OCI_SERVICE_NAME = "\${service.prefix:}docker\${service.suffix:}" const val JOB_SERVICE_NAME = "\${service.prefix:}job\${service.suffix:}" +const val SCHEDULE_SERVICE_NAME = "\${service.prefix:}job-schedule\${service.suffix:}" const val FS_SERVER_SERVICE_NAME = "\${service.prefix:}fs-server\${service.suffix:}" const val MAVEN_SERVICE_NAME = "\${service.prefix:}maven\${service.suffix:}" const val ARCHIVE_SERVICE_NAME = "\${service.prefix:}archive\${service.suffix:}" diff --git a/src/backend/job/api-schedule/build.gradle.kts b/src/backend/job/api-schedule/build.gradle.kts new file mode 100644 index 0000000000..88886a2cdc --- /dev/null +++ b/src/backend/job/api-schedule/build.gradle.kts @@ -0,0 +1,4 @@ +dependencies { + implementation(project(":common:common-api")) + compileOnly("org.springframework.cloud:spring-cloud-openfeign-core") +} \ No newline at end of file diff --git a/src/backend/job/api-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/api/JobScheduleClient.kt b/src/backend/job/api-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/api/JobScheduleClient.kt new file mode 100644 index 0000000000..f2a10ebf98 --- /dev/null +++ b/src/backend/job/api-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/api/JobScheduleClient.kt @@ -0,0 +1,20 @@ +package com.tencent.bkrepo.job.schedule.api + +import com.tencent.bkrepo.common.api.constant.SCHEDULE_SERVICE_NAME +import org.springframework.cloud.openfeign.FeignClient +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping + +@FeignClient(SCHEDULE_SERVICE_NAME, contextId = "JobScheduleClient") +@RequestMapping("/service/job") +interface JobScheduleClient { + /** + * 触发任务执行 + * @param id 任务id + * @param executorParam 执行参数 + * */ + @PostMapping("/trigger/{id}") + fun triggerJob(@PathVariable id: String, @RequestBody executorParam: String) +} diff --git a/src/backend/job/boot-job-schedule/build.gradle.kts b/src/backend/job/boot-job-schedule/build.gradle.kts new file mode 100644 index 0000000000..573582b513 --- /dev/null +++ b/src/backend/job/boot-job-schedule/build.gradle.kts @@ -0,0 +1,6 @@ +dependencies { + implementation(project(":common:common-service")) + implementation(project(":common:common-security")) + implementation(project(":job:api-schedule")) + implementation("com.tencent.devops:devops-boot-starter-schedule-server") +} diff --git a/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/JobScheduleApplication.kt b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/JobScheduleApplication.kt new file mode 100644 index 0000000000..18856121c1 --- /dev/null +++ b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/JobScheduleApplication.kt @@ -0,0 +1,11 @@ +package com.tencent.bkrepo.job.schedule + +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication + +@SpringBootApplication +class JobScheduleApplication + +fun main(args: Array) { + runApplication(*args) +} diff --git a/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/config/JobScheduleConfiguration.kt b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/config/JobScheduleConfiguration.kt new file mode 100644 index 0000000000..ef33848eec --- /dev/null +++ b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/config/JobScheduleConfiguration.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.job.schedule.config + +import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class JobScheduleConfiguration { + @Bean + fun httpAuthSecurity(): HttpAuthSecurity { + return HttpAuthSecurity() + .withPrefix("/schedule") + } +} diff --git a/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/controller/JobScheduleController.kt b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/controller/JobScheduleController.kt new file mode 100644 index 0000000000..468a16ea0a --- /dev/null +++ b/src/backend/job/boot-job-schedule/src/main/kotlin/com/tencent/bkrepo/job/schedule/controller/JobScheduleController.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.job.schedule.controller + +import com.tencent.bkrepo.job.schedule.api.JobScheduleClient +import com.tencent.devops.schedule.manager.JobManager +import org.springframework.web.bind.annotation.RestController + +@RestController +class JobScheduleController( + private val jobManager: JobManager, +) : JobScheduleClient { + override fun triggerJob(id: String, executorParam: String) { + jobManager.triggerJob(id, executorParam) + } +} diff --git a/src/backend/job/boot-job-schedule/src/main/resources/bootstrap.yaml b/src/backend/job/boot-job-schedule/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..4c70d39e21 --- /dev/null +++ b/src/backend/job/boot-job-schedule/src/main/resources/bootstrap.yaml @@ -0,0 +1,8 @@ +server.port: 25913 +spring.application.name: job-schedule + +devops: + schedule: + server: + auth: + access-token: xxx \ No newline at end of file diff --git a/src/backend/job/boot-job-worker/build.gradle.kts b/src/backend/job/boot-job-worker/build.gradle.kts new file mode 100644 index 0000000000..afadbb6124 --- /dev/null +++ b/src/backend/job/boot-job-worker/build.gradle.kts @@ -0,0 +1,4 @@ +dependencies { + implementation(project(":common:common-service")) + implementation("com.tencent.devops:devops-boot-starter-schedule-worker") +} \ No newline at end of file diff --git a/src/backend/job/boot-job-worker/shell/media-process.sh b/src/backend/job/boot-job-worker/shell/media-process.sh new file mode 100644 index 0000000000..38c0e8ce68 --- /dev/null +++ b/src/backend/job/boot-job-worker/shell/media-process.sh @@ -0,0 +1,44 @@ +#!/bin/bash +get_field() { + local field_name=$1 + field_value=$(echo $DEVOPS_SCHEDULE_JOB_PARAMETERS| jq -r ".$field_name") + if [ -z "$field_value" ]; then + echo "Error: Field '$field_name' not found in input file" + exit 1 + fi + echo "$field_value" +} + +echo "1. 获取参数" +inputUrl=$(get_field inputUrl) +callbackUrl=$(get_field callbackUrl) +inputFileName=$(get_field inputFileName) +scale=$(get_field scale) +videoCodec=$(get_field videoCodec) +outputFileName=$(get_field outputFileName) + +echo "2. 下载待转码文件 - $inputFileName" +# 使用 -s 参数静默执行,-w "%{http_code}" 输出HTTP状态码 +http_status=$(curl -s -w "%{http_code}" -o $inputFileName $inputUrl) +if [ $http_status -ne 200 ];then + echo "文件下载失败[$http_status],Url: $inputUrl" + exit 1 +fi +ls -l + +echo "3. 开始转码 - $inputFileName > $outputFileName" +echo "ffmpeg -i $inputFileName -vf scale=$scale -c:a copy -c:v $videoCodec $outputFileName" +ffmpeg -i $inputFileName -vf scale=$scale -c:a copy -c:v $videoCodec $outputFileName +if [ $? -ne 0 ];then + echo "转码失败" + exit 1 +fi + +echo "4. 上传转码后的文件 - $outputFileName" +http_status=$(curl -s -w "%{http_code}" -X PUT -T $outputFileName "$callbackUrl") +if [ $http_status -ne 200 ];then + echo 文件上传失败[$http_status],Url: $callbackUrl + exit 1 +fi + +echo "转码完成 - $inputFileName" \ No newline at end of file diff --git a/src/backend/job/boot-job-worker/src/main/kotlin/com/tencent/bkrepo/job/worker/JobWorkerApplication.kt b/src/backend/job/boot-job-worker/src/main/kotlin/com/tencent/bkrepo/job/worker/JobWorkerApplication.kt new file mode 100644 index 0000000000..ad9d318bde --- /dev/null +++ b/src/backend/job/boot-job-worker/src/main/kotlin/com/tencent/bkrepo/job/worker/JobWorkerApplication.kt @@ -0,0 +1,11 @@ +package com.tencent.bkrepo.job.worker + +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication + +@SpringBootApplication +class JobWorkerApplication + +fun main(args: Array) { + runApplication(*args) +} diff --git a/src/backend/job/boot-job-worker/src/main/resources/bootstrap.yaml b/src/backend/job/boot-job-worker/src/main/resources/bootstrap.yaml new file mode 100644 index 0000000000..bdeef8c529 --- /dev/null +++ b/src/backend/job/boot-job-worker/src/main/resources/bootstrap.yaml @@ -0,0 +1,10 @@ +server.port: 25821 +spring.application.name: job-worker + +devops: + schedule: + worker: + mode: DISCOVERY + server: + address: http://${service.prefix}job-schedule + access-token: xxx diff --git a/src/backend/media/biz-media/build.gradle.kts b/src/backend/media/biz-media/build.gradle.kts index d0fabdc34f..08e58450fa 100644 --- a/src/backend/media/biz-media/build.gradle.kts +++ b/src/backend/media/biz-media/build.gradle.kts @@ -1,5 +1,6 @@ dependencies { api(project(":common:common-artifact:artifact-service")) + api(project(":job:api-schedule")) implementation("org.bytedeco:javacpp:${Versions.JavaCpp}") implementation("org.bytedeco:ffmpeg:${Versions.FFmpegPlatform}") implementation("org.bytedeco:javacpp:${Versions.JavaCpp}:windows-x86_64") diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt index e006fece4a..7addbad8fb 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/artifact/repository/MediaLocalRepository.kt @@ -1,7 +1,16 @@ package com.tencent.bkrepo.media.artifact.repository +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository +import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import org.springframework.stereotype.Component @Component -class MediaLocalRepository : LocalRepository() +class MediaLocalRepository : LocalRepository() { + override fun remove(context: ArtifactRemoveContext) { + with(context.artifactInfo) { + val nodeDeleteRequest = NodeDeleteRequest(projectId, repoName, getArtifactFullPath(), context.userId) + nodeClient.deleteNode(nodeDeleteRequest) + } + } +} diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/MediaProperties.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/MediaProperties.kt index 6160993ae5..b7cb147b01 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/MediaProperties.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/config/MediaProperties.kt @@ -1,11 +1,13 @@ package com.tencent.bkrepo.media.config import cn.hutool.core.io.unit.DataSize +import com.tencent.bkrepo.media.stream.TranscodeConfig import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "media") data class MediaProperties( var maxRecordFileSize: DataSize = DataSize.ofGigabytes(100), var serverAddress: String = "", - var fileExpireDays: Int = 180, + var transcodeConfig: Map = mutableMapOf(), + var repoHost: String = "", ) diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/TranscodeController.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/TranscodeController.kt new file mode 100644 index 0000000000..6cb3ed47a6 --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/controller/TranscodeController.kt @@ -0,0 +1,79 @@ +package com.tencent.bkrepo.media.controller + +import com.tencent.bkrepo.auth.pojo.token.TokenType +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.media.artifact.MediaArtifactInfo +import com.tencent.bkrepo.media.artifact.MediaArtifactInfo.Companion.DEFAULT_STREAM_MAPPING_URI +import com.tencent.bkrepo.media.service.TokenService +import com.tencent.bkrepo.media.service.TranscodeService +import com.tencent.bkrepo.media.stream.TranscodeConfig +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestAttribute +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +/** + * 转码资源控制器 + * 用于临时下载文件,和上传转码后的文件 + * */ +@RestController +@RequestMapping("/user/transcode") +class TranscodeController( + private val tokenService: TokenService, + private val transcodeService: TranscodeService, +) { + /** + * 下载源文件 + * @param artifactInfo 要下载的源文件 + * @param token 下载token + * */ + @GetMapping("/download/$DEFAULT_STREAM_MAPPING_URI") + fun download( + artifactInfo: MediaArtifactInfo, + @RequestParam token: String, + ) { + val tokenInfo = tokenService.validateToken(token, artifactInfo, TokenType.DOWNLOAD) + transcodeService.download(artifactInfo) + tokenService.decrementPermits(tokenInfo) + } + + /** + * 回传转码后文件 + * @param artifactInfo 转码后的文件信息 + * @param file 转码后的文件 + * @param token 上传token + * @param origin 转码源文件完整路径 + * */ + @PutMapping("/upload/$DEFAULT_STREAM_MAPPING_URI") + fun callback( + artifactInfo: MediaArtifactInfo, + file: ArtifactFile, + @RequestParam token: String, + @RequestParam origin: String, + @RequestParam originToken: String, + ) { + val tokenInfo = tokenService.validateToken(token, artifactInfo, TokenType.UPLOAD) + val originArtifactInfo = MediaArtifactInfo(artifactInfo.projectId, artifactInfo.repoName, origin) + val originTokenInfo = tokenService.validateToken(originToken, originArtifactInfo, TokenType.UPLOAD) + transcodeService.transcodeCallback(artifactInfo, file, originArtifactInfo) + tokenService.decrementPermits(tokenInfo) + tokenService.decrementPermits(originTokenInfo) + } + + /** + * 视频转码 + * @param artifactInfo 待转码文件 + * @param transcodeConfig 转码配置 + * */ + @PutMapping(DEFAULT_STREAM_MAPPING_URI) + fun transcode( + artifactInfo: MediaArtifactInfo, + @RequestAttribute userId: String, + @RequestBody transcodeConfig: TranscodeConfig, + ) { + transcodeService.transcode(artifactInfo, transcodeConfig, userId) + } +} diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt index 8b7a539d70..2df5645909 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/StreamService.kt @@ -13,13 +13,14 @@ import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.media.STREAM_PATH import com.tencent.bkrepo.media.artifact.MediaArtifactInfo import com.tencent.bkrepo.media.config.MediaProperties -import com.tencent.bkrepo.media.stream.ArtifactFileConsumer +import com.tencent.bkrepo.media.stream.MediaArtifactFileConsumer import com.tencent.bkrepo.media.stream.ArtifactFileRecordingListener import com.tencent.bkrepo.media.stream.ClientStream import com.tencent.bkrepo.media.stream.MediaType import com.tencent.bkrepo.media.stream.RemuxRecordingListener import com.tencent.bkrepo.media.stream.StreamManger import com.tencent.bkrepo.media.stream.StreamMode +import com.tencent.bkrepo.media.stream.TranscodeConfig import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -38,6 +39,7 @@ class StreamService( private val storageManager: StorageManager, private val storageProperties: StorageProperties, private val streamManger: StreamManger, + private val transcodeService: TranscodeService, ) : ArtifactService() { /** @@ -74,7 +76,7 @@ class StreamService( fullPathSet = setOf(STREAM_PATH), type = TokenType.UPLOAD, ) - val token = tokenService.createToken(temporaryTokenRequest) + val token = tokenService.createToken(temporaryTokenRequest).firstOrNull() return "${mediaProperties.serverAddress}/$projectId/$repoName$STREAM_PATH?token=$token" } @@ -100,12 +102,13 @@ class StreamService( val repoId = ArtifactContextHolder.RepositoryId(projectId, repoName) val repo = ArtifactContextHolder.getRepoDetail(repoId) val credentials = repo.storageCredentials ?: storageProperties.defaultStorageCredentials() - val fileConsumer = ArtifactFileConsumer( + val fileConsumer = MediaArtifactFileConsumer( storageManager, + transcodeService, repo, userId, STREAM_PATH, - mediaProperties.fileExpireDays, + getTranscodeConfig(projectId), ) val recordingListener = if (remux) { RemuxRecordingListener(credentials.upload.location, scheduler, saveType, fileConsumer) @@ -130,7 +133,13 @@ class StreamService( repository.download(context) } + private fun getTranscodeConfig(projectId: String): TranscodeConfig? { + val transcodeConfig = mediaProperties.transcodeConfig + return transcodeConfig[projectId] ?: transcodeConfig[DEFAULT] + } + companion object { private val logger = LoggerFactory.getLogger(StreamService::class.java) + private const val DEFAULT = "default" } } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt index 9d7a2c4547..6d58a9daf8 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TokenService.kt @@ -27,8 +27,8 @@ class TokenService( private val temporaryTokenClient: ServiceTemporaryTokenClient, ) { - fun createToken(tokenCreateRequest: TemporaryTokenCreateRequest): String { - return temporaryTokenClient.createToken(tokenCreateRequest).data?.firstOrNull()?.token.orEmpty() + fun createToken(tokenCreateRequest: TemporaryTokenCreateRequest): List { + return temporaryTokenClient.createToken(tokenCreateRequest).data?.map { it.token }.orEmpty() } /** diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt new file mode 100644 index 0000000000..d04f2ad82f --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/service/TranscodeService.kt @@ -0,0 +1,137 @@ +package com.tencent.bkrepo.media.service + +import com.tencent.bkrepo.auth.pojo.token.TemporaryTokenCreateRequest +import com.tencent.bkrepo.auth.pojo.token.TokenType +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactRemoveContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.media.artifact.MediaArtifactInfo +import com.tencent.bkrepo.media.config.MediaProperties +import com.tencent.bkrepo.media.stream.TranscodeParam +import com.tencent.bkrepo.media.stream.TranscodeConfig +import com.tencent.bkrepo.media.stream.TranscodeHelper +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service + +/** + * 视频转码服务 + * */ +@Service +class TranscodeService(private val tokenService: TokenService, private val mediaProperties: MediaProperties) : + ArtifactService() { + + /** + * 视频转码 + * */ + fun transcode(artifactInfo: ArtifactInfo, transcodeConfig: TranscodeConfig, userId: String) { + val transcodeParam = generateTranscodeParam(artifactInfo, transcodeConfig, userId) + TranscodeHelper.addTask(transcodeConfig.jobId, transcodeParam) + logger.info("Add transcode task for artifact[$artifactInfo]") + } + + /** + * 下载视频 + * */ + fun download(artifactInfo: MediaArtifactInfo) { + with(artifactInfo) { + val repo = ArtifactContextHolder.getRepoDetail() + ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) + val context = ArtifactDownloadContext(repo, artifactInfo) + repository.download(context) + } + } + + /** + * 转码成功后回调 + * 删除原视频,保留转码后的视频 + * */ + fun transcodeCallback( + newArtifactInfo: MediaArtifactInfo, + newArtifactFile: ArtifactFile, + originArtifactInfo: MediaArtifactInfo, + ) { + with(newArtifactInfo) { + val repo = ArtifactContextHolder.getRepoDetail() + ?: throw ErrorCodeException(ArtifactMessageCode.REPOSITORY_NOT_FOUND, repoName) + logger.info("File[$originArtifactInfo] transcode successful") + val context = ArtifactUploadContext(repo, newArtifactFile, newArtifactInfo) + repository.upload(context) + logger.info("Upload new file[$newArtifactInfo]") + val removeContext = ArtifactRemoveContext(repo, originArtifactInfo) + repository.remove(removeContext) + logger.info("Delete origin file[$originArtifactInfo]") + } + } + + private fun generateTranscodeParam( + artifactInfo: ArtifactInfo, + transcodeConfig: TranscodeConfig, + userId: String, + ): TranscodeParam { + with(transcodeConfig) { + val outputArtifactInfo = covertOutputArtifactInfo(artifactInfo, scale) + val (inputUrl, callbackUrl) = generateUrl( + artifactInfo, + outputArtifactInfo, + mediaProperties.repoHost, + userId, + ) + return TranscodeParam( + inputUrl = inputUrl, + callbackUrl = callbackUrl, + scale = scale, + videoCodec = videoCodec, + audioCodec = audioCodec, + inputFileName = artifactInfo.getResponseName(), + outputFileName = outputArtifactInfo.getResponseName(), + ) + } + } + + private fun covertOutputArtifactInfo(input: ArtifactInfo, scale: String): ArtifactInfo { + with(input) { + val name = getResponseName() + val (fileName, fileType) = name.split(".") + val outFileName = "${fileName}_$scale.$fileType" + val outputFilePath = getArtifactFullPath().replace(name, outFileName) + return ArtifactInfo(projectId, repoName, outputFilePath) + } + } + + private fun generateUrl( + input: ArtifactInfo, + output: ArtifactInfo, + host: String, + userId: String, + ): Pair { + val inputToken = createAccessToken(input, userId) + val outputToken = createAccessToken(output, userId) + val downloadUrl = "$host/media/user/transcode/download$input?token=$inputToken" + val callbackUrl = "$host/media/user/transcode/upload$output?token=$outputToken" + + "&origin=${input.getArtifactFullPath()}&originToken=$inputToken" + return Pair(downloadUrl, callbackUrl) + } + + private fun createAccessToken(artifactInfo: ArtifactInfo, userId: String): String { + with(artifactInfo) { + val tokenRequest = TemporaryTokenCreateRequest( + projectId = projectId, + repoName = repoName, + fullPathSet = setOf(getArtifactFullPath()), + type = TokenType.ALL, + createdBy = userId, + ) + return tokenService.createToken(tokenRequest).firstOrNull().orEmpty() + } + } + + companion object { + private val logger = LoggerFactory.getLogger(TranscodeService::class.java) + } +} diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt index 7e05764d04..3ec2fe39d9 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileRecordingListener.kt @@ -8,7 +8,7 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler * */ class ArtifactFileRecordingListener( private val artifactFile: ChunkedArtifactFile, - private val artifactFileConsumer: ArtifactFileConsumer, + private val fileConsumer: FileConsumer, private val type: MediaType, scheduler: ThreadPoolTaskScheduler, ) : AsyncStreamListener(scheduler) { @@ -31,7 +31,7 @@ class ArtifactFileRecordingListener( private fun storeFile() { try { artifactFile.finish() - artifactFileConsumer.accept(artifactFile, name) + fileConsumer.accept(artifactFile, name) } finally { artifactFile.delete() } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/FileConsumer.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/FileConsumer.kt index af1556cbcb..8e8ec1025f 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/FileConsumer.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/FileConsumer.kt @@ -1,5 +1,6 @@ package com.tencent.bkrepo.media.stream +import com.tencent.bkrepo.common.artifact.api.ArtifactFile import java.io.File import java.util.function.Consumer @@ -14,4 +15,6 @@ interface FileConsumer : Consumer { * @param name 文件名 * */ fun accept(file: File, name: String) + + fun accept(file: ArtifactFile, name: String) } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileConsumer.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt similarity index 80% rename from src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileConsumer.kt rename to src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt index 064a32faa0..6d7ed3f1e7 100644 --- a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/ArtifactFileConsumer.kt +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/MediaArtifactFileConsumer.kt @@ -4,20 +4,23 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.toArtifactFile import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.media.service.TranscodeService import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import org.slf4j.LoggerFactory import java.io.File /** * 将文件保存为制品构件 * */ -class ArtifactFileConsumer( +class MediaArtifactFileConsumer( private val storageManager: StorageManager, + private val transcodeService: TranscodeService, private val repo: RepositoryDetail, private val userId: String, private val path: String, - private val expireDays: Int, + private val transcodeConfig: TranscodeConfig? = null, ) : FileConsumer { private val startTime = System.currentTimeMillis() @@ -29,11 +32,15 @@ class ArtifactFileConsumer( accept(file.toArtifactFile(), name) } - fun accept(file: ArtifactFile, name: String) { + override fun accept(file: ArtifactFile, name: String) { val filePath = "$path/$name" val artifactInfo = ArtifactInfo(repo.projectId, repo.name, filePath) val nodeCreateRequest = buildNodeCreateRequest(artifactInfo, file, userId) storageManager.storeArtifactFile(nodeCreateRequest, file, repo.storageCredentials) + if (transcodeConfig != null) { + transcodeService.transcode(artifactInfo, transcodeConfig, userId) + logger.info("Add transcode task for artifact[$artifactInfo]") + } } private fun buildNodeCreateRequest( @@ -51,7 +58,6 @@ class ArtifactFileConsumer( size = file.getSize(), sha256 = file.getFileSha256(), md5 = file.getFileMd5(), - expires = expireDays.toLong(), operator = userId, nodeMetadata = listOf( MetadataModel(key = METADATA_KEY_MEDIA_START_TIME, value = startTime, system = true), @@ -62,6 +68,7 @@ class ArtifactFileConsumer( } companion object { + private val logger = LoggerFactory.getLogger(MediaArtifactFileConsumer::class.java) private const val METADATA_KEY_MEDIA_START_TIME = "media.startTime" private const val METADATA_KEY_MEDIA_STOP_TIME = "media.stopTime" } diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt new file mode 100644 index 0000000000..bc785920ce --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeConfig.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.media.stream + +data class TranscodeConfig( + var scale: String = "", // 分辨率,比如1280x720 + var videoCodec: String = "", // 视频编码 + var audioCodec: String = "", // 音频编码 + var jobId: String = "", // 转码任务id +) diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeHelper.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeHelper.kt new file mode 100644 index 0000000000..a880635e30 --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeHelper.kt @@ -0,0 +1,27 @@ +package com.tencent.bkrepo.media.stream + +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.job.schedule.api.JobScheduleClient +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component + +@Component +class TranscodeHelper(jobScheduleClient: JobScheduleClient) { + init { + Companion.jobScheduleClient = jobScheduleClient + } + + companion object { + private val logger = LoggerFactory.getLogger(TranscodeHelper::class.java) + private lateinit var jobScheduleClient: JobScheduleClient + + fun addTask( + jobId: String, + transcodeParam: TranscodeParam, + ) { + val jobParam = transcodeParam.toJsonString() + jobScheduleClient.triggerJob(jobId, jobParam) + logger.debug("Add transcode task {}", transcodeParam) + } + } +} diff --git a/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt new file mode 100644 index 0000000000..fdf6c9ce85 --- /dev/null +++ b/src/backend/media/biz-media/src/main/kotlin/com/tencent/bkrepo/media/stream/TranscodeParam.kt @@ -0,0 +1,11 @@ +package com.tencent.bkrepo.media.stream + +data class TranscodeParam( + val inputUrl: String, // 源文件下载路径 + val callbackUrl: String, // 转码后回调地址,上传转码后的文件 + val scale: String? = null, // 分辨率,比如1280x720 + val videoCodec: String? = null, // 视频编码 + val audioCodec: String? = null, // 音频编码 + var inputFileName: String, // 源文件名 + var outputFileName: String, // 输出文件名 +)