diff --git a/.azure/pipelines/azure-pipelines-external-release.yml b/.azure/pipelines/azure-pipelines-external-release.yml index f11210c27c..be0833dd3a 100644 --- a/.azure/pipelines/azure-pipelines-external-release.yml +++ b/.azure/pipelines/azure-pipelines-external-release.yml @@ -42,18 +42,28 @@ jobs: displayName: NuGet Authenticate - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' + - task: DotNetCoreCLI@2 displayName: dotnet build + enabled: True inputs: projects: '**/Garnet.*.csproj' arguments: -c Release + + - task: PowerShell@2 + displayName: 'Publish the GarnetServer binaries' + inputs: + filePath: .azure/pipelines/createbinaries.ps1 + arguments: 1 + workingDirectory: .azure/pipelines + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@2 - displayName: Sign the binaries + displayName: Sign the binaries for nuget and zipped files enabled: True inputs: ConnectedServiceName: Garnet Code Signing FolderPath: . - Pattern: Garnet.server.dll,Garnet.client.dll,Garnet.common.dll,Garnet.cluster.dll,Garnet.host.dll,HdrHistogram.dll,Tsavorite.core.dll,Tsavorite.devices.AzureStorageDevice.dll,native_device.dll + Pattern: Garnet.server.dll,Garnet.client.dll,Garnet.common.dll,Garnet.cluster.dll,Garnet.host.dll,HdrHistogram.dll,Tsavorite.core.dll,Tsavorite.devices.AzureStorageDevice.dll,native_device.dll,GarnetServer.exe signConfigType: inlineSignParams inlineOperation: >- [ @@ -93,23 +103,38 @@ jobs: "toolVersion": "1.0" } ] - - task: CmdLine@2 - displayName: Command Line Script - inputs: - script: dir + - task: CopyFiles@2 - displayName: 'Copy Files to Artifacts dir: $(build.artifactstagingdirectory)' + displayName: 'Copy Files for Nuget package to Artifacts dir: $(build.artifactstagingdirectory)' + enabled: True inputs: Contents: '**/bin/AnyCPU/$(BuildConfiguration)/**/*' TargetFolder: $(build.artifactstagingdirectory) + - task: NuGetCommand@2 displayName: nuget pack Garnet enabled: True inputs: command: custom arguments: pack Garnet.nuspec -OutputDirectory $(Build.ArtifactStagingDirectory) -Properties Configuration=$(BuildConfiguration) -Symbols -SymbolPackageFormat snupkg -version $(Build.BuildNumber) -Verbosity Detailed + + # Do after Nuget Pack so not part of Nuget Pack + - task: PowerShell@2 + displayName: 'Zip the GarnetServer binaries' + inputs: + filePath: .azure/pipelines/createbinaries.ps1 + arguments: 2 + workingDirectory: .azure/pipelines + + - task: CopyFiles@2 + displayName: 'Copy Zipped Files to Artifacts dir: $(Build.artifactstagingdirectory)' + inputs: + Contents: '**' + SourceFolder: '$(Build.SourcesDirectory)/main/GarnetServer/bin/Release/net8.0/publish/output' + TargetFolder: $(build.artifactstagingdirectory) + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@2 - displayName: Sign the NuGet Package + displayName: Sign the NuGet Package enabled: True inputs: ConnectedServiceName: Garnet Code Signing @@ -133,13 +158,16 @@ jobs: "toolVersion": "1.0" } ] + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: drop' enabled: True + - task: GitHubRelease@0 displayName: 'Create the GitHub release' - enabled: true + enabled: True inputs: + action: 'create' gitHubConnection: ADO_to_Github_ServiceConnection tagSource: manual tag: 'v$(Build.BuildNumber)' @@ -154,10 +182,15 @@ jobs: * https://github.com/microsoft/garnet * https://www.microsoft.com/en-us/research/project/garnet - assets: '$(Build.ArtifactStagingDirectory)/*.nupkg' + assets: | + $(Build.ArtifactStagingDirectory)/*.nupkg + $(Build.ArtifactStagingDirectory)/*.zip + $(Build.ArtifactStagingDirectory)/*.tar.xz + $(Build.ArtifactStagingDirectory)/*.7z + - task: NuGetCommand@2 displayName: 'Push to NuGet.org' - enabled: true + enabled: True inputs: command: push packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg' diff --git a/.azure/pipelines/createbinaries.ps1 b/.azure/pipelines/createbinaries.ps1 new file mode 100644 index 0000000000..2b8e4e9aca --- /dev/null +++ b/.azure/pipelines/createbinaries.ps1 @@ -0,0 +1,136 @@ +<#$f +.SYNOPSIS + This script is designed to publish GarnetServer into various platforms. + +.DESCRIPTION + + Script to publish the GarnetServer executable from various Profiles. It will clean up all the files not needed and by default it will zip them all up in the output directory. + + Parameter: mode + 0 (default) = do both publish and zip + 1 = publish only + 2 = zip only + + The mode allows for special case when running from a pipeline that it only does one of the two parts (publish or zip) at a time. + Doing this allows actions (code signing etc) on the files before it is zipped. Running the script after that can zip up everything (using mode 2). + +.EXAMPLE + ./createbinaries.ps1 + ./createbinaries.ps1 0 + ./createbinaries.ps1 -mode 1 +#> + +# Send the config file for the benchmark. Defaults to a simple one +param ( + [string]$mode = 0 +) + + + +# ******** FUNCTION DEFINITIONS ********* + +################## CleanUpFiles ##################### +# +# Publishes the files and clean it up so only the necessary files will be ready to be zipped +# +###################################################### +function CleanUpFiles { + param ($publishFolder, $platform) + + $publishPath = "$basePath/main/GarnetServer/bin/Release/net8.0/publish/$publishFolder" + $garnetServerEXE = "$publishPath/GarnetServer.exe" + $excludeGarnetServerPDB = 'GarnetServer.pdb' + + # Native binary is different based on OS by default + $nativeFile = "libnative_device.so" + $garnetServerEXE = "$publishPath/GarnetServer" + + if ($platform -match "win-x64") { + $nativeFile = "native_device.dll" + $garnetServerEXE = "$publishPath/GarnetServer.exe" + } + + $nativeRuntimePathFile = "$publishPath/runtimes/$platform/native/$nativeFile" + + if (Test-Path $garnetServerEXE) { + Get-ChildItem -Path $publishPath -Filter '*.xml' | Remove-Item -Force + Get-ChildItem -Path $publishPath -Filter '*.pfx' | Remove-Item -Force + Get-ChildItem -Path $publishPath -Filter *.pdb | Where-Object { $_.Name -ne $excludeGarnetServerPDB } | Remove-Item + + # Copy proper native run time to publish directory + Copy-Item -Path $nativeRuntimePathFile -Destination $publishPath + + # Confirm the files are there + if (Test-Path "$publishPath/$nativeFile") { + + # Delete RunTimes folder + Remove-Item -Path "$publishPath/runtimes" -Recurse -Force + + } else { + Write-Error "$publishPath/$nativeFile does not exist." + } + } else { + Write-Error "$garnetServerEXE was not found." + } +} + +$lastPwd = $pwd + +# Get base path since paths can differ from machine to machine +$string = $pwd.Path +$position = $string.IndexOf(".azure") +$basePath = $string.Substring(0,$position-1) # take off slash off end as well +Set-Location $basePath/main/GarnetServer + +if ($mode -eq 0 -or $mode -eq 1) { + Write-Host "** Publish ... **" + dotnet publish GarnetServer.csproj -p:PublishProfile=linux-arm64-based -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=linux-x64-based -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=osx-arm64-based -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=osx-x64-based -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=portable -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=win-arm64-based-readytorun -f:net8.0 + dotnet publish GarnetServer.csproj -p:PublishProfile=win-x64-based-readytorun -f:net8.0 + + # Clean up all the extra files + CleanUpFiles "linux-arm64" "linux-x64" + CleanUpFiles "linux-x64" "linux-x64" + CleanUpFiles "osx-arm64" "linux-x64" + CleanUpFiles "osx-x64" "linux-x64" + #CleanUpFiles "portable" "win-x64" # don't clean up all files for portable ... leave as is + CleanUpFiles "win-x64" "win-x64" + CleanUpFiles "win-arm64" "win-x64" +} + +if ($mode -eq 0 -or $mode -eq 2) { + + # Make sure the publish folder exists as basic check files are actually published before trying to zip + $publishedFilesFolder = "$basePath/main/GarnetServer/bin/Release/net8.0/publish" + if (!(Test-Path $publishedFilesFolder)) { + Write-Error "$publishedFilesFolder does not exist. Run .\CreateBinaries 1 to publish the binaries first." + exit + } + + # Create the directories + if (!(Test-Path $basePath/main/GarnetServer/bin/Release/net8.0/publish/output)) { + mkdir $basePath/main/GarnetServer/bin/Release/net8.0/publish/output + } + Set-Location $basePath/main/GarnetServer/bin/Release/net8.0/publish/output + + # Compress the files + Write-Host "** Compressing the files ... **" + 7z a -mmt20 -mx5 -scsWIN win-x64-based-readytorun.zip ../win-x64/* + 7z a -mmt20 -mx5 -scsWIN win-arm64-based-readytorun.zip ../win-arm64/* + 7z a -scsUTF-8 linux-x64-based.tar ../linux-x64/* + 7z a -scsUTF-8 linux-arm64-based.tar ../linux-arm64/* + 7z a -scsUTF-8 osx-x64-based.tar ../osx-x64/* + 7z a -scsUTF-8 osx-arm64-based.tar ../osx-arm64/* + 7z a -mmt20 -mx5 -sdel linux-x64-based.tar.xz linux-x64-based.tar + 7z a -mmt20 -mx5 -sdel linux-arm64-based.tar.xz linux-arm64-based.tar + 7z a -mmt20 -mx5 -sdel osx-x64-based.tar.xz osx-x64-based.tar + 7z a -mmt20 -mx5 -sdel osx-arm64-based.tar.xz osx-arm64-based.tar + 7z a -mmt20 -mx5 -scsUTF-8 portable.7z ../portable/* +} + +Write-Host "** DONE! **" +Set-Location $lastPwd diff --git a/main/GarnetServer/Properties/PublishProfiles/linux-arm64-based.pubxml b/main/GarnetServer/Properties/PublishProfiles/linux-arm64-based.pubxml new file mode 100644 index 0000000000..f3fcef046b --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/linux-arm64-based.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\linux-arm64\ + FileSystem + <_TargetId>Folder + net8.0 + linux-arm64 + false + true + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/linux-x64-based.pubxml b/main/GarnetServer/Properties/PublishProfiles/linux-x64-based.pubxml new file mode 100644 index 0000000000..49c61f6121 --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/linux-x64-based.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\linux-x64\ + FileSystem + <_TargetId>Folder + net8.0 + linux-x64 + false + true + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/osx-arm64-based.pubxml b/main/GarnetServer/Properties/PublishProfiles/osx-arm64-based.pubxml new file mode 100644 index 0000000000..a8ede61b7a --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/osx-arm64-based.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\osx-arm64\ + FileSystem + <_TargetId>Folder + net8.0 + osx-arm64 + false + true + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/osx-x64-based.pubxml b/main/GarnetServer/Properties/PublishProfiles/osx-x64-based.pubxml new file mode 100644 index 0000000000..14715051d3 --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/osx-x64-based.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\osx-x64\ + FileSystem + <_TargetId>Folder + net8.0 + osx-x64 + false + true + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/portable.pubxml b/main/GarnetServer/Properties/PublishProfiles/portable.pubxml new file mode 100644 index 0000000000..a48466510a --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/portable.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\portable\ + FileSystem + <_TargetId>Folder + net8.0 + false + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/win-arm64-based-readytorun.pubxml b/main/GarnetServer/Properties/PublishProfiles/win-arm64-based-readytorun.pubxml new file mode 100644 index 0000000000..c02cd8414f --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/win-arm64-based-readytorun.pubxml @@ -0,0 +1,18 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\win-arm64\ + FileSystem + <_TargetId>Folder + net8.0 + win-arm64 + false + true + true + + \ No newline at end of file diff --git a/main/GarnetServer/Properties/PublishProfiles/win-x64-based-readytorun.pubxml b/main/GarnetServer/Properties/PublishProfiles/win-x64-based-readytorun.pubxml new file mode 100644 index 0000000000..0dbb866b72 --- /dev/null +++ b/main/GarnetServer/Properties/PublishProfiles/win-x64-based-readytorun.pubxml @@ -0,0 +1,18 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\win-x64\ + FileSystem + <_TargetId>Folder + net8.0 + win-x64 + false + true + true + + \ No newline at end of file