Skip to content

Commit

Permalink
Increased CDN cache duration from 90 to 180 days. Use beta releases o…
Browse files Browse the repository at this point in the history
…f Unfucked and DataSizeUnits libraries instead of local builds. Update some MIME types for PGP files.
  • Loading branch information
Aldaviva committed Oct 1, 2024
1 parent e2de7a1 commit e84b4f3
Show file tree
Hide file tree
Showing 9 changed files with 210 additions and 189 deletions.
2 changes: 1 addition & 1 deletion RaspberryPiDotnetRepository/Azure/BlobStorageClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public interface BlobStorageClient {
public class BlobStorageClientImpl(BlobContainerClient container, UploadProgressFactory uploadProgress, IOptions<Options> options, ILogger<BlobStorageClientImpl> logger)
: BlobStorageClient, IDisposable {

private static readonly TimeSpan CACHE_DURATION = TimeSpan.FromDays(90);
private static readonly TimeSpan CACHE_DURATION = TimeSpan.FromDays(180); // the max Azure CDN cache duration is 366 days

private readonly SemaphoreSlim uploadSemaphore = new(options.Value.storageParallelUploads);

Expand Down
72 changes: 36 additions & 36 deletions RaspberryPiDotnetRepository/Azure/CdnClient.cs
Original file line number Diff line number Diff line change
@@ -1,37 +1,37 @@
using Azure;
using Azure.ResourceManager.Cdn;
using Azure.ResourceManager.Cdn.Models;

namespace RaspberryPiDotnetRepository.Azure;

/*
* There seems to be no upper limit to the duration of a certificate that Azure apps can use, unlike client secrets, which must be rotated at least once every 2 years.
*
* You can generate a new certificate using PowerShell:
*
* New-SelfSignedCertificate -KeyAlgorithm RSA -KeyLength 2048 -CertStoreLocation "Cert:\CurrentUser\My" -KeyExportPolicy Exportable -KeySpec Signature -Subject "CN=RaspberryPiDotnetRepository" -NotAfter (Get-Date).AddYears(100)
*
* Then, use certmgr.msc to export this certificate as a CER file without the private key, and upload it to portal.azure.com > App registrations > your app > Certificates & secrets.
*
* Next, export the same certificate as a PFX file with the private key and a password, and pass its absolute path as certFilePath.
*
* You may now delete the cert from the Personal store if you want.
*/
public interface CdnClient {

Task purge();

}

public class CdnClientImpl(CdnEndpointResource? cdnEndpoint, ILogger<CdnClientImpl> logger): CdnClient {

public async Task purge() {
if (cdnEndpoint != null) {
await cdnEndpoint.PurgeContentAsync(WaitUntil.Started, new PurgeContent(["/dists/*", "/badges/*"]));
logger.LogInformation("Starting CDN purge, will finish asynchronously later");
} else {
logger.LogInformation("No CDN configured, not purging");
}
}

using Azure;
using Azure.ResourceManager.Cdn;
using Azure.ResourceManager.Cdn.Models;

namespace RaspberryPiDotnetRepository.Azure;

/*
* There seems to be no upper limit to the duration of a certificate that Azure apps can use, unlike client secrets, which must be rotated at least once every 2 years.
*
* You can generate a new certificate using PowerShell:
*
* New-SelfSignedCertificate -KeyAlgorithm RSA -KeyLength 2048 -CertStoreLocation "Cert:\CurrentUser\My" -KeyExportPolicy Exportable -KeySpec Signature -Subject "CN=RaspberryPiDotnetRepository" -NotAfter (Get-Date).AddYears(100)
*
* Then, use certmgr.msc to export this certificate as a CER file without the private key, and upload it to portal.azure.com > App registrations > your app > Certificates & secrets.
*
* Next, export the same certificate as a PFX file with the private key and a password, and pass its absolute path as certFilePath.
*
* You may now delete the cert from the Personal store if you want.
*/
public interface CdnClient {

Task purge(IEnumerable<string> paths);

}

public class CdnClientImpl(CdnEndpointResource? cdnEndpoint, ILogger<CdnClientImpl> logger): CdnClient {

public async Task purge(IEnumerable<string> paths) {
if (cdnEndpoint != null) {
await cdnEndpoint.PurgeContentAsync(WaitUntil.Started, new PurgeContent(paths));
logger.LogInformation("Starting CDN purge, will finish asynchronously later");
} else {
logger.LogInformation("No CDN configured, not purging");
}
}

}
173 changes: 81 additions & 92 deletions RaspberryPiDotnetRepository/Debian/Package/PackageBuilder.cs
Original file line number Diff line number Diff line change
@@ -1,93 +1,82 @@
using LibObjectFile.Ar;
using RaspberryPiDotnetRepository.Data.ControlMetadata;
using SharpCompress.Common;
using SharpCompress.Compressors;
using SharpCompress.Compressors.Deflate;
using SharpCompress.IO;
using SharpCompress.Writers;
using SharpCompress.Writers.GZip;
using SharpCompress.Writers.Tar;
using TarWriter = Unfucked.Compression.Writers.Tar.TarWriter;

namespace RaspberryPiDotnetRepository.Debian.Package;

//TODO update comments for new Control type
/// <summary>
/// Create a Debian package with given files to install and control metadata.
///
/// <list type="number">
/// <item><description>Construct a new <see cref="PackageBuilderImpl"/> instance</description></item>
/// <item><description>Set <see cref="control"/> to be the control metadata (see <see href="https://www.debian.org/doc/debian-policy/ch-controlfields.html"/>)</description></item>
/// <item><description>Add files to install by getting <see cref="data"/> and calling <see cref="TarWriter.WriteFile"/>, <see cref="TarWriter.WriteDirectory"/>, or <see cref="TarWriter.WriteSymLink"/> as many times as you want on it</description></item>
/// <item><description>Create a destination stream (like a <see cref="FileStream"/>) and call <see cref="build"/> to save the package to a .deb file</description></item>
/// </list>
/// </summary>
public interface PackageBuilder: IAsyncDisposable, IDisposable {

CompressionLevel gzipCompressionLevel { get; set; }
TarWriter data { get; }

Task build(Control control, Stream output);

}

public class PackageBuilderImpl: PackageBuilder {

public const string CONTROL_ARCHIVE_FILENAME = "control.tar.gz";

public CompressionLevel gzipCompressionLevel { get; set; } = CompressionLevel.Default;
public TarWriter data { get; }

private readonly Stream dataArchiveStream = new MemoryStream();
private readonly GZipStream dataGzipStream;

public PackageBuilderImpl() {
dataGzipStream = new GZipStream(NonDisposingStream.Create(dataArchiveStream), CompressionMode.Compress, gzipCompressionLevel);
data = new TarWriter(dataGzipStream, new TarWriterOptions(CompressionType.None, true));
}

public async Task build(Control control, Stream output) {
data.Dispose();
await dataGzipStream.DisposeAsync();
dataArchiveStream.Position = 0;

await using Stream controlArchiveStream = new MemoryStream();
using (IWriter controlArchiveWriter = WriterFactory.Open(controlArchiveStream, ArchiveType.Tar, new GZipWriterOptions { CompressionLevel = gzipCompressionLevel })) {
await using Stream controlFileBuffer = control.serialize().ToStream();
controlArchiveWriter.Write("./control", controlFileBuffer);
}

controlArchiveStream.Position = 0;

ArArchiveFile debArchive = new() { Kind = ArArchiveKind.Common };
debArchive.AddFile(new ArBinaryFile {
Name = "debian-binary",
Stream = "2.0\n".ToStream()
});
debArchive.AddFile(new ArBinaryFile {
Name = CONTROL_ARCHIVE_FILENAME,
Stream = controlArchiveStream
});
debArchive.AddFile(new ArBinaryFile {
Name = "data.tar.gz",
Stream = dataArchiveStream
});

debArchive.Write(output);
}

public void Dispose() {
data.Dispose();
dataGzipStream.Dispose();
dataArchiveStream.Dispose();
GC.SuppressFinalize(this);
}

public async ValueTask DisposeAsync() {
data.Dispose();
await dataGzipStream.DisposeAsync();
await dataArchiveStream.DisposeAsync();
GC.SuppressFinalize(this);
}

using LibObjectFile.Ar;
using RaspberryPiDotnetRepository.Data.ControlMetadata;
using SharpCompress.Common;
using SharpCompress.Compressors;
using SharpCompress.Compressors.Deflate;
using SharpCompress.IO;
using SharpCompress.Writers;
using SharpCompress.Writers.GZip;
using SharpCompress.Writers.Tar;
using TarWriter = Unfucked.Compression.Writers.Tar.TarWriter;

namespace RaspberryPiDotnetRepository.Debian.Package;

public interface PackageBuilder: IAsyncDisposable, IDisposable {

CompressionLevel gzipCompressionLevel { get; set; }
TarWriter data { get; }

Task build(Control control, Stream output);

}

public class PackageBuilderImpl: PackageBuilder {

public const string CONTROL_ARCHIVE_FILENAME = "control.tar.gz";

public CompressionLevel gzipCompressionLevel { get; set; } = CompressionLevel.Default;
public TarWriter data { get; }

private readonly Stream dataArchiveStream = new MemoryStream();
private readonly GZipStream dataGzipStream;

public PackageBuilderImpl() {
dataGzipStream = new GZipStream(NonDisposingStream.Create(dataArchiveStream), CompressionMode.Compress, gzipCompressionLevel);
data = new TarWriter(dataGzipStream, new TarWriterOptions(CompressionType.None, true));
}

public async Task build(Control control, Stream output) {
data.Dispose();
await dataGzipStream.DisposeAsync();
dataArchiveStream.Position = 0;

await using Stream controlArchiveStream = new MemoryStream();
using (IWriter controlArchiveWriter = WriterFactory.Open(controlArchiveStream, ArchiveType.Tar, new GZipWriterOptions { CompressionLevel = gzipCompressionLevel })) {
await using Stream controlFileBuffer = control.serialize().ToByteStream();
controlArchiveWriter.Write("./control", controlFileBuffer);
}

controlArchiveStream.Position = 0;

ArArchiveFile debArchive = new() { Kind = ArArchiveKind.Common };
debArchive.AddFile(new ArBinaryFile {
Name = "debian-binary",
Stream = "2.0\n".ToByteStream()
});
debArchive.AddFile(new ArBinaryFile {
Name = CONTROL_ARCHIVE_FILENAME,
Stream = controlArchiveStream
});
debArchive.AddFile(new ArBinaryFile {
Name = "data.tar.gz",
Stream = dataArchiveStream
});

debArchive.Write(output);
}

public void Dispose() {
data.Dispose();
dataGzipStream.Dispose();
dataArchiveStream.Dispose();
GC.SuppressFinalize(this);
}

public async ValueTask DisposeAsync() {
data.Dispose();
await dataGzipStream.DisposeAsync();
await dataArchiveStream.DisposeAsync();
GC.SuppressFinalize(this);
}

}
7 changes: 4 additions & 3 deletions RaspberryPiDotnetRepository/Orchestrator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ await Task.WhenAll(generatedPackages.Where(p => !p.isUpToDateInBlobStorage).Sele
// Upload InRelease index files to Azure Blob Storage
Task<BlobContentInfo?[]> releaseIndexUploads = Task.WhenAll(releaseIndexFiles.Where(file => !file.isUpToDateInBlobStorage).SelectMany(file =>
new[] { file.inreleaseFilePathRelativeToRepo, file.releaseFilePathRelativeToRepo, file.releaseGpgFilePathRelativeToRepo }.Select(relativeFilePath =>
blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath, "text/plain", ct))));
blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath, Path.GetExtension(relativeFilePath) == ".gpg" ? "application/pgp-signature" : "text/plain",
ct))));

await packageIndexUploads;
await releaseIndexUploads;
Expand All @@ -82,10 +83,10 @@ await Task.WhenAll(generatedPackages.Where(p => !p.isUpToDateInBlobStorage).Sele
await Task.WhenAll(badgeFiles.Where(file => !file.isUpToDateInBlobStorage)
.Select(file => blobStorage.uploadFile(Path.Combine(repoBaseDir, file.filePathRelativeToRepo), file.filePathRelativeToRepo, "application/json", ct)));
await blobStorage.uploadFile(Path.Combine(repoBaseDir, readmeFilename), readmeFilename, "text/plain", ct);
await blobStorage.uploadFile(Path.Combine(repoBaseDir, gpgPublicKeyFile), gpgPublicKeyFile, "application/octet-stream", ct);
await blobStorage.uploadFile(Path.Combine(repoBaseDir, gpgPublicKeyFile), gpgPublicKeyFile, "application/pgp-keys", ct);

// Clear CDN cache
await cdnClient.purge();
await cdnClient.purge(["/dists/*", "/badges/*", "/manifest.json"]);

// Upload manifest.json file to Azure Blob Storage
await blobStorage.uploadFile(manifestManager.manifestFilePath, manifestManager.manifestFilename, "application/json", ct);
Expand Down
32 changes: 10 additions & 22 deletions RaspberryPiDotnetRepository/RaspberryPiDotnetRepository.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<NoWarn>8524</NoWarn>
<Version>1.0.0</Version>
<Version>1.0.1</Version>
<Authors>Ben Hutchison</Authors>
<Copyright>© 2024 $(Authors)</Copyright>
<Company>$(Authors)</Company>
Expand All @@ -24,13 +24,19 @@
<ItemGroup>
<PackageReference Include="Azure.Identity" Version="1.12.0" />
<PackageReference Include="Azure.ResourceManager.Cdn" Version="1.3.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.21.2" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.22.0" />
<PackageReference Include="Bom.Squad" Version="0.3.0" />
<PackageReference Include="DataSizeUnits" Version="3.0.0-beta1" />
<PackageReference Include="LibObjectFile" Version="0.6.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="PgpCore" Version="6.5.0" />
<PackageReference Include="SharpCompress" Version="0.37.2" />
<PackageReference Include="PgpCore" Version="6.5.1" />
<PackageReference Include="SharpCompress" Version="0.38.0" />
<PackageReference Include="System.Text.Json" Version="8.0.4" /> <!-- pinned to non-vulnerable version -->
<PackageReference Include="ThrottleDebounce" Version="2.0.0" />
<PackageReference Include="Unfucked" Version="0.0.0-beta3" />
<PackageReference Include="Unfucked.Compression" Version="0.0.0-beta2" />
<PackageReference Include="Unfucked.DI" Version="0.0.0-beta2" />
<PackageReference Include="Unfucked.PGP" Version="0.0.0-beta2" />
</ItemGroup>

<ItemGroup>
Expand All @@ -42,24 +48,6 @@
</None>
</ItemGroup>

<ItemGroup>
<Reference Include="DataSizeUnits">
<HintPath>..\..\DataSizeUnits\DataSizeUnits\bin\Debug\net5.0\DataSizeUnits.dll</HintPath>
</Reference>
<Reference Include="Unfucked">
<HintPath>..\..\Unfucked\Unfucked\bin\Debug\net8.0\Unfucked.dll</HintPath>
</Reference>
<Reference Include="Unfucked.Compression">
<HintPath>..\..\Unfucked\Unfucked.Compression\bin\Debug\netstandard2.0\Unfucked.Compression.dll</HintPath>
</Reference>
<Reference Include="Unfucked.DependencyInjection">
<HintPath>..\..\Unfucked\Unfucked.DependencyInjection\bin\Debug\net6.0\Unfucked.DependencyInjection.dll</HintPath>
</Reference>
<Reference Include="Unfucked.PGP">
<HintPath>..\..\Unfucked\Unfucked.PGP\bin\Debug\netstandard2.0\Unfucked.PGP.dll</HintPath>
</Reference>
</ItemGroup>

<PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true' or '$(Configuration)' == 'Release'">
<ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
</PropertyGroup>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using DataSizeUnits;
using System.Diagnostics;

namespace RaspberryPiDotnetRepository.Debian.Package;
namespace RaspberryPiDotnetRepository;

public interface StatisticsService {

Expand Down
1 change: 1 addition & 0 deletions RaspberryPiDotnetRepository/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"cdnTenantId": null,

// Insert the application/client ID of an Azure OAuth app registration (Azure Portal > App registrations > your app > Overview > Essentials > Application (client) ID)
// This application must be granted a role with both Microsoft.Cdn/profiles/endpoints/read and Microsoft.Cdn/profiles/endpoints/Purge/action permissions on the Resource Group, such as the built-in CDN Endpoint Contributor role, or a custom role that has those two permissions.
"cdnClientId": null,

/*
Expand Down
Loading

0 comments on commit e84b4f3

Please sign in to comment.