feat: gRPC support (#2)

.github/workflows/deploy.yml

@@ -3,6 +3,8 @@ on:
   release:
     types: [published]
   push:
+    branches:
+      - 'main'
 
 jobs:
   docker:

.github/workflows/typings.yml

@@ -0,0 +1,45 @@
+name: Deploy Client Package
+on:
+  push:
+    branches:
+      - 'main'
+  workflow_dispatch:
+
+jobs:
+  package:
+    name: Publish Client Package
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Checkout submodules
+        shell: bash
+        run: |
+          git config --global url."https://github.com/".insteadOf "git@github.com:"
+          auth_header="$(git config --local --get http.https://github.com/.extraheader)"
+          git submodule sync --recursive
+          git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
+
+      - name: Install Node v16
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+          registry-url: https://registry.npmjs.org/
+
+      - name: Enter Directory
+        run: cd typings
+
+      - name: Install dependencies
+        run: yarn install --immutable
+
+      - name: Build Client and Types
+        run: yarn suite
+
+      - name: Deploy to Github Package Registry
+        run: yarn npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}

ExportAPI/ExportAPI.csproj

@@ -6,10 +6,16 @@
 
   <ItemGroup>
     <ProjectReference Include="..\DiscordChatExporter\DiscordChatExporter.Core\DiscordChatExporter.Core.csproj" />
+    <Protobuf Include="Protos\export.proto" GrpcServices="Server" />
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Calzolari.Grpc.AspNetCore.Validation" Version="6.1.1" />
+    <PackageReference Include="FluentValidation" Version="10.3.6" />
+    <PackageReference Include="Grpc.AspNetCore" Version="2.43.0-pre1" />
+    <PackageReference Include="Grpc.AspNetCore.Server.Reflection" Version="2.43.0-pre1" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Gress" Version="2.0.1" />
   </ItemGroup>
 
 </Project>

ExportAPI/Interceptors/ExceptionInterceptor.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Grpc.Core.Interceptors;
+using Microsoft.Extensions.Logging;
+
+public class ExceptionInterceptor:  Interceptor
+{
+    private readonly ILogger<ExceptionInterceptor> _logger;
+
+    public ExceptionInterceptor(ILogger<ExceptionInterceptor> logger)
+    {
+        _logger = logger;
+    }
+
+    public override async Task<TResponse> UnaryServerHandler<TRequest, TResponse>(
+        TRequest request,
+        ServerCallContext context,
+        UnaryServerMethod<TRequest, TResponse> continuation)
+    {
+        try
+        {
+            return await continuation(request, context);
+        }
+        catch (Exception exception)
+        {
+            throw new RpcException(new Status(StatusCode.Internal, exception.ToString()));
+        }
+    }
+}

ExportAPI/Program.cs

@@ -1,5 +1,6 @@
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
 
 namespace ExportAPI
 {

ExportAPI/Protos/export.proto

@@ -0,0 +1,36 @@
+syntax = "proto3";
+
+option csharp_namespace = "ExportAPI.Proto";
+
+service Exporter {
+  rpc CreateExport (CreateExportRequest) returns (stream CreateExportResponse);
+}
+
+enum ExportFormat {
+	PlainText = 0;
+	HtmlDark = 1;
+	HtmlLight = 2;
+	CSV = 3;
+	JSON = 4;
+}
+
+message CreateExportRequest {
+  string token = 1;
+  string channel_id = 2;
+  ExportFormat export_format = 3;
+  string date_format = 4;
+  string after = 5;
+  string before = 6;
+}
+
+message CreateExportResponse {
+  oneof ResponseType {
+    double progress = 1;
+    ExportComplete data = 2;
+  }
+}
+
+message ExportComplete {
+  int32 message_count = 1;
+  bytes data = 2;
+}

ExportAPI/Services/ExportService.cs

@@ -0,0 +1,142 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Microsoft.Extensions.Logging;
+using ExportAPI.Proto;
+using DiscordChatExporter.Core.Discord;
+using DiscordChatExporter.Core.Exceptions;
+using DiscordChatExporter.Core.Exporting;
+using DiscordChatExporter.Core.Exporting.Partitioning;
+using DiscordChatExporter.Core.Exporting.Filtering;
+using DiscordChatExporter.Core.Discord.Data;
+using Gress;
+using Google.Protobuf;
+using Google.Protobuf.WellKnownTypes;
+
+namespace ExportAPI.Services;
+
+public class ExporterService : Exporter.ExporterBase
+{
+	private readonly ILogger _logger;
+
+	private const int ChunkSize = 1024 * 32; // 32 KB
+
+	public ExporterService(ILoggerFactory loggerFactory)
+	{
+		_logger = loggerFactory.CreateLogger<ExporterService>();
+
+		if (!Directory.Exists("/tmp/exports"))
+		{
+			Directory.CreateDirectory("/tmp/exports");
+		}
+
+		var nowhex = DateTime.Now.Ticks.ToString("X2");
+		if (!Directory.Exists($"/tmp/exports/{nowhex}"))
+		{
+			Directory.CreateDirectory($"/tmp/exports/{nowhex}");
+		}
+	}
+
+	internal static string GetPath(string channelId, DiscordChatExporter.Core.Exporting.ExportFormat exportType)
+	{
+		var nowhex = DateTime.Now.Ticks.ToString("X2");
+		return $"/tmp/exports/{nowhex}/{channelId}.{exportType.GetFileExtension()}";
+	}
+
+	internal void deleteFile(string path)
+	{
+		var exists = System.IO.File.Exists(path);
+		if (exists)
+		{
+			try
+			{
+				System.IO.File.Delete(path);
+				_logger.LogInformation($"Deleted {path}");
+			}
+			catch { }
+		}
+
+	}
+
+	public override async Task CreateExport(CreateExportRequest options, IServerStreamWriter<CreateExportResponse> responseStream, ServerCallContext context)
+	{
+
+		var ef = options.ExportFormat;
+		var exportFormat = (DiscordChatExporter.Core.Exporting.ExportFormat)ef;
+
+		var parsed = Snowflake.TryParse(options.ChannelId);
+		var channelId = parsed ?? Snowflake.Zero;
+
+		var client = new DiscordClient(options.Token);
+		client._tokenKind = TokenKind.Bot;
+		Channel channel;
+		try
+		{
+			channel = await client.GetChannelAsync(channelId);
+		}
+		catch (DiscordChatExporterException e)
+		{
+			if (e.Message.Contains("Authentication"))
+			{
+				throw new RpcException(new Status(StatusCode.PermissionDenied, "An invalid Discord token was provided."));
+			}
+			if (e.Message.Contains("Requested resource does not exist"))
+			{
+				throw new RpcException(new Status(StatusCode.NotFound, "A channel with the provided ID was not found."));
+			}
+			throw new RpcException(new Status(StatusCode.Unknown, $"An unknown error occurred: {e.Message}"));
+		}
+
+		var guild = await client.GetGuildAsync(channel.GuildId);
+		var res = await client.GetJsonResponseAsync("users/@me");
+		var me = DiscordChatExporter.Core.Discord.Data.User.Parse(res);
+
+		var path = GetPath(channel.Id.ToString(), exportFormat);
+		_logger.LogInformation($"[{me.FullName} ({me.Id})] Exporting #{channel.Name} ({channel.Id}) within {guild.Name} ({guild.Id}) to {path}");
+		var request = new ExportRequest(
+			guild,
+			channel,
+			path,
+			exportFormat,
+			Snowflake.TryParse(options.After),
+			Snowflake.TryParse(options.Before),
+			PartitionLimit.Null,
+			MessageFilter.Null,
+			false,
+			false,
+			options.DateFormat
+		);
+
+		var exporter = new ChannelExporter(client);
+
+		_logger.LogInformation("Starting export");
+		var progress = new Progress<double>(p => responseStream.WriteAsync(new CreateExportResponse { Progress = p }));
+		var messageCount = await exporter.ExportChannelAsync(request, progress);
+		_logger.LogInformation("Finished exporting");
+
+
+		var buffer = new byte[ChunkSize];
+		await using var readStream = File.OpenRead(path);
+		while (true)
+		{
+			var count = await readStream.ReadAsync(buffer);
+
+			if (count == 0)
+			{
+				break;
+			}
+
+			Console.WriteLine("Sending file data chunk of length " + count);
+			await responseStream.WriteAsync(new CreateExportResponse
+			{
+				Data = new ExportComplete {
+					MessageCount = messageCount,
+					Data = UnsafeByteOperations.UnsafeWrap(buffer.AsMemory(0, count))
+				}
+			});
+		}
+
+		deleteFile(path);
+	}
+}

ExportAPI/Startup.cs

@@ -2,44 +2,62 @@
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Calzolari.Grpc.AspNetCore.Validation;
 using Microsoft.Extensions.Hosting;
+using ExportAPI.Services;
+using ExportAPI.Validators;
 
 namespace ExportAPI
 {
-    public class Startup
-    {
-        public Startup(IConfiguration configuration)
-        {
-            Configuration = configuration;
-        }
-
-        public IConfiguration Configuration { get; }
-
-        // This method gets called by the runtime. Use this method to add services to the container.
-        public void ConfigureServices(IServiceCollection services)
-        {
-            services.AddControllers();
-            // services.AddControllers().AddJsonOptions(jo => {
-            //     jo.JsonSerializerOptions.PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance;
-            // });
-        }
-
-        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
-        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
-        {
-            if (env.IsDevelopment())
-            {
-                app.UseDeveloperExceptionPage();
-            } else {
-                app.UseHsts();
-            }
-
-            app.UseRouting();
-
-            app.UseEndpoints(endpoints =>
-            {
-                endpoints.MapControllers();
-            });
-        }
-    }
+	public class Startup
+	{
+		public Startup(IConfiguration configuration)
+		{
+			Configuration = configuration;
+		}
+
+		public IConfiguration Configuration { get; }
+
+		// This method gets called by the runtime. Use this method to add services to the container.
+		public void ConfigureServices(IServiceCollection services)
+		{
+			services.AddControllers();
+
+			services.AddGrpc(options =>
+			{
+				options.Interceptors.Add<ExceptionInterceptor>();
+				options.EnableMessageValidation();
+			});
+			services.AddGrpcReflection();
+
+			services.AddValidator<CreateExportRequestValidator>();
+			services.AddGrpcValidation();
+		}
+
+		// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
+		public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+		{
+			if (env.IsDevelopment())
+			{
+				app.UseDeveloperExceptionPage();
+			}
+			else
+			{
+				app.UseHsts();
+			}
+
+			app.UseRouting();
+
+			app.UseEndpoints(endpoints =>
+			{
+				endpoints.MapGrpcService<ExporterService>();
+				endpoints.MapControllers();
+
+				if (env.IsDevelopment())
+				{
+					endpoints.MapGrpcReflectionService();
+				}
+			});
+		}
+	}
 }