Feature/add pgsql database connection (#1121)

* Add entityframework + PGSql nuget packages

* Add PGSql database context + initial database creation

.github/workflows/build.yml

@@ -28,11 +28,11 @@ jobs:
       working-directory: src/
 
     - name: Restore dependencies
-      run: dotnet restore
+      run: dotnet restore AzureIoTHub.Portal.sln 
       working-directory: src/
 
     - name: Build
-      run: dotnet build --no-restore -p:ClientAssetsRestoreCommand="npm ci"
+      run: dotnet build AzureIoTHub.Portal.sln --no-restore -p:ClientAssetsRestoreCommand="npm ci"
       working-directory: src/
 
     - name: Generate Open API documentation

.github/workflows/codeql.yml

@@ -45,7 +45,7 @@ jobs:
         queries: +security-and-quality,security-extended
 
     - name: Build
-      run: dotnet build --configuration Release
+      run: dotnet build AzureIoTHub.Portal.sln --configuration Release
       working-directory: src/
 
     - name: Perform CodeQL Analysis

src/AzureIoTHub.Portal.Tests.Unit/Server/DevelopmentConfigHandlerTests.cs

@@ -46,6 +46,7 @@ private DevelopmentConfigHandler CreateDevelopmentConfigHandler()
         [TestCase(ConfigHandler.IoTHubConnectionStringKey, nameof(ConfigHandler.IoTHubConnectionString))]
         [TestCase(ConfigHandler.DPSConnectionStringKey, nameof(ConfigHandler.DPSConnectionString))]
         [TestCase(ConfigHandler.StorageAccountConnectionStringKey, nameof(ConfigHandler.StorageAccountConnectionString))]
+        [TestCase(ConfigHandler.PostgreSQLConnectionStringKey, nameof(ConfigHandler.PostgreSQLConnectionString))]
         public void SettingsShouldGetValueFromAppSettings(string configKey, string configPropertyName)
         {
             // Arrange

src/AzureIoTHub.Portal.Tests.Unit/Server/ProductionConfigHandlerTests.cs

@@ -36,6 +36,7 @@ private ProductionConfigHandler CreateProductionConfigHandler()
         [TestCase(ConfigHandler.DPSConnectionStringKey, nameof(ConfigHandler.DPSConnectionString))]
         [TestCase(ConfigHandler.StorageAccountConnectionStringKey, nameof(ConfigHandler.StorageAccountConnectionString))]
         [TestCase(ConfigHandler.LoRaKeyManagementCodeKey, nameof(ConfigHandler.LoRaKeyManagementCode))]
+        [TestCase(ConfigHandler.PostgreSQLConnectionStringKey, nameof(ConfigHandler.PostgreSQLConnectionString))]
         public void SecretsShouldGetValueFromConnectionStrings(string configKey, string configPropertyName)
         {
             // Arrange

src/AzureIoTHub.Portal.sln

@@ -54,6 +54,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ISSUE_TEMPLATES", "ISSUE_TE
 		..\.github\ISSUE_TEMPLATE\user_story.md = ..\.github\ISSUE_TEMPLATE\user_story.md
 	EndProjectSection
 EndProject
+Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{B9D2DE01-84DE-461F-998C-20B57E4AA021}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -76,6 +78,10 @@ Global
 		{51FD5B90-B422-47BF-83F2-516520CFB124}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{51FD5B90-B422-47BF-83F2-516520CFB124}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{51FD5B90-B422-47BF-83F2-516520CFB124}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B9D2DE01-84DE-461F-998C-20B57E4AA021}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B9D2DE01-84DE-461F-998C-20B57E4AA021}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B9D2DE01-84DE-461F-998C-20B57E4AA021}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B9D2DE01-84DE-461F-998C-20B57E4AA021}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/AzureIoTHub.Portal/Server/AzureIoTHub.Portal.Server.csproj

@@ -44,13 +44,20 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.8">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="6.0.8" />
+    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="6.0.8" />
     <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.8" />
     <PackageReference Include="Microsoft.Graph" Version="4.38.0" />
     <PackageReference Include="Microsoft.OpenApi" Version="1.3.2" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.17.0" />
     <PackageReference Include="MudBlazor" Version="6.0.15" />
     <PackageReference Include="NetEscapades.AspNetCore.SecurityHeaders" Version="0.17.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.6" />
     <PackageReference Include="prometheus-net.AspNetCore" Version="6.0.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
@@ -86,5 +93,8 @@
   <PropertyGroup>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <NoWarn>$(NoWarn);</NoWarn>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+    <DockerfileContext>..\..</DockerfileContext>
+    <DockerComposeProjectPath>..\..\docker-compose.dcproj</DockerComposeProjectPath>
   </PropertyGroup>
 </Project>

src/AzureIoTHub.Portal/Server/ConfigHandler.cs

@@ -16,6 +16,7 @@ public abstract class ConfigHandler
         internal const string DPSServiceEndpointKey = "IoTDPS:ServiceEndpoint";
         internal const string DPSIDScopeKey = "IoTDPS:IDScope";
         internal const string UseSecurityHeadersKey = "UseSecurityHeaders";
+        internal const string PostgreSQLConnectionStringKey = "PostgreSQL:ConnectionString";
 
         internal const string OIDCScopeKey = "OIDC:Scope";
         internal const string OIDCAuthorityKey = "OIDC:Authority";
@@ -116,5 +117,7 @@ internal static ConfigHandler Create(IWebHostEnvironment env, IConfiguration con
         internal abstract string IdeasAuthenticationHeader { get; }
 
         internal abstract string IdeasAuthenticationToken { get; }
+
+        internal abstract string PostgreSQLConnectionString { get; }
     }
 }

src/AzureIoTHub.Portal/Server/DevelopmentConfigHandler.cs

@@ -68,5 +68,7 @@ internal DevelopmentConfigHandler(IConfiguration config)
         internal override string IdeasUrl => this.config.GetValue(IdeasUrlKey, string.Empty);
         internal override string IdeasAuthenticationHeader => this.config.GetValue(IdeasAuthenticationHeaderKey, "Ocp-Apim-Subscription-Key");
         internal override string IdeasAuthenticationToken => this.config.GetValue(IdeasAuthenticationTokenKey, string.Empty);
+
+        internal override string PostgreSQLConnectionString => this.config[PostgreSQLConnectionStringKey];
     }
 }

src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.cs

@@ -0,0 +1,20 @@
+// Copyright (c) CGI France. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace AzureIoTHub.Portal.Server.Migrations
+{
+    using Microsoft.EntityFrameworkCore.Migrations;
+
+    public partial class InitialCreate : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+
+        }
+    }
+}

src/AzureIoTHub.Portal/Server/Migrations/PortalDbContextModelSnapshot.cs

@@ -0,0 +1,26 @@
+// <auto-generated />
+using AzureIoTHub.Portal.Server.Model;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace AzureIoTHub.Portal.Server.Migrations
+{
+    [DbContext(typeof(PortalDbContext))]
+    partial class PortalDbContextModelSnapshot : ModelSnapshot
+    {
+        protected override void BuildModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "6.0.8")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+#pragma warning restore 612, 618
+        }
+    }
+}

src/AzureIoTHub.Portal/Server/Model/PortalDbContext.cs

@@ -0,0 +1,14 @@
+// Copyright (c) CGI France. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace AzureIoTHub.Portal.Server.Model
+{
+    using Microsoft.EntityFrameworkCore;
+
+    public class PortalDbContext : DbContext
+    {
+        public PortalDbContext(DbContextOptions<PortalDbContext> options)
+            : base(options)
+        { }
+    }
+}

src/AzureIoTHub.Portal/Server/ProductionConfigHandler.cs

@@ -30,6 +30,8 @@ internal ProductionConfigHandler(IConfiguration config)
 
         internal override string StorageAccountConnectionString => this.config.GetConnectionString(StorageAccountConnectionStringKey);
 
+        internal override string PostgreSQLConnectionString => this.config.GetConnectionString(PostgreSQLConnectionStringKey);
+
         internal override int StorageAccountDeviceModelImageMaxAge => this.config.GetValue(StorageAccountDeviceModelImageMaxAgeKey, 86400);
 
         internal override bool UseSecurityHeaders => this.config.GetValue(UseSecurityHeadersKey, true);

src/AzureIoTHub.Portal/Server/Startup.cs

@@ -11,6 +11,7 @@ namespace AzureIoTHub.Portal.Server
     using AutoMapper;
     using Azure;
     using Azure.Storage.Blobs;
+    using AzureIoTHub.Portal.Server.Model;
     using Exceptions;
     using Extensions;
     using Factories;
@@ -28,6 +29,7 @@ namespace AzureIoTHub.Portal.Server
     using Microsoft.AspNetCore.Mvc.Versioning;
     using Microsoft.Azure.Devices;
     using Microsoft.Azure.Devices.Provisioning.Service;
+    using Microsoft.EntityFrameworkCore;
     using Microsoft.Extensions.Configuration;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Hosting;
@@ -95,6 +97,9 @@ public void ConfigureServices(IServiceCollection services)
                     opts.TokenValidationParameters.ValidateTokenReplay = configuration.OIDCValidateTokenReplay;
                 });
 
+            _ = services
+                .AddDbContext<PortalDbContext>(opts => opts.UseNpgsql(configuration.PostgreSQLConnectionString));
+
             _ = services.AddSingleton(configuration);
             _ = services.AddSingleton(new PortalMetric());
 
@@ -259,6 +264,7 @@ Specify the authorization token got from your IDP as a header.
             _ = services.AddSingleton(mapper);
 
             _ = services.AddHealthChecks()
+                .AddDbContextCheck<PortalDbContext>()
                 .AddCheck<IoTHubHealthCheck>("iothubHealth")
                 .AddCheck<StorageAccountHealthCheck>("storageAccountHealth")
                 .AddCheck<TableStorageHealthCheck>("tableStorageHealth")
@@ -365,11 +371,12 @@ public async void Configure(IApplicationBuilder app, IWebHostEnvironment env)
                 });
             });
 
-
             var deviceModelImageManager = app.ApplicationServices.GetService<IDeviceModelImageManager>();
 
             await deviceModelImageManager?.InitializeDefaultImageBlob()!;
             await deviceModelImageManager?.SyncImagesCacheControl()!;
+
+            await EnsureDatabaseCreatedAndUpToDate(app)!;
         }
 
         private static void UseApiExceptionMiddleware(IApplicationBuilder app)
@@ -392,5 +399,15 @@ private Task HandleApiFallback(HttpContext context)
             context.Response.StatusCode = StatusCodes.Status404NotFound;
             return Task.CompletedTask;
         }
+
+        private static async Task EnsureDatabaseCreatedAndUpToDate(IApplicationBuilder app)
+        {
+            using var scope = app.ApplicationServices.CreateScope();
+
+            using var context = scope.ServiceProvider.GetRequiredService<PortalDbContext>();
+
+            // Create the database if not exists and migrate it using the database bigration scripts.
+            await context.Database.MigrateAsync();
+        }
     }
 }

src/docker-compose.dcproj

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" Sdk="Microsoft.Docker.Sdk">
+  <PropertyGroup Label="Globals">
+    <ProjectVersion>2.1</ProjectVersion>
+    <DockerTargetOS>Linux</DockerTargetOS>
+    <ProjectGuid>b9d2de01-84de-461f-998c-20b57e4aa021</ProjectGuid>
+    <DockerLaunchAction>LaunchBrowser</DockerLaunchAction>
+    <DockerServiceUrl>{Scheme}://localhost:{ServicePort}/{Scheme}://{ServiceHost}:{ServicePort}</DockerServiceUrl>
+    <DockerServiceName>azureiothub.portal.server</DockerServiceName>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Include="docker-compose.override.yml">
+      <DependentUpon>docker-compose.yml</DependentUpon>
+    </None>
+    <None Include="docker-compose.yml" />
+    <None Include=".dockerignore" />
+  </ItemGroup>
+</Project>

src/docker-compose.override.yml

@@ -0,0 +1,13 @@
+version: '3.4'
+
+services:
+  azureiothub.portal.server:
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
+      - ASPNETCORE_URLS=https://+:443;http://+:80
+    ports:
+      - "80"
+      - "8001:443"
+    volumes:
+      - ${APPDATA}/Microsoft/UserSecrets:/root/.microsoft/usersecrets:ro
+      - ${APPDATA}/ASP.NET/Https:/root/.aspnet/https:ro

src/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.4'
+
+services:
+  database:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: postgrePassword
+  azureiothub.portal.server:
+    image: ${DOCKER_REGISTRY-}azureiothubportalserver
+    build:
+      context: .
+      dockerfile: AzureIoTHub.Portal/Server/Dockerfile

src/launchSettings.json

@@ -0,0 +1,15 @@
+{
+  "profiles": {
+    "Docker Compose": {
+      "commandName": "DockerCompose",
+      "commandVersion": "1.0",
+      "composeLaunchAction": "LaunchBrowser",
+      "composeLaunchServiceName": "azureiothub.portal.server",
+      "composeLaunchUrl": "https://localhost:8001",
+      "serviceActions": {
+        "azureiothub.portal.server": "StartDebugging",
+        "database": "StartWithoutDebugging"
+      }
+    }
+  }
+}