Initialize VMR

.vault-config/product-construction-dev.yaml

@@ -0,0 +1,38 @@
+storageLocation:
+  type: azure-key-vault
+  parameters:
+    subscription: cab65fc3-d077-467d-931f-3932eabf36d3
+    name: ProductConstructionDev
+
+references:
+  helixkv:
+    type: azure-key-vault
+    parameters:
+      subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1
+      name: helixkv
+
+  engkeyvault:
+    type: azure-key-vault
+    parameters:
+      subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1
+      name: engkeyvault
+
+secrets:
+  BotAccount-dotnet-bot-repo-PAT:
+    type: github-access-token
+    parameters:
+      gitHubBotAccountSecret: 
+        location: engkeyvault
+        name: BotAccount-dotnet-bot
+      gitHubBotAccountName: dotnet-bot
+
+  dn-bot-all-orgs-code-r:
+    type: azure-devops-access-token
+    parameters:
+      domainAccountName: dn-bot
+      domainAccountSecret:
+          location: helixkv
+          name: dn-bot-account-redmond
+      name: dn-bot-all-orgs-code-r
+      organizations: devdiv dnceng
+      scopes: code

.vault-config/product-construction-int.yaml

@@ -0,0 +1,54 @@
+storageLocation:
+  type: azure-key-vault
+  parameters:
+    subscription: cab65fc3-d077-467d-931f-3932eabf36d3
+    name: ProductConstructionInt
+
+references:
+  maestroint:
+    type: azure-key-vault
+    parameters:
+      subscription: cab65fc3-d077-467d-931f-3932eabf36d3
+      name: maestroint
+
+  helixkv:
+    type: azure-key-vault
+    parameters:
+      subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1
+      name: helixkv
+
+  engkeyvault:
+    type: azure-key-vault
+    parameters:
+      subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1
+      name: engkeyvault
+
+secrets:
+  build-asset-registry-sql-connection-string:
+    type: sql-connection-string
+    parameters:
+      adminConnection:
+        location: maestroint 
+        name: build-asset-registry-admin-connection-string
+      dataSource: tcp:maestro-int-server.database.windows.net,1433
+      database: BuildAssetRegistry
+      permissions: r
+
+  BotAccount-dotnet-bot-repo-PAT:
+    type: github-access-token
+    parameters:
+      gitHubBotAccountSecret: 
+        location: engkeyvault
+        name: BotAccount-dotnet-bot
+      gitHubBotAccountName: dotnet-bot
+
+  dn-bot-all-orgs-code-r:
+    type: azure-devops-access-token
+    parameters:
+      domainAccountName: dn-bot
+      domainAccountSecret:
+          location: helixkv
+          name: dn-bot-account-redmond
+      name: dn-bot-all-orgs-code-r
+      organizations: devdiv dnceng
+      scopes: code

azure-pipelines-product-construction-service.yml

@@ -43,8 +43,9 @@ stages:
   - job: Publish
     displayName: Publish Product Construction Service
     pool:
-      name: NetCore1ESPool-Internal
-      demands: ImageOverride -equals 1es-ubuntu-2004
+      # name: NetCore1ESPool-Internal
+      # demands: ImageOverride -equals 1es-ubuntu-2004
+      vmImage: 'ubuntu-latest'
 
     steps:
     - checkout: self

eng/deployment/product-construction-service-deploy.ps1

@@ -43,7 +43,7 @@ function StopAndWait([string]$pcsStatusUrl, [string]$pcsStopUrl) {
         } While ($pcsStateResponse.Content -notmatch "Stopped")
     }
     catch {
-        Write-Error "An error occurred: $($_.Exception.Message).  Deploying the new revision without stopping the service."
+        Write-Warning "An error occurred: $($_.Exception.Message).  Deploying the new revision without stopping the service."
     }
     return
 }
@@ -119,7 +119,7 @@ try
         Write-Host "All traffic has been redirected to label $inactiveLabel"
     }
     else {
-        Write-Error "New revision is not running. Check revision $newRevisionName logs in the inactive revisions. Deactivating the new revision"
+        Write-Warning "New revision is not running. Check revision $newRevisionName logs in the inactive revisions. Deactivating the new revision"
         az containerapp revision deactivate --revision $newRevisionName --name $containerappName --resource-group $resourceGroupName
         exit 1
     }

eng/service-templates/ProductConstructionService/provision.bicep

@@ -170,6 +170,18 @@ var env = [
         name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
         value: applicationInsights.properties.ConnectionString
     }
+    {
+        name: 'VmrPath'
+        value: '/mnt/datadir/vmr'
+    }
+    {
+        name: 'TmpPath'
+        value: '/mnt/datadir/tmp'
+    }
+    {
+        name: 'VmrUri'
+        value: 'https://github.com/maestro-auth-test/dnceng-vmr'
+    }
 ]
 
 // container app hosting the Product Construction Service
@@ -223,7 +235,7 @@ resource containerapp 'Microsoft.App/containerApps@2023-04-01-preview' = {
                     probes: [
                         {
                             httpGet: {
-                                path: '/health'
+                                path: '/alive'
                                 port: 8080
                                 scheme: 'HTTP'
                             }
@@ -233,6 +245,18 @@ resource containerapp 'Microsoft.App/containerApps@2023-04-01-preview' = {
                             failureThreshold: 3
                             type: 'Startup'
                         }
+                        {
+                            httpGet: {
+                                path: '/vmrReady'
+                                port: 8080
+                                scheme: 'HTTP'
+                            }
+                            initialDelaySeconds: 60
+                            failureThreshold: 10
+                            successThreshold: 1
+                            periodSeconds: 30
+                            type: 'Readiness'
+                        }
                     ]
                 } 
             ]

src/Microsoft.DotNet.Darc/DarcLib/Constants.cs

@@ -27,4 +27,6 @@ public class Constants
     public static readonly LibGit2Sharp.Identity DotnetBotIdentity = new(DarcBotName, DarcBotEmail);
 
     public const string CommonScriptFilesPath = "eng/common";
+
+    public const string VmrFolderName = "vmr";
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/RepositoryCloneManager.cs

@@ -59,6 +59,11 @@ Task<ILocalGitRepo> PrepareCloneAsync(
         IReadOnlyCollection<string> requestedRefs,
         string checkoutRef,
         CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Prepares a clone of the VMR.
+    /// </summary>
+    Task<ILocalGitRepo> PrepareVmrCloneAsync(string vmrUri, CancellationToken cancellationToken);
 }
 
 /// <summary>
@@ -200,6 +205,19 @@ public async Task<ILocalGitRepo> PrepareCloneAsync(
         return repo;
     }
 
+    public async Task<ILocalGitRepo> PrepareVmrCloneAsync(string vmrUri, CancellationToken cancellationToken)
+    {
+        // The vmr directory won't use a hash for its name, so we don't accidentally overwrite it
+        var path = await PrepareCloneInternal(vmrUri, Constants.VmrFolderName, cancellationToken);
+        var repo = _localGitRepoFactory.Create(path);
+        await repo.CheckoutAsync("main");
+
+        _vmrInfo.VmrPath = path;
+        _vmrInfo.VmrUri = vmrUri;
+
+        return repo;
+    }
+
     private async Task<NativePath> PrepareCloneInternal(string remoteUri, string dirName, CancellationToken cancellationToken)
     {
         cancellationToken.ThrowIfCancellationRequested();

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs

@@ -21,6 +21,11 @@ public interface IVmrInfo
     /// </summary>
     NativePath VmrPath { get; set; }
 
+    /// <summary>
+    /// Uri from which the VMR has been cloned
+    /// </summary>
+    string VmrUri { get; set; }
+
     /// <summary>
     /// Path within the VMR where VMR patches are stored.
     /// These patches are applied on top of the synchronized content.
@@ -79,21 +84,38 @@ public class VmrInfo : IVmrInfo
 
     public static UnixPath DefaultRelativeSourceManifestPath { get; } = RelativeSourcesDir / SourceManifestFileName;
 
-    public NativePath VmrPath { get; set; }
+    private NativePath _vmrPath;
+
+    public NativePath VmrPath
+    {
+        get
+        {
+            return _vmrPath;
+        }
+
+        set
+        {
+            _vmrPath = value;
+            SourceManifestPath = value / DefaultRelativeSourceManifestPath;
+        }
+    }
 
     public NativePath TmpPath { get; set; }
 
     public string? PatchesPath { get; set; }
 
     public string? SourceMappingsPath { get; set; }
 
+    public string VmrUri { get; set; }
+
     public IReadOnlyCollection<(string Source, string? Destination)> AdditionalMappings { get; set; } = Array.Empty<(string, string?)>();
 
     public VmrInfo(NativePath vmrPath, NativePath tmpPath)
     {
-        VmrPath = vmrPath;
+        _vmrPath = vmrPath;
         TmpPath = tmpPath;
         SourceManifestPath = vmrPath / SourcesDir / SourceManifestFileName;
+        VmrUri = Constants.DefaultVmrUri;
     }
 
     public VmrInfo(string vmrPath, string tmpPath) : this(new NativePath(vmrPath), new NativePath(tmpPath))
@@ -108,5 +130,5 @@ public VmrInfo(NativePath vmrPath, NativePath tmpPath)
 
     public static UnixPath GetRelativeRepoSourcesPath(string mappingName) => RelativeSourcesDir / mappingName;
 
-    public NativePath SourceManifestPath { get; }
+    public NativePath SourceManifestPath { get; private set; }
 }

src/ProductConstructionService/ProductConstructionService.Api/Controllers/StatusController.cs

@@ -8,29 +8,34 @@
 namespace ProductConstructionService.Api.Controllers;
 
 [Route("status")]
-public class StatusController(JobsProcessorScopeManager jobsProcessorScopeManager) : Controller
+public class StatusController(JobProcessorScopeManager jobProcessorScopeManager) : Controller
 {
-    private readonly JobsProcessorScopeManager _jobsProcessorScopeManager = jobsProcessorScopeManager;
+    private readonly JobProcessorScopeManager _jobProcessorScopeManager = jobProcessorScopeManager;
 
     [HttpPut("stop")]
-    public IActionResult StopPcsJobsProcessor()
+    public IActionResult StopPcsJobProcessor()
     {
-        _jobsProcessorScopeManager.FinishJobAndStop();
+        _jobProcessorScopeManager.FinishJobAndStop();
 
-        return GetPcsJobsProcessorStatus();
+        return GetPcsJobProcessorStatus();
     }
 
     [HttpPut("start")]
-    public IActionResult StartPcsJobsProcessor()
+    public IActionResult StartPcsJobProcessor()
     {
-        _jobsProcessorScopeManager.Start();
+        if (_jobProcessorScopeManager.State == JobsProcessorState.WaitingForVmrClone)
+        {
+            return BadRequest("The JobProcessor can't be started until the VMR is cloned");
+        }
 
-        return GetPcsJobsProcessorStatus();
+        _jobProcessorScopeManager.Start();
+
+        return GetPcsJobProcessorStatus();
     }
 
     [HttpGet]
-    public IActionResult GetPcsJobsProcessorStatus()
+    public IActionResult GetPcsJobProcessorStatus()
     {
-        return Ok(_jobsProcessorScopeManager.State.GetDisplayName());
+        return Ok(_jobProcessorScopeManager.State.GetDisplayName());
     }
 }

src/ProductConstructionService/ProductConstructionService.Api/Dockerfile

@@ -1,35 +1,51 @@
 #See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
 
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
-USER app
+RUN apt-get update
+RUN apt-get install git -y
 WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
 
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
+
 COPY ["Directory.Packages.props", "Directory.Build.props", "Directory.Build.targets", "NuGet.config", "global.json", "./"]
 COPY ["eng/Versions.props", "./eng/"]
-COPY ["src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj", "./ProductConstructionService/ProductConstructionService.Api/"]
-COPY ["src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj", "./ProductConstructionService/ProductConstructionService.ServiceDefaults/"]
+
 COPY ["src/Maestro/Client/src/Microsoft.DotNet.Maestro.Client.csproj", "./Maestro/Client/src/"]
+COPY ["src/Maestro/Maestro.AzureDevOps/Maestro.AzureDevOps.csproj", "./Maestro/Maestro.AzureDevOps/"]
 COPY ["src/Maestro/Maestro.Data/Maestro.Data.csproj", "./Maestro/Maestro.Data/"]
+COPY ["src/Maestro/Maestro.DataProviders/Maestro.DataProviders.csproj", "./Maestro/Maestro.DataProviders/"]
 COPY ["src/Maestro/Maestro.MergePolicyEvaluation/Maestro.MergePolicyEvaluation.csproj", "./Maestro/Maestro.MergePolicyEvaluation/"]
+
 COPY ["src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj", "./Microsoft.DotNet.Darc/DarcLib/"]
+
+COPY ["src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj", "./ProductConstructionService/ProductConstructionService.Api/"]
+COPY ["src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj", "./ProductConstructionService/ProductConstructionService.ServiceDefaults/"]
+
 RUN dotnet restore "./ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj"
-COPY ["src/ProductConstructionService", "./ProductConstructionService"]
+
 COPY ["src/Maestro/Client/src", "./Maestro/Client/src"]
+COPY ["src/Maestro/Maestro.AzureDevOps", "./Maestro/Maestro.AzureDevOps"]
 COPY ["src/Maestro/Maestro.Data", "./Maestro/Maestro.Data"]
+COPY ["src/Maestro/Maestro.DataProviders", "./Maestro/Maestro.DataProviders"]
 COPY ["src/Maestro/Maestro.MergePolicyEvaluation", "./Maestro/Maestro.MergePolicyEvaluation"]
+
 COPY ["src/Microsoft.DotNet.Darc/DarcLib", "./Microsoft.DotNet.Darc/DarcLib"]
+
+COPY ["src/ProductConstructionService", "./ProductConstructionService"]
+
 RUN dotnet build "./ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj" -c $BUILD_CONFIGURATION -o /app/build --no-restore
 
 FROM build AS publish
 ARG BUILD_CONFIGURATION=Release
 RUN dotnet publish "./ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
 
 FROM base AS final
+RUN git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" \
+ && git config --global user.name "dotnet-maestro[bot]"
 WORKDIR /app
 COPY --from=publish /app/publish .
-ENTRYPOINT ["dotnet", "ProductConstructionService.Api.dll"]
+ENTRYPOINT ["dotnet", "ProductConstructionService.Api.dll"]