KafkaSinkConnector

eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml

@@ -318,7 +318,8 @@ the main ServiceBusClientBuilder. -->
   <suppress checks="ConstantName" files="com.azure.spring.cloud.config.AppConfigurationPropertySourceLocator"/>
   <suppress checks="com.azure.tools.checkstyle.checks.GoodLoggingCheck"
             files="[/\\]azure-cosmos-kafka-connect[/\\]"/>
-  <suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosDBSourceConnector"/>
+  <suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosSourceConnector"/>
+  <suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosSinkConnector"/>
 
   <!-- Checkstyle suppressions for resource manager package -->
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.resourcemanager.*"/>

eng/common/pipelines/templates/steps/validate-all-packages.yml

@@ -0,0 +1,34 @@
+parameters:
+  ArtifactPath: $(Build.ArtifactStagingDirectory)
+  Artifacts: []
+  ConfigFileDir: $(Build.ArtifactStagingDirectory)/PackageInfo
+
+steps:
+ - ${{ if and(ne(variables['Skip.PackageValidation'], 'true'), eq(variables['System.TeamProject'], 'internal')) }}:
+    - pwsh: |
+        echo "##vso[task.setvariable variable=SetAsReleaseBuild]false"
+      displayName: "Set as release build"
+      condition: and(succeeded(), eq(variables['SetAsReleaseBuild'], ''))
+
+    - task: Powershell@2
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/scripts/Validate-All-Packages.ps1
+        arguments: >
+          -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name)
+          -ArtifactPath ${{ parameters.ArtifactPath }}
+          -RepoRoot $(Build.SourcesDirectory)
+          -APIKey $(azuresdk-apiview-apikey)
+          -ConfigFileDir '${{ parameters.ConfigFileDir }}'
+          -BuildDefinition $(System.CollectionUri)$(System.TeamProject)/_build?definitionId=$(System.DefinitionId)
+          -PipelineUrl $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)
+          -Devops_pat '$(azuresdk-azure-sdk-devops-release-work-item-pat)'
+          -IsReleaseBuild $$(SetAsReleaseBuild)
+        pwsh: true
+        workingDirectory: $(Pipeline.Workspace)
+      displayName: Validate packages and update work items
+      continueOnError: true
+      condition: >-
+        and(
+          succeededOrFailed(),
+          not(endsWith(variables['Build.Repository.Name'], '-pr'))
+        )

eng/common/scripts/ChangeLog-Operations.ps1

@@ -138,14 +138,27 @@ function Confirm-ChangeLogEntry {
     [Parameter(Mandatory = $true)]
     [String]$VersionString,
     [boolean]$ForRelease = $false,
-    [Switch]$SantizeEntry
+    [Switch]$SantizeEntry,
+    [PSCustomObject]$ChangeLogStatus = $null
   )
 
+  if (!$ChangeLogStatus) {
+    $ChangeLogStatus = [PSCustomObject]@{
+      IsValid = $false
+      Message = ""
+    }
+  }
+  else {
+    # Do not stop the script on error when status object is passed as param
+    $ErrorActionPreference = 'Continue'
+  }
   $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
   $changeLogEntry = $changeLogEntries[$VersionString]
 
   if (!$changeLogEntry) {
-    LogError "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    $ChangeLogStatus.Message = "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}."
+    $ChangeLogStatus.IsValid = $false
+    LogError "$($ChangeLogStatus.Message)"
     return $false
   }
 
@@ -161,24 +174,28 @@ function Confirm-ChangeLogEntry {
   Write-Host "-----"
 
   if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) {
-    LogError "Entry does not have a correct release status. Please ensure the status is set to a date '($CHANGELOG_DATE_FORMAT)' or '$CHANGELOG_UNRELEASED_STATUS' if not yet released. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    $ChangeLogStatus.Message = "Entry does not have a release status. Please ensure the status is set to a date '($CHANGELOG_DATE_FORMAT)' or '$CHANGELOG_UNRELEASED_STATUS' if not yet released. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    $ChangeLogStatus.IsValid = $false
+    LogError "$($ChangeLogStatus.Message)"
     return $false
   }
 
   if ($ForRelease -eq $True)
   {
     LogDebug "Verifying as a release build because ForRelease parameter is set to true"
-    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries
+    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus
   }
 
   # If the release status is a valid date then verify like its about to be released
   $status = $changeLogEntry.ReleaseStatus.Trim().Trim("()")
   if ($status -as [DateTime])
   {
     LogDebug "Verifying as a release build because the changelog entry has a valid date."
-    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries
+    return Confirm-ChangeLogForRelease -changeLogEntry $changeLogEntry -changeLogEntries $changeLogEntries -ChangeLogStatus $ChangeLogStatus
   }
 
+  $ChangeLogStatus.Message = "ChangeLog[${ChangeLogLocation}] has an entry for version ${VersionString}."
+  $ChangeLogStatus.IsValid = $true
   return $true
 }
 
@@ -338,41 +355,53 @@ function Confirm-ChangeLogForRelease {
     [Parameter(Mandatory = $true)]
     $changeLogEntry,
     [Parameter(Mandatory = $true)]
-    $changeLogEntries
+    $changeLogEntries,
+    $ChangeLogStatus = $null
   )
 
+  if (!$ChangeLogStatus) {
+    $ChangeLogStatus = [PSCustomObject]@{
+      IsValid = $false
+      Message = ""
+    }
+  }
   $entries = Sort-ChangeLogEntries -changeLogEntries $changeLogEntries
 
-  $isValid = $true
+  $ChangeLogStatus.IsValid = $true
   if ($changeLogEntry.ReleaseStatus -eq $CHANGELOG_UNRELEASED_STATUS) {
-    LogError "Entry has no release date set. Please ensure to set a release date with format '$CHANGELOG_DATE_FORMAT'. See https://aka.ms/azsdk/guideline/changelogs for more info."
-    $isValid = $false
+    $ChangeLogStatus.Message = "Entry has no release date set. Please ensure to set a release date with format '$CHANGELOG_DATE_FORMAT'. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    $ChangeLogStatus.IsValid = $false
+    LogError "$($ChangeLogStatus.Message)"
   }
   else {
     $status = $changeLogEntry.ReleaseStatus.Trim().Trim("()")
     try {
       $releaseDate = [DateTime]$status
       if ($status -ne ($releaseDate.ToString($CHANGELOG_DATE_FORMAT)))
       {
-        LogError "Date must be in the format $($CHANGELOG_DATE_FORMAT). See https://aka.ms/azsdk/guideline/changelogs for more info."
-        $isValid = $false
+        $ChangeLogStatus.Message = "Date must be in the format $($CHANGELOG_DATE_FORMAT). See https://aka.ms/azsdk/guideline/changelogs for more info."
+        $ChangeLogStatus.IsValid = $false
+        LogError "$($ChangeLogStatus.Message)"
       }
 
       if (@($entries.ReleaseStatus)[0] -ne $changeLogEntry.ReleaseStatus)
       {
-        LogError "Invalid date [ $status ]. The date for the changelog being released must be the latest in the file."
-        $isValid = $false
+        $ChangeLogStatus.Message = "Invalid date [ $status ]. The date for the changelog being released must be the latest in the file."
+        $ChangeLogStatus.IsValid = $false
+        LogError "$($ChangeLogStatus.Message)"
       }
     }
     catch {
-        LogError "Invalid date [ $status ] passed as status for Version [$($changeLogEntry.ReleaseVersion)]. See https://aka.ms/azsdk/guideline/changelogs for more info."
-        $isValid = $false
+        $ChangeLogStatus.Message = "Invalid date [ $status ] passed as status for Version [$($changeLogEntry.ReleaseVersion)]. See https://aka.ms/azsdk/guideline/changelogs for more info."
+        $ChangeLogStatus.IsValid = $false
+        LogError "$($ChangeLogStatus.Message)"
     }
   }
 
   if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) {
-    LogError "Entry has no content. Please ensure to provide some content of what changed in this version. See https://aka.ms/azsdk/guideline/changelogs for more info."
-    $isValid = $false
+    $ChangeLogStatus.Message = "Entry has no content. Please ensure to provide some content of what changed in this version. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    $ChangeLogStatus.IsValid = $false
+    LogError "$($ChangeLogStatus.Message)"
   }
 
   $foundRecommendedSection = $false
@@ -391,12 +420,14 @@ function Confirm-ChangeLogForRelease {
   }
   if ($emptySections.Count -gt 0)
   {
-    LogError "The changelog entry has the following sections with no content ($($emptySections -join ', ')). Please ensure to either remove the empty sections or add content to the section."
-    $isValid = $false
+    $ChangeLogStatus.Message = "The changelog entry has the following sections with no content ($($emptySections -join ', ')). Please ensure to either remove the empty sections or add content to the section."
+    $ChangeLogStatus.IsValid = $false
+    LogError "$($ChangeLogStatus.Message)"
   }
   if (!$foundRecommendedSection)
   {
-    LogWarning "The changelog entry did not contain any of the recommended sections ($($RecommendedSectionHeaders -join ', ')), please add at least one. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    $ChangeLogStatus.Message = "The changelog entry did not contain any of the recommended sections ($($RecommendedSectionHeaders -join ', ')), please add at least one. See https://aka.ms/azsdk/guideline/changelogs for more info."
+    LogWarning "$($ChangeLogStatus.Message)"
   }
-  return $isValid
+  return $ChangeLogStatus.IsValid
 }

eng/common/scripts/Helpers/ApiView-Helpers.ps1

@@ -1,12 +1,12 @@
-function MapLanguageName($language)
+function MapLanguageToRequestParam($language)
 {
     $lang = $language
     # Update language name to match those in API cosmos DB. Cosmos SQL is case sensitive and handling this within the query makes it slow.
     if($lang -eq 'javascript'){
         $lang = "JavaScript"
     }
     elseif ($lang -eq "dotnet"){
-        $lang = "C#"
+        $lang = "C%23"
     }
     elseif ($lang -eq "java"){
         $lang = "Java"
@@ -23,17 +23,12 @@ function MapLanguageName($language)
 function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $apiKey, $apiApprovalStatus = $null, $packageNameStatus = $null)
 {
   # Get API view URL and API Key to check status
-  Write-Host "Checking API review status"
-  $lang = MapLanguageName -language $language
+  Write-Host "Checking API review status for package: ${packageName}"
+  $lang = MapLanguageToRequestParam -language $language
   if ($lang -eq $null) {
     return
   }
   $headers = @{ "ApiKey" = $apiKey }
-  $body = @{
-    language = $lang
-    packageName = $packageName
-    packageVersion = $packageVersion
-  }
 
   if (!$apiApprovalStatus) {
     $apiApprovalStatus = [PSCustomObject]@{
@@ -51,7 +46,10 @@ function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $
 
   try
   {
-    $response = Invoke-WebRequest $url -Method 'GET' -Headers $headers -Body $body
+    $requestUrl = "${url}?language=${lang}&packageName=${packageName}&packageVersion=${packageVersion}"
+    Write-Host "Request to APIView: [${requestUrl}]"
+    $response = Invoke-WebRequest $requestUrl -Method 'GET' -Headers $headers
+    Write-Host "Response: $($response.StatusCode)"
     Process-ReviewStatusCode -statusCode $response.StatusCode -packageName $packageName -apiApprovalStatus $apiApprovalStatus -packageNameStatus $packageNameStatus
     if ($apiApprovalStatus.IsApproved) {
       Write-Host $($apiApprovalStatus.Details)

eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1

@@ -985,4 +985,46 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions)
     -Uri "https://dev.azure.com/azure-sdk/_apis/wit/workitems/${id}?api-version=6.0" `
     -Headers (Get-DevOpsRestHeaders) -Body $body -ContentType "application/json-patch+json" | ConvertTo-Json -Depth 10 | ConvertFrom-Json -AsHashTable
   return $response
+}
+
+function UpdateValidationStatus($pkgvalidationDetails, $BuildDefinition, $PipelineUrl)
+{
+    $pkgName = $pkgValidationDetails.Name
+    $versionString = $pkgValidationDetails.Version
+
+    $parsedNewVersion = [AzureEngSemanticVersion]::new($versionString)
+    $versionMajorMinor = "" + $parsedNewVersion.Major + "." + $parsedNewVersion.Minor
+    $workItem = FindPackageWorkItem -lang $LanguageDisplayName -packageName $pkgName -version $versionMajorMinor -includeClosed $true -outputCommand $false
+
+    if (!$workItem)
+    {
+        Write-Host"No work item found for package [$pkgName]."
+        return $false
+    }
+
+    $changeLogStatus = $pkgValidationDetails.ChangeLogValidation.Status
+    $changeLogDetails  = $pkgValidationDetails.ChangeLogValidation.Message
+    $apiReviewStatus = $pkgValidationDetails.APIReviewValidation.Status
+    $apiReviewDetails = $pkgValidationDetails.APIReviewValidation.Message
+    $packageNameStatus = $pkgValidationDetails.PackageNameValidation.Status
+    $packageNameDetails = $pkgValidationDetails.PackageNameValidation.Message
+
+    $fields = @()
+    $fields += "`"PackageVersion=${versionString}`""
+    $fields += "`"ChangeLogStatus=${changeLogStatus}`""
+    $fields += "`"ChangeLogValidationDetails=${changeLogDetails}`""
+    $fields += "`"APIReviewStatus=${apiReviewStatus}`""
+    $fields += "`"APIReviewStatusDetails=${apiReviewDetails}`""
+    $fields += "`"PackageNameApprovalStatus=${packageNameStatus}`""
+    $fields += "`"PackageNameApprovalDetails=${packageNameDetails}`""
+    if ($BuildDefinition) {
+        $fields += "`"PipelineDefinition=$BuildDefinition`""
+    }
+    if ($PipelineUrl) {
+        $fields += "`"LatestPipelineRun=$PipelineUrl`""
+    }
+
+    $workItem = UpdateWorkItem -id $workItem.id -fields $fields
+    Write-Host "[$($workItem.id)]$LanguageDisplayName - $pkgName($versionMajorMinor) - Updated"
+    return $true
 }

eng/common/scripts/Update-DevOps-Release-WorkItem.ps1

@@ -15,7 +15,8 @@ param(
   [string]$packageNewLibrary = "true",
   [string]$relatedWorkItemId = $null,
   [string]$tag = $null,
-  [string]$devops_pat = $env:DEVOPS_PAT
+  [string]$devops_pat = $env:DEVOPS_PAT,
+  [bool]$inRelease = $true
 )
 #Requires -Version 6.0
 Set-StrictMode -Version 3
@@ -97,8 +98,11 @@ Write-Host "  PackageDisplayName: $($workItem.fields['Custom.PackageDisplayName'
 Write-Host "  ServiceName: $($workItem.fields['Custom.ServiceName'])"
 Write-Host "  PackageType: $($workItem.fields['Custom.PackageType'])"
 Write-Host ""
-Write-Host "Marking item [$($workItem.id)]$($workItem.fields['System.Title']) as '$state' for '$releaseType'"
-$updatedWI = UpdatePackageWorkItemReleaseState -id $workItem.id -state "In Release" -releaseType $releaseType -outputCommand $false
+if ($inRelease)
+{
+  Write-Host "Marking item [$($workItem.id)]$($workItem.fields['System.Title']) as '$state' for '$releaseType'"
+  $updatedWI = UpdatePackageWorkItemReleaseState -id $workItem.id -state "In Release" -releaseType $releaseType -outputCommand $false
+}
 $updatedWI = UpdatePackageVersions $workItem -plannedVersions $plannedVersions
 
 Write-Host "Release tracking item is at https://dev.azure.com/azure-sdk/Release/_workitems/edit/$($updatedWI.id)/"

eng/common/scripts/Validate-All-Packages.ps1

@@ -0,0 +1,52 @@
+[CmdletBinding()]
+Param (
+  [Parameter(Mandatory=$True)]
+  [array]$ArtifactList,
+  [Parameter(Mandatory=$True)]
+  [string]$ArtifactPath,
+  [Parameter(Mandatory=$True)]
+  [string]$RepoRoot,
+  [Parameter(Mandatory=$True)]
+  [string]$APIKey,
+  [string]$ConfigFileDir,
+  [string]$BuildDefinition,
+  [string]$PipelineUrl,
+  [string]$APIViewUri  = "https://apiview.dev/AutoReview/GetReviewStatus",
+  [string]$Devops_pat = $env:DEVOPS_PAT,
+  [bool] $IsReleaseBuild = $false
+)
+
+Set-StrictMode -Version 3
+. (Join-Path $PSScriptRoot common.ps1)
+
+function ProcessPackage($PackageName, $ConfigFileDir)
+{
+    Write-Host "Artifact path: $($ArtifactPath)"
+    Write-Host "Package Name: $($PackageName)"
+    Write-Host "Config File directory: $($ConfigFileDir)"
+
+    &$EngCommonScriptsDir/Validate-Package.ps1 `
+        -PackageName $PackageName `
+        -ArtifactPath $ArtifactPath `
+        -RepoRoot $RepoRoot `
+        -APIViewUri $APIViewUri `
+        -APIKey $APIKey `
+        -BuildDefinition $BuildDefinition `
+        -PipelineUrl $PipelineUrl `
+        -ConfigFileDir $ConfigFileDir `
+        -Devops_pat $Devops_pat
+    if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to validate package $PackageName"
+        exit 1
+    }
+}
+
+# Check if package config file is present. This file has package version, SDK type etc info.
+if (-not $ConfigFileDir) {
+    $ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo"
+}
+foreach ($artifact in $ArtifactList)
+{
+    Write-Host "Processing $($artifact.name)"
+    ProcessPackage -PackageName $artifact.name -ConfigFileDir $ConfigFileDir
+}