Avoid creating duplicate entries in Sw360 (#166)

* Initial

* Duplicate check

* Unique entries

* UT Added

* UT Added

* Sonar Issues fixed

* Log added

* Review comments

* Update CreatorHelper.cs

* Review comments

* Nuspec update

* BuG fix

* Update CA_UsageDocument.md

* Sumanth's changes

* Logging while linking to components

* Update compile.yml

* Update compile.yml

---------

Co-authored-by: Sumanth Kb <sumanth.k-b@siemens.com>
Co-authored-by: Karthika Geethanand  ¯\_(ツ)_/¯ <40568919+karthika-g@users.noreply.github.com>
Co-authored-by: karthika <karthikag1810@gmail.com>

.github/workflows/compile.yml

@@ -96,7 +96,7 @@ jobs:
         $sourceFolder = Join-Path $env:GITHUB_WORKSPACE "out" | Join-Path -ChildPath "*"
         $outFolder = Join-Path $env:GITHUB_WORKSPACE "out" | Join-Path -ChildPath "continuous-clearing" 
         New-Item -ItemType Directory -Force -Path $outFolder
-        $fileName = "continuous-clearing-v6.2.0.zip"
+        $fileName = "continuous-clearing-v6.2.1.zip"
         Write-Host "Filename: '$fileName'"
         Write-Host "sourceFolder: '$sourceFolder'"
         Write-Host "Outfolder: '$outFolder'"
@@ -164,9 +164,9 @@ jobs:
       #if: ${{ false }}  # disable for now
       run: |
         echo "Files in directory:"
-        docker build . --file Dockerfile --tag ${{ github.repository }}:continuous-clearing-v6.2.0
-        docker save ${{ github.repository }}:continuous-clearing-v6.2.0 -o continuous-clearing-v6.2.0.tar
-        echo "::set-output name=docker-LicenseClearingTool::continuous-clearing-v6.2.0.tar"
+        docker build . --file Dockerfile --tag ${{ github.repository }}:continuous-clearing-v6.2.1
+        docker save ${{ github.repository }}:continuous-clearing-v6.2.1 -o continuous-clearing-v6.2.1.tar
+        echo "::set-output name=docker-LicenseClearingTool::continuous-clearing-v6.2.1.tar"
         
     - name: Archive docker image      
       #if: ${{ false }}  # disable for now
@@ -214,8 +214,8 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
         with:
-          tag_name: v6.2.0
-          release_name: Release v6.2.0
+          tag_name: v6.2.1
+          release_name: Release v6.2.1
           body: |
             ${{ github.event.head_commit.message }}
           draft: true

CA.nuspec

@@ -4,7 +4,7 @@
 <package >
   <metadata>
     <id>continuous-clearing</id>
-    <version>6.2.0</version>
+    <version>6.2.1</version>
     <authors>Siemens AG</authors>
     <owners>continuous-clearing contributors</owners>
     <projectUrl>https://github.com/siemens/continuous-clearing</projectUrl>

doc/UsageDoc/CA_UsageDocument.md

@@ -180,7 +180,7 @@ Currently LTA support is not provided for SBOM, hence until that is implemented
       
       - **Project Type :** **Nuget**
       
-          * .Net core/.Net standard type project's input file repository should contain **package.lock.json** file. If not present do a `dotnet restore --use-lock-file`.
+          * .Net core/.Net standard type project's input file repository should contain **project.assets.json** file. If not present do a `dotnet restore`.
           
           * .Net Framework projects, input file repository should contain a **packages.config** file.
 

src/LCT.APICommunications/SW360Apicommunication.cs

@@ -127,12 +127,34 @@ public async Task<string> GetReleases()
             var result = string.Empty;
             try
             {
-                return await httpClient.GetStringAsync(sw360ReleaseApi);
+                HttpResponseMessage responseMessage = await httpClient.GetAsync(sw360ReleaseApi);
+                if (responseMessage != null && responseMessage.StatusCode.Equals(HttpStatusCode.OK))
+                {
+                    return await responseMessage.Content.ReadAsStringAsync();
+                }
+                else
+                {
+                    Logger.Error("SW360 server is not accessible while getting All Releases,Please wait for sometime and re run the pipeline again." +
+                        " StatusCode:" + responseMessage?.StatusCode + " & ReasonPharse :" + responseMessage?.ReasonPhrase);
+                    Environment.Exit(-1);
+                }
             }
             catch (TaskCanceledException ex)
             {
-                Logger.Debug($"{ex.Message}");
-                Logger.Error("A timeout error is thrown from SW360 server,Please wait for sometime and re run the pipeline again");
+                Logger.Debug($"GetReleases():TaskCanceledException Error : {ex.Message}", ex);
+                Logger.Error("TaskCanceledException error has error while getting all releases from the SW360 server,Please wait for sometime and re run the pipeline again. Error :" + ex.Message);
+                Environment.Exit(-1);
+            }
+            catch (HttpRequestException ex)
+            {
+                Logger.Debug($"GetReleases():HttpRequestException Error : {ex.Message}", ex);
+                Logger.Error("HttpRequestException error has error while getting all releases from the SW360 server,Please wait for sometime and re run the pipeline again. Error :" + ex.Message);
+                Environment.Exit(-1);
+            }
+            catch (InvalidOperationException ex)
+            {
+                Logger.Debug($"GetReleases():InvalidOperationException Error : {ex.Message}", ex);
+                Logger.Error("InvalidOperationException error has error while getting all releases from the SW360 server,Please wait for sometime and re run the pipeline again. Error :" + ex.Message);
                 Environment.Exit(-1);
             }
             return result;

src/LCT.PackageIdentifier/NpmProcessor.cs

@@ -74,7 +74,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings)
             return bom;
         }
 
-        public List<Component> ParsePackageLockJson(string filepath, CommonAppSettings appSettings)
+        public static List<Component> ParsePackageLockJson(string filepath, CommonAppSettings appSettings)
         {
             List<BundledComponents> bundledComponents = new List<BundledComponents>();
             List<Component> lstComponentForBOM = new List<Component>();
@@ -247,7 +247,7 @@ private static void GetComponentsForBom(string filepath, CommonAppSettings appSe
 
                 components.Description = folderPath;
                 components.Version = Convert.ToString(properties[Version]);
-                components.Author = prop?.Value[Requires]?.ToString();
+                components.Author = prop.Value[Requires]?.ToString();
                 components.Purl = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}";
                 components.BomRef = $"{ApiConstant.NPMExternalID}{componentName}@{components.Version}";
                 components.Properties = new List<Property>();
@@ -399,7 +399,7 @@ public static void GetdependencyDetails(List<Component> componentsForBOM, List<D
                 if ((component.Author?.Split(",")) != null)
                 {
                     List<Dependency> subDependencies = new();
-                    foreach (var item in (component?.Author?.Split(",")).Where(item => item.Contains(':')))
+                    foreach (var item in (component.Author?.Split(",")).Where(item => item.Contains(':')))
                     {
                         var componentDetails = item.Split(":");
                         var name = StringFormat(componentDetails[0]);
@@ -540,7 +540,7 @@ private static List<Component> GetExcludedComponentsList(List<Component> compone
                 else
                 {
                     BomCreator.bomKpiData.ComponentsExcluded++;
-                    Logger.Debug($"GetExcludedComponentsList():InvalidComponent For NPM : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}");
+                    Logger.Debug($"GetExcludedComponentsList():InvalidComponent For NPM : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}");
                 }
             }
             return components;

src/LCT.SW360PackageCreator.UTest/ComponentCreatorTest.cs

@@ -234,7 +234,6 @@ public async Task CycloneDxBomParser_PassingFilePath_ReturnsSuccess()
             Assert.That(list.Count > 0);
         }
 
-
         [Test]
         public async Task CycloneDxBomParser_PassingFilePath_ReturnsComponentsExcludingDev()
         {

src/LCT.SW360PackageCreator.UTest/CreatorHelperTest.cs

@@ -19,6 +19,10 @@
 using System.Diagnostics;
 using LCT.Common;
 using Castle.Core.Internal;
+using LCT.Facade.Interfaces;
+using System.Net.Http;
+using System.Net;
+using System.Text;
 
 namespace SW360ComponentCreator.UTest
 {
@@ -169,6 +173,53 @@ public async Task SetContentsForComparisonBOM_ProvidedValidBomDetails_ReturnsLis
             Assert.That(data.Count > 0);
         }
 
+        [Test]
+        public async Task SetContentsForComparisonBOM_ProvidedValidBomDetailsWithSw360InvalidCred_ReturnsEmpty()
+        {
+            //Arrange
+            var debianPatcher = new Mock<IDebianPatcher>();
+            IDictionary<string, IPackageDownloader> _packageDownloderList = new Dictionary<string, IPackageDownloader>
+            {
+                { "DEBIAN", new DebianPackageDownloader(debianPatcher.Object) }
+            };
+            var creatorHelper = new CreatorHelper(_packageDownloderList);
+
+            ReleasesInfo releasesInfo = new ReleasesInfo();
+            releasesInfo.SourceCodeDownloadUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xz";
+
+            List<Components> componentsAvailableInSw360 = new List<Components>();
+            List<Components> comparisonBomData = new List<Components>
+            {
+                new Components()
+                {
+                    Name = "adduser",
+                    Version = "3.118",
+                    ComponentExternalId = "pkg:deb/debian/adduser?arch=source",
+                    ReleaseExternalId = "pkg:deb/debian/adduser@3.118?arch=source",
+                    SourceUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xz",
+                    DownloadUrl = "https://snapshot.debian.org/archive/debian/20180915T211528Z/pool/main/a/adduser/adduser_3.118.tar.xz"
+                }
+            };
+            var iSW360Service = new Mock<ISW360Service>();
+
+            //Mocking the Sw360 result as Empty with SuccessCode
+            HttpResponseMessage responseMessage = new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent("", Encoding.UTF8)
+            };
+            var iSW360ApicommunicationFacade = new Mock<ISW360ApicommunicationFacade>();
+            iSW360ApicommunicationFacade.Setup(x => x.GetReleases()).ReturnsAsync(await responseMessage.Content.ReadAsStringAsync());
+            iSW360Service.Setup(x => x.GetAvailableReleasesInSw360(comparisonBomData)).ReturnsAsync(componentsAvailableInSw360);
+            iSW360Service.Setup(x => x.GetReleaseDataOfComponent(comparisonBomData[0].ReleaseLink)).ReturnsAsync(releasesInfo);
+
+            //Act
+            var data = await creatorHelper.SetContentsForComparisonBOM(comparisonBomData, iSW360Service.Object);
+
+            //Assert
+            Assert.That(data.Count.Equals(0));
+        }
+
         [Test]
         public async Task GetUpdatedComponentsDetails_ProvidedValidBomDetails_ReturnsUpdatedBom()
         {

src/LCT.SW360PackageCreator/ComponentCreator.cs

@@ -86,7 +86,7 @@ private async Task<List<Components>> GetListOfBomData(List<Component> components
 
                     Components component = await GetSourceUrl(componentsData.Name, componentsData.Version, componentsData.ProjectType, item.BomRef);
                     componentsData.SourceUrl = component.SourceUrl;
-                    
+
                     if (componentsData.ProjectType.ToUpperInvariant() == "ALPINE")
                     {
                         componentsData.AlpineSourceData = component.AlpineSourceData;
@@ -202,7 +202,7 @@ private static async Task<Components> GetSourceUrl(string name, string version,
                     componentsData.SourceUrl = await UrlHelper.Instance.GetSourceUrlForConanPackage(name, version);
                     break;
                 case "ALPINE":
-                    Components alpComponentData = await UrlHelper.Instance.GetSourceUrlForAlpinePackage(name, version,bomRef);
+                    Components alpComponentData = await UrlHelper.Instance.GetSourceUrlForAlpinePackage(name, version, bomRef);
                     componentsData = alpComponentData;
                     componentsData.ProjectType = projectType;
                     break;
@@ -276,7 +276,7 @@ private async Task CreateComponent(ICreatorHelper creatorHelper,
                     await CreateComponentAndRealease(creatorHelper, sw360CreatorService, item, sw360Url, appSettings);
                 }
 
-                if (appSettings.ProjectType.ToUpperInvariant()=="ALPINE")
+                if (appSettings.ProjectType.ToUpperInvariant() == "ALPINE")
                 {
                     string localPathforSourceRepo = UrlHelper.GetDownloadPathForAlpineRepo();
                     if (Directory.GetDirectories(localPathforSourceRepo).Length != 0)

src/LCT.SW360PackageCreator/CreatorHelper.cs

@@ -198,13 +198,27 @@ private static async Task<string> GetAttachmentUrlList(ComparisonBomData compone
             return downloadPath;
         }
 
+
         public async Task<List<ComparisonBomData>> SetContentsForComparisonBOM(List<Components> lstComponentForBOM, ISW360Service sw360Service)
         {
             Logger.Debug($"SetContentsForComparisonBOM():Start");
             List<ComparisonBomData> comparisonBomData = new List<ComparisonBomData>();
             Logger.Logger.Log(null, Level.Notice, $"Collecting comparison BOM Data...", null);
             componentsAvailableInSw360 = await sw360Service.GetAvailableReleasesInSw360(lstComponentForBOM);
 
+            //Checking components count before getting status of individual comp details
+            if (componentsAvailableInSw360?.Count > 0)
+            {
+                comparisonBomData = await GetComparisionBomItems(lstComponentForBOM, sw360Service);
+            }
+
+            Logger.Debug($"SetContentsForComparisonBOM():End");
+            return comparisonBomData;
+        }
+
+        private async Task<List<ComparisonBomData>> GetComparisionBomItems(List<Components> lstComponentForBOM, ISW360Service sw360Service)
+        {
+            List<ComparisonBomData> comparisonBomData = new();
             ComparisonBomData mapper;
             foreach (Components item in lstComponentForBOM)
             {
@@ -236,11 +250,11 @@ public async Task<List<ComparisonBomData>> SetContentsForComparisonBOM(List<Comp
                 {
                     mapper.DownloadUrl = GetMavenDownloadUrl(mapper, item, releasesInfo);
                 }
-                else if (!string.IsNullOrEmpty(item.ReleaseExternalId) && 
+                else if (!string.IsNullOrEmpty(item.ReleaseExternalId) &&
                             (item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["PYTHON"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["CONAN"]) || item.ReleaseExternalId.Contains(Dataconstant.PurlCheck()["ALPINE"])))
                 {
-                    mapper.DownloadUrl = mapper.SourceUrl;                    
-                }                
+                    mapper.DownloadUrl = mapper.SourceUrl;
+                }
                 else
                 {
                     mapper.DownloadUrl = GetComponentDownloadUrl(mapper, item, repo, releasesInfo);
@@ -251,10 +265,11 @@ public async Task<List<ComparisonBomData>> SetContentsForComparisonBOM(List<Comp
                 mapper.FossologyUploadStatus = GetFossologyUploadStatus(mapper.ApprovedStatus);
                 mapper.ReleaseAttachmentLink = string.Empty;
                 mapper.ReleaseLink = GetReleaseLink(componentsAvailableInSw360, item.Name, item.Version);
+
+                Logger.Debug($"Sw360 avilability status for Name " + mapper.Name + ":" + mapper.ComponentExternalId + "=" + mapper.ComponentStatus +
+                    "-Version " + mapper.Version + ":" + mapper.ReleaseExternalId + "=" + mapper.ReleaseStatus);
                 comparisonBomData.Add(mapper);
             }
-
-            Logger.Debug($"SetContentsForComparisonBOM():End");
             return comparisonBomData;
         }
 

src/LCT.SW360PackageCreator/PackageDownloader.cs

@@ -145,7 +145,7 @@ private static Result ListTagsOfComponent(ComparisonBomData component)
             const int timeOutMs = 200 * 60 * 1000;
             var processResult = ProcessAsyncHelper.RunAsync(p.StartInfo, timeOutMs);
             Result result = processResult?.Result ?? new Result();
-            Logger.Debug($"GetCorrectVersion:{gitCommand}:{result.ExitCode}, output:{result?.StdOut}, Error:{result.StdErr}");
+            Logger.Debug($"GetCorrectVersion:{gitCommand}:{result.ExitCode}, output:{result.StdOut}, Error:{result.StdErr}");
             return result;
         }