From 641b99f5952d188e5c0006547fb82d39a51c004d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jul 2024 21:01:47 +0800 Subject: [PATCH 01/22] chore(deps): bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /src (#20721) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.64.0 to 1.64.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.64.0...v1.64.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: indirect ... Signed-off-by: dependabot[bot] Signed-off-by: Shengwen YU Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shengwen YU Signed-off-by: kunal-511 --- src/go.mod | 2 +- src/go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 652335ca8e6..cb3fc840e97 100644 --- a/src/go.mod +++ b/src/go.mod @@ -179,7 +179,7 @@ require ( google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/src/go.sum b/src/go.sum index a355858092b..3f704380ad5 100644 --- a/src/go.sum +++ b/src/go.sum @@ -783,8 +783,10 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go. google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 8696cf0383979df66bfc3ef66d6b2262bebadb3a Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Tue, 30 Jul 2024 17:29:16 +0530 Subject: [PATCH 02/22] Resolved the bug The banner message is causing the search bar to be obscured #20608 Signed-off-by: kunal-511 --- .../app/shared/components/global-search/search.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portal/src/app/shared/components/global-search/search.component.scss b/src/portal/src/app/shared/components/global-search/search.component.scss index 4757a4a1650..6501822af0c 100644 --- a/src/portal/src/app/shared/components/global-search/search.component.scss +++ b/src/portal/src/app/shared/components/global-search/search.component.scss @@ -8,7 +8,7 @@ z-index: 999; box-sizing: border-box; background: #fafafa; - top: 60px; + top: 95px; left: 0; padding-left: 36px; padding-right: 36px; From fac99347f2449bb46c6879442406e3b01b5da16e Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 1 Aug 2024 09:48:43 +0530 Subject: [PATCH 03/22] Updated the search-result.component.ts file to dynamically update the top value depending on banner message Signed-off-by: kunal-511 --- .../global-search/search-result.component.ts | 48 +++++++++++++++++-- .../global-search/search.component.scss | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.ts b/src/portal/src/app/shared/components/global-search/search-result.component.ts index f5a32850130..24ab9221eb8 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.ts +++ b/src/portal/src/app/shared/components/global-search/search-result.component.ts @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { Component, OnInit, OnDestroy } from '@angular/core'; +import { Component, OnInit, OnDestroy, Renderer2, ElementRef } from '@angular/core'; import { Observable, Subscription } from 'rxjs'; import { GlobalSearchService } from './global-search.service'; @@ -21,6 +21,7 @@ import { SearchTriggerService } from './search-trigger.service'; import { AppConfigService } from '../../../services/app-config.service'; import { MessageHandlerService } from '../../services/message-handler.service'; import { filter, switchMap } from 'rxjs/operators'; +import { BannerMessage } from 'src/app/base/left-side-nav/config/config'; @Component({ selector: 'search-result', @@ -50,8 +51,12 @@ export class SearchResultComponent implements OnInit, OnDestroy { constructor( private search: GlobalSearchService, private msgHandler: MessageHandlerService, - private searchTrigger: SearchTriggerService - ) {} + private searchTrigger: SearchTriggerService, + private renderer: Renderer2, + private el: ElementRef, + private appConfigService: AppConfigService + + ) { } ngOnInit() { this.searchSub = this.searchTrigger.searchTriggerChan$ @@ -79,6 +84,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { this.onGoing = false; this.originalCopy = searchResults; // Keep the original data this.searchResults = this.clone(searchResults); + this.adjustOverlayTop(); }, error => { this.onGoing = false; @@ -133,6 +139,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { // Show the results show(): void { this.stateIndicator = true; + this.adjustOverlayTop(); } // Close the result page @@ -169,6 +176,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { this.onGoing = false; this.originalCopy = searchResults; // Keep the original data this.searchResults = this.clone(searchResults); + this.adjustOverlayTop(); }, error => { this.onGoing = false; @@ -176,4 +184,38 @@ export class SearchResultComponent implements OnInit, OnDestroy { } ); } + hasValidBannerMessage(): boolean { + const current: Date = this.appConfigService.getConfig()?.current_time + ? new Date(this.appConfigService.getConfig()?.current_time) + : new Date(); + if (this.appConfigService.getConfig()?.banner_message) { + const bm = JSON.parse( + this.appConfigService.getConfig()?.banner_message + ) as BannerMessage; + if (bm?.fromDate && bm?.toDate) { + return ( + new Date(current) <= new Date(bm.toDate) && + new Date(current) >= new Date(bm.fromDate) + ); + } + if (bm?.fromDate && !bm?.toDate) { + return new Date(current) >= new Date(bm.fromDate); + } + + if (!bm?.fromDate && bm?.toDate) { + return new Date(current) <= new Date(bm.toDate); + } + } + return false; + } + + private adjustOverlayTop(): void { + const hasBannerMessage = this.hasValidBannerMessage(); + const topValue = hasBannerMessage ? '95px' : '60px'; + + const overlayElement = this.el.nativeElement.querySelector('.search-overlay'); + if (overlayElement) { + this.renderer.setStyle(overlayElement, 'top', topValue); + } + } } diff --git a/src/portal/src/app/shared/components/global-search/search.component.scss b/src/portal/src/app/shared/components/global-search/search.component.scss index 6501822af0c..4757a4a1650 100644 --- a/src/portal/src/app/shared/components/global-search/search.component.scss +++ b/src/portal/src/app/shared/components/global-search/search.component.scss @@ -8,7 +8,7 @@ z-index: 999; box-sizing: border-box; background: #fafafa; - top: 95px; + top: 60px; left: 0; padding-left: 36px; padding-right: 36px; From 1bc7e5238573209a0bca6c722a427de44aa880e5 Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sat, 3 Aug 2024 09:50:50 +0530 Subject: [PATCH 04/22] Check for the other case of baneers also Signed-off-by: kunal-511 --- .../search-result.component.html | 2 +- .../global-search/search-result.component.ts | 31 ++++--------------- .../global-search/search.component.scss | 1 - 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.html b/src/portal/src/app/shared/components/global-search/search-result.component.html index 240891d047f..b05652ea6d4 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.html +++ b/src/portal/src/app/shared/components/global-search/search-result.component.html @@ -1,4 +1,4 @@ -
+
= new Date(bm.fromDate) - ); - } - if (bm?.fromDate && !bm?.toDate) { - return new Date(current) >= new Date(bm.fromDate); - } - - if (!bm?.fromDate && bm?.toDate) { - return new Date(current) <= new Date(bm.toDate); - } - } - return false; + getTopValue(): number { + const headerHeight: number = document.querySelector('navigator')?.clientHeight || 10; + const bannerHeight: number = document.querySelector('app-app-level-alerts')?.clientHeight || 10; + return headerHeight + bannerHeight; } private adjustOverlayTop(): void { - const hasBannerMessage = this.hasValidBannerMessage(); - const topValue = hasBannerMessage ? '95px' : '60px'; + const topValue = `${this.getTopValue()}px`; const overlayElement = this.el.nativeElement.querySelector('.search-overlay'); if (overlayElement) { this.renderer.setStyle(overlayElement, 'top', topValue); } } + } diff --git a/src/portal/src/app/shared/components/global-search/search.component.scss b/src/portal/src/app/shared/components/global-search/search.component.scss index 4757a4a1650..bb5c00fdb66 100644 --- a/src/portal/src/app/shared/components/global-search/search.component.scss +++ b/src/portal/src/app/shared/components/global-search/search.component.scss @@ -8,7 +8,6 @@ z-index: 999; box-sizing: border-box; background: #fafafa; - top: 60px; left: 0; padding-left: 36px; padding-right: 36px; From a605a5f50e75312d1a9cde7ba6e82c221e5c0e3a Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Wed, 7 Aug 2024 09:37:49 +0530 Subject: [PATCH 05/22] Replace inline style binding with topValue property in search-result.component Signed-off-by: kunal-511 --- .../global-search/search-result.component.html | 14 +++++--------- .../global-search/search-result.component.ts | 16 ++++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.html b/src/portal/src/app/shared/components/global-search/search-result.component.html index b05652ea6d4..ac41739b73b 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.html +++ b/src/portal/src/app/shared/components/global-search/search-result.component.html @@ -1,9 +1,7 @@ -
+
- < {{ 'SEARCH.BACK' | translate }} + < {{ 'SEARCH.BACK' | translate }}
+
\ No newline at end of file diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.ts b/src/portal/src/app/shared/components/global-search/search-result.component.ts index 5e4f9864cbd..01bd33d1616 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.ts +++ b/src/portal/src/app/shared/components/global-search/search-result.component.ts @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { Component, OnInit, OnDestroy, Renderer2, ElementRef } from '@angular/core'; +import { Component, OnInit, OnDestroy, Renderer2, ElementRef, HostListener } from '@angular/core'; import { Observable, Subscription } from 'rxjs'; import { GlobalSearchService } from './global-search.service'; @@ -185,18 +185,18 @@ export class SearchResultComponent implements OnInit, OnDestroy { ); } getTopValue(): number { - const headerHeight: number = document.querySelector('navigator')?.clientHeight || 10; - const bannerHeight: number = document.querySelector('app-app-level-alerts')?.clientHeight || 10; + const headerHeight: number = document.querySelector('navigator')?.clientHeight || 0; + const bannerHeight: number = document.querySelector('app-app-level-alerts')?.clientHeight || 0; return headerHeight + bannerHeight; } private adjustOverlayTop(): void { - const topValue = `${this.getTopValue()}px`; + const topValue = this.getTopValue(); - const overlayElement = this.el.nativeElement.querySelector('.search-overlay'); - if (overlayElement) { - this.renderer.setStyle(overlayElement, 'top', topValue); - } + } + @HostListener('window:resize', ['$event']) + onResize(event) { + this.adjustOverlayTop(); } } From 2b0f37af491d84ad5c6f1644e55c76d20477330e Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Sat, 17 Aug 2024 14:57:29 +0530 Subject: [PATCH 06/22] Refactored search overlay component bindings. Signed-off-by: kunal-511 --- .../search-result.component.html | 14 ++++++++----- .../global-search/search-result.component.ts | 20 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.html b/src/portal/src/app/shared/components/global-search/search-result.component.html index ac41739b73b..b05652ea6d4 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.html +++ b/src/portal/src/app/shared/components/global-search/search-result.component.html @@ -1,7 +1,9 @@ -
+
- < {{ 'SEARCH.BACK' | translate }} + < {{ 'SEARCH.BACK' | translate }}
\ No newline at end of file +
diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.ts b/src/portal/src/app/shared/components/global-search/search-result.component.ts index 01bd33d1616..9b0cdcf8029 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.ts +++ b/src/portal/src/app/shared/components/global-search/search-result.component.ts @@ -11,7 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { Component, OnInit, OnDestroy, Renderer2, ElementRef, HostListener } from '@angular/core'; +import { + Component, + OnInit, + OnDestroy, + Renderer2, + ElementRef, + HostListener, +} from '@angular/core'; import { Observable, Subscription } from 'rxjs'; import { GlobalSearchService } from './global-search.service'; @@ -55,8 +62,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { private renderer: Renderer2, private el: ElementRef, private appConfigService: AppConfigService - - ) { } + ) {} ngOnInit() { this.searchSub = this.searchTrigger.searchTriggerChan$ @@ -185,18 +191,18 @@ export class SearchResultComponent implements OnInit, OnDestroy { ); } getTopValue(): number { - const headerHeight: number = document.querySelector('navigator')?.clientHeight || 0; - const bannerHeight: number = document.querySelector('app-app-level-alerts')?.clientHeight || 0; + const headerHeight: number = + document.querySelector('navigator')?.clientHeight || 0; + const bannerHeight: number = + document.querySelector('app-app-level-alerts')?.clientHeight || 0; return headerHeight + bannerHeight; } private adjustOverlayTop(): void { const topValue = this.getTopValue(); - } @HostListener('window:resize', ['$event']) onResize(event) { this.adjustOverlayTop(); } - } From e8875d2e27af8ea3ca892ce67a824a416a5f4bbc Mon Sep 17 00:00:00 2001 From: kunal-511 Date: Thu, 22 Aug 2024 22:24:59 +0530 Subject: [PATCH 07/22] Removed the listener and clean unused components Signed-off-by: kunal-511 --- .../components/global-search/search-result.component.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/portal/src/app/shared/components/global-search/search-result.component.ts b/src/portal/src/app/shared/components/global-search/search-result.component.ts index 9b0cdcf8029..eb0c1c1190b 100644 --- a/src/portal/src/app/shared/components/global-search/search-result.component.ts +++ b/src/portal/src/app/shared/components/global-search/search-result.component.ts @@ -58,10 +58,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { constructor( private search: GlobalSearchService, private msgHandler: MessageHandlerService, - private searchTrigger: SearchTriggerService, - private renderer: Renderer2, - private el: ElementRef, - private appConfigService: AppConfigService + private searchTrigger: SearchTriggerService ) {} ngOnInit() { @@ -201,7 +198,7 @@ export class SearchResultComponent implements OnInit, OnDestroy { private adjustOverlayTop(): void { const topValue = this.getTopValue(); } - @HostListener('window:resize', ['$event']) + onResize(event) { this.adjustOverlayTop(); } From 72d7e647acdebd4b7d2a33274c6a9c2e29b193ad Mon Sep 17 00:00:00 2001 From: Shengwen YU Date: Thu, 1 Aug 2024 16:05:39 +0800 Subject: [PATCH 08/22] add Test Case for SBOM feature (#20797) Signed-off-by: Shengwen Yu Signed-off-by: kunal-511 --- tests/apitests/python/library/artifact.py | 25 +++++ tests/apitests/python/library/scan.py | 15 +++ tests/apitests/python/library/scan_stop.py | 15 +++ .../python/test_project_permission.py | 15 ++- .../test_sbom_generation_of_image_artifact.py | 87 ++++++++++++++++++ ..._stop_sbom_generation_of_image_artifact.py | 91 +++++++++++++++++++ .../Harbor-Pages/Project-Repository.robot | 17 +++- .../Project-Repository_Elements.robot | 5 +- .../Harbor-Pages/Vulnerability.robot | 35 +++++++ tests/resources/TestCaseBody.robot | 43 +++++++++ tests/robot-cases/Group0-BAT/API_DB.robot | 8 ++ tests/robot-cases/Group1-Nightly/Trivy.robot | 12 +++ 12 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 tests/apitests/python/test_sbom_generation_of_image_artifact.py create mode 100644 tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py diff --git a/tests/apitests/python/library/artifact.py b/tests/apitests/python/library/artifact.py index d8b6905b966..eca6a37c82f 100644 --- a/tests/apitests/python/library/artifact.py +++ b/tests/apitests/python/library/artifact.py @@ -30,6 +30,8 @@ def get_reference_info(self, project_name, repo_name, reference, expect_status_c if "with_scan_overview" in kwargs: params["with_scan_overview"] = kwargs["with_scan_overview"] params["x_accept_vulnerabilities"] = ",".join(report_mime_types) + if "with_sbom_overview" in kwargs: + params["with_sbom_overview"] = kwargs["with_sbom_overview"] if "with_immutable_status" in kwargs: params["with_immutable_status"] = kwargs["with_immutable_status"] if "with_accessory" in kwargs: @@ -140,6 +142,29 @@ def check_image_scan_result(self, project_name, repo_name, reference, expected_s return raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status)) + def check_image_sbom_generation_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs): + timeout_count = 30 + scan_status="" + while True: + time.sleep(5) + timeout_count = timeout_count - 1 + if (timeout_count == 0): + break + artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs) + if expected_scan_status in ["Not Scanned", "No SBOM Overview"]: + if artifact.sbom_overview is None: + if (timeout_count > 24): + continue + print("artifact SBOM is not generated.") + return + else: + raise Exception("Artifact SBOM should not be generated {}.".format(artifact.sbom_overview)) + + scan_status = artifact.sbom_overview.scan_status + if scan_status == expected_scan_status: + return + raise Exception("Generate image SBOM result is {}, not as expected {}.".format(scan_status, expected_scan_status)) + def check_reference_exist(self, project_name, repo_name, reference, ignore_not_found = False, **kwargs): artifact = self.get_reference_info( project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs) return { diff --git a/tests/apitests/python/library/scan.py b/tests/apitests/python/library/scan.py index a9260e2d646..6af92200fa7 100644 --- a/tests/apitests/python/library/scan.py +++ b/tests/apitests/python/library/scan.py @@ -21,3 +21,18 @@ def scan_artifact(self, project_name, repo_name, reference, expect_status_code = base._assert_status_code(expect_status_code, status_code) return data + + def sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs): + try: + req_param = dict(scan_type = {"scan_type":"sbom"}) + data, status_code, _ = self._get_client(**kwargs).scan_artifact_with_http_info(project_name, repo_name, reference, **req_param) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + + base._assert_status_code(expect_status_code, status_code) + + return data + diff --git a/tests/apitests/python/library/scan_stop.py b/tests/apitests/python/library/scan_stop.py index e002239b277..80507ebbb55 100644 --- a/tests/apitests/python/library/scan_stop.py +++ b/tests/apitests/python/library/scan_stop.py @@ -22,4 +22,19 @@ def stop_scan_artifact(self, project_name, repo_name, reference, expect_status_c base._assert_status_code(expect_status_code, status_code) + return data + + def stop_sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs): + try: + scanType = v2_swagger_client.ScanType() + scanType.scan_type = "sbom" + data, status_code, _ = self._get_client(**kwargs).stop_scan_artifact_with_http_info(project_name, repo_name, reference, scanType) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + + base._assert_status_code(expect_status_code, status_code) + return data \ No newline at end of file diff --git a/tests/apitests/python/test_project_permission.py b/tests/apitests/python/test_project_permission.py index 491eba2dcb2..a7533c72297 100644 --- a/tests/apitests/python/test_project_permission.py +++ b/tests/apitests/python/test_project_permission.py @@ -73,7 +73,7 @@ def call(self): read_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "GET", 200, metadata_payload) metadata_payload_for_update = { "auto_scan": "false" } update_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "PUT", 200, metadata_payload_for_update) -delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload) +delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload_for_update) # 4. Resource: repository actions: ['read', 'list', 'update', 'delete', 'pull', 'push'] # note: pull and push are for docker cli, no API needs them @@ -89,12 +89,17 @@ def call(self): delete_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts/{}".format(harbor_base_url, project_name, source_artifact_tag), "DELETE", 200) # 6. Resource scan actions: ['read', 'create', 'stop'] -stop_scan_payload = { +vulnerability_scan_payload = { "scan_type": "vulnerability" } -create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202) -stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, stop_scan_payload) +create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload) +stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload) read_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/83be44fd-1234-5678-b49f-4b6d6e8f5730/log".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "get", 404) +sbom_gen_payload = { + "scan_type": "sbom" +} +create_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload) +stop_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload) # 7. Resource tag actions: ['list', 'create', 'delete'] tag_payload = { "name": "test-{}".format(int(random.randint(1000, 9999))) } @@ -240,7 +245,7 @@ def call(self): "metadata": [create_metadata, list_metadata, read_metadata, update_metadata, delete_metadata], "repository": [list_repo, read_repo, update_repo, delete_repo], "artifact": [list_artifact, read_artifact, copy_artifact, delete_artifact], - "scan": [create_scan, stop_scan, read_scan], + "scan": [create_scan, stop_scan, read_scan, create_sbom_generation, stop_sbom_generation], "tag": [create_tag, list_tag, delete_tag], "accessory": [list_accessory], "artifact-addition": [read_artifact_addition_vul, read_artifact_addition_dependencies], diff --git a/tests/apitests/python/test_sbom_generation_of_image_artifact.py b/tests/apitests/python/test_sbom_generation_of_image_artifact.py new file mode 100644 index 00000000000..8025fafe3b0 --- /dev/null +++ b/tests/apitests/python/test_sbom_generation_of_image_artifact.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import +import unittest +import sys + +from testutils import harbor_server, suppress_urllib3_warning +from testutils import TEARDOWN +from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_self_build_image_to_project +from library.artifact import Artifact +from library.scan import Scan + +class TestSBOMGeneration(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.project= Project() + self.user= User() + self.artifact = Artifact() + self.repo = Repository() + self.scan = Scan() + + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True) + + + #2. Create a new private project(PA) by user(UA); + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT) + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def do_tearDown(self): + #1. Delete repository(RA) by user(UA); + self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT) + + #2. Delete project(PA); + self.project.delete_project(self.project_id, **self.USER_CLIENT) + + #3. Delete user(UA); + self.user.delete_user(self.user_id, **ADMIN_CLIENT) + + def testGenerateSBOMOfImageArtifact(self): + """ + Test case: + Generate an SBOM of An Image Artifact + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished; + Tear down: + 1. Delete repository(RA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA); + """ + + #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + self.project.projects_should_exist(dict(public=False), expected_count = 1, + expected_project_id = self.project_id, **self.USER_CLIENT) + + #Note: Please make sure that this Image has never been pulled before by any other cases, + # so it is a not-scanned image right after repository creation. + image = "docker" + src_tag = "1.13" + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) + + #6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished; + self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + self.artifact.check_image_sbom_generation_result(self.project_name, image, tag, **self.USER_CLIENT) + + self.do_tearDown() + + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestSBOMGeneration)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"SBOM generation test failed: {}".format(result)) diff --git a/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py b/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py new file mode 100644 index 00000000000..232e5e85dd7 --- /dev/null +++ b/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import +import unittest +import sys + +from testutils import harbor_server, suppress_urllib3_warning +from testutils import TEARDOWN +from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_self_build_image_to_project +from library.artifact import Artifact +from library.scan import Scan +from library.scan_stop import StopScan + +class TestStopSBOMGeneration(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.project= Project() + self.user= User() + self.artifact = Artifact() + self.repo = Repository() + self.scan = Scan() + self.stop_scan = StopScan() + + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True) + + + #2. Create a new private project(PA) by user(UA); + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT) + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def do_tearDown(self): + #1. Delete repository(RA) by user(UA); + self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT) + + #2. Delete project(PA); + self.project.delete_project(self.project_id, **self.USER_CLIENT) + + #3. Delete user(UA); + self.user.delete_user(self.user_id, **ADMIN_CLIENT) + + def testStopSBOMGenerationOfImageArtifact(self): + """ + Test case: + Stop SBOM Generation Of An Image Artifact + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Send SBOM generation of an image command; + 7. Send stop SBOM generation of an image command. + Tear down: + 1. Delete repository(RA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA); + """ + + #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + self.project.projects_should_exist(dict(public=False), expected_count = 1, + expected_project_id = self.project_id, **self.USER_CLIENT) + + #Note: Please make sure that this Image has never been pulled before by any other cases, + # so it is a not-scanned image right after repository creation. + image = "docker" + src_tag = "1.13" + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) + + #6. Send SBOM generation of an image command; + self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + + #7. Send stop SBOM generation of an image command. + self.stop_scan.stop_sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + + self.do_tearDown() + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestStopSBOMGeneration)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"Stop SBOM generation test failed: {}".format(result)) diff --git a/tests/resources/Harbor-Pages/Project-Repository.robot b/tests/resources/Harbor-Pages/Project-Repository.robot index 04c0b355b32..1eef8e0685f 100644 --- a/tests/resources/Harbor-Pages/Project-Repository.robot +++ b/tests/resources/Harbor-Pages/Project-Repository.robot @@ -41,9 +41,20 @@ Stop Scan Artifact Retry Element Click ${stop_scan_artifact_btn} Check Scan Artifact Job Status Is Stopped - Wait Until Element Is Visible ${stopped_label} - ${job_status}= Get Text ${stopped_label} - Should Be Equal As Strings '${job_status}' 'Scan stopped' + Wait Until Element Is Visible ${scan_stopped_label} + +Generate Artifact SBOM + [Arguments] ${project} ${repo} ${label_xpath}=//clr-dg-row//label[contains(@class,'clr-control-label')][1] + Go Into Repo ${project} ${repo} + Retry Element Click ${label_xpath} + Retry Element Click ${gen_artifact_sbom_btn} + +Stop Gen Artifact SBOM + Retry Element Click ${artifact_action_xpath} + Retry Element Click ${stop_gen_artifact_sbom_btn} + +Check Gen Artifact SBOM Job Status Is Stopped + Wait Until Element Is Visible ${gen_sbom_stopped_label} Refresh Repositories Retry Element Click ${refresh_repositories_xpath} diff --git a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot index 6cb3440586d..66c6aa7ffef 100644 --- a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot @@ -24,5 +24,8 @@ ${build_history_data} //clr-dg-row ${push_image_command_btn} //hbr-push-image-button//button ${scan_artifact_btn} //button[@id='scan-btn'] ${stop_scan_artifact_btn} //button[@id='stop-scan'] -${stopped_label} //span[@class='label stopped'] +${scan_stopped_label} //span[normalize-space()='Scan stopped'] +${gen_sbom_stopped_label} //span[normalize-space()='Generation stopped'] +${gen_artifact_sbom_btn} //button[@id='generate-sbom-btn'] +${stop_gen_artifact_sbom_btn} //button[@id='stop-sbom-btn'] ${refresh_repositories_xpath} //hbr-repository-gridview//span[contains(@class,'refresh-btn')] \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Vulnerability.robot b/tests/resources/Harbor-Pages/Vulnerability.robot index b7fb6d430fc..03ed48eefa0 100644 --- a/tests/resources/Harbor-Pages/Vulnerability.robot +++ b/tests/resources/Harbor-Pages/Vulnerability.robot @@ -59,6 +59,41 @@ Enable Scan On Push Checkbox Should Be Selected //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input Retry Element Click ${project_config_save_btn} +Generate Repo SBOM + [Arguments] ${tagname} ${status} + Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//label[contains(@class,'clr-control-label')] + Retry Element Click //button[@id='generate-sbom-btn'] + Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //a[@title='SBOM details'] 300 + +Checkout And Review SBOM Details + [Arguments] ${tagname} + Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//a[@title='SBOM details'] + # Download SBOM file + Retry Element Click //button[@id='sbom-btn'] + ${sbom_artifact_short_sha256}= Get Text //span[@class='margin-left-10px'] + ${sbom_filename_raw}= Get Text //clr-dg-cell[contains(text(),'${sbom_artifact_short_sha256}')] + ${sbom_filename}= Replace String ${sbom_filename_raw} : _ count=-1 + ${sbom_filename}= Replace String ${sbom_filename} / _ count=-1 + ${sbom_json_path}= Set Variable ${download_directory}/${sbom_filename}.json + Retry File Should Exist ${sbom_json_path} + # Load the downloaded SBOM json file and verify the first N package records + ${sbom_json_content}= Load Json From File ${sbom_json_path} + ${items}= Get Value From JSON ${sbom_json_content} packages + ${items_length}= Get Length ${items} + ${first_n_records}= Evaluate min(5, ${items_length}) + FOR ${idx} IN RANGE 1 ${first_n_records} + ${item}= Get From List ${items} ${idx} + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.name}'] + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.versionInfo}'] + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.licenseConcluded}'] + END + +Enable Generating SBOM On Push + Checkbox Should Not Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input + Retry Element Click //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//label[contains(@class,'clr-control-label')] + Checkbox Should Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input + Retry Element Click ${project_config_save_btn} + Vulnerability Not Ready Project Hint Sleep 2 ${element}= Set Variable xpath=//span[contains(@class, 'db-status-warning')] diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index fc785a1858b..9c721d8deb0 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -417,6 +417,49 @@ Stop Scan All Stop Scan All Artifact Retry Action Keyword Check Scan All Artifact Job Status Is Stopped +Body Of Generate SBOM of An Image In The Repo + [Arguments] ${image_argument} ${tag_argument} + Init Chrome Driver + + ${d}= get current date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_argument}:${tag_argument} + Go Into Repo project${d} ${image_argument} + Generate Repo SBOM ${tag_argument} Succeed + Checkout And Review SBOM Details ${tag_argument} + Close Browser + +Body Of Generate Image SBOM On Push + Init Chrome Driver + ${d}= get current date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Goto Project Config + Enable Generating SBOM On Push + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} memcached + Go Into Repo project${d} memcached + Checkout And Review SBOM Details latest + Close Browser + +Body Of Stop SBOM Manual Generation + Init Chrome Driver + ${d}= get current date result_format=%m%s + ${repo}= Set Variable goharbor/harbor-e2e-engine + ${tag}= Set Variable test-ui + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${repo} ${tag} ${tag} + # stop generate sbom of an artifact + Retry Action Keyword Stop SBOM Generation project${d} ${repo} + Close Browser + +Stop SBOM Generation + [Arguments] ${project_name} ${repo} + Generate Artifact SBOM ${project_name} ${repo} + Stop Gen Artifact SBOM + Retry Action Keyword Check Gen Artifact SBOM Job Status Is Stopped + Prepare Image Package Test Files [Arguments] ${files_path} ${rc} ${output}= Run And Return Rc And Output bash tests/robot-cases/Group0-Util/prepare_imgpkg_test_files.sh ${files_path} diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index f1a92b7205a..7e5bae3c238 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -120,6 +120,14 @@ Test Case - Stop Scan All Images [Tags] stop_scan_all Harbor API Test ./tests/apitests/python/test_system_level_stop_scan_all.py +Test Case - Generate SBOM Of An Image + [Tags] generate_sbom + Harbor API Test ./tests/apitests/python/test_sbom_generation_of_image_artifact.py + +Test Case - Stop Generating SBOM Of An Image + [Tags] stop_generating_sbom + Harbor API Test ./tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py + Test Case - Registry API [Tags] reg_api Harbor API Test ./tests/apitests/python/test_registry_api.py diff --git a/tests/robot-cases/Group1-Nightly/Trivy.robot b/tests/robot-cases/Group1-Nightly/Trivy.robot index bcfff07d24a..4d976ace388 100644 --- a/tests/robot-cases/Group1-Nightly/Trivy.robot +++ b/tests/robot-cases/Group1-Nightly/Trivy.robot @@ -164,6 +164,18 @@ Test Case - Stop Scan And Stop Scan All [Tags] stop_scan_job Body Of Stop Scan And Stop Scan All +Test Case - Verify SBOM Manual Generation + [Tags] sbom_manual_gen + Body Of Generate SBOM of An Image In The Repo alpine 3.10 + +Test Case - Generate Image SBOM On Push + [Tags] run-once + Body Of Generate Image SBOM On Push + +Test Case - Stop SBOM Manual Generation + [Tags] stop_sbom_gen + Body Of Stop SBOM Manual Generation + Test Case - External Scanner CRUD [Tags] external_scanner_crud need_scanner_endpoint ${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY} From 4179f6ac8f633dad66ac13b7cae7e3c1de374ea7 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Mon, 5 Aug 2024 19:11:05 +0800 Subject: [PATCH 09/22] refactor: unify the mock file generation (#20765) 1. Mock remote interface for distribution manifest by mockery package feature. 2. Refactor hand-generated mock files to automated management generation. 3. Clean useless mocks. Signed-off-by: chlins Signed-off-by: kunal-511 --- src/.mockery.yaml | 81 ++++-- src/controller/artifact/controller_test.go | 8 +- .../artifact/processor/chart/chart_test.go | 6 +- src/controller/p2p/preheat/controllor_test.go | 8 +- src/controller/p2p/preheat/enforcer_test.go | 4 +- src/controller/scan/base_controller_test.go | 5 +- src/controller/scan/callback_test.go | 4 +- src/controller/securityhub/controller_test.go | 4 +- src/controller/tag/controller_test.go | 50 ++-- .../job/impl/gc/garbage_collection_test.go | 20 +- src/pkg/scan/vulnerability/vul_test.go | 7 +- src/testing/lib/libcache/cache.go | 136 ---------- src/testing/pkg/artifactrash/manager.go | 129 +++++++-- src/testing/pkg/chart/operator.go | 94 +++++-- src/testing/pkg/distribution/manifest.go | 19 +- src/testing/pkg/immutable/matcher.go | 54 +++- .../pkg/p2p/preheat/instance/manager.go | 90 ++++-- src/testing/pkg/p2p/preheat/policy/manager.go | 108 ++++++-- src/testing/pkg/parser/parser.go | 24 +- src/testing/pkg/processor/processor.go | 40 ++- .../scan/dao/scan/vulnerability_record_dao.go | 256 ------------------ .../native_scan_report_converter.go | 91 +++++++ .../scan/postprocessors/report_converters.go | 23 -- src/testing/pkg/tag/manager.go | 235 ++++++++++++---- src/testing/registryctl/client.go | 80 +++++- 25 files changed, 924 insertions(+), 652 deletions(-) delete mode 100644 src/testing/lib/libcache/cache.go delete mode 100644 src/testing/pkg/scan/dao/scan/vulnerability_record_dao.go create mode 100644 src/testing/pkg/scan/postprocessors/native_scan_report_converter.go delete mode 100644 src/testing/pkg/scan/postprocessors/report_converters.go diff --git a/src/.mockery.yaml b/src/.mockery.yaml index c0d0b35a6c9..7c5e8569710 100644 --- a/src/.mockery.yaml +++ b/src/.mockery.yaml @@ -9,6 +9,17 @@ packages: Controller: config: dir: testing/controller/artifact + github.com/goharbor/harbor/src/controller/artifact/processor: + interfaces: + Processor: + config: + dir: testing/pkg/processor + github.com/goharbor/harbor/src/controller/artifact/annotation: + interfaces: + Parser: + config: + dir: testing/pkg/parser + outpkg: parser github.com/goharbor/harbor/src/controller/blob: interfaces: Controller: @@ -188,6 +199,11 @@ packages: Manager: config: dir: testing/pkg/artifact + github.com/goharbor/harbor/src/pkg/artifactrash: + interfaces: + Manager: + config: + dir: testing/pkg/artifactrash github.com/goharbor/harbor/src/pkg/blob: interfaces: Manager: @@ -218,6 +234,11 @@ packages: Handler: config: dir: testing/pkg/scan + github.com/goharbor/harbor/src/pkg/scan/postprocessors: + interfaces: + NativeScanReportConverter: + config: + dir: testing/pkg/scan/postprocessors github.com/goharbor/harbor/src/pkg/scan/report: interfaces: Manager: @@ -238,7 +259,7 @@ packages: dir: pkg/scheduler outpkg: scheduler mockname: mockDAO - filename: mock_dao_test.go + filename: mock_dao_test.go inpackage: True Scheduler: config: @@ -342,6 +363,14 @@ packages: DAO: config: dir: testing/pkg/immutable/dao + github.com/goharbor/harbor/src/pkg/immutable/match: + interfaces: + ImmutableTagMatcher: + config: + dir: testing/pkg/immutable + filename: matcher.go + outpkg: immutable + mockname: FakeMatcher github.com/goharbor/harbor/src/pkg/ldap: interfaces: Manager: @@ -505,20 +534,36 @@ packages: Manager: config: dir: testing/pkg/securityhub - - - - - - - - - - - - - - - - - + github.com/goharbor/harbor/src/pkg/tag: + interfaces: + Manager: + config: + dir: testing/pkg/tag + github.com/goharbor/harbor/src/pkg/p2p/preheat/policy: + interfaces: + Manager: + config: + dir: testing/pkg/p2p/preheat/policy + github.com/goharbor/harbor/src/pkg/p2p/preheat/instance: + interfaces: + Manager: + config: + dir: testing/pkg/p2p/preheat/instance + github.com/goharbor/harbor/src/pkg/chart: + interfaces: + Operator: + config: + dir: testing/pkg/chart + # registryctl related mocks + github.com/goharbor/harbor/src/registryctl/client: + interfaces: + Client: + config: + dir: testing/registryctl + outpkg: registryctl + # remote interfaces + github.com/docker/distribution: + interfaces: + Manifest: + config: + dir: testing/pkg/distribution diff --git a/src/controller/artifact/controller_test.go b/src/controller/artifact/controller_test.go index 0e6ed458a26..6521b1f19e4 100644 --- a/src/controller/artifact/controller_test.go +++ b/src/controller/artifact/controller_test.go @@ -67,7 +67,7 @@ type controllerTestSuite struct { ctl *controller repoMgr *repotesting.Manager artMgr *arttesting.Manager - artrashMgr *artrashtesting.FakeManager + artrashMgr *artrashtesting.Manager blobMgr *blob.Manager tagCtl *tagtesting.FakeController labelMgr *label.Manager @@ -80,7 +80,7 @@ type controllerTestSuite struct { func (c *controllerTestSuite) SetupTest() { c.repoMgr = &repotesting.Manager{} c.artMgr = &arttesting.Manager{} - c.artrashMgr = &artrashtesting.FakeManager{} + c.artrashMgr = &artrashtesting.Manager{} c.blobMgr = &blob.Manager{} c.tagCtl = &tagtesting.FakeController{} c.labelMgr = &label.Manager{} @@ -476,7 +476,7 @@ func (c *controllerTestSuite) TestDeleteDeeply() { }, }, nil) c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{}, nil) - c.artrashMgr.On("Create").Return(0, nil) + c.artrashMgr.On("Create", mock.Anything, mock.Anything).Return(int64(0), nil) c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil) err = c.ctl.deleteDeeply(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, false, false) c.Require().Nil(err) @@ -534,7 +534,7 @@ func (c *controllerTestSuite) TestDeleteDeeply() { c.blobMgr.On("List", mock.Anything, mock.Anything).Return(nil, nil) c.blobMgr.On("CleanupAssociationsForProject", mock.Anything, mock.Anything, mock.Anything).Return(nil) c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{}, nil) - c.artrashMgr.On("Create").Return(0, nil) + c.artrashMgr.On("Create", mock.Anything, mock.Anything).Return(int64(0), nil) err = c.ctl.deleteDeeply(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, true, true) c.Require().Nil(err) diff --git a/src/controller/artifact/processor/chart/chart_test.go b/src/controller/artifact/processor/chart/chart_test.go index e8e208dd99f..637da7bf6f0 100644 --- a/src/controller/artifact/processor/chart/chart_test.go +++ b/src/controller/artifact/processor/chart/chart_test.go @@ -64,12 +64,12 @@ type processorTestSuite struct { suite.Suite processor *processor regCli *registry.Client - chartOptr *chart.FakeOpertaor + chartOptr *chart.Operator } func (p *processorTestSuite) SetupTest() { p.regCli = ®istry.Client{} - p.chartOptr = &chart.FakeOpertaor{} + p.chartOptr = &chart.Operator{} p.processor = &processor{ chartOperator: p.chartOptr, } @@ -106,7 +106,7 @@ func (p *processorTestSuite) TestAbstractAddition() { p.Require().Nil(err) p.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) p.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), io.NopCloser(strings.NewReader(chartYaml)), nil) - p.chartOptr.On("GetDetails").Return(chartDetails, nil) + p.chartOptr.On("GetDetails", mock.Anything).Return(chartDetails, nil) // values.yaml addition, err := p.processor.AbstractAddition(nil, artifact, AdditionTypeValues) diff --git a/src/controller/p2p/preheat/controllor_test.go b/src/controller/p2p/preheat/controllor_test.go index b06af2672e8..c57802dd980 100644 --- a/src/controller/p2p/preheat/controllor_test.go +++ b/src/controller/p2p/preheat/controllor_test.go @@ -31,8 +31,8 @@ type preheatSuite struct { suite.Suite ctx context.Context controller Controller - fakeInstanceMgr *instance.FakeManager - fakePolicyMgr *pmocks.FakeManager + fakeInstanceMgr *instance.Manager + fakePolicyMgr *pmocks.Manager fakeScheduler *smocks.Scheduler mockInstanceServer *httptest.Server fakeExecutionMgr *tmocks.ExecutionManager @@ -40,8 +40,8 @@ type preheatSuite struct { func TestPreheatSuite(t *testing.T) { t.Log("Start TestPreheatSuite") - fakeInstanceMgr := &instance.FakeManager{} - fakePolicyMgr := &pmocks.FakeManager{} + fakeInstanceMgr := &instance.Manager{} + fakePolicyMgr := &pmocks.Manager{} fakeScheduler := &smocks.Scheduler{} fakeExecutionMgr := &tmocks.ExecutionManager{} diff --git a/src/controller/p2p/preheat/enforcer_test.go b/src/controller/p2p/preheat/enforcer_test.go index b789bd3ab52..9d6ed5f2434 100644 --- a/src/controller/p2p/preheat/enforcer_test.go +++ b/src/controller/p2p/preheat/enforcer_test.go @@ -70,7 +70,7 @@ func (suite *EnforcerTestSuite) SetupSuite() { suite.server.StartTLS() fakePolicies := mockPolicies() - fakePolicyManager := &policy.FakeManager{} + fakePolicyManager := &policy.Manager{} fakePolicyManager.On("Get", context.TODO(), mock.AnythingOfType("int64")). @@ -130,7 +130,7 @@ func (suite *EnforcerTestSuite) SetupSuite() { }, }, nil) - fakeInstanceMgr := &instance.FakeManager{} + fakeInstanceMgr := &instance.Manager{} fakeInstanceMgr.On("Get", context.TODO(), mock.AnythingOfType("int64"), diff --git a/src/controller/scan/base_controller_test.go b/src/controller/scan/base_controller_test.go index 28811ce68b0..ca12196d7d3 100644 --- a/src/controller/scan/base_controller_test.go +++ b/src/controller/scan/base_controller_test.go @@ -82,7 +82,7 @@ type ControllerTestSuite struct { reportMgr *reporttesting.Manager ar artifact.Controller c *basicController - reportConverter *postprocessorstesting.ScanReportV1ToV2Converter + reportConverter *postprocessorstesting.NativeScanReportConverter cache *mockcache.Cache } @@ -339,7 +339,7 @@ func (suite *ControllerTestSuite) SetupSuite() { execMgr: suite.execMgr, taskMgr: suite.taskMgr, - reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{}, + reportConverter: &postprocessorstesting.NativeScanReportConverter{}, cache: func() cache.Cache { return suite.cache }, } mock.OnAnything(suite.scanHandler, "JobVendorType").Return("IMAGE_SCAN") @@ -486,6 +486,7 @@ func (suite *ControllerTestSuite) TestScanControllerGetReport() { {ExtraAttrs: suite.makeExtraAttrs(int64(1), "rp-uuid-001")}, }, nil).Once() mock.OnAnything(suite.accessoryMgr, "List").Return(nil, nil) + mock.OnAnything(suite.c.reportConverter, "FromRelationalSchema").Return("", nil) rep, err := suite.c.GetReport(ctx, suite.artifact, []string{v1.MimeTypeNativeReport}) require.NoError(suite.T(), err) assert.Equal(suite.T(), 1, len(rep)) diff --git a/src/controller/scan/callback_test.go b/src/controller/scan/callback_test.go index e1fe6c4e287..165524e458f 100644 --- a/src/controller/scan/callback_test.go +++ b/src/controller/scan/callback_test.go @@ -51,7 +51,7 @@ type CallbackTestSuite struct { scanCtl Controller taskMgr *tasktesting.Manager - reportConverter *postprocessorstesting.ScanReportV1ToV2Converter + reportConverter *postprocessorstesting.NativeScanReportConverter } func (suite *CallbackTestSuite) SetupSuite() { @@ -69,7 +69,7 @@ func (suite *CallbackTestSuite) SetupSuite() { suite.taskMgr = &tasktesting.Manager{} taskMgr = suite.taskMgr - suite.reportConverter = &postprocessorstesting.ScanReportV1ToV2Converter{} + suite.reportConverter = &postprocessorstesting.NativeScanReportConverter{} suite.scanCtl = &basicController{ makeCtx: context.TODO, diff --git a/src/controller/securityhub/controller_test.go b/src/controller/securityhub/controller_test.go index 600c692bf89..68140d2f6b5 100644 --- a/src/controller/securityhub/controller_test.go +++ b/src/controller/securityhub/controller_test.go @@ -44,7 +44,7 @@ type ControllerTestSuite struct { c *controller scannerMgr *scannerMock.Manager secHubMgr *securityMock.Manager - tagMgr *tagMock.FakeManager + tagMgr *tagMock.Manager } // TestController is the entry of controller test suite @@ -56,7 +56,7 @@ func TestController(t *testing.T) { func (suite *ControllerTestSuite) SetupTest() { suite.secHubMgr = &securityMock.Manager{} suite.scannerMgr = &scannerMock.Manager{} - suite.tagMgr = &tagMock.FakeManager{} + suite.tagMgr = &tagMock.Manager{} suite.c = &controller{ secHubMgr: suite.secHubMgr, diff --git a/src/controller/tag/controller_test.go b/src/controller/tag/controller_test.go index bd8d1e7ecdc..9755bbd607a 100644 --- a/src/controller/tag/controller_test.go +++ b/src/controller/tag/controller_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/goharbor/harbor/src/lib/errors" @@ -27,6 +26,7 @@ import ( _ "github.com/goharbor/harbor/src/pkg/config/inmemory" "github.com/goharbor/harbor/src/pkg/tag/model/tag" ormtesting "github.com/goharbor/harbor/src/testing/lib/orm" + "github.com/goharbor/harbor/src/testing/mock" "github.com/goharbor/harbor/src/testing/pkg/artifact" "github.com/goharbor/harbor/src/testing/pkg/immutable" "github.com/goharbor/harbor/src/testing/pkg/repository" @@ -38,14 +38,14 @@ type controllerTestSuite struct { ctl *controller repoMgr *repository.Manager artMgr *artifact.Manager - tagMgr *tagtesting.FakeManager + tagMgr *tagtesting.Manager immutableMtr *immutable.FakeMatcher } func (c *controllerTestSuite) SetupTest() { c.repoMgr = &repository.Manager{} c.artMgr = &artifact.Manager{} - c.tagMgr = &tagtesting.FakeManager{} + c.tagMgr = &tagtesting.Manager{} c.immutableMtr = &immutable.FakeMatcher{} c.ctl = &controller{ tagMgr: c.tagMgr, @@ -56,7 +56,7 @@ func (c *controllerTestSuite) SetupTest() { func (c *controllerTestSuite) TestEnsureTag() { // the tag already exists under the repository and is attached to the artifact - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { ID: 1, RepositoryID: 1, @@ -67,7 +67,7 @@ func (c *controllerTestSuite) TestEnsureTag() { c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err := c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -76,7 +76,7 @@ func (c *controllerTestSuite) TestEnsureTag() { c.SetupTest() // the tag exists under the repository, but it is attached to other artifact - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { ID: 1, RepositoryID: 1, @@ -84,11 +84,11 @@ func (c *controllerTestSuite) TestEnsureTag() { Name: "latest", }, }, nil) - c.tagMgr.On("Update").Return(nil) + c.tagMgr.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err = c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -97,26 +97,26 @@ func (c *controllerTestSuite) TestEnsureTag() { c.SetupTest() // the tag doesn't exist under the repository, create it - c.tagMgr.On("List").Return([]*tag.Tag{}, nil) - c.tagMgr.On("Create").Return(1, nil) + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{}, nil) + c.tagMgr.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err = c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) } func (c *controllerTestSuite) TestCount() { - c.tagMgr.On("Count").Return(1, nil) + c.tagMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil) total, err := c.ctl.Count(nil, nil) c.Require().Nil(err) c.Equal(int64(1), total) } func (c *controllerTestSuite) TestList() { - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { RepositoryID: 1, Name: "testlist", @@ -134,7 +134,7 @@ func (c *controllerTestSuite) TestGet() { getTest.RepositoryID = 1 getTest.Name = "testget" - c.tagMgr.On("Get").Return(getTest, nil) + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(getTest, nil) tag, err := c.ctl.Get(nil, 1, nil) c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -143,36 +143,36 @@ func (c *controllerTestSuite) TestGet() { } func (c *controllerTestSuite) TestDelete() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, Name: "test", }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) err := c.ctl.Delete(nil, 1) c.Require().Nil(err) } func (c *controllerTestSuite) TestDeleteImmutable() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, Name: "test", }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(true, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(true, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) err := c.ctl.Delete(nil, 1) c.Require().NotNil(err) c.True(errors.IsErr(err, errors.PreconditionCode)) } func (c *controllerTestSuite) TestUpdate() { - c.tagMgr.On("Update").Return(nil) + mock.OnAnything(c.tagMgr, "Update").Return(nil) err := c.ctl.Update(nil, &Tag{ Tag: tag.Tag{ RepositoryID: 1, @@ -184,14 +184,14 @@ func (c *controllerTestSuite) TestUpdate() { } func (c *controllerTestSuite) TestDeleteTags() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) ids := []int64{1, 2, 3, 4} err := c.ctl.DeleteTags(nil, ids) c.Require().Nil(err) @@ -218,7 +218,7 @@ func (c *controllerTestSuite) TestAssembleTag() { } c.artMgr.On("Get", mock.Anything, mock.Anything).Return(art, nil) - c.immutableMtr.On("Match").Return(true, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(true, nil) tag := c.ctl.assembleTag(nil, tg, option) c.Require().NotNil(tag) c.Equal(tag.ID, tg.ID) diff --git a/src/jobservice/job/impl/gc/garbage_collection_test.go b/src/jobservice/job/impl/gc/garbage_collection_test.go index 124cff7f983..5aacd737e7d 100644 --- a/src/jobservice/job/impl/gc/garbage_collection_test.go +++ b/src/jobservice/job/impl/gc/garbage_collection_test.go @@ -42,8 +42,8 @@ import ( type gcTestSuite struct { htesting.Suite artifactCtl *artifacttesting.Controller - artrashMgr *trashtesting.FakeManager - registryCtlClient *registryctl.Mockclient + artrashMgr *trashtesting.Manager + registryCtlClient *registryctl.Client projectCtl *projecttesting.Controller blobMgr *blob.Manager @@ -54,8 +54,8 @@ type gcTestSuite struct { func (suite *gcTestSuite) SetupTest() { suite.artifactCtl = &artifacttesting.Controller{} - suite.artrashMgr = &trashtesting.FakeManager{} - suite.registryCtlClient = ®istryctl.Mockclient{} + suite.artrashMgr = &trashtesting.Manager{} + suite.registryCtlClient = ®istryctl.Client{} suite.blobMgr = &blob.Manager{} suite.projectCtl = &projecttesting.Controller{} @@ -98,7 +98,7 @@ func (suite *gcTestSuite) TestDeletedArt() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{ + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{ { ID: 1, Digest: suite.DigestString(), @@ -163,6 +163,8 @@ func (suite *gcTestSuite) TestInit() { "time_window": 1, "workers": float64(3), } + + mock.OnAnything(gc.registryCtlClient, "Health").Return(nil) suite.Nil(gc.init(ctx, params)) suite.True(gc.deleteUntagged) suite.Equal(3, gc.workers) @@ -230,7 +232,7 @@ func (suite *gcTestSuite) TestRun() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{}, nil) + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{}, nil) mock.OnAnything(suite.projectCtl, "List").Return([]*proModels.Project{ { @@ -271,6 +273,8 @@ func (suite *gcTestSuite) TestRun() { mock.OnAnything(suite.blobMgr, "Delete").Return(nil) + mock.OnAnything(suite.registryCtlClient, "Health").Return(nil) + gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, @@ -284,6 +288,7 @@ func (suite *gcTestSuite) TestRun() { "workers": 3, } + mock.OnAnything(gc.registryCtlClient, "DeleteBlob").Return(nil) suite.Nil(gc.Run(ctx, params)) } @@ -302,7 +307,7 @@ func (suite *gcTestSuite) TestMark() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{ + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{ { ID: 1, Digest: suite.DigestString(), @@ -381,6 +386,7 @@ func (suite *gcTestSuite) TestSweep() { workers: 3, } + mock.OnAnything(gc.registryCtlClient, "DeleteBlob").Return(nil) suite.Nil(gc.sweep(ctx)) } diff --git a/src/pkg/scan/vulnerability/vul_test.go b/src/pkg/scan/vulnerability/vul_test.go index 2d0dcbe983e..be0eb50c8f9 100644 --- a/src/pkg/scan/vulnerability/vul_test.go +++ b/src/pkg/scan/vulnerability/vul_test.go @@ -62,12 +62,12 @@ func TestPostScan(t *testing.T) { origRp := &scan.Report{} rawReport := "" - mocker := &postprocessorstesting.ScanReportV1ToV2Converter{} - mocker.On("ToRelationalSchema", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, "original report", nil) + mocker := &postprocessorstesting.NativeScanReportConverter{} + mocker.On("ToRelationalSchema", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", "original report", nil) postprocessors.Converter = mocker sr := &v1.ScanRequest{Artifact: artifact} refreshedReport, err := v.PostScan(ctx, sr, origRp, rawReport, time.Now(), &model.Robot{}) - assert.Equal(t, "", refreshedReport, "PostScan should return the refreshed report") + assert.Equal(t, "original report", refreshedReport, "PostScan should return the refreshed report") assert.Nil(t, err, "PostScan should not return an error") } @@ -209,6 +209,7 @@ func (suite *VulHandlerTestSuite) TestMakeReportPlaceHolder() { mock.OnAnything(suite.reportMgr, "Create").Return("uuid", nil).Once() mock.OnAnything(suite.reportMgr, "Delete").Return(nil).Once() mock.OnAnything(suite.taskMgr, "ListScanTasksByReportUUID").Return([]*task.Task{{Status: "Success"}}, nil) + mock.OnAnything(suite.handler.reportConverter, "FromRelationalSchema").Return("", nil) rps, err := suite.handler.MakePlaceHolder(ctx, art, r) require.NoError(suite.T(), err) assert.Equal(suite.T(), 1, len(rps)) diff --git a/src/testing/lib/libcache/cache.go b/src/testing/lib/libcache/cache.go deleted file mode 100644 index 8eb215e52d9..00000000000 --- a/src/testing/lib/libcache/cache.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. - -package libcache - -import ( - context "context" - - cache "github.com/goharbor/harbor/src/lib/cache" - - mock "github.com/stretchr/testify/mock" - - time "time" -) - -// Cache is an autogenerated mock type for the Cache type -type Cache struct { - mock.Mock -} - -// Contains provides a mock function with given fields: ctx, key -func (_m *Cache) Contains(ctx context.Context, key string) bool { - ret := _m.Called(ctx, key) - - var r0 bool - if rf, ok := ret.Get(0).(func(context.Context, string) bool); ok { - r0 = rf(ctx, key) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Delete provides a mock function with given fields: ctx, key -func (_m *Cache) Delete(ctx context.Context, key string) error { - ret := _m.Called(ctx, key) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(ctx, key) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Fetch provides a mock function with given fields: ctx, key, value -func (_m *Cache) Fetch(ctx context.Context, key string, value interface{}) error { - ret := _m.Called(ctx, key, value) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, interface{}) error); ok { - r0 = rf(ctx, key, value) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Ping provides a mock function with given fields: ctx -func (_m *Cache) Ping(ctx context.Context) error { - ret := _m.Called(ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Save provides a mock function with given fields: ctx, key, value, expiration -func (_m *Cache) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error { - _va := make([]interface{}, len(expiration)) - for _i := range expiration { - _va[_i] = expiration[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, key, value) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, interface{}, ...time.Duration) error); ok { - r0 = rf(ctx, key, value, expiration...) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Scan provides a mock function with given fields: ctx, match -func (_m *Cache) Scan(ctx context.Context, match string) (cache.Iterator, error) { - ret := _m.Called(ctx, match) - - var r0 cache.Iterator - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) (cache.Iterator, error)); ok { - return rf(ctx, match) - } - if rf, ok := ret.Get(0).(func(context.Context, string) cache.Iterator); ok { - r0 = rf(ctx, match) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(cache.Iterator) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, match) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewCache interface { - mock.TestingT - Cleanup(func()) -} - -// NewCache creates a new instance of Cache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewCache(t mockConstructorTestingTNewCache) *Cache { - mock := &Cache{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/src/testing/pkg/artifactrash/manager.go b/src/testing/pkg/artifactrash/manager.go index c1a43680321..de70e91a2fc 100644 --- a/src/testing/pkg/artifactrash/manager.go +++ b/src/testing/pkg/artifactrash/manager.go @@ -1,38 +1,123 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + package artifactrash import ( - "context" - - "github.com/stretchr/testify/mock" + context "context" - "github.com/goharbor/harbor/src/pkg/artifactrash/model" + model "github.com/goharbor/harbor/src/pkg/artifactrash/model" + mock "github.com/stretchr/testify/mock" ) -// FakeManager is a fake tag manager that implement the src/pkg/tag.Manager interface -type FakeManager struct { +// Manager is an autogenerated mock type for the Manager type +type Manager struct { mock.Mock } -// Create ... -func (f *FakeManager) Create(ctx context.Context, artifactrsh *model.ArtifactTrash) (id int64, err error) { - args := f.Called() - return int64(args.Int(0)), args.Error(1) +// Create provides a mock function with given fields: ctx, artifactrsh +func (_m *Manager) Create(ctx context.Context, artifactrsh *model.ArtifactTrash) (int64, error) { + ret := _m.Called(ctx, artifactrsh) + + if len(ret) == 0 { + panic("no return value specified for Create") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *model.ArtifactTrash) (int64, error)); ok { + return rf(ctx, artifactrsh) + } + if rf, ok := ret.Get(0).(func(context.Context, *model.ArtifactTrash) int64); ok { + r0 = rf(ctx, artifactrsh) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, *model.ArtifactTrash) error); ok { + r1 = rf(ctx, artifactrsh) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *Manager) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for Delete") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 } -// Delete ... -func (f *FakeManager) Delete(ctx context.Context, id int64) error { - args := f.Called() - return args.Error(0) +// Filter provides a mock function with given fields: ctx, timeWindow +func (_m *Manager) Filter(ctx context.Context, timeWindow int64) ([]model.ArtifactTrash, error) { + ret := _m.Called(ctx, timeWindow) + + if len(ret) == 0 { + panic("no return value specified for Filter") + } + + var r0 []model.ArtifactTrash + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) ([]model.ArtifactTrash, error)); ok { + return rf(ctx, timeWindow) + } + if rf, ok := ret.Get(0).(func(context.Context, int64) []model.ArtifactTrash); ok { + r0 = rf(ctx, timeWindow) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]model.ArtifactTrash) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, timeWindow) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// Filter ... -func (f *FakeManager) Filter(ctx context.Context, timeWindow int64) (arts []model.ArtifactTrash, err error) { - args := f.Called() - return args.Get(0).([]model.ArtifactTrash), args.Error(1) +// Flush provides a mock function with given fields: ctx, timeWindow +func (_m *Manager) Flush(ctx context.Context, timeWindow int64) error { + ret := _m.Called(ctx, timeWindow) + + if len(ret) == 0 { + panic("no return value specified for Flush") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, timeWindow) + } else { + r0 = ret.Error(0) + } + + return r0 } -// Flush ... -func (f *FakeManager) Flush(ctx context.Context, timeWindow int64) (err error) { - args := f.Called() - return args.Error(0) +// NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewManager(t interface { + mock.TestingT + Cleanup(func()) +}) *Manager { + mock := &Manager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/src/testing/pkg/chart/operator.go b/src/testing/pkg/chart/operator.go index eecaab4ef09..4ed9e281613 100644 --- a/src/testing/pkg/chart/operator.go +++ b/src/testing/pkg/chart/operator.go @@ -1,33 +1,89 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + package chart import ( - "github.com/stretchr/testify/mock" - helm_chart "helm.sh/helm/v3/pkg/chart" + mock "github.com/stretchr/testify/mock" + chart "helm.sh/helm/v3/pkg/chart" - chartserver "github.com/goharbor/harbor/src/pkg/chart" + pkgchart "github.com/goharbor/harbor/src/pkg/chart" ) -// FakeOpertaor ... -type FakeOpertaor struct { +// Operator is an autogenerated mock type for the Operator type +type Operator struct { mock.Mock } -// GetDetails ... -func (f *FakeOpertaor) GetDetails(content []byte) (*chartserver.VersionDetails, error) { - args := f.Called() - var chartDetails *chartserver.VersionDetails - if args.Get(0) != nil { - chartDetails = args.Get(0).(*chartserver.VersionDetails) +// GetData provides a mock function with given fields: content +func (_m *Operator) GetData(content []byte) (*chart.Chart, error) { + ret := _m.Called(content) + + if len(ret) == 0 { + panic("no return value specified for GetData") + } + + var r0 *chart.Chart + var r1 error + if rf, ok := ret.Get(0).(func([]byte) (*chart.Chart, error)); ok { + return rf(content) + } + if rf, ok := ret.Get(0).(func([]byte) *chart.Chart); ok { + r0 = rf(content) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*chart.Chart) + } + } + + if rf, ok := ret.Get(1).(func([]byte) error); ok { + r1 = rf(content) + } else { + r1 = ret.Error(1) } - return chartDetails, args.Error(1) + + return r0, r1 } -// GetData ... -func (f *FakeOpertaor) GetData(content []byte) (*helm_chart.Chart, error) { - args := f.Called() - var chartData *helm_chart.Chart - if args.Get(0) != nil { - chartData = args.Get(0).(*helm_chart.Chart) +// GetDetails provides a mock function with given fields: content +func (_m *Operator) GetDetails(content []byte) (*pkgchart.VersionDetails, error) { + ret := _m.Called(content) + + if len(ret) == 0 { + panic("no return value specified for GetDetails") } - return chartData, args.Error(1) + + var r0 *pkgchart.VersionDetails + var r1 error + if rf, ok := ret.Get(0).(func([]byte) (*pkgchart.VersionDetails, error)); ok { + return rf(content) + } + if rf, ok := ret.Get(0).(func([]byte) *pkgchart.VersionDetails); ok { + r0 = rf(content) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pkgchart.VersionDetails) + } + } + + if rf, ok := ret.Get(1).(func([]byte) error); ok { + r1 = rf(content) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewOperator creates a new instance of Operator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewOperator(t interface { + mock.TestingT + Cleanup(func()) +}) *Operator { + mock := &Operator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/src/testing/pkg/distribution/manifest.go b/src/testing/pkg/distribution/manifest.go index 6250248ecf4..7d96ca80e65 100644 --- a/src/testing/pkg/distribution/manifest.go +++ b/src/testing/pkg/distribution/manifest.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package distribution @@ -16,6 +16,10 @@ type Manifest struct { func (_m *Manifest) Payload() (string, []byte, error) { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Payload") + } + var r0 string var r1 []byte var r2 error @@ -49,6 +53,10 @@ func (_m *Manifest) Payload() (string, []byte, error) { func (_m *Manifest) References() []distribution.Descriptor { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for References") + } + var r0 []distribution.Descriptor if rf, ok := ret.Get(0).(func() []distribution.Descriptor); ok { r0 = rf() @@ -61,13 +69,12 @@ func (_m *Manifest) References() []distribution.Descriptor { return r0 } -type mockConstructorTestingTNewManifest interface { +// NewManifest creates a new instance of Manifest. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewManifest(t interface { mock.TestingT Cleanup(func()) -} - -// NewManifest creates a new instance of Manifest. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewManifest(t mockConstructorTestingTNewManifest) *Manifest { +}) *Manifest { mock := &Manifest{} mock.Mock.Test(t) diff --git a/src/testing/pkg/immutable/matcher.go b/src/testing/pkg/immutable/matcher.go index 2e00b917da9..f14c04e2fa0 100644 --- a/src/testing/pkg/immutable/matcher.go +++ b/src/testing/pkg/immutable/matcher.go @@ -1,20 +1,58 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + package immutable import ( - "context" + context "context" - "github.com/stretchr/testify/mock" + mock "github.com/stretchr/testify/mock" - "github.com/goharbor/harbor/src/lib/selector" + selector "github.com/goharbor/harbor/src/lib/selector" ) -// FakeMatcher ... +// FakeMatcher is an autogenerated mock type for the ImmutableTagMatcher type type FakeMatcher struct { mock.Mock } -// Match ... -func (f *FakeMatcher) Match(ctx context.Context, pid int64, c selector.Candidate) (bool, error) { - args := f.Called() - return args.Bool(0), args.Error(1) +// Match provides a mock function with given fields: ctx, pid, c +func (_m *FakeMatcher) Match(ctx context.Context, pid int64, c selector.Candidate) (bool, error) { + ret := _m.Called(ctx, pid, c) + + if len(ret) == 0 { + panic("no return value specified for Match") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, selector.Candidate) (bool, error)); ok { + return rf(ctx, pid, c) + } + if rf, ok := ret.Get(0).(func(context.Context, int64, selector.Candidate) bool); ok { + r0 = rf(ctx, pid, c) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, int64, selector.Candidate) error); ok { + r1 = rf(ctx, pid, c) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewFakeMatcher creates a new instance of FakeMatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFakeMatcher(t interface { + mock.TestingT + Cleanup(func()) +}) *FakeMatcher { + mock := &FakeMatcher{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/src/testing/pkg/p2p/preheat/instance/manager.go b/src/testing/pkg/p2p/preheat/instance/manager.go index 3c18aada7f1..6e0e5233b18 100644 --- a/src/testing/pkg/p2p/preheat/instance/manager.go +++ b/src/testing/pkg/p2p/preheat/instance/manager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package instance @@ -7,27 +7,35 @@ import ( mock "github.com/stretchr/testify/mock" - q "github.com/goharbor/harbor/src/lib/q" provider "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/provider" + + q "github.com/goharbor/harbor/src/lib/q" ) -// FakeManager is an autogenerated mock type for the Manager type -type FakeManager struct { +// Manager is an autogenerated mock type for the Manager type +type Manager struct { mock.Mock } // Count provides a mock function with given fields: ctx, query -func (_m *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) { +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { ret := _m.Called(ctx, query) + if len(ret) == 0 { + panic("no return value specified for Count") + } + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) (int64, error)); ok { + return rf(ctx, query) + } if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { r0 = rf(ctx, query) } else { r0 = ret.Get(0).(int64) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { r1 = rf(ctx, query) } else { @@ -38,9 +46,13 @@ func (_m *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) } // Delete provides a mock function with given fields: ctx, id -func (_m *FakeManager) Delete(ctx context.Context, id int64) error { +func (_m *Manager) Delete(ctx context.Context, id int64) error { ret := _m.Called(ctx, id) + if len(ret) == 0 { + panic("no return value specified for Delete") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { r0 = rf(ctx, id) @@ -52,10 +64,18 @@ func (_m *FakeManager) Delete(ctx context.Context, id int64) error { } // Get provides a mock function with given fields: ctx, id -func (_m *FakeManager) Get(ctx context.Context, id int64) (*provider.Instance, error) { +func (_m *Manager) Get(ctx context.Context, id int64) (*provider.Instance, error) { ret := _m.Called(ctx, id) + if len(ret) == 0 { + panic("no return value specified for Get") + } + var r0 *provider.Instance + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) (*provider.Instance, error)); ok { + return rf(ctx, id) + } if rf, ok := ret.Get(0).(func(context.Context, int64) *provider.Instance); ok { r0 = rf(ctx, id) } else { @@ -64,7 +84,6 @@ func (_m *FakeManager) Get(ctx context.Context, id int64) (*provider.Instance, e } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { r1 = rf(ctx, id) } else { @@ -75,10 +94,18 @@ func (_m *FakeManager) Get(ctx context.Context, id int64) (*provider.Instance, e } // GetByName provides a mock function with given fields: ctx, name -func (_m *FakeManager) GetByName(ctx context.Context, name string) (*provider.Instance, error) { +func (_m *Manager) GetByName(ctx context.Context, name string) (*provider.Instance, error) { ret := _m.Called(ctx, name) + if len(ret) == 0 { + panic("no return value specified for GetByName") + } + var r0 *provider.Instance + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*provider.Instance, error)); ok { + return rf(ctx, name) + } if rf, ok := ret.Get(0).(func(context.Context, string) *provider.Instance); ok { r0 = rf(ctx, name) } else { @@ -87,7 +114,6 @@ func (_m *FakeManager) GetByName(ctx context.Context, name string) (*provider.In } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { r1 = rf(ctx, name) } else { @@ -98,10 +124,18 @@ func (_m *FakeManager) GetByName(ctx context.Context, name string) (*provider.In } // List provides a mock function with given fields: ctx, query -func (_m *FakeManager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) { +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*provider.Instance, error) { ret := _m.Called(ctx, query) + if len(ret) == 0 { + panic("no return value specified for List") + } + var r0 []*provider.Instance + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) ([]*provider.Instance, error)); ok { + return rf(ctx, query) + } if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*provider.Instance); ok { r0 = rf(ctx, query) } else { @@ -110,7 +144,6 @@ func (_m *FakeManager) List(ctx context.Context, query *q.Query) ([]*provider.In } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { r1 = rf(ctx, query) } else { @@ -121,17 +154,24 @@ func (_m *FakeManager) List(ctx context.Context, query *q.Query) ([]*provider.In } // Save provides a mock function with given fields: ctx, inst -func (_m *FakeManager) Save(ctx context.Context, inst *provider.Instance) (int64, error) { +func (_m *Manager) Save(ctx context.Context, inst *provider.Instance) (int64, error) { ret := _m.Called(ctx, inst) + if len(ret) == 0 { + panic("no return value specified for Save") + } + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance) (int64, error)); ok { + return rf(ctx, inst) + } if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance) int64); ok { r0 = rf(ctx, inst) } else { r0 = ret.Get(0).(int64) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *provider.Instance) error); ok { r1 = rf(ctx, inst) } else { @@ -142,7 +182,7 @@ func (_m *FakeManager) Save(ctx context.Context, inst *provider.Instance) (int64 } // Update provides a mock function with given fields: ctx, inst, props -func (_m *FakeManager) Update(ctx context.Context, inst *provider.Instance, props ...string) error { +func (_m *Manager) Update(ctx context.Context, inst *provider.Instance, props ...string) error { _va := make([]interface{}, len(props)) for _i := range props { _va[_i] = props[_i] @@ -152,6 +192,10 @@ func (_m *FakeManager) Update(ctx context.Context, inst *provider.Instance, prop _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for Update") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, *provider.Instance, ...string) error); ok { r0 = rf(ctx, inst, props...) @@ -161,3 +205,17 @@ func (_m *FakeManager) Update(ctx context.Context, inst *provider.Instance, prop return r0 } + +// NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewManager(t interface { + mock.TestingT + Cleanup(func()) +}) *Manager { + mock := &Manager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/testing/pkg/p2p/preheat/policy/manager.go b/src/testing/pkg/p2p/preheat/policy/manager.go index e83524dd893..7c79fcfb383 100644 --- a/src/testing/pkg/p2p/preheat/policy/manager.go +++ b/src/testing/pkg/p2p/preheat/policy/manager.go @@ -1,33 +1,40 @@ -// Code generated by mockery v2.0.3. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package policy import ( context "context" + modelspolicy "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" mock "github.com/stretchr/testify/mock" q "github.com/goharbor/harbor/src/lib/q" - modelspolicy "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" ) -// FakeManager is an autogenerated mock type for the Manager type -type FakeManager struct { +// Manager is an autogenerated mock type for the Manager type +type Manager struct { mock.Mock } // Count provides a mock function with given fields: ctx, query -func (_m *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) { +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { ret := _m.Called(ctx, query) + if len(ret) == 0 { + panic("no return value specified for Count") + } + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) (int64, error)); ok { + return rf(ctx, query) + } if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { r0 = rf(ctx, query) } else { r0 = ret.Get(0).(int64) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { r1 = rf(ctx, query) } else { @@ -38,17 +45,24 @@ func (_m *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) } // Create provides a mock function with given fields: ctx, schema -func (_m *FakeManager) Create(ctx context.Context, schema *modelspolicy.Schema) (int64, error) { +func (_m *Manager) Create(ctx context.Context, schema *modelspolicy.Schema) (int64, error) { ret := _m.Called(ctx, schema) + if len(ret) == 0 { + panic("no return value specified for Create") + } + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *modelspolicy.Schema) (int64, error)); ok { + return rf(ctx, schema) + } if rf, ok := ret.Get(0).(func(context.Context, *modelspolicy.Schema) int64); ok { r0 = rf(ctx, schema) } else { r0 = ret.Get(0).(int64) } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *modelspolicy.Schema) error); ok { r1 = rf(ctx, schema) } else { @@ -59,9 +73,13 @@ func (_m *FakeManager) Create(ctx context.Context, schema *modelspolicy.Schema) } // Delete provides a mock function with given fields: ctx, id -func (_m *FakeManager) Delete(ctx context.Context, id int64) error { +func (_m *Manager) Delete(ctx context.Context, id int64) error { ret := _m.Called(ctx, id) + if len(ret) == 0 { + panic("no return value specified for Delete") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { r0 = rf(ctx, id) @@ -73,10 +91,18 @@ func (_m *FakeManager) Delete(ctx context.Context, id int64) error { } // Get provides a mock function with given fields: ctx, id -func (_m *FakeManager) Get(ctx context.Context, id int64) (*modelspolicy.Schema, error) { +func (_m *Manager) Get(ctx context.Context, id int64) (*modelspolicy.Schema, error) { ret := _m.Called(ctx, id) + if len(ret) == 0 { + panic("no return value specified for Get") + } + var r0 *modelspolicy.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) (*modelspolicy.Schema, error)); ok { + return rf(ctx, id) + } if rf, ok := ret.Get(0).(func(context.Context, int64) *modelspolicy.Schema); ok { r0 = rf(ctx, id) } else { @@ -85,7 +111,6 @@ func (_m *FakeManager) Get(ctx context.Context, id int64) (*modelspolicy.Schema, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { r1 = rf(ctx, id) } else { @@ -95,22 +120,29 @@ func (_m *FakeManager) Get(ctx context.Context, id int64) (*modelspolicy.Schema, return r0, r1 } -// GetByName provides a mock function with given fields: ctx, projectId, name -func (_m *FakeManager) GetByName(ctx context.Context, projectId int64, name string) (*modelspolicy.Schema, error) { - ret := _m.Called(ctx, projectId, name) +// GetByName provides a mock function with given fields: ctx, projectID, name +func (_m *Manager) GetByName(ctx context.Context, projectID int64, name string) (*modelspolicy.Schema, error) { + ret := _m.Called(ctx, projectID, name) + + if len(ret) == 0 { + panic("no return value specified for GetByName") + } var r0 *modelspolicy.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, string) (*modelspolicy.Schema, error)); ok { + return rf(ctx, projectID, name) + } if rf, ok := ret.Get(0).(func(context.Context, int64, string) *modelspolicy.Schema); ok { - r0 = rf(ctx, projectId, name) + r0 = rf(ctx, projectID, name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*modelspolicy.Schema) } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, int64, string) error); ok { - r1 = rf(ctx, projectId, name) + r1 = rf(ctx, projectID, name) } else { r1 = ret.Error(1) } @@ -119,10 +151,18 @@ func (_m *FakeManager) GetByName(ctx context.Context, projectId int64, name stri } // ListPolicies provides a mock function with given fields: ctx, query -func (_m *FakeManager) ListPolicies(ctx context.Context, query *q.Query) ([]*modelspolicy.Schema, error) { +func (_m *Manager) ListPolicies(ctx context.Context, query *q.Query) ([]*modelspolicy.Schema, error) { ret := _m.Called(ctx, query) + if len(ret) == 0 { + panic("no return value specified for ListPolicies") + } + var r0 []*modelspolicy.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) ([]*modelspolicy.Schema, error)); ok { + return rf(ctx, query) + } if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*modelspolicy.Schema); ok { r0 = rf(ctx, query) } else { @@ -131,7 +171,6 @@ func (_m *FakeManager) ListPolicies(ctx context.Context, query *q.Query) ([]*mod } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { r1 = rf(ctx, query) } else { @@ -142,10 +181,18 @@ func (_m *FakeManager) ListPolicies(ctx context.Context, query *q.Query) ([]*mod } // ListPoliciesByProject provides a mock function with given fields: ctx, project, query -func (_m *FakeManager) ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) ([]*modelspolicy.Schema, error) { +func (_m *Manager) ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) ([]*modelspolicy.Schema, error) { ret := _m.Called(ctx, project, query) + if len(ret) == 0 { + panic("no return value specified for ListPoliciesByProject") + } + var r0 []*modelspolicy.Schema + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64, *q.Query) ([]*modelspolicy.Schema, error)); ok { + return rf(ctx, project, query) + } if rf, ok := ret.Get(0).(func(context.Context, int64, *q.Query) []*modelspolicy.Schema); ok { r0 = rf(ctx, project, query) } else { @@ -154,7 +201,6 @@ func (_m *FakeManager) ListPoliciesByProject(ctx context.Context, project int64, } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, int64, *q.Query) error); ok { r1 = rf(ctx, project, query) } else { @@ -165,7 +211,7 @@ func (_m *FakeManager) ListPoliciesByProject(ctx context.Context, project int64, } // Update provides a mock function with given fields: ctx, schema, props -func (_m *FakeManager) Update(ctx context.Context, schema *modelspolicy.Schema, props ...string) error { +func (_m *Manager) Update(ctx context.Context, schema *modelspolicy.Schema, props ...string) error { _va := make([]interface{}, len(props)) for _i := range props { _va[_i] = props[_i] @@ -175,6 +221,10 @@ func (_m *FakeManager) Update(ctx context.Context, schema *modelspolicy.Schema, _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for Update") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, *modelspolicy.Schema, ...string) error); ok { r0 = rf(ctx, schema, props...) @@ -184,3 +234,17 @@ func (_m *FakeManager) Update(ctx context.Context, schema *modelspolicy.Schema, return r0 } + +// NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewManager(t interface { + mock.TestingT + Cleanup(func()) +}) *Manager { + mock := &Manager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/testing/pkg/parser/parser.go b/src/testing/pkg/parser/parser.go index 5144b79445d..f533da44e8f 100644 --- a/src/testing/pkg/parser/parser.go +++ b/src/testing/pkg/parser/parser.go @@ -1,13 +1,13 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package parser import ( context "context" - mock "github.com/stretchr/testify/mock" - artifact "github.com/goharbor/harbor/src/pkg/artifact" + + mock "github.com/stretchr/testify/mock" ) // Parser is an autogenerated mock type for the Parser type @@ -19,6 +19,10 @@ type Parser struct { func (_m *Parser) Parse(ctx context.Context, _a1 *artifact.Artifact, manifest []byte) error { ret := _m.Called(ctx, _a1, manifest) + if len(ret) == 0 { + panic("no return value specified for Parse") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, []byte) error); ok { r0 = rf(ctx, _a1, manifest) @@ -28,3 +32,17 @@ func (_m *Parser) Parse(ctx context.Context, _a1 *artifact.Artifact, manifest [] return r0 } + +// NewParser creates a new instance of Parser. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewParser(t interface { + mock.TestingT + Cleanup(func()) +}) *Parser { + mock := &Parser{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/testing/pkg/processor/processor.go b/src/testing/pkg/processor/processor.go index 98dadbff033..9f7896a70b5 100644 --- a/src/testing/pkg/processor/processor.go +++ b/src/testing/pkg/processor/processor.go @@ -1,14 +1,15 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. +// Code generated by mockery v2.43.2. DO NOT EDIT. package processor import ( context "context" + artifact "github.com/goharbor/harbor/src/pkg/artifact" + mock "github.com/stretchr/testify/mock" processor "github.com/goharbor/harbor/src/controller/artifact/processor" - artifact "github.com/goharbor/harbor/src/pkg/artifact" ) // Processor is an autogenerated mock type for the Processor type @@ -20,7 +21,15 @@ type Processor struct { func (_m *Processor) AbstractAddition(ctx context.Context, _a1 *artifact.Artifact, additionType string) (*processor.Addition, error) { ret := _m.Called(ctx, _a1, additionType) + if len(ret) == 0 { + panic("no return value specified for AbstractAddition") + } + var r0 *processor.Addition + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, string) (*processor.Addition, error)); ok { + return rf(ctx, _a1, additionType) + } if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, string) *processor.Addition); ok { r0 = rf(ctx, _a1, additionType) } else { @@ -29,7 +38,6 @@ func (_m *Processor) AbstractAddition(ctx context.Context, _a1 *artifact.Artifac } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *artifact.Artifact, string) error); ok { r1 = rf(ctx, _a1, additionType) } else { @@ -43,6 +51,10 @@ func (_m *Processor) AbstractAddition(ctx context.Context, _a1 *artifact.Artifac func (_m *Processor) AbstractMetadata(ctx context.Context, _a1 *artifact.Artifact, manifest []byte) error { ret := _m.Called(ctx, _a1, manifest) + if len(ret) == 0 { + panic("no return value specified for AbstractMetadata") + } + var r0 error if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact, []byte) error); ok { r0 = rf(ctx, _a1, manifest) @@ -57,6 +69,10 @@ func (_m *Processor) AbstractMetadata(ctx context.Context, _a1 *artifact.Artifac func (_m *Processor) GetArtifactType(ctx context.Context, _a1 *artifact.Artifact) string { ret := _m.Called(ctx, _a1) + if len(ret) == 0 { + panic("no return value specified for GetArtifactType") + } + var r0 string if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact) string); ok { r0 = rf(ctx, _a1) @@ -71,6 +87,10 @@ func (_m *Processor) GetArtifactType(ctx context.Context, _a1 *artifact.Artifact func (_m *Processor) ListAdditionTypes(ctx context.Context, _a1 *artifact.Artifact) []string { ret := _m.Called(ctx, _a1) + if len(ret) == 0 { + panic("no return value specified for ListAdditionTypes") + } + var r0 []string if rf, ok := ret.Get(0).(func(context.Context, *artifact.Artifact) []string); ok { r0 = rf(ctx, _a1) @@ -82,3 +102,17 @@ func (_m *Processor) ListAdditionTypes(ctx context.Context, _a1 *artifact.Artifa return r0 } + +// NewProcessor creates a new instance of Processor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewProcessor(t interface { + mock.TestingT + Cleanup(func()) +}) *Processor { + mock := &Processor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/testing/pkg/scan/dao/scan/vulnerability_record_dao.go b/src/testing/pkg/scan/dao/scan/vulnerability_record_dao.go deleted file mode 100644 index 5c9c7715271..00000000000 --- a/src/testing/pkg/scan/dao/scan/vulnerability_record_dao.go +++ /dev/null @@ -1,256 +0,0 @@ -// Code generated by mockery v2.1.0. DO NOT EDIT. - -package scan - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" - - q "github.com/goharbor/harbor/src/lib/q" - scan "github.com/goharbor/harbor/src/pkg/scan/dao/scan" -) - -// VulnerabilityRecordDao is an autogenerated mock type for the VulnerabilityRecordDao type -type VulnerabilityRecordDao struct { - mock.Mock -} - -// Create provides a mock function with given fields: ctx, vr -func (_m *VulnerabilityRecordDao) Create(ctx context.Context, vr *scan.VulnerabilityRecord) (int64, error) { - ret := _m.Called(ctx, vr) - - var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, *scan.VulnerabilityRecord) int64); ok { - r0 = rf(ctx, vr) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *scan.VulnerabilityRecord) error); ok { - r1 = rf(ctx, vr) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Delete provides a mock function with given fields: ctx, vr -func (_m *VulnerabilityRecordDao) Delete(ctx context.Context, vr *scan.VulnerabilityRecord) error { - ret := _m.Called(ctx, vr) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *scan.VulnerabilityRecord) error); ok { - r0 = rf(ctx, vr) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteForDigests provides a mock function with given fields: ctx, digests -func (_m *VulnerabilityRecordDao) DeleteForDigests(ctx context.Context, digests ...string) (int64, error) { - _va := make([]interface{}, len(digests)) - for _i := range digests { - _va[_i] = digests[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, ...string) int64); ok { - r0 = rf(ctx, digests...) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, ...string) error); ok { - r1 = rf(ctx, digests...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// DeleteForReport provides a mock function with given fields: ctx, reportUUID -func (_m *VulnerabilityRecordDao) DeleteForReport(ctx context.Context, reportUUID string) (int64, error) { - ret := _m.Called(ctx, reportUUID) - - var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, string) int64); ok { - r0 = rf(ctx, reportUUID) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, reportUUID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// DeleteForScanner provides a mock function with given fields: ctx, registrationUUID -func (_m *VulnerabilityRecordDao) DeleteForScanner(ctx context.Context, registrationUUID string) (int64, error) { - ret := _m.Called(ctx, registrationUUID) - - var r0 int64 - if rf, ok := ret.Get(0).(func(context.Context, string) int64); ok { - r0 = rf(ctx, registrationUUID) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, registrationUUID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetForReport provides a mock function with given fields: ctx, reportUUID -func (_m *VulnerabilityRecordDao) GetForReport(ctx context.Context, reportUUID string) ([]*scan.VulnerabilityRecord, error) { - ret := _m.Called(ctx, reportUUID) - - var r0 []*scan.VulnerabilityRecord - if rf, ok := ret.Get(0).(func(context.Context, string) []*scan.VulnerabilityRecord); ok { - r0 = rf(ctx, reportUUID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*scan.VulnerabilityRecord) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, reportUUID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetForScanner provides a mock function with given fields: ctx, registrationUUID -func (_m *VulnerabilityRecordDao) GetForScanner(ctx context.Context, registrationUUID string) ([]*scan.VulnerabilityRecord, error) { - ret := _m.Called(ctx, registrationUUID) - - var r0 []*scan.VulnerabilityRecord - if rf, ok := ret.Get(0).(func(context.Context, string) []*scan.VulnerabilityRecord); ok { - r0 = rf(ctx, registrationUUID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*scan.VulnerabilityRecord) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, registrationUUID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetRecordIdsForScanner provides a mock function with given fields: ctx, registrationUUID -func (_m *VulnerabilityRecordDao) GetRecordIdsForScanner(ctx context.Context, registrationUUID string) ([]int, error) { - ret := _m.Called(ctx, registrationUUID) - - var r0 []int - if rf, ok := ret.Get(0).(func(context.Context, string) []int); ok { - r0 = rf(ctx, registrationUUID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]int) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, registrationUUID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// InsertForReport provides a mock function with given fields: ctx, reportUUID, vulnerabilityRecordIDs -func (_m *VulnerabilityRecordDao) InsertForReport(ctx context.Context, reportUUID string, vulnerabilityRecordIDs ...int64) error { - _va := make([]interface{}, len(vulnerabilityRecordIDs)) - for _i := range vulnerabilityRecordIDs { - _va[_i] = vulnerabilityRecordIDs[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, reportUUID) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, ...int64) error); ok { - r0 = rf(ctx, reportUUID, vulnerabilityRecordIDs...) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// List provides a mock function with given fields: ctx, query -func (_m *VulnerabilityRecordDao) List(ctx context.Context, query *q.Query) ([]*scan.VulnerabilityRecord, error) { - ret := _m.Called(ctx, query) - - var r0 []*scan.VulnerabilityRecord - if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*scan.VulnerabilityRecord); ok { - r0 = rf(ctx, query) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*scan.VulnerabilityRecord) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { - r1 = rf(ctx, query) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Update provides a mock function with given fields: ctx, vr, cols -func (_m *VulnerabilityRecordDao) Update(ctx context.Context, vr *scan.VulnerabilityRecord, cols ...string) error { - _va := make([]interface{}, len(cols)) - for _i := range cols { - _va[_i] = cols[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, vr) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *scan.VulnerabilityRecord, ...string) error); ok { - r0 = rf(ctx, vr, cols...) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/src/testing/pkg/scan/postprocessors/native_scan_report_converter.go b/src/testing/pkg/scan/postprocessors/native_scan_report_converter.go new file mode 100644 index 00000000000..377f6f29e02 --- /dev/null +++ b/src/testing/pkg/scan/postprocessors/native_scan_report_converter.go @@ -0,0 +1,91 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package postprocessors + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// NativeScanReportConverter is an autogenerated mock type for the NativeScanReportConverter type +type NativeScanReportConverter struct { + mock.Mock +} + +// FromRelationalSchema provides a mock function with given fields: ctx, reportUUID, artifactDigest, reportSummary +func (_m *NativeScanReportConverter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string) (string, error) { + ret := _m.Called(ctx, reportUUID, artifactDigest, reportSummary) + + if len(ret) == 0 { + panic("no return value specified for FromRelationalSchema") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) (string, error)); ok { + return rf(ctx, reportUUID, artifactDigest, reportSummary) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) string); ok { + r0 = rf(ctx, reportUUID, artifactDigest, reportSummary) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { + r1 = rf(ctx, reportUUID, artifactDigest, reportSummary) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ToRelationalSchema provides a mock function with given fields: ctx, reportUUID, registrationUUID, digest, reportData +func (_m *NativeScanReportConverter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) { + ret := _m.Called(ctx, reportUUID, registrationUUID, digest, reportData) + + if len(ret) == 0 { + panic("no return value specified for ToRelationalSchema") + } + + var r0 string + var r1 string + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) (string, string, error)); ok { + return rf(ctx, reportUUID, registrationUUID, digest, reportData) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) string); ok { + r0 = rf(ctx, reportUUID, registrationUUID, digest, reportData) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string) string); ok { + r1 = rf(ctx, reportUUID, registrationUUID, digest, reportData) + } else { + r1 = ret.Get(1).(string) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string, string, string) error); ok { + r2 = rf(ctx, reportUUID, registrationUUID, digest, reportData) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// NewNativeScanReportConverter creates a new instance of NativeScanReportConverter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNativeScanReportConverter(t interface { + mock.TestingT + Cleanup(func()) +}) *NativeScanReportConverter { + mock := &NativeScanReportConverter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/src/testing/pkg/scan/postprocessors/report_converters.go b/src/testing/pkg/scan/postprocessors/report_converters.go deleted file mode 100644 index 16944a26f58..00000000000 --- a/src/testing/pkg/scan/postprocessors/report_converters.go +++ /dev/null @@ -1,23 +0,0 @@ -package postprocessors - -import ( - "context" - - mock "github.com/stretchr/testify/mock" -) - -// ScanReportV1ToV2Converter is an auto-generated mock type for converting native Harbor report in JSON -// to relational schema -type ScanReportV1ToV2Converter struct { - mock.Mock -} - -// ToRelationalSchema is a mock implementation of the scan report conversion -func (_c *ScanReportV1ToV2Converter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) { - return "mockId", reportData, nil -} - -// ToRelationalSchema is a mock implementation of the scan report conversion -func (_c *ScanReportV1ToV2Converter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportData string) (string, error) { - return "mockId", nil -} diff --git a/src/testing/pkg/tag/manager.go b/src/testing/pkg/tag/manager.go index 75938f8756e..23af9f07c29 100644 --- a/src/testing/pkg/tag/manager.go +++ b/src/testing/pkg/tag/manager.go @@ -1,79 +1,208 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Code generated by mockery v2.43.2. DO NOT EDIT. package tag import ( - "context" + context "context" - "github.com/stretchr/testify/mock" + modeltag "github.com/goharbor/harbor/src/pkg/tag/model/tag" + mock "github.com/stretchr/testify/mock" - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/tag/model/tag" + q "github.com/goharbor/harbor/src/lib/q" ) -// FakeManager is a fake tag manager that implement the src/pkg/tag.Manager interface -type FakeManager struct { +// Manager is an autogenerated mock type for the Manager type +type Manager struct { mock.Mock } -// Count ... -func (f *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) { - args := f.Called() - return int64(args.Int(0)), args.Error(1) +// Count provides a mock function with given fields: ctx, query +func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) { + ret := _m.Called(ctx, query) + + if len(ret) == 0 { + panic("no return value specified for Count") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) (int64, error)); ok { + return rf(ctx, query) + } + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok { + r0 = rf(ctx, query) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// List ... -func (f *FakeManager) List(ctx context.Context, query *q.Query) ([]*tag.Tag, error) { - args := f.Called() - var tags []*tag.Tag - if args.Get(0) != nil { - tags = args.Get(0).([]*tag.Tag) +// Create provides a mock function with given fields: ctx, _a1 +func (_m *Manager) Create(ctx context.Context, _a1 *modeltag.Tag) (int64, error) { + ret := _m.Called(ctx, _a1) + + if len(ret) == 0 { + panic("no return value specified for Create") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *modeltag.Tag) (int64, error)); ok { + return rf(ctx, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *modeltag.Tag) int64); ok { + r0 = rf(ctx, _a1) + } else { + r0 = ret.Get(0).(int64) } - return tags, args.Error(1) + + if rf, ok := ret.Get(1).(func(context.Context, *modeltag.Tag) error); ok { + r1 = rf(ctx, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Delete provides a mock function with given fields: ctx, id +func (_m *Manager) Delete(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for Delete") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 } -// Get ... -func (f *FakeManager) Get(ctx context.Context, id int64) (*tag.Tag, error) { - args := f.Called() - var tg *tag.Tag - if args.Get(0) != nil { - tg = args.Get(0).(*tag.Tag) +// DeleteOfArtifact provides a mock function with given fields: ctx, artifactID +func (_m *Manager) DeleteOfArtifact(ctx context.Context, artifactID int64) error { + ret := _m.Called(ctx, artifactID) + + if len(ret) == 0 { + panic("no return value specified for DeleteOfArtifact") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, artifactID) + } else { + r0 = ret.Error(0) } - return tg, args.Error(1) + + return r0 } -// Create ... -func (f *FakeManager) Create(ctx context.Context, tag *tag.Tag) (int64, error) { - args := f.Called() - return int64(args.Int(0)), args.Error(1) +// Get provides a mock function with given fields: ctx, id +func (_m *Manager) Get(ctx context.Context, id int64) (*modeltag.Tag, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 *modeltag.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) (*modeltag.Tag, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, int64) *modeltag.Tag); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*modeltag.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// Update ... -func (f *FakeManager) Update(ctx context.Context, tag *tag.Tag, props ...string) error { - args := f.Called() - return args.Error(0) +// List provides a mock function with given fields: ctx, query +func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*modeltag.Tag, error) { + ret := _m.Called(ctx, query) + + if len(ret) == 0 { + panic("no return value specified for List") + } + + var r0 []*modeltag.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) ([]*modeltag.Tag, error)); ok { + return rf(ctx, query) + } + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*modeltag.Tag); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*modeltag.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// Delete ... -func (f *FakeManager) Delete(ctx context.Context, id int64) error { - args := f.Called() - return args.Error(0) +// Update provides a mock function with given fields: ctx, _a1, props +func (_m *Manager) Update(ctx context.Context, _a1 *modeltag.Tag, props ...string) error { + _va := make([]interface{}, len(props)) + for _i := range props { + _va[_i] = props[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, _a1) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for Update") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *modeltag.Tag, ...string) error); ok { + r0 = rf(ctx, _a1, props...) + } else { + r0 = ret.Error(0) + } + + return r0 } -// DeleteOfArtifact ... -func (f *FakeManager) DeleteOfArtifact(ctx context.Context, artifactID int64) error { - args := f.Called() - return args.Error(0) +// NewManager creates a new instance of Manager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewManager(t interface { + mock.TestingT + Cleanup(func()) +}) *Manager { + mock := &Manager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/src/testing/registryctl/client.go b/src/testing/registryctl/client.go index 084716ce42f..eca5728f165 100644 --- a/src/testing/registryctl/client.go +++ b/src/testing/registryctl/client.go @@ -1,24 +1,78 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + package registryctl -import ( - "github.com/stretchr/testify/mock" -) +import mock "github.com/stretchr/testify/mock" -type Mockclient struct { +// Client is an autogenerated mock type for the Client type +type Client struct { mock.Mock } -// Health ... -func (c *Mockclient) Health() error { - return nil +// DeleteBlob provides a mock function with given fields: reference +func (_m *Client) DeleteBlob(reference string) error { + ret := _m.Called(reference) + + if len(ret) == 0 { + panic("no return value specified for DeleteBlob") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(reference) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteManifest provides a mock function with given fields: repository, reference +func (_m *Client) DeleteManifest(repository string, reference string) error { + ret := _m.Called(repository, reference) + + if len(ret) == 0 { + panic("no return value specified for DeleteManifest") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(repository, reference) + } else { + r0 = ret.Error(0) + } + + return r0 } -// DeleteBlob ... -func (c *Mockclient) DeleteBlob(reference string) (err error) { - return nil +// Health provides a mock function with given fields: +func (_m *Client) Health() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Health") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 } -// DeleteManifest ... -func (c *Mockclient) DeleteManifest(repository, reference string) (err error) { - return nil +// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClient(t interface { + mock.TestingT + Cleanup(func()) +}) *Client { + mock := &Client{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } From 4acf375d123b37c05ec67442cfd5ccc0d418cebf Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Tue, 6 Aug 2024 18:29:13 +0800 Subject: [PATCH 10/22] add list project arifacts API (#20803) * add list project arifacts API This API supports listing all artifacts belonging to a specified project. It also allows fetching the latest artifact in each repositry, with the option to filter by either media_type or artifact_type. Signed-off-by: wang yan * resolve the comments Signed-off-by: wang yan --------- Signed-off-by: wang yan Signed-off-by: kunal-511 --- api/v2.0/swagger.yaml | 88 +++++++++++++++++++ src/controller/artifact/controller.go | 15 ++++ src/controller/artifact/controller_test.go | 38 ++++++++ src/controller/artifact/model.go | 9 +- src/pkg/artifact/dao/dao.go | 49 +++++++++++ src/pkg/artifact/dao/dao_test.go | 69 +++++++++++++++ src/pkg/artifact/manager.go | 18 ++++ src/pkg/artifact/manager_test.go | 27 ++++++ src/pkg/cached/artifact/redis/manager.go | 4 + src/server/v2.0/handler/artifact.go | 13 +-- src/server/v2.0/handler/model/artifact.go | 2 + src/server/v2.0/handler/project.go | 80 +++++++++++++++++ src/testing/controller/artifact/controller.go | 30 +++++++ src/testing/pkg/artifact/manager.go | 30 +++++++ 14 files changed, 462 insertions(+), 10 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index c9e1e8a50ff..b219902e310 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -1548,6 +1548,88 @@ paths: $ref: '#/responses/409' '500': $ref: '#/responses/500' + /projects/{project_name_or_id}/artifacts: + get: + summary: List artifacts + description: List artifacts of the specified project + tags: + - project + operationId: listArtifactsOfProject + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/isResourceName' + - $ref: '#/parameters/projectNameOrId' + - $ref: '#/parameters/query' + - $ref: '#/parameters/sort' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/acceptVulnerabilities' + - name: with_tag + in: query + description: Specify whether the tags are included inside the returning artifacts + type: boolean + required: false + default: true + - name: with_label + in: query + description: Specify whether the labels are included inside the returning artifacts + type: boolean + required: false + default: false + - name: with_scan_overview + in: query + description: Specify whether the scan overview is included inside the returning artifacts + type: boolean + required: false + default: false + - name: with_sbom_overview + in: query + description: Specify whether the SBOM overview is included in returning artifacts, when this option is true, the SBOM overview will be included in the response + type: boolean + required: false + default: false + - name: with_immutable_status + in: query + description: Specify whether the immutable status is included inside the tags of the returning artifacts. Only works when setting "with_immutable_status=true" + type: boolean + required: false + default: false + - name: with_accessory + in: query + description: Specify whether the accessories are included of the returning artifacts. Only works when setting "with_accessory=true" + type: boolean + required: false + default: false + - name: latest_in_repository + in: query + description: Specify whether only the latest pushed artifact of each repository is included inside the returning artifacts. Only works when either artifact_type or media_type is included in the query. + type: boolean + required: false + default: false + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of artifacts + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/Artifact' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' '/projects/{project_name_or_id}/scanner': get: summary: Get project level scanner @@ -6586,6 +6668,9 @@ definitions: manifest_media_type: type: string description: The manifest media type of the artifact + artifact_type: + type: string + description: The artifact_type in the manifest of the artifact project_id: type: integer format: int64 @@ -6594,6 +6679,9 @@ definitions: type: integer format: int64 description: The ID of the repository that the artifact belongs to + repository_name: + type: string + description: The name of the repository that the artifact belongs to digest: type: string description: The digest of the artifact diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index 45f6e4f59c2..0a288826291 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -118,6 +118,8 @@ type Controller interface { Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error // HasUnscannableLayer check artifact with digest if has unscannable layer HasUnscannableLayer(ctx context.Context, dgst string) (bool, error) + // ListWithLatest list the artifacts when the latest_in_repository in the query was set + ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) } // NewController creates an instance of the default artifact controller @@ -782,3 +784,16 @@ func (c *controller) HasUnscannableLayer(ctx context.Context, dgst string) (bool } return false, nil } + +// ListWithLatest ... +func (c *controller) ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) { + arts, err := c.artMgr.ListWithLatest(ctx, query) + if err != nil { + return nil, err + } + var res []*Artifact + for _, art := range arts { + res = append(res, c.assembleArtifact(ctx, art, option)) + } + return res, nil +} diff --git a/src/controller/artifact/controller_test.go b/src/controller/artifact/controller_test.go index 6521b1f19e4..d4b6743db7f 100644 --- a/src/controller/artifact/controller_test.go +++ b/src/controller/artifact/controller_test.go @@ -323,6 +323,44 @@ func (c *controllerTestSuite) TestList() { c.Equal(0, len(artifacts[0].Accessories)) } +func (c *controllerTestSuite) TestListWithLatest() { + query := &q.Query{} + option := &Option{ + WithTag: true, + WithAccessory: true, + } + c.artMgr.On("ListWithLatest", mock.Anything, mock.Anything).Return([]*artifact.Artifact{ + { + ID: 1, + RepositoryID: 1, + }, + }, nil) + c.tagCtl.On("List").Return([]*tag.Tag{ + { + Tag: model_tag.Tag{ + ID: 1, + RepositoryID: 1, + ArtifactID: 1, + Name: "latest", + }, + }, + }, nil) + c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{ + Name: "library/hello-world", + }, nil) + c.repoMgr.On("List", mock.Anything, mock.Anything).Return([]*repomodel.RepoRecord{ + {RepositoryID: 1, Name: "library/hello-world"}, + }, nil) + c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil) + artifacts, err := c.ctl.ListWithLatest(nil, query, option) + c.Require().Nil(err) + c.Require().Len(artifacts, 1) + c.Equal(int64(1), artifacts[0].ID) + c.Require().Len(artifacts[0].Tags, 1) + c.Equal(int64(1), artifacts[0].Tags[0].ID) + c.Equal(0, len(artifacts[0].Accessories)) +} + func (c *controllerTestSuite) TestGet() { c.artMgr.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{ ID: 1, diff --git a/src/controller/artifact/model.go b/src/controller/artifact/model.go index 6bf731dd754..d24369c9868 100644 --- a/src/controller/artifact/model.go +++ b/src/controller/artifact/model.go @@ -102,8 +102,9 @@ type AdditionLink struct { // Option is used to specify the properties returned when listing/getting artifacts type Option struct { - WithTag bool - TagOption *tag.Option // only works when WithTag is set to true - WithLabel bool - WithAccessory bool + WithTag bool + TagOption *tag.Option // only works when WithTag is set to true + WithLabel bool + WithAccessory bool + LatestInRepository bool } diff --git a/src/pkg/artifact/dao/dao.go b/src/pkg/artifact/dao/dao.go index 0e2e79a41cc..c59baeb7f58 100644 --- a/src/pkg/artifact/dao/dao.go +++ b/src/pkg/artifact/dao/dao.go @@ -54,6 +54,8 @@ type DAO interface { DeleteReference(ctx context.Context, id int64) (err error) // DeleteReferences deletes the references referenced by the artifact specified by parent ID DeleteReferences(ctx context.Context, parentID int64) (err error) + // ListWithLatest ... + ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) } const ( @@ -282,6 +284,53 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error { return err } +func (d *dao) ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) { + ormer, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + + sql := `SELECT a.* + FROM artifact a + JOIN ( + SELECT repository_name, MAX(push_time) AS latest_push_time + FROM artifact + WHERE project_id = ? and %s = ? + GROUP BY repository_name + ) latest ON a.repository_name = latest.repository_name AND a.push_time = latest.latest_push_time` + + queryParam := make([]interface{}, 0) + var ok bool + var pid interface{} + if pid, ok = query.Keywords["ProjectID"]; !ok { + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage(`the value of "ProjectID" must be set`) + } + queryParam = append(queryParam, pid) + + var attributionValue interface{} + if attributionValue, ok = query.Keywords["media_type"]; ok { + sql = fmt.Sprintf(sql, "media_type") + } else if attributionValue, ok = query.Keywords["artifact_type"]; ok { + sql = fmt.Sprintf(sql, "artifact_type") + } + + if attributionValue == "" { + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage(`the value of "media_type" or "artifact_type" must be set`) + } + queryParam = append(queryParam, attributionValue) + + sql, queryParam = orm.PaginationOnRawSQL(query, sql, queryParam) + arts := []*Artifact{} + _, err = ormer.Raw(sql, queryParam...).QueryRows(&arts) + if err != nil { + return nil, err + } + + return arts, nil +} + func querySetter(ctx context.Context, query *q.Query, options ...orm.Option) (beegoorm.QuerySeter, error) { qs, err := orm.QuerySetter(ctx, &Artifact{}, query, options...) if err != nil { diff --git a/src/pkg/artifact/dao/dao_test.go b/src/pkg/artifact/dao/dao_test.go index adefcfff725..08b448fba83 100644 --- a/src/pkg/artifact/dao/dao_test.go +++ b/src/pkg/artifact/dao/dao_test.go @@ -472,6 +472,75 @@ func (d *daoTestSuite) TestDeleteReferences() { d.True(errors.IsErr(err, errors.NotFoundCode)) } +func (d *daoTestSuite) TestListWithLatest() { + now := time.Now() + art := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1234, + RepositoryName: "library2/hello-world1", + Digest: "digest", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id, err := d.dao.Create(d.ctx, art) + d.Require().Nil(err) + + time.Sleep(1 * time.Second) + now = time.Now() + + art2 := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1235, + RepositoryName: "library2/hello-world2", + Digest: "digest", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id1, err := d.dao.Create(d.ctx, art2) + d.Require().Nil(err) + + time.Sleep(1 * time.Second) + now = time.Now() + + art3 := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1235, + RepositoryName: "library2/hello-world2", + Digest: "digest2", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id2, err := d.dao.Create(d.ctx, art3) + d.Require().Nil(err) + + latest, err := d.dao.ListWithLatest(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ProjectID": 1234, + "media_type": v1.MediaTypeImageConfig, + }, + }) + + d.Require().Nil(err) + d.Require().Equal(2, len(latest)) + d.Equal("library2/hello-world1", latest[0].RepositoryName) + + defer d.dao.Delete(d.ctx, id) + defer d.dao.Delete(d.ctx, id1) + defer d.dao.Delete(d.ctx, id2) +} + func TestDaoTestSuite(t *testing.T) { suite.Run(t, &daoTestSuite{}) } diff --git a/src/pkg/artifact/manager.go b/src/pkg/artifact/manager.go index b28fd8b253e..ec133c341b7 100644 --- a/src/pkg/artifact/manager.go +++ b/src/pkg/artifact/manager.go @@ -48,6 +48,8 @@ type Manager interface { ListReferences(ctx context.Context, query *q.Query) (references []*Reference, err error) // DeleteReference specified by ID DeleteReference(ctx context.Context, id int64) (err error) + // ListWithLatest list the artifacts when the latest_in_repository in the query was set + ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) } // NewManager returns an instance of the default manager @@ -147,6 +149,22 @@ func (m *manager) DeleteReference(ctx context.Context, id int64) error { return m.dao.DeleteReference(ctx, id) } +func (m *manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*Artifact, error) { + arts, err := m.dao.ListWithLatest(ctx, query) + if err != nil { + return nil, err + } + var artifacts []*Artifact + for _, art := range arts { + artifact, err := m.assemble(ctx, art) + if err != nil { + return nil, err + } + artifacts = append(artifacts, artifact) + } + return artifacts, nil +} + // assemble the artifact with references populated func (m *manager) assemble(ctx context.Context, art *dao.Artifact) (*Artifact, error) { artifact := &Artifact{} diff --git a/src/pkg/artifact/manager_test.go b/src/pkg/artifact/manager_test.go index 9418705cfdf..af434d7c225 100644 --- a/src/pkg/artifact/manager_test.go +++ b/src/pkg/artifact/manager_test.go @@ -80,6 +80,11 @@ func (f *fakeDao) DeleteReferences(ctx context.Context, parentID int64) error { return args.Error(0) } +func (f *fakeDao) ListWithLatest(ctx context.Context, query *q.Query) ([]*dao.Artifact, error) { + args := f.Called() + return args.Get(0).([]*dao.Artifact), args.Error(1) +} + type managerTestSuite struct { suite.Suite mgr *manager @@ -135,6 +140,28 @@ func (m *managerTestSuite) TestAssemble() { m.Equal(2, len(artifact.References)) } +func (m *managerTestSuite) TestListWithLatest() { + art := &dao.Artifact{ + ID: 1, + Type: "IMAGE", + MediaType: "application/vnd.oci.image.config.v1+json", + ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + ProjectID: 1, + RepositoryID: 1, + Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", + Size: 1024, + PushTime: time.Now(), + PullTime: time.Now(), + ExtraAttrs: `{"attr1":"value1"}`, + Annotations: `{"anno1":"value1"}`, + } + m.dao.On("ListWithLatest", mock.Anything).Return([]*dao.Artifact{art}, nil) + artifacts, err := m.mgr.ListWithLatest(nil, nil) + m.Require().Nil(err) + m.Equal(1, len(artifacts)) + m.Equal(art.ID, artifacts[0].ID) +} + func (m *managerTestSuite) TestList() { art := &dao.Artifact{ ID: 1, diff --git a/src/pkg/cached/artifact/redis/manager.go b/src/pkg/cached/artifact/redis/manager.go index c5371a3a743..11879affdee 100644 --- a/src/pkg/cached/artifact/redis/manager.go +++ b/src/pkg/cached/artifact/redis/manager.go @@ -65,6 +65,10 @@ func (m *Manager) List(ctx context.Context, query *q.Query) ([]*artifact.Artifac return m.delegator.List(ctx, query) } +func (m *Manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) { + return m.delegator.ListWithLatest(ctx, query) +} + func (m *Manager) Create(ctx context.Context, artifact *artifact.Artifact) (int64, error) { return m.delegator.Create(ctx, artifact) } diff --git a/src/server/v2.0/handler/artifact.go b/src/server/v2.0/handler/artifact.go index dfc457047de..79cf3162231 100644 --- a/src/server/v2.0/handler/artifact.go +++ b/src/server/v2.0/handler/artifact.go @@ -95,7 +95,7 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr // set option option := option(params.WithTag, params.WithImmutableStatus, - params.WithLabel, params.WithAccessory) + params.WithLabel, params.WithAccessory, nil) // get the total count of artifacts total, err := a.artCtl.Count(ctx, query) @@ -129,7 +129,7 @@ func (a *artifactAPI) GetArtifact(ctx context.Context, params operation.GetArtif } // set option option := option(params.WithTag, params.WithImmutableStatus, - params.WithLabel, params.WithAccessory) + params.WithLabel, params.WithAccessory, nil) // get the artifact artifact, err := a.artCtl.GetByReference(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName), params.Reference, option) @@ -501,11 +501,12 @@ func (a *artifactAPI) RequireLabelInProject(ctx context.Context, projectID, labe return nil } -func option(withTag, withImmutableStatus, withLabel, withAccessory *bool) *artifact.Option { +func option(withTag, withImmutableStatus, withLabel, withAccessory *bool, latestInRepository *bool) *artifact.Option { option := &artifact.Option{ - WithTag: true, // return the tag by default - WithLabel: lib.BoolValue(withLabel), - WithAccessory: true, // return the accessory by default + WithTag: true, // return the tag by default + WithLabel: lib.BoolValue(withLabel), + WithAccessory: true, // return the accessory by default + LatestInRepository: lib.BoolValue(latestInRepository), } if withTag != nil { diff --git a/src/server/v2.0/handler/model/artifact.go b/src/server/v2.0/handler/model/artifact.go index f931c0abf29..78d4d11bafc 100644 --- a/src/server/v2.0/handler/model/artifact.go +++ b/src/server/v2.0/handler/model/artifact.go @@ -49,6 +49,8 @@ func (a *Artifact) ToSwagger() *models.Artifact { PushTime: strfmt.DateTime(a.PushTime), ExtraAttrs: a.ExtraAttrs, Annotations: a.Annotations, + ArtifactType: a.ArtifactType, + RepositoryName: a.RepositoryName, } for _, reference := range a.References { diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index 29286f8ec40..6a133fb30fc 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -29,6 +29,7 @@ import ( "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" robotSec "github.com/goharbor/harbor/src/common/security/robot" + "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/p2p/preheat" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/quota" @@ -52,6 +53,7 @@ import ( "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/robot" userModels "github.com/goharbor/harbor/src/pkg/user/models" + "github.com/goharbor/harbor/src/server/v2.0/handler/assembler" "github.com/goharbor/harbor/src/server/v2.0/handler/model" "github.com/goharbor/harbor/src/server/v2.0/models" operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/project" @@ -63,6 +65,7 @@ const defaultDaysToRetentionForProxyCacheProject = 7 func newProjectAPI() *projectAPI { return &projectAPI{ auditMgr: audit.Mgr, + artCtl: artifact.Ctl, metadataMgr: pkg.ProjectMetaMgr, userCtl: user.Ctl, repositoryCtl: repository.Ctl, @@ -79,6 +82,7 @@ func newProjectAPI() *projectAPI { type projectAPI struct { BaseAPI auditMgr audit.Manager + artCtl artifact.Controller metadataMgr metadata.Manager userCtl user.Controller repositoryCtl repository.Controller @@ -660,6 +664,82 @@ func (a *projectAPI) SetScannerOfProject(ctx context.Context, params operation.S return operation.NewSetScannerOfProjectOK() } +func (a *projectAPI) ListArtifactsOfProject(ctx context.Context, params operation.ListArtifactsOfProjectParams) middleware.Responder { + if err := a.RequireAuthenticated(ctx); err != nil { + return a.SendError(ctx, err) + } + projectNameOrID := parseProjectNameOrID(params.ProjectNameOrID, params.XIsResourceName) + if err := a.RequireProjectAccess(ctx, projectNameOrID, rbac.ActionList, rbac.ResourceArtifact); err != nil { + return a.SendError(ctx, err) + } + // set query + pro, err := a.projectCtl.Get(ctx, projectNameOrID) + if err != nil { + return a.SendError(ctx, err) + } + query, err := a.BuildQuery(ctx, params.Q, params.Sort, params.Page, params.PageSize) + if err != nil { + return a.SendError(ctx, err) + } + query.Keywords["ProjectID"] = pro.ProjectID + + // set option + option := option(params.WithTag, params.WithImmutableStatus, + params.WithLabel, params.WithAccessory, params.LatestInRepository) + + var total int64 + // list artifacts according to the query and option + var arts []*artifact.Artifact + if option.LatestInRepository { + // ignore page & page_size + _, hasMediaType := query.Keywords["media_type"] + _, hasArtifactType := query.Keywords["artifact_type"] + if hasMediaType == hasArtifactType { + return a.SendError(ctx, errors.BadRequestError(fmt.Errorf("either 'media_type' or 'artifact_type' must be specified, but not both, when querying with latest_in_repository"))) + } + + getCount := func() (int64, error) { + var countQ *q.Query + if query != nil { + countQ = q.New(query.Keywords) + } + allArts, err := a.artCtl.ListWithLatest(ctx, countQ, nil) + if err != nil { + return int64(0), err + } + return int64(len(allArts)), nil + } + total, err = getCount() + if err != nil { + return a.SendError(ctx, err) + } + arts, err = a.artCtl.ListWithLatest(ctx, query, option) + } else { + total, err = a.artCtl.Count(ctx, query) + if err != nil { + return a.SendError(ctx, err) + } + arts, err = a.artCtl.List(ctx, query, option) + } + if err != nil { + return a.SendError(ctx, err) + } + overviewOpts := model.NewOverviewOptions(model.WithSBOM(lib.BoolValue(params.WithSbomOverview)), model.WithVuln(lib.BoolValue(params.WithScanOverview))) + assembler := assembler.NewScanReportAssembler(overviewOpts, parseScanReportMimeTypes(params.XAcceptVulnerabilities)) + var artifacts []*models.Artifact + for _, art := range arts { + artifact := &model.Artifact{} + artifact.Artifact = *art + _ = assembler.WithArtifacts(artifact).Assemble(ctx) + artifacts = append(artifacts, artifact.ToSwagger()) + } + + return operation.NewListArtifactsOfProjectOK(). + WithXTotalCount(total). + WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(artifacts) +} + func (a *projectAPI) deletable(ctx context.Context, projectNameOrID interface{}) (*project.Project, *models.ProjectDeletable, error) { p, err := a.getProject(ctx, projectNameOrID) if err != nil { diff --git a/src/testing/controller/artifact/controller.go b/src/testing/controller/artifact/controller.go index 23ed6396f0f..3ec563103d5 100644 --- a/src/testing/controller/artifact/controller.go +++ b/src/testing/controller/artifact/controller.go @@ -296,6 +296,36 @@ func (_m *Controller) List(ctx context.Context, query *q.Query, option *artifact return r0, r1 } +// ListWithLatest provides a mock function with given fields: ctx, query, option +func (_m *Controller) ListWithLatest(ctx context.Context, query *q.Query, option *artifact.Option) ([]*artifact.Artifact, error) { + ret := _m.Called(ctx, query, option) + + if len(ret) == 0 { + panic("no return value specified for ListWithLatest") + } + + var r0 []*artifact.Artifact + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query, *artifact.Option) ([]*artifact.Artifact, error)); ok { + return rf(ctx, query, option) + } + if rf, ok := ret.Get(0).(func(context.Context, *q.Query, *artifact.Option) []*artifact.Artifact); ok { + r0 = rf(ctx, query, option) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*artifact.Artifact) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *q.Query, *artifact.Option) error); ok { + r1 = rf(ctx, query, option) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // RemoveLabel provides a mock function with given fields: ctx, artifactID, labelID func (_m *Controller) RemoveLabel(ctx context.Context, artifactID int64, labelID int64) error { ret := _m.Called(ctx, artifactID, labelID) diff --git a/src/testing/pkg/artifact/manager.go b/src/testing/pkg/artifact/manager.go index 208bfa040a6..a189685cc65 100644 --- a/src/testing/pkg/artifact/manager.go +++ b/src/testing/pkg/artifact/manager.go @@ -231,6 +231,36 @@ func (_m *Manager) ListReferences(ctx context.Context, query *q.Query) ([]*artif return r0, r1 } +// ListWithLatest provides a mock function with given fields: ctx, query +func (_m *Manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) { + ret := _m.Called(ctx, query) + + if len(ret) == 0 { + panic("no return value specified for ListWithLatest") + } + + var r0 []*artifact.Artifact + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) ([]*artifact.Artifact, error)); ok { + return rf(ctx, query) + } + if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*artifact.Artifact); ok { + r0 = rf(ctx, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*artifact.Artifact) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok { + r1 = rf(ctx, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Update provides a mock function with given fields: ctx, _a1, props func (_m *Manager) Update(ctx context.Context, _a1 *artifact.Artifact, props ...string) error { _va := make([]interface{}, len(props)) From cd698c4e2d2c213e2733f7bfb2f12136b749035f Mon Sep 17 00:00:00 2001 From: Lichao Xue <68891670+xuelichao@users.noreply.github.com> Date: Wed, 7 Aug 2024 14:51:39 +0800 Subject: [PATCH 11/22] Fixes-20799 can't remove artifact labels (#20816) Signed-off-by: xuelichao Signed-off-by: kunal-511 --- .../artifact-list-tab/artifact-list-tab.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html b/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html index ad601cf6d67..d5516f10725 100644 --- a/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html +++ b/src/portal/src/app/base/project/repository/artifact/artifact-list-page/artifact-list/artifact-list-tab/artifact-list-tab.component.html @@ -139,6 +139,7 @@ }} Date: Thu, 8 Aug 2024 19:02:54 +0800 Subject: [PATCH 13/22] feat: implement bandwidth limit for proxy-cache (#20812) Signed-off-by: Shengwen Yu Signed-off-by: kunal-511 --- api/v2.0/swagger.yaml | 4 ++ src/controller/proxy/controller.go | 2 +- src/controller/proxy/options.go | 37 +++++++++++++++++++ src/controller/proxy/options_test.go | 33 +++++++++++++++++ src/controller/proxy/remote.go | 17 +++++++-- .../replication/transfer/image/transfer.go | 5 ++- .../transfer => lib}/iothrottler.go | 2 +- src/pkg/project/models/pro_meta.go | 1 + src/pkg/project/models/project.go | 13 +++++++ src/server/middleware/repoproxy/proxy.go | 10 ++--- src/server/middleware/repoproxy/tag.go | 4 +- src/server/v2.0/handler/project.go | 17 +++++++++ src/server/v2.0/handler/project_metadata.go | 6 +++ 13 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 src/controller/proxy/options.go create mode 100644 src/controller/proxy/options_test.go rename src/{controller/replication/transfer => lib}/iothrottler.go (98%) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index b219902e310..681d0e354c5 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -7340,6 +7340,10 @@ definitions: type: string description: 'The ID of the tag retention policy for the project' x-nullable: true + proxy_speed_kb: + type: string + description: 'The bandwidth limit of proxy cache, in Kbps (kilobits per second). It limits the communication between Harbor and the upstream registry, not the client and the Harbor.' + x-nullable: true ProjectSummary: type: object properties: diff --git a/src/controller/proxy/controller.go b/src/controller/proxy/controller.go index 38d55397c37..7138b6f0b0e 100644 --- a/src/controller/proxy/controller.go +++ b/src/controller/proxy/controller.go @@ -264,7 +264,7 @@ func (c *controller) HeadManifest(_ context.Context, art lib.ArtifactInfo, remot func (c *controller) ProxyBlob(ctx context.Context, p *proModels.Project, art lib.ArtifactInfo) (int64, io.ReadCloser, error) { remoteRepo := getRemoteRepo(art) log.Debugf("The blob doesn't exist, proxy the request to the target server, url:%v", remoteRepo) - rHelper, err := NewRemoteHelper(ctx, p.RegistryID) + rHelper, err := NewRemoteHelper(ctx, p.RegistryID, WithSpeed(p.ProxyCacheSpeed())) if err != nil { return 0, nil, err } diff --git a/src/controller/proxy/options.go b/src/controller/proxy/options.go new file mode 100644 index 00000000000..2f81bfc3ff1 --- /dev/null +++ b/src/controller/proxy/options.go @@ -0,0 +1,37 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +type Option func(*Options) + +type Options struct { + // Speed is the data transfer speed for proxy cache from Harbor to upstream registry, no limit by default. + Speed int32 +} + +func NewOptions(opts ...Option) *Options { + o := &Options{} + for _, opt := range opts { + opt(o) + } + + return o +} + +func WithSpeed(speed int32) Option { + return func(o *Options) { + o.Speed = speed + } +} diff --git a/src/controller/proxy/options_test.go b/src/controller/proxy/options_test.go new file mode 100644 index 00000000000..2b0a4ef8018 --- /dev/null +++ b/src/controller/proxy/options_test.go @@ -0,0 +1,33 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewOptions(t *testing.T) { + // test default options + o := NewOptions() + assert.Equal(t, int32(0), o.Speed) + + // test with options + // with speed + withSpeed := WithSpeed(1024) + o = NewOptions(withSpeed) + assert.Equal(t, int32(1024), o.Speed) +} diff --git a/src/controller/proxy/remote.go b/src/controller/proxy/remote.go index ac7f23f28b8..4143b817099 100644 --- a/src/controller/proxy/remote.go +++ b/src/controller/proxy/remote.go @@ -21,6 +21,7 @@ import ( "github.com/docker/distribution" + "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/pkg/reg" "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/model" @@ -43,13 +44,16 @@ type remoteHelper struct { regID int64 registry adapter.ArtifactRegistry registryMgr reg.Manager + opts *Options } // NewRemoteHelper create a remote interface -func NewRemoteHelper(ctx context.Context, regID int64) (RemoteInterface, error) { +func NewRemoteHelper(ctx context.Context, regID int64, opts ...Option) (RemoteInterface, error) { r := &remoteHelper{ regID: regID, - registryMgr: reg.Mgr} + registryMgr: reg.Mgr, + opts: NewOptions(opts...), + } if err := r.init(ctx); err != nil { return nil, err } @@ -83,7 +87,14 @@ func (r *remoteHelper) init(ctx context.Context) error { } func (r *remoteHelper) BlobReader(repo, dig string) (int64, io.ReadCloser, error) { - return r.registry.PullBlob(repo, dig) + sz, bReader, err := r.registry.PullBlob(repo, dig) + if err != nil { + return 0, nil, err + } + if r.opts != nil && r.opts.Speed > 0 { + bReader = lib.NewReader(bReader, r.opts.Speed) + } + return sz, bReader, err } func (r *remoteHelper) Manifest(repo string, ref string) (distribution.Manifest, string, error) { diff --git a/src/controller/replication/transfer/image/transfer.go b/src/controller/replication/transfer/image/transfer.go index 55bda964699..09c1bea27e9 100644 --- a/src/controller/replication/transfer/image/transfer.go +++ b/src/controller/replication/transfer/image/transfer.go @@ -30,6 +30,7 @@ import ( common_http "github.com/goharbor/harbor/src/common/http" trans "github.com/goharbor/harbor/src/controller/replication/transfer" + "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/model" @@ -380,7 +381,7 @@ func (t *transfer) copyBlobByMonolithic(srcRepo, dstRepo, digest string, sizeFro return err } if speed > 0 { - data = trans.NewReader(data, speed) + data = lib.NewReader(data, speed) } defer data.Close() // get size 0 from PullBlob, use size from distribution.Descriptor instead. @@ -435,7 +436,7 @@ func (t *transfer) copyBlobByChunk(srcRepo, dstRepo, digest string, sizeFromDesc } if speed > 0 { - data = trans.NewReader(data, speed) + data = lib.NewReader(data, speed) } // failureEnd will only be used for adjusting content range when issue happened during push the chunk. var failureEnd int64 diff --git a/src/controller/replication/transfer/iothrottler.go b/src/lib/iothrottler.go similarity index 98% rename from src/controller/replication/transfer/iothrottler.go rename to src/lib/iothrottler.go index 828c4403577..b0853e0e58c 100644 --- a/src/controller/replication/transfer/iothrottler.go +++ b/src/lib/iothrottler.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transfer +package lib import ( "fmt" diff --git a/src/pkg/project/models/pro_meta.go b/src/pkg/project/models/pro_meta.go index a8a0659fe8a..25f7e41bee1 100644 --- a/src/pkg/project/models/pro_meta.go +++ b/src/pkg/project/models/pro_meta.go @@ -24,4 +24,5 @@ const ( ProMetaAutoScan = "auto_scan" ProMetaReuseSysCVEAllowlist = "reuse_sys_cve_allowlist" ProMetaAutoSBOMGen = "auto_sbom_generation" + ProMetaProxySpeed = "proxy_speed_kb" ) diff --git a/src/pkg/project/models/project.go b/src/pkg/project/models/project.go index e533bd91124..80318aa21a4 100644 --- a/src/pkg/project/models/project.go +++ b/src/pkg/project/models/project.go @@ -156,6 +156,19 @@ func (p *Project) AutoSBOMGen() bool { return isTrue(auto) } +// ProxyCacheSpeed ... +func (p *Project) ProxyCacheSpeed() int32 { + speed, exist := p.GetMetadata(ProMetaProxySpeed) + if !exist { + return 0 + } + speedInt, err := strconv.ParseInt(speed, 10, 32) + if err != nil { + return 0 + } + return int32(speedInt) +} + // FilterByPublic returns orm.QuerySeter with public filter func (p *Project) FilterByPublic(_ context.Context, qs orm.QuerySeter, _ string, value interface{}) orm.QuerySeter { subQuery := `SELECT project_id FROM project_metadata WHERE name = 'public' AND value = '%s'` diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go index ddb2017843f..641638766e7 100644 --- a/src/server/middleware/repoproxy/proxy.go +++ b/src/server/middleware/repoproxy/proxy.go @@ -60,7 +60,7 @@ func BlobGetMiddleware() func(http.Handler) http.Handler { func handleBlob(w http.ResponseWriter, r *http.Request, next http.Handler) error { ctx := r.Context() - art, p, proxyCtl, err := preCheck(ctx) + art, p, proxyCtl, err := preCheck(ctx, true) if err != nil { return err } @@ -96,14 +96,14 @@ func handleBlob(w http.ResponseWriter, r *http.Request, next http.Handler) error return nil } -func preCheck(ctx context.Context) (art lib.ArtifactInfo, p *proModels.Project, ctl proxy.Controller, err error) { +func preCheck(ctx context.Context, withProjectMetadata bool) (art lib.ArtifactInfo, p *proModels.Project, ctl proxy.Controller, err error) { none := lib.ArtifactInfo{} art = lib.GetArtifactInfo(ctx) if art == none { return none, nil, nil, errors.New("artifactinfo is not found").WithCode(errors.NotFoundCode) } ctl = proxy.ControllerInstance() - p, err = project.Ctl.GetByName(ctx, art.ProjectName, project.Metadata(false)) + p, err = project.Ctl.GetByName(ctx, art.ProjectName, project.Metadata(withProjectMetadata)) return } @@ -155,7 +155,7 @@ func defaultBlobURL(projectName string, name string, digest string) string { func handleManifest(w http.ResponseWriter, r *http.Request, next http.Handler) error { ctx := r.Context() - art, p, proxyCtl, err := preCheck(ctx) + art, p, proxyCtl, err := preCheck(ctx, true) if err != nil { return err } @@ -174,7 +174,7 @@ func handleManifest(w http.ResponseWriter, r *http.Request, next http.Handler) e next.ServeHTTP(w, r) return nil } - remote, err := proxy.NewRemoteHelper(r.Context(), p.RegistryID) + remote, err := proxy.NewRemoteHelper(r.Context(), p.RegistryID, proxy.WithSpeed(p.ProxyCacheSpeed())) if err != nil { return err } diff --git a/src/server/middleware/repoproxy/tag.go b/src/server/middleware/repoproxy/tag.go index 6e0d3070d41..005a5f7774b 100644 --- a/src/server/middleware/repoproxy/tag.go +++ b/src/server/middleware/repoproxy/tag.go @@ -35,7 +35,7 @@ func TagsListMiddleware() func(http.Handler) http.Handler { return middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { ctx := r.Context() - art, p, _, err := preCheck(ctx) + art, p, _, err := preCheck(ctx, false) if err != nil { libhttp.SendError(w, err) return @@ -69,7 +69,7 @@ func TagsListMiddleware() func(http.Handler) http.Handler { util.SendListTagsResponse(w, r, tags) }() - remote, err := proxy.NewRemoteHelper(ctx, p.RegistryID) + remote, err := proxy.NewRemoteHelper(ctx, p.RegistryID, proxy.WithSpeed(p.ProxyCacheSpeed())) if err != nil { logger.Warningf("failed to get remote interface, error: %v, fallback to local tags", err) return diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index 6a133fb30fc..0479929540c 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -159,6 +159,11 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP } } + // ignore metadata.proxy_speed_kb for non-proxy-cache project + if req.RegistryID == nil { + req.Metadata.ProxySpeedKb = nil + } + // ignore enable_content_trust metadata for proxy cache project // see https://github.com/goharbor/harbor/issues/12940 to get more info if req.RegistryID != nil { @@ -551,6 +556,11 @@ func (a *projectAPI) UpdateProject(ctx context.Context, params operation.UpdateP } } + // ignore metadata.proxy_speed_kb for non-proxy-cache project + if params.Project.Metadata != nil && !p.IsProxy() { + params.Project.Metadata.ProxySpeedKb = nil + } + // ignore enable_content_trust metadata for proxy cache project // see https://github.com/goharbor/harbor/issues/12940 to get more info if params.Project.Metadata != nil && p.IsProxy() { @@ -792,6 +802,13 @@ func (a *projectAPI) validateProjectReq(ctx context.Context, req *models.Project if !permitted { return errors.BadRequestError(fmt.Errorf("unsupported registry type %s", string(registry.Type))) } + + // validate metadata.proxy_speed_kb. It should be an int32 + if ps := req.Metadata.ProxySpeedKb; ps != nil { + if _, err := strconv.ParseInt(*ps, 10, 32); err != nil { + return errors.BadRequestError(nil).WithMessage(fmt.Sprintf("metadata.proxy_speed_kb should by an int32, but got: '%s', err: %s", *ps, err)) + } + } } if req.StorageLimit != nil { diff --git a/src/server/v2.0/handler/project_metadata.go b/src/server/v2.0/handler/project_metadata.go index a5bde2e4538..7fa297b7238 100644 --- a/src/server/v2.0/handler/project_metadata.go +++ b/src/server/v2.0/handler/project_metadata.go @@ -155,6 +155,12 @@ func (p *projectMetadataAPI) validate(metas map[string]string) (map[string]strin return nil, errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("invalid value: %s", value) } metas[proModels.ProMetaSeverity] = strings.ToLower(severity.String()) + case proModels.ProMetaProxySpeed: + v, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("invalid value: %s", value) + } + metas[proModels.ProMetaProxySpeed] = strconv.FormatInt(v, 10) default: return nil, errors.New(nil).WithCode(errors.BadRequestCode).WithMessage("invalid key: %s", key) } From 05fbeb486c3ac01e6d70f9dbf620df6e7aa8c539 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 9 Aug 2024 15:24:25 +0800 Subject: [PATCH 14/22] Parallel attach ldap group (#20705) Parallel attach LDAP group Add configure LDAP group attach parallel UI Change the /c/login timeout from 60 (nginx default) to 900 seconds in nginx.conf Signed-off-by: stonezdj Signed-off-by: kunal-511 --- api/v2.0/swagger.yaml | 8 ++ .../templates/nginx/nginx.http.conf.jinja | 3 + src/common/const.go | 1 + src/core/auth/ldap/ldap.go | 95 +++++++++++++++++-- src/lib/config/metadata/metadatalist.go | 1 + src/lib/config/models/model.go | 1 + src/lib/config/userconfig.go | 1 + .../config/auth/config-auth.component.html | 31 ++++++ .../config/auth/config-auth.component.ts | 7 +- .../app/base/left-side-nav/config/config.ts | 1 + src/portal/src/i18n/lang/en-us-lang.json | 4 +- src/portal/src/i18n/lang/zh-cn-lang.json | 5 +- 12 files changed, 144 insertions(+), 14 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 681d0e354c5..07beb01e9ae 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -8989,6 +8989,9 @@ definitions: ldap_group_search_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' + ldap_group_attach_parallel: + $ref: '#/definitions/BoolConfigItem' + description: Attach LDAP user group information in parallel. ldap_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' @@ -9179,6 +9182,11 @@ definitions: description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' x-omitempty: true x-isnullable: true + ldap_group_attach_parallel: + type: boolean + description: Attach LDAP user group information in parallel, the parallel worker count is 5 + x-omitempty: true + x-isnullable: true ldap_scope: type: integer description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' diff --git a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja index 6022b3ac941..7e55e72ded9 100644 --- a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja @@ -101,6 +101,9 @@ http { proxy_buffering off; proxy_request_buffering off; + + proxy_send_timeout 900; + proxy_read_timeout 900; } location /api/ { diff --git a/src/common/const.go b/src/common/const.go index aaa3c3fbe0d..224a2e4f35c 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -134,6 +134,7 @@ const ( OIDCGroupType = 3 LDAPGroupAdminDn = "ldap_group_admin_dn" LDAPGroupMembershipAttribute = "ldap_group_membership_attribute" + LDAPGroupAttachParallel = "ldap_group_attach_parallel" DefaultRegistryControllerEndpoint = "http://registryctl:8080" DefaultPortalURL = "http://portal:8080" DefaultRegistryCtlURL = "http://registryctl:8080" diff --git a/src/core/auth/ldap/ldap.go b/src/core/auth/ldap/ldap.go index 38fa5a6f93f..56533b28754 100644 --- a/src/core/auth/ldap/ldap.go +++ b/src/core/auth/ldap/ldap.go @@ -21,6 +21,7 @@ import ( "strings" goldap "github.com/go-ldap/ldap/v3" + "golang.org/x/sync/errgroup" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" @@ -38,6 +39,10 @@ import ( ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model" ) +const ( + workerCount = 5 +) + // Auth implements AuthenticateHelper interface to authenticate against LDAP type Auth struct { auth.DefaultAuthenticateHelper @@ -117,22 +122,92 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m return } userGroups := make([]ugModel.UserGroup, 0) + if groupCfg.AttachParallel { + log.Debug("Attach LDAP group in parallel") + l.attachGroupParallel(ctx, ldapUsers, u) + return + } + // Attach LDAP group sequencially for _, dn := range ldapUsers[0].GroupDNList { - lGroups, err := sess.SearchGroupByDN(dn) - if err != nil { - log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err) - continue + if lgroup, exist := verifyGroupInLDAP(dn, sess); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } - if len(lGroups) == 0 { - log.Warningf("Can not get the ldap group name with DN %v", dn) - continue - } - userGroups = append(userGroups, ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } u.GroupIDs, err = ugCtl.Ctl.Populate(ctx, userGroups) if err != nil { - log.Warningf("Failed to fetch ldap group configuration:%v", err) + log.Warningf("Failed to populate ldap group, error: %v", err) + } +} + +func (l *Auth) attachGroupParallel(ctx context.Context, ldapUsers []model.User, u *models.User) { + userGroupsList := make([][]ugModel.UserGroup, workerCount) + gdsList := make([][]string, workerCount) + // Divide the groupDNs into workerCount parts + for index, dn := range ldapUsers[0].GroupDNList { + idx := index % workerCount + gdsList[idx] = append(gdsList[idx], dn) + } + g := new(errgroup.Group) + g.SetLimit(workerCount) + + for i := 0; i < workerCount; i++ { + curIndex := i + g.Go(func() error { + userGroups := make([]ugModel.UserGroup, 0) + groups := gdsList[curIndex] + if len(groups) == 0 { + return nil + } + // use different ldap session for each go routine + ldapSession, err := ldapCtl.Ctl.Session(ctx) + if err != nil { + return err + } + if err = ldapSession.Open(); err != nil { + return err + } + defer ldapSession.Close() + log.Debugf("Current worker index is %v", curIndex) + // verify and populate group + for _, dn := range groups { + if lgroup, exist := verifyGroupInLDAP(dn, ldapSession); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) + } + } + userGroupsList[curIndex] = userGroups + + return nil + }) + } + if err := g.Wait(); err != nil { + log.Warningf("failed to verify and populate ldap group parallel, error %v", err) + } + ugs := make([]ugModel.UserGroup, 0) + for _, userGroups := range userGroupsList { + ugs = append(ugs, userGroups...) + } + + groupIDsList, err := ugCtl.Ctl.Populate(ctx, ugs) + if err != nil { + log.Warningf("Failed to populate user groups :%v", err) + } + u.GroupIDs = groupIDsList +} + +func verifyGroupInLDAP(groupDN string, sess *ldap.Session) (*model.Group, bool) { + if _, err := goldap.ParseDN(groupDN); err != nil { + return nil, false + } + lGroups, err := sess.SearchGroupByDN(groupDN) + if err != nil { + log.Warningf("Can not get the ldap group name with DN %v, error %v", groupDN, err) + return nil, false + } + if len(lGroups) == 0 { + log.Warningf("Can not get the ldap group name with DN %v", groupDN) + return nil, false } + return &lGroups[0], true } func (l *Auth) syncUserInfoFromDB(ctx context.Context, u *models.User) { diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index dd2a7f67cde..aab4919fd89 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -96,6 +96,7 @@ var ( {Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`}, {Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`}, {Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`}, + {Name: common.LDAPGroupAttachParallel, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_ATTACH_PARALLEL", DefaultValue: "false", ItemType: &BoolType{}, Editable: true, Description: `Attach LDAP group information to Harbor in parallel`}, {Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false}, {Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`}, diff --git a/src/lib/config/models/model.go b/src/lib/config/models/model.go index 51a9c0c2cbc..25b41388ace 100644 --- a/src/lib/config/models/model.go +++ b/src/lib/config/models/model.go @@ -94,6 +94,7 @@ type GroupConf struct { SearchScope int `json:"ldap_group_search_scope"` AdminDN string `json:"ldap_group_admin_dn,omitempty"` MembershipAttribute string `json:"ldap_group_membership_attribute,omitempty"` + AttachParallel bool `json:"ldap_group_attach_parallel,omitempty"` } type GDPRSetting struct { diff --git a/src/lib/config/userconfig.go b/src/lib/config/userconfig.go index a0fd5f90aee..4012097c9e3 100644 --- a/src/lib/config/userconfig.go +++ b/src/lib/config/userconfig.go @@ -81,6 +81,7 @@ func LDAPGroupConf(ctx context.Context) (*cfgModels.GroupConf, error) { SearchScope: mgr.Get(ctx, common.LDAPGroupSearchScope).GetInt(), AdminDN: mgr.Get(ctx, common.LDAPGroupAdminDn).GetString(), MembershipAttribute: mgr.Get(ctx, common.LDAPGroupMembershipAttribute).GetString(), + AttachParallel: mgr.Get(ctx, common.LDAPGroupAttachParallel).GetBool(), }, nil } diff --git a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html index b64ba7cf512..2b1401d1b5f 100644 --- a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html +++ b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html @@ -498,6 +498,37 @@ + + + + + +