Parallel attach ldap group (#20705)

Parallel attach LDAP group

  Add configure LDAP group attach parallel UI
  Change the /c/login timeout from 60 (nginx default) to 900 seconds in nginx.conf

Signed-off-by: stonezdj <stone.zhang@broadcom.com>

api/v2.0/swagger.yaml

@@ -8989,6 +8989,9 @@ definitions:
       ldap_group_search_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
+      ldap_group_attach_parallel:
+        $ref: '#/definitions/BoolConfigItem'
+        description: Attach LDAP user group information in parallel.
       ldap_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'
@@ -9179,6 +9182,11 @@ definitions:
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
         x-omitempty: true
         x-isnullable: true
+      ldap_group_attach_parallel:
+        type: boolean
+        description: Attach LDAP user group information in parallel, the parallel worker count is 5
+        x-omitempty: true
+        x-isnullable: true
       ldap_scope:
         type: integer
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'

make/photon/prepare/templates/nginx/nginx.http.conf.jinja

@@ -101,6 +101,9 @@ http {
 
       proxy_buffering off;
       proxy_request_buffering off;
+
+      proxy_send_timeout 900;
+      proxy_read_timeout 900;
     }
 
     location /api/ {

src/common/const.go

@@ -134,6 +134,7 @@ const (
 	OIDCGroupType                     = 3
 	LDAPGroupAdminDn                  = "ldap_group_admin_dn"
 	LDAPGroupMembershipAttribute      = "ldap_group_membership_attribute"
+	LDAPGroupAttachParallel           = "ldap_group_attach_parallel"
 	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
 	DefaultPortalURL                  = "http://portal:8080"
 	DefaultRegistryCtlURL             = "http://registryctl:8080"

src/core/auth/ldap/ldap.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	goldap "github.com/go-ldap/ldap/v3"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/models"
@@ -38,6 +39,10 @@ import (
 	ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model"
 )
 
+const (
+	workerCount = 5
+)
+
 // Auth implements AuthenticateHelper interface to authenticate against LDAP
 type Auth struct {
 	auth.DefaultAuthenticateHelper
@@ -117,22 +122,92 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m
 		return
 	}
 	userGroups := make([]ugModel.UserGroup, 0)
+	if groupCfg.AttachParallel {
+		log.Debug("Attach LDAP group in parallel")
+		l.attachGroupParallel(ctx, ldapUsers, u)
+		return
+	}
+	// Attach LDAP group sequencially
 	for _, dn := range ldapUsers[0].GroupDNList {
-		lGroups, err := sess.SearchGroupByDN(dn)
-		if err != nil {
-			log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err)
-			continue
+		if lgroup, exist := verifyGroupInLDAP(dn, sess); exist {
+			userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType})
 		}
-		if len(lGroups) == 0 {
-			log.Warningf("Can not get the ldap group name with DN %v", dn)
-			continue
-		}
-		userGroups = append(userGroups, ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType})
 	}
 	u.GroupIDs, err = ugCtl.Ctl.Populate(ctx, userGroups)
 	if err != nil {
-		log.Warningf("Failed to fetch ldap group configuration:%v", err)
+		log.Warningf("Failed to populate ldap group, error: %v", err)
+	}
+}
+
+func (l *Auth) attachGroupParallel(ctx context.Context, ldapUsers []model.User, u *models.User) {
+	userGroupsList := make([][]ugModel.UserGroup, workerCount)
+	gdsList := make([][]string, workerCount)
+	// Divide the groupDNs into workerCount parts
+	for index, dn := range ldapUsers[0].GroupDNList {
+		idx := index % workerCount
+		gdsList[idx] = append(gdsList[idx], dn)
+	}
+	g := new(errgroup.Group)
+	g.SetLimit(workerCount)
+
+	for i := 0; i < workerCount; i++ {
+		curIndex := i
+		g.Go(func() error {
+			userGroups := make([]ugModel.UserGroup, 0)
+			groups := gdsList[curIndex]
+			if len(groups) == 0 {
+				return nil
+			}
+			// use different ldap session for each go routine
+			ldapSession, err := ldapCtl.Ctl.Session(ctx)
+			if err != nil {
+				return err
+			}
+			if err = ldapSession.Open(); err != nil {
+				return err
+			}
+			defer ldapSession.Close()
+			log.Debugf("Current worker index is %v", curIndex)
+			// verify and populate group
+			for _, dn := range groups {
+				if lgroup, exist := verifyGroupInLDAP(dn, ldapSession); exist {
+					userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType})
+				}
+			}
+			userGroupsList[curIndex] = userGroups
+
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		log.Warningf("failed to verify and populate ldap group parallel, error %v", err)
+	}
+	ugs := make([]ugModel.UserGroup, 0)
+	for _, userGroups := range userGroupsList {
+		ugs = append(ugs, userGroups...)
+	}
+
+	groupIDsList, err := ugCtl.Ctl.Populate(ctx, ugs)
+	if err != nil {
+		log.Warningf("Failed to populate user groups :%v", err)
+	}
+	u.GroupIDs = groupIDsList
+}
+
+func verifyGroupInLDAP(groupDN string, sess *ldap.Session) (*model.Group, bool) {
+	if _, err := goldap.ParseDN(groupDN); err != nil {
+		return nil, false
+	}
+	lGroups, err := sess.SearchGroupByDN(groupDN)
+	if err != nil {
+		log.Warningf("Can not get the ldap group name with DN %v, error %v", groupDN, err)
+		return nil, false
+	}
+	if len(lGroups) == 0 {
+		log.Warningf("Can not get the ldap group name with DN %v", groupDN)
+		return nil, false
 	}
+	return &lGroups[0], true
 }
 
 func (l *Auth) syncUserInfoFromDB(ctx context.Context, u *models.User) {

src/lib/config/metadata/metadatalist.go

@@ -96,6 +96,7 @@ var (
 		{Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`},
 		{Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`},
 		{Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`},
+		{Name: common.LDAPGroupAttachParallel, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_ATTACH_PARALLEL", DefaultValue: "false", ItemType: &BoolType{}, Editable: true, Description: `Attach LDAP group information to Harbor in parallel`},
 
 		{Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false},
 		{Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`},

src/lib/config/models/model.go

@@ -94,6 +94,7 @@ type GroupConf struct {
 	SearchScope         int    `json:"ldap_group_search_scope"`
 	AdminDN             string `json:"ldap_group_admin_dn,omitempty"`
 	MembershipAttribute string `json:"ldap_group_membership_attribute,omitempty"`
+	AttachParallel      bool   `json:"ldap_group_attach_parallel,omitempty"`
 }
 
 type GDPRSetting struct {

src/lib/config/userconfig.go

@@ -81,6 +81,7 @@ func LDAPGroupConf(ctx context.Context) (*cfgModels.GroupConf, error) {
 		SearchScope:         mgr.Get(ctx, common.LDAPGroupSearchScope).GetInt(),
 		AdminDN:             mgr.Get(ctx, common.LDAPGroupAdminDn).GetString(),
 		MembershipAttribute: mgr.Get(ctx, common.LDAPGroupMembershipAttribute).GetString(),
+		AttachParallel:      mgr.Get(ctx, common.LDAPGroupAttachParallel).GetBool(),
 	}, nil
 }
 

src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html

@@ -498,6 +498,37 @@
                 </option>
             </select>
         </clr-select-container>
+        <clr-checkbox-container>
+            <label for="ldapGroupAttachParallel">
+                {{ 'CONFIG.LDAP.GROUP_ATTACH_PARALLEL' | translate }}
+                <clr-tooltip>
+                    <clr-icon
+                        clrTooltipTrigger
+                        shape="info-circle"
+                        size="24"></clr-icon>
+                    <clr-tooltip-content
+                        *clrIfOpen
+                        clrPosition="top-right"
+                        clrSize="lg">
+                        <span>{{
+                            'CONFIG.LDAP.GROUP_ATTACH_PARALLEL_INFO' | translate
+                        }}</span>
+                    </clr-tooltip-content>
+                </clr-tooltip>
+            </label>
+            <clr-checkbox-wrapper>
+                <input
+                    (ngModelChange)="setLdapGroupAttachParallelValue($event)"
+                    [disabled]="
+                        disabled(currentConfig.ldap_group_attach_parallel)
+                    "
+                    [ngModel]="currentConfig.ldap_group_attach_parallel.value"
+                    clrCheckbox
+                    id="ldapGroupAttachParallel"
+                    name="ldapGroupAttachParallel"
+                    type="checkbox" />
+            </clr-checkbox-wrapper>
+        </clr-checkbox-container>
     </section>
     <clr-checkbox-container *ngIf="showSelfReg">
         <label for="selfReg"

src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.ts

@@ -11,15 +11,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-import { Component, ViewChild, OnInit } from '@angular/core';
+import { Component, OnInit, ViewChild } from '@angular/core';
 import { NgForm } from '@angular/forms';
 import { MessageHandlerService } from '../../../../shared/services/message-handler.service';
 import { AppConfigService } from '../../../../services/app-config.service';
 import { ConfigurationService } from '../../../../services/config.service';
 import { SystemInfoService } from '../../../../shared/services';
 import {
-    isEmpty,
     getChanges as getChangesFunc,
+    isEmpty,
 } from '../../../../shared/units/utils';
 import { CONFIG_AUTH_MODE } from '../../../../shared/entities/shared.const';
 import { errorHandler } from '../../../../shared/units/shared.utils';
@@ -132,6 +132,9 @@ export class ConfigurationAuthComponent implements OnInit {
         this.currentConfig.ldap_verify_cert.value = $event;
     }
 
+    setLdapGroupAttachParallelValue($event: any) {
+        this.currentConfig.ldap_group_attach_parallel.value = $event;
+    }
     public pingTestServer(): void {
         if (this.testingOnGoing) {
             return; // Should not come here

src/portal/src/app/base/left-side-nav/config/config.ts

@@ -72,6 +72,7 @@ export class Configuration {
     ldap_group_search_scope: NumberValueItem;
     ldap_group_membership_attribute: StringValueItem;
     ldap_group_admin_dn: StringValueItem;
+    ldap_group_attach_parallel: BoolValueItem;
     uaa_client_id: StringValueItem;
     uaa_client_secret?: StringValueItem;
     uaa_endpoint: StringValueItem;

src/portal/src/i18n/lang/en-us-lang.json

@@ -928,7 +928,9 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP Group Membership",
             "LDAP_GROUP_MEMBERSHIP_INFO": "The attribute indicates the membership of LDAP group, default value is memberof, in some LDAP server it could be \"ismemberof\". This field cannot be empty if you need to enable the LDAP group related feature.",
             "GROUP_SCOPE": "LDAP Group Search Scope",
-            "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default."
+          "GROUP_SCOPE_INFO": "The scope to search for groups, select Subtree by default.",
+          "GROUP_ATTACH_PARALLEL": "LDAP Group Attached In Parallel",
+          "GROUP_ATTACH_PARALLEL_INFO": "Enable this option to attach group in parallel to avoid timeout when there are too many groups. If disabled, the LDAP group information will be attached sequentially."
 
         },
         "UAA": {

src/portal/src/i18n/lang/zh-cn-lang.json

@@ -926,7 +926,10 @@
             "LDAP_GROUP_MEMBERSHIP": "LDAP 组成员",
             "LDAP_GROUP_MEMBERSHIP_INFO": "LDAP组成员的membership属性，默认为 memberof, 在某些LDAP服务器会变为 ismemberof。如果要开启LDAP组功能，则此项必填",
             "GROUP_SCOPE": "LDAP组搜索范围",
-            "GROUP_SCOPE_INFO": "搜索组的范围，默认值为\"子树\""
+          "GROUP_SCOPE_INFO": "搜索组的范围，默认值为\"子树\"",
+          "GROUP_ATTACH_PARALLEL": "LDAP组并行同步",
+          "GROUP_ATTACH_PARALLEL_INFO": "打开这个选项时，LDAP组的信息是并行同步到Harbor, 这样可以防止用户组太多时造成的登录超时，如果关闭这个选项，LDAP组信息是顺序同步到Harbor"
+
         },
         "UAA": {
             "ENDPOINT": "UAA Endpoint",