Scalability: Handle large number of apps in cf dashboards

src/frontend/app/features/cloud-foundry/cf.helpers.ts

@@ -9,10 +9,10 @@ import { pathGet } from '../../core/utils.service';
 import { SetClientFilter } from '../../store/actions/pagination.actions';
 import { RouterNav } from '../../store/actions/router.actions';
 import { AppState } from '../../store/app-state';
-import { applicationSchemaKey, endpointSchemaKey } from '../../store/helpers/entity-factory';
+import { applicationSchemaKey, endpointSchemaKey, entityFactory } from '../../store/helpers/entity-factory';
 import { selectPaginationState } from '../../store/selectors/pagination.selectors';
 import { APIResource } from '../../store/types/api.types';
-import { PaginationEntityState } from '../../store/types/pagination.types';
+import { PaginationEntityState, PaginatedAction } from '../../store/types/pagination.types';
 import {
   CfUser,
   CfUserRoleParams,
@@ -27,7 +27,12 @@ import { ICfRolesState } from '../../store/types/current-user-roles.types';
 import { getCurrentUserCFEndpointRolesState } from '../../store/selectors/current-user-roles-permissions-selectors/role.selectors';
 import { EndpointModel } from '../../store/types/endpoint.types';
 import { selectEntities } from '../../store/selectors/api.selectors';
+import { Headers, Http, Request, RequestOptions, URLSearchParams } from '@angular/http';
+import { environment } from '../../../environments/environment';
+import { getPaginationObservables } from '../../store/reducers/pagination-reducer/pagination-reducer.helper';
+import { PaginationMonitorFactory } from '../../shared/monitors/pagination-monitor.factory';
 
+const { proxyAPIVersion, cfAPIVersion } = environment;
 
 export interface IUserRole<T> {
   string: string;
@@ -265,3 +270,36 @@ export function haveMultiConnectedCfs(store: Store<AppState>): Observable<boolea
 export function filterEntitiesByGuid<T>(guid: string, array?: Array<APIResource<T>>): Array<APIResource<T>> {
   return array ? array.filter(entity => entity.metadata.guid === guid) : null;
 }
+
+export function createFetchTotalResultsPagKey(standardActionKey: string): string {
+  return standardActionKey + '-totalResults';
+}
+
+export function fetchTotalResults(
+  action: PaginatedAction,
+  store: Store<AppState>,
+  paginationMonitorFactory: PaginationMonitorFactory
+): Observable<number> {
+
+  action.paginationKey = createFetchTotalResultsPagKey(action.paginationKey);
+  action.initialParams['results-per-page'] = 1;
+  action.flattenPagination = false;
+
+  const pagObs = getPaginationObservables({
+    store,
+    action,
+    paginationMonitor: paginationMonitorFactory.create(
+      action.paginationKey,
+      entityFactory(action.entityKey)
+    )
+  });
+  // Ensure the request is made by sub'ing to the entities observable
+  pagObs.entities$.pipe(
+    first(),
+  ).subscribe();
+
+  return pagObs.pagination$.pipe(
+    filter(pagination => !!pagination && !!pagination.pageRequests && !!pagination.pageRequests[1] && !pagination.pageRequests[1].busy),
+    map(pag => pag.totalResults)
+  );
+}

src/frontend/app/features/cloud-foundry/services/cloud-foundry-endpoint.service.ts

@@ -1,20 +1,20 @@
 import { Injectable } from '@angular/core';
 import { Store } from '@ngrx/store';
-import { Observable } from 'rxjs';
-import { first, map, publishReplay, refCount } from 'rxjs/operators';
+import { Observable, of as observableOf } from 'rxjs';
+import { filter, first, map, publishReplay, refCount, switchMap } from 'rxjs/operators';
 
 import { IApp, ICfV2Info, IOrganization, ISpace } from '../../../core/cf-api.types';
 import { EntityService } from '../../../core/entity-service';
 import { EntityServiceFactory } from '../../../core/entity-service-factory.service';
 import { CfUserService } from '../../../shared/data-services/cf-user.service';
 import { PaginationMonitorFactory } from '../../../shared/monitors/pagination-monitor.factory';
+import { GetAllApplications } from '../../../store/actions/application.actions';
 import { GetCFInfo } from '../../../store/actions/cloud-foundry.actions';
 import { FetchAllDomains } from '../../../store/actions/domains.actions';
 import { GetAllEndpoints } from '../../../store/actions/endpoint.actions';
 import { DeleteOrganization, GetAllOrganizations } from '../../../store/actions/organization.actions';
 import { AppState } from '../../../store/app-state';
 import {
-  applicationSchemaKey,
   cfInfoSchemaKey,
   domainSchemaKey,
   endpointSchemaKey,
@@ -34,8 +34,10 @@ import { getPaginationObservables } from '../../../store/reducers/pagination-red
 import { APIResource, EntityInfo } from '../../../store/types/api.types';
 import { CfApplicationState } from '../../../store/types/application.types';
 import { EndpointModel, EndpointUser } from '../../../store/types/endpoint.types';
+import { QParam } from '../../../store/types/pagination.types';
 import { CfUser } from '../../../store/types/user.types';
 import { ActiveRouteCfOrgSpace } from '../cf-page.types';
+import { fetchTotalResults } from '../cf.helpers';
 
 export function appDataSort(app1: APIResource<IApp>, app2: APIResource<IApp>): number {
   const app1Date = new Date(app1.metadata.updated_at);
@@ -57,6 +59,8 @@ export class CloudFoundryEndpointService {
   totalMem$: Observable<number>;
   paginationSubscription: any;
   allApps$: Observable<APIResource<IApp>[]>;
+  hasAllApps$: Observable<boolean>;
+  totalApps$: Observable<number>;
   users$: Observable<APIResource<CfUser>[]>;
   orgs$: Observable<APIResource<IOrganization>[]>;
   info$: Observable<EntityInfo<APIResource<ICfV2Info>>>;
@@ -80,7 +84,6 @@ export class CloudFoundryEndpointService {
         createEntityRelationKey(organizationSchemaKey, domainSchemaKey),
         createEntityRelationKey(organizationSchemaKey, quotaDefinitionSchemaKey),
         createEntityRelationKey(organizationSchemaKey, privateDomainsSchemaKey),
-        createEntityRelationKey(spaceSchemaKey, applicationSchemaKey),
         createEntityRelationKey(spaceSchemaKey, serviceInstancesSchemaKey),
         createEntityRelationKey(spaceSchemaKey, routeSchemaKey), // Not really needed at top level, but if we drop down into an org with
         // lots of spaces it saves n x routes requests
@@ -97,12 +100,35 @@ export class CloudFoundryEndpointService {
       ]);
   }
 
+  public static fetchAppCount(
+    store: Store<AppState>,
+    pmf: PaginationMonitorFactory,
+    cfGuid: string,
+    orgGuid?: string,
+    spaceGuid?: string)
+    : Observable<number> {
+    const parentSchemaKey = spaceGuid ? spaceSchemaKey : orgGuid ? organizationSchemaKey : 'cf';
+    const uniqueKey = spaceGuid || orgGuid || cfGuid;
+    const action = new GetAllApplications(createEntityRelationPaginationKey(parentSchemaKey, uniqueKey), cfGuid);
+    action.initialParams = {
+      q: []
+    };
+    if (orgGuid) {
+      action.initialParams.q.push(new QParam('organization_guid', orgGuid, ' IN '));
+    }
+    if (spaceGuid) {
+      action.initialParams.q.push(new QParam('space_guid', spaceGuid, ' IN '));
+    }
+    return fetchTotalResults(action, store, pmf);
+  }
+
   constructor(
     public activeRouteCfOrgSpace: ActiveRouteCfOrgSpace,
     private store: Store<AppState>,
     private entityServiceFactory: EntityServiceFactory,
     private cfUserService: CfUserService,
-    private paginationMonitorFactory: PaginationMonitorFactory
+    private paginationMonitorFactory: PaginationMonitorFactory,
+    private pmf: PaginationMonitorFactory
   ) {
     this.cfGuid = activeRouteCfOrgSpace.cfGuid;
     this.getAllOrgsAction = CloudFoundryEndpointService.createGetAllOrganizations(this.cfGuid);
@@ -127,7 +153,7 @@ export class CloudFoundryEndpointService {
 
   }
 
-  constructCoreObservables() {
+  private constructCoreObservables() {
     this.endpoint$ = this.cfEndpointEntityService.waitForEntity$;
 
     this.orgs$ = getPaginationObservables<APIResource<IOrganization>>({
@@ -143,23 +169,44 @@ export class CloudFoundryEndpointService {
 
     this.info$ = this.cfInfoEntityService.waitForEntity$;
 
-    this.allApps$ = this.orgs$.pipe(
-      map(orgs => [].concat(...orgs.map(org => org.entity.spaces))),
-      map((spaces: APIResource<ISpace>[]) => [].concat(...spaces.map(space => space ? space.entity.apps : [])))
-    );
+    this.constructAppObservables();
 
     this.fetchDomains();
   }
 
-  constructSecondaryObservable() {
+  constructAppObservables() {
+    const action = new GetAllApplications(createEntityRelationPaginationKey('cf', this.cfGuid), this.cfGuid);
+
+    const pagObs = getPaginationObservables<APIResource<IApp>>({
+      store: this.store,
+      action,
+      paginationMonitor: this.pmf.create(action.paginationKey, entityFactory(action.entityKey))
+    });
+
+    this.allApps$ = pagObs.entities$.pipe(// Ensure we sub to entities to kick off fetch process
+      switchMap(() => pagObs.pagination$),
+      filter(pagination => !!pagination && !!pagination.pageRequests && !!pagination.pageRequests[1] && !pagination.pageRequests[1].busy),
+      switchMap(pagination => pagination.maxedResults ? observableOf(null) : pagObs.entities$)
+    );
+
+    this.hasAllApps$ = this.allApps$.pipe(
+      map((allApps: APIResource<IApp>[]) => !!allApps)
+    );
+
+    this.totalApps$ = pagObs.pagination$.pipe(
+      map(pag => pag.totalResults)
+    );
+  }
+
+  private constructSecondaryObservable() {
 
     this.hasSSHAccess$ = this.info$.pipe(
       map(p => !!(p.entity.entity &&
         p.entity.entity.app_ssh_endpoint &&
         p.entity.entity.app_ssh_host_key_fingerprint &&
         p.entity.entity.app_ssh_oauth_client))
     );
-    this.totalMem$ = this.allApps$.pipe(map(a => this.getMetricFromApps(a, 'memory')));
+    this.totalMem$ = this.allApps$.pipe(map(apps => this.getMetricFromApps(apps, 'memory')));
 
     this.connected$ = this.endpoint$.pipe(
       map(p => p.entity.connectionStatus === 'connected')
@@ -169,35 +216,29 @@ export class CloudFoundryEndpointService {
 
   }
 
-  getAppsInOrg(
+  public getAppsInOrgViaAllApps(
     org: APIResource<IOrganization>
   ): Observable<APIResource<IApp>[]> {
     return this.allApps$.pipe(
+      filter(allApps => !!allApps),
       map(allApps => {
         const orgSpaces = org.entity.spaces.map(s => s.metadata.guid);
         return allApps.filter(a => orgSpaces.indexOf(a.entity.space_guid) !== -1);
       })
     );
   }
 
-  getAppsInSpace(
+  public getAppsInSpaceViaAllApps(
     space: APIResource<ISpace>
   ): Observable<APIResource<IApp>[]> {
     return this.allApps$.pipe(
+      filter(allApps => !!allApps),
       map(apps => {
         return apps.filter(a => a.entity.space_guid === space.metadata.guid);
       })
     );
   }
 
-  getAggregateStat(
-    org: APIResource<IOrganization>,
-    statMetric: string
-  ): Observable<number> {
-    return this.getAppsInOrg(org).pipe(
-      map(apps => this.getMetricFromApps(apps, statMetric))
-    );
-  }
   public getMetricFromApps(
     apps: APIResource<IApp>[],
     statMetric: string
@@ -208,7 +249,7 @@ export class CloudFoundryEndpointService {
       .reduce((a, t) => a + t, 0) : 0;
   }
 
-  fetchDomains = () => {
+  public fetchDomains = () => {
     const action = new FetchAllDomains(this.cfGuid);
     this.paginationSubscription = getPaginationObservables<APIResource>(
       {
@@ -223,7 +264,7 @@ export class CloudFoundryEndpointService {
     ).entities$.subscribe();
   }
 
-  deleteOrg(orgGuid: string, endpointGuid: string) {
+  public deleteOrg(orgGuid: string, endpointGuid: string) {
     this.store.dispatch(new DeleteOrganization(orgGuid, endpointGuid));
   }
 }

src/frontend/app/features/cloud-foundry/services/cloud-foundry-organization.service.ts

@@ -14,7 +14,6 @@ import { GetAllOrgUsers, GetOrganization } from '../../../store/actions/organiza
 import { DeleteSpace } from '../../../store/actions/space.actions';
 import { AppState } from '../../../store/app-state';
 import {
-  applicationSchemaKey,
   cfUserSchemaKey,
   domainSchemaKey,
   entityFactory,
@@ -36,6 +35,18 @@ import { ActiveRouteCfOrgSpace } from '../cf-page.types';
 import { getOrgRolesString } from '../cf.helpers';
 import { CloudFoundryEndpointService } from './cloud-foundry-endpoint.service';
 
+export const createQuotaDefinition = (orgGuid: string): APIResource<IQuotaDefinition> => ({
+  entity: {
+    memory_limit: -1,
+    app_instance_limit: -1,
+    instance_memory_limit: -1,
+    name: 'None assigned',
+    organization_guid: orgGuid,
+    total_services: -1,
+    total_routes: -1
+  },
+  metadata: null
+});
 
 @Injectable()
 export class CloudFoundryOrganizationService {
@@ -50,6 +61,7 @@ export class CloudFoundryOrganizationService {
   spaces$: Observable<APIResource<ISpace>[]>;
   appInstances$: Observable<number>;
   apps$: Observable<APIResource<IApp>[]>;
+  appCount$: Observable<number>;
   org$: Observable<EntityInfo<APIResource<IOrganization>>>;
   allOrgUsers$: Observable<APIResource<CfUser>[]>;
   usersPaginationKey: string;
@@ -60,8 +72,7 @@ export class CloudFoundryOrganizationService {
     private entityServiceFactory: EntityServiceFactory,
     private cfUserService: CfUserService,
     private paginationMonitorFactory: PaginationMonitorFactory,
-    private cfEndpointService: CloudFoundryEndpointService
-
+    private cfEndpointService: CloudFoundryEndpointService,
   ) {
     this.orgGuid = activeRouteCfOrgSpace.orgGuid;
     this.cfGuid = activeRouteCfOrgSpace.cfGuid;
@@ -83,7 +94,6 @@ export class CloudFoundryOrganizationService {
           createEntityRelationKey(organizationSchemaKey, quotaDefinitionSchemaKey),
           createEntityRelationKey(organizationSchemaKey, privateDomainsSchemaKey),
           createEntityRelationKey(spaceSchemaKey, serviceInstancesSchemaKey),
-          createEntityRelationKey(spaceSchemaKey, applicationSchemaKey),
           createEntityRelationKey(spaceSchemaKey, routeSchemaKey),
         ];
         if (!isAdmin) {
@@ -128,13 +138,34 @@ export class CloudFoundryOrganizationService {
   }
 
   private initialiseAppObservables() {
-    this.apps$ = this.spaces$.pipe(this.getFlattenedList('apps'));
+    this.apps$ = this.org$.pipe(
+      switchMap(org => this.cfEndpointService.getAppsInOrgViaAllApps(org.entity))
+    );
     this.appInstances$ = this.apps$.pipe(
       filter($apps => !!$apps),
       map(getStartedAppInstanceCount)
     );
 
     this.totalMem$ = this.apps$.pipe(map(a => this.cfEndpointService.getMetricFromApps(a, 'memory')));
+
+    this.appCount$ = this.cfEndpointService.hasAllApps$.pipe(
+      switchMap(hasAllApps => hasAllApps ? this.countExistingApps() : this.fetchAppCount()),
+    );
+  }
+
+  private countExistingApps(): Observable<number> {
+    return this.apps$.pipe(
+      map(apps => apps.length)
+    );
+  }
+
+  private fetchAppCount(): Observable<number> {
+    return CloudFoundryEndpointService.fetchAppCount(
+      this.store,
+      this.paginationMonitorFactory,
+      this.activeRouteCfOrgSpace.cfGuid,
+      this.activeRouteCfOrgSpace.orgGuid
+    );
   }
 
   private initialiseOrgObservables() {