From 537eb579b9c6d95d600f455a187e6d7b9bf700ef Mon Sep 17 00:00:00 2001 From: John Kaster Date: Wed, 30 Jun 2021 11:40:15 -0700 Subject: [PATCH] feat: paging results for the Typescript SDK (#698) This PR is for the first version of the SDK response paging prototype. This supports the "page at a time" retrieval. The "row at a time while paged behind the scenes" iterator will come later. See `doc/paging.md` for the documentation. --- docs/paging.md | 108 +++++ .../src/authToken/extensionProxyTransport.ts | 11 + packages/sdk-node/src/nodeTransport.spec.ts | 4 +- packages/sdk-node/src/nodeTransport.ts | 156 ++++--- packages/sdk-node/test/methods.spec.ts | 83 +++- packages/sdk-rtl/src/baseTransport.ts | 14 +- packages/sdk-rtl/src/browserTransport.ts | 91 ++-- packages/sdk-rtl/src/extensionTransport.ts | 16 +- packages/sdk-rtl/src/index.ts | 5 +- packages/sdk-rtl/src/paging.spec.ts | 268 ++++++++++++ packages/sdk-rtl/src/paging.ts | 403 ++++++++++++++++++ packages/sdk-rtl/src/transport.ts | 52 +-- 12 files changed, 1070 insertions(+), 141 deletions(-) create mode 100644 docs/paging.md create mode 100644 packages/sdk-rtl/src/paging.spec.ts create mode 100644 packages/sdk-rtl/src/paging.ts diff --git a/docs/paging.md b/docs/paging.md new file mode 100644 index 000000000..5736082ae --- /dev/null +++ b/docs/paging.md @@ -0,0 +1,108 @@ +# API response paging + +Looker is adding [alpha-level](#alpha-support-level) support for API response paging in Looker API 4.0. + +Any endpoint that accepts `limit` and `offset` parameters can support generic paging. Starting with Looker release 21.12, Looker is adding paging support for API 4.0 endpoints (until all endpoints that accept `limit` and `offset` provide the headers). + +| Parameter | Description | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `limit` | If provided, this value sets the number of results to return per _page_ and triggers paging headers to be provided. | +| `offset` | This value sets the starting position of the results to return. A value of `0` (zero) is used for the first result. `offset` defaults to 0 if `limit` is provided and `offset` is not. | + +Some endpoints have `page` and `per_page` parameters instead of, or in addition to, `limit` and `offset`. The `limit` and `offset` parameters take precedence over the `page` and `per_page` parameters for endpoints that support both. +Only API calls specifying `limit` will produce paging headers for those endpoints that provide paging headers. + +**NOTE**: The `page` and `per_page` parameters may be removed for API 4.0. Also, the Looker API does not support cursor-based paging. + +## Paging headers + +The [`X-Total-Count`](https://stackoverflow.com/a/43968710) and [`Link`](https://datatracker.ietf.org/doc/html/rfc5988) headers provide all the information required for an SDK to generically page API calls that return a collection of items. + +### X-Total-Count header + +If the `total count` of items can be known, the value of this header is that count. If `total count` is unknown, this header is not in the endpoint response. + +Because many Looker endpoints restrict the user's ability to view individual items of a collection based on complex access constraints, sometimes calculating the total count degrades performance too much to calculate it. + +### Link header + +The Looker API adopts the [GitHub Link header values](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#link-header). + +Paging responses always include `Link` headers. Different **Link Relation Type** (`rel`) values may or may not exist in the Link header. + +The table below explains Looker's use of the `rel` values adopted from GitHub. + +| Rel | Description | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `first` | The URI to the first page of results. This link is always provided. | +| `next` | The URI to the next page of results. This link is provided when `total count` is known or the number of items returned == the `limit` | +| `prev` | The URI to the previous page of results. This link is provided when there **is** a previous page. | +| `last` | The URI to the last page of results. This link can only be provided when `total count` is known. | + +Here is an example of a "full" Link header's content: + +``` +; rel="first", +; rel="last", +; rel="next", +; rel="prev" +``` + +## SDK Paging + +Thanks to the adoption of the "standard" paging headers shown above, the SDKs can implement API result paging generically. + +The current SDK-based paging pattern prototype is in the `@looker/sdk-rtl` TypeScript/Javascript package. + +### Paging interface + +The main routines that initialize SDK paging are the functions `pager` and `pageAll`, and the class `Paging` defined in [paging.ts](/packages/sdk-rtl/src/paging.ts). + + +### Page iteration example + +Results can be retrieved a page at a time with code like this sample: + +```ts +// "Monolithic" SDK search function +async function dashboardSearchResultsByPage(inTitle: string, limit: number = 100) { + const sdk = new Looker40SDK(session) + return await pager(sdk, () => + sdk.search_dashboards({title: inTitle, limit}) + ) +} + +const pagedDashboards = await dashboardSearchResultsByPage('JOEL') +for (const dash of pagedDashboards.items) { + console.log(dash.title) +} +while (pagedDashboards.more()) { + for (const dash of await pagedDashboards.nextPage()) { + console.log(dash.title) + } +} +``` + +For the functional SDK, the syntax is almost identical (the imports will vary). The search function can be changed to: + +```ts +// Functional SDK search function +async function dashboardSearchResultsByPage(inTitle: string, limit: number = 100) { + const sdk = new Looker40SDK(session) + return await pager(sdk, () => + search_dashboards(sdk, {title: inTitle, limit}) + ) +} + +``` + +**Note** The above examples will only work correctly when a Looker release with paging headers for the API 4.0 implementation of `search_dashboards` is available. + +## Alpha support level + +Support for paging headers is currently at alpha level. This means that: + +- Not all endpoints with `limit` and `offset` parameters provide paging headers. +- Paging performance may vary for large results sets. We recommend making the `limit` size a larger value (half or a quarter of the total count, perhaps) to reduce paging if performance degradation is noticed as the `offset` grows larger. +- Currently, SDK support for paging is only available in the Typescript SDK prototype. +- While SDK paging routines **should** work for API endpoints that provide paging headers, reliability is not guaranteed, and SDK paging routines are only "community supported." This means that issues can be filed in this repository and Looker engineering will attempt to address them, but no timeframe or response is guaranteed. diff --git a/packages/hackathon/src/authToken/extensionProxyTransport.ts b/packages/hackathon/src/authToken/extensionProxyTransport.ts index 5a38c7823..a57d1a893 100644 --- a/packages/hackathon/src/authToken/extensionProxyTransport.ts +++ b/packages/hackathon/src/authToken/extensionProxyTransport.ts @@ -90,6 +90,16 @@ export class ExtensionProxyTransport extends BaseTransport { return props } + parseResponse( + _raw: IRawResponse + ): Promise> { + const result: SDKResponse = { + ok: false, + error: new Error('Should not be called!') as unknown as TError, + } + return Promise.resolve(result) + } + async rawRequest( method: HttpMethod, path: string, @@ -131,6 +141,7 @@ export class ExtensionProxyTransport extends BaseTransport { ok: true, statusCode: res.status, statusMessage: `${res.status} fetched`, + headers: res.headers, } } diff --git a/packages/sdk-node/src/nodeTransport.spec.ts b/packages/sdk-node/src/nodeTransport.spec.ts index d6049943d..2bb414022 100644 --- a/packages/sdk-node/src/nodeTransport.spec.ts +++ b/packages/sdk-node/src/nodeTransport.spec.ts @@ -57,9 +57,7 @@ describe('NodeTransport', () => { expect(response.ok).toEqual(false) expect(response.statusCode).toEqual(404) expect(response.body).toBeDefined() - expect(response.body.indexOf(errorMessage)).toEqual(0) - expect(response.body.length).toBeGreaterThan(0) - expect(response.statusMessage.indexOf('"type":"Buffer":')).toEqual(-1) + expect(response.statusMessage.indexOf('"type":"Buffer"')).toEqual(-1) expect(response.statusMessage.indexOf(errorMessage)).toEqual(0) }) }) diff --git a/packages/sdk-node/src/nodeTransport.ts b/packages/sdk-node/src/nodeTransport.ts index 5ed335c31..2b4e1f804 100644 --- a/packages/sdk-node/src/nodeTransport.ts +++ b/packages/sdk-node/src/nodeTransport.ts @@ -31,25 +31,39 @@ import rp from 'request-promise-native' import { PassThrough, Readable } from 'readable-stream' import { StatusCodeError } from 'request-promise-native/errors' import { - Authenticator, + BaseTransport, + ResponseMode, defaultTimeout, + responseMode, + trace, + LookerAppId, + agentPrefix, + safeBase64, +} from '@looker/sdk-rtl' +import type { + Authenticator, HttpMethod, ISDKError, ITransportSettings, - responseMode, - ResponseMode, SDKResponse, - trace, Values, IRequestHeaders, - LookerAppId, IRawResponse, - agentPrefix, - safeBase64, - BaseTransport, ICryptoHash, } from '@looker/sdk-rtl' +const utf8 = 'utf8' + +const asString = (value: any): string => { + if (value instanceof Buffer) { + return Buffer.from(value).toString(utf8) + } + if (value instanceof Object) { + return JSON.stringify(value) + } + return value.toString() +} + export class NodeCryptoHash implements ICryptoHash { secureRandom(byteCount: number): string { return nodeCrypto.randomBytes(byteCount).toString('hex') @@ -62,38 +76,9 @@ export class NodeCryptoHash implements ICryptoHash { } } -export type RequestOptions = rq.RequiredUriUrl & rp.RequestPromiseOptions - -async function parseResponse(res: IRawResponse) { - const mode = responseMode(res.contentType) - const utf8 = 'utf8' - let result = await res.body - if (mode === ResponseMode.string) { - if (res.contentType.match(/^application\/.*\bjson\b/g)) { - try { - if (result instanceof Buffer) { - result = (result as Buffer).toString(utf8) - } - if (result instanceof Object) { - return result - } - return JSON.parse(result.toString()) - } catch (error) { - return Promise.reject(error) - } - } - if (result instanceof Buffer) { - result = (result as Buffer).toString(utf8) - } - return result.toString() - } else { - try { - return (result as Buffer).toString('binary') - } catch (error) { - return Promise.reject(error) - } - } -} +export type RequestOptions = rq.RequiredUriUrl & + rp.RequestPromiseOptions & + rq.OptionsWithUrl export class NodeTransport extends BaseTransport { constructor(protected readonly options: ITransportSettings) { @@ -108,7 +93,7 @@ export class NodeTransport extends BaseTransport { authenticator?: Authenticator, options?: Partial ): Promise { - const init = await this.initRequest( + const init: RequestOptions = await this.initRequest( method, path, queryParams, @@ -117,51 +102,97 @@ export class NodeTransport extends BaseTransport { options ) const req = rp(init).promise() + let rawResponse: IRawResponse try { const res = await req const resTyped = res as rq.Response - return { - url: resTyped.url || '', + rawResponse = { + url: resTyped.url || init.url.toString() || '', body: await resTyped.body, contentType: String(resTyped.headers['content-type']), ok: true, statusCode: resTyped.statusCode, statusMessage: resTyped.statusMessage, + headers: res.headers, } } catch (e) { - const statusMessage = `${method} ${path}` + let statusMessage = `${method} ${path}` let statusCode = 404 let contentType = 'text' - let body: string + let body if (e instanceof StatusCodeError) { statusCode = e.statusCode - const text = e.message - body = e.message - // Need to re-parse the error message - const matches = /^\d+\s*-\s*({.*})/gim.exec(text) - if (matches && matches.length > 1) { - const json = matches[1] - const obj = JSON.parse(json) - if (obj.data) { - body = Buffer.from(obj.data).toString('utf8') - } + if (e.error instanceof Buffer) { + body = asString(e.error) + statusMessage += `: ${statusCode}` + } else if (e.error instanceof Object) { + // Capture error object as body + body = e.error + statusMessage += `: ${e.message}` + // Clarify the error message + body.message = statusMessage + contentType = 'application/json' } - body = `${statusMessage} ${body}` } else if (e.error instanceof Buffer) { - body = Buffer.from(e.error).toString('utf8') + body = asString(e.error) } else { body = JSON.stringify(e) contentType = 'application/json' } - return { - url: this.makeUrl(path, { ...this.options, ...options }, queryParams), + rawResponse = { + url: init.url.toString(), body, contentType, ok: false, statusCode, statusMessage, + headers: {}, + } + } + return this.observer ? this.observer(rawResponse) : rawResponse + } + + async parseResponse(res: IRawResponse) { + const mode = responseMode(res.contentType) + let response: SDKResponse + let error + if (!res.ok) { + // Raw request had an error. Make sure it's a string before parsing the result + error = res.body + if (typeof error === 'string') error = JSON.parse(error) + response = { ok: false, error } + return response + } + let result = await res.body + if (mode === ResponseMode.string) { + if (res.contentType.match(/^application\/.*\bjson\b/g)) { + try { + if (result instanceof Buffer) { + result = Buffer.from(result).toString(utf8) + } + if (!(result instanceof Object)) { + result = JSON.parse(result.toString()) + } + } catch (err) { + error = err + } + } else if (!error) { + // Convert to string otherwise + result = asString(result) + } + } else { + try { + result = Buffer.from(result).toString('binary') + } catch (err) { + error = err } } + if (!error) { + response = { ok: true, value: result } + } else { + response = { ok: false, error } + } + return response } async request( @@ -181,12 +212,7 @@ export class NodeTransport extends BaseTransport { authenticator, options ) - const parsed = await parseResponse(res) - if (this.ok(res)) { - return { ok: true, value: parsed } - } else { - return { error: parsed, ok: false } - } + return await this.parseResponse(res) } catch (e) { const error: ISDKError = { message: diff --git a/packages/sdk-node/test/methods.spec.ts b/packages/sdk-node/test/methods.spec.ts index acec12533..396a55752 100644 --- a/packages/sdk-node/test/methods.spec.ts +++ b/packages/sdk-node/test/methods.spec.ts @@ -39,21 +39,24 @@ import { Looker40SDK, Looker31SDKStream, Looker40SDKStream, + IDashboard, } from '@looker/sdk' import { DelimArray, boolDefault, defaultTimeout, ApiConfigMap, + pageAll, + pager, } from '@looker/sdk-rtl' import { NodeSettings, NodeSettingsIniFile, NodeSession, + LookerNodeSDK, readIniConfig, } from '../src' import { TestConfig } from '../../sdk-rtl/src/testUtils' -import { LookerNodeSDK } from '../src/nodeSdk' const envKey = ApiConfigMap(environmentPrefix) const strLookerBaseUrl = envKey.base_url @@ -641,6 +644,84 @@ describe('LookerNodeSDK', () => { expect(actual).toEqual(task) }) }) + + // TODO remove skip after Looker 21.12 is available + describe.skip('paging alpha', () => { + describe('pager', () => { + test( + 'getRel can override limit and offset', + async () => { + const sdk = new LookerSDK(session) + const limit = 2 + const all = await sdk.ok(sdk.search_dashboards({ fields: 'id' })) + const paged = await pager(sdk, () => + sdk.search_dashboards({ fields: 'id', limit }) + ) + const full = await sdk.ok(paged.getRel('first', all.length)) + expect(full).toEqual(all) + }, + testTimeout + ) + }) + describe('pageAll', () => { + test( + 'search_dashboard', + async () => { + const sdk = new LookerSDK(session) + // Use a small limit to test paging for a small number of dashboards + const limit = 2 + let count = 0 + let actual: IDashboard[] = [] + const aggregate = (page: IDashboard[]) => { + console.log(`Page ${++count} has ${page.length} items`) + actual = actual.concat(page) + return page + } + const paged = await pageAll( + sdk, + () => sdk.search_dashboards({ fields: 'id,title', limit }), + aggregate + ) + expect(paged.limit).toEqual(limit) + expect(paged.more()).toEqual(false) + + const all = await sdk.ok( + sdk.search_dashboards({ fields: 'id, title' }) + ) + expect(actual.length).toEqual(all.length) + expect(actual).toEqual(all) + }, + testTimeout + ) + test( + 'all_dashboards pageAll returns non-paged results', + async () => { + const sdk = new LookerSDK(session) + // Use a small limit to test paging for a small number of dashboards + let count = 0 + let actual: IDashboard[] = [] + const aggregate = (page: IDashboard[]) => { + console.log(`Page ${++count} has ${page.length} items`) + actual = actual.concat(page) + return page + } + const paged = await pageAll( + sdk, + () => sdk.all_dashboards('id,title'), + aggregate + ) + expect(paged.limit).toEqual(-1) + expect(paged.more()).toEqual(false) + + const all = await sdk.ok(sdk.all_dashboards('id, title')) + expect(actual.length).toEqual(all.length) + expect(actual).toEqual(all) + }, + testTimeout + ) + }) + }) + describe('Query calls', () => { it( 'create and run query', diff --git a/packages/sdk-rtl/src/baseTransport.ts b/packages/sdk-rtl/src/baseTransport.ts index d1d0a7f6f..73f97c4a2 100644 --- a/packages/sdk-rtl/src/baseTransport.ts +++ b/packages/sdk-rtl/src/baseTransport.ts @@ -25,16 +25,16 @@ */ import { Readable } from 'readable-stream' -import { - addQueryParams, +import { addQueryParams, StatusCode } from './transport' +import type { Authenticator, HttpMethod, IRawResponse, ITransport, ITransportSettings, SDKResponse, - StatusCode, Values, + RawObserver, } from './transport' export abstract class BaseTransport implements ITransport { @@ -42,7 +42,13 @@ export abstract class BaseTransport implements ITransport { this.options = options } - protected ok(res: IRawResponse) { + observer: RawObserver | undefined = undefined + + abstract parseResponse( + raw: IRawResponse + ): Promise> + + protected ok(res: IRawResponse): boolean { return ( res.statusCode >= StatusCode.OK && res.statusCode <= StatusCode.IMUsed ) diff --git a/packages/sdk-rtl/src/browserTransport.ts b/packages/sdk-rtl/src/browserTransport.ts index a2466659d..87038e556 100644 --- a/packages/sdk-rtl/src/browserTransport.ts +++ b/packages/sdk-rtl/src/browserTransport.ts @@ -199,7 +199,9 @@ export class BrowserTransport extends BaseTransport { // Request will markEnd, so don't mark the end here BrowserTransport.markEnd(requestPath, started) } - return { + const headers = {} + res.headers.forEach((value, key) => (headers[key] = value)) + const response: IRawResponse = { url: requestPath, body: responseBody, contentType, @@ -207,7 +209,51 @@ export class BrowserTransport extends BaseTransport { statusCode: res.status, statusMessage: res.statusText, startMark: started, + headers, + } + return this.observer ? this.observer(response) : response + } + + /** + * Process the response based on content type + * @param res response to process + */ + async parseResponse( + res: IRawResponse + ): Promise> { + const perfMark = res.startMark || '' + let value + let error + if (res.contentType.match(/application\/json/g)) { + try { + value = JSON.parse(await res.body) + BrowserTransport.markEnd(res.url, perfMark) + } catch (err) { + error = err + BrowserTransport.markEnd(res.url, perfMark) + } + } else if ( + res.contentType === 'text' || + res.contentType.startsWith('text/') + ) { + value = res.body.toString() + BrowserTransport.markEnd(res.url, perfMark) + } else { + try { + BrowserTransport.markEnd(res.url, perfMark) + value = res.body + } catch (err) { + BrowserTransport.markEnd(res.url, perfMark) + error = err + } + } + let result: SDKResponse + if (error) { + result = { ok: false, error } + } else { + result = { ok: true, value } } + return result } async request( @@ -231,12 +277,10 @@ export class BrowserTransport extends BaseTransport { options ) // eslint-disable-next-line @typescript-eslint/no-use-before-define - const parsed = await parseResponse(res) - if (this.ok(res)) { - return { ok: true, value: parsed } - } else { - return { error: parsed, ok: false } - } + const result: SDKResponse = await this.parseResponse( + res + ) + return result } catch (e) { const error: ISDKError = { message: @@ -375,36 +419,3 @@ export class BrowserTransport extends BaseTransport { */ } } - -/** - * Process the response based on content type - * @param res response to process - */ -export const parseResponse = async (res: IRawResponse) => { - const perfMark = res.startMark || '' - if (res.contentType.match(/application\/json/g)) { - try { - const result = JSON.parse(await res.body) - BrowserTransport.markEnd(res.url, perfMark) - return result - } catch (error) { - BrowserTransport.markEnd(res.url, perfMark) - return Promise.reject(error) - } - } else if ( - res.contentType === 'text' || - res.contentType.startsWith('text/') - ) { - const result = res.body.toString() - BrowserTransport.markEnd(res.url, perfMark) - return result - } else { - try { - BrowserTransport.markEnd(res.url, perfMark) - return res.body - } catch (error) { - BrowserTransport.markEnd(res.url, perfMark) - return Promise.reject(error) - } - } -} diff --git a/packages/sdk-rtl/src/extensionTransport.ts b/packages/sdk-rtl/src/extensionTransport.ts index 30a8bcbe7..212193035 100644 --- a/packages/sdk-rtl/src/extensionTransport.ts +++ b/packages/sdk-rtl/src/extensionTransport.ts @@ -33,6 +33,7 @@ import { ITransportSettings, HttpMethod, IRawResponse, + RawObserver, } from './transport' export interface IHostConnection { @@ -74,6 +75,8 @@ export class ExtensionTransport implements ITransport { this.hostConnection = hostConnection } + observer: RawObserver | undefined + async rawRequest( method: HttpMethod, path: string, @@ -82,7 +85,7 @@ export class ExtensionTransport implements ITransport { authenticator?: any, options?: Partial ): Promise { - return this.hostConnection.rawRequest( + const response = await this.hostConnection.rawRequest( method, path, body, @@ -90,6 +93,7 @@ export class ExtensionTransport implements ITransport { authenticator, options ) + return this.observer ? this.observer(response) : response } async request( @@ -130,4 +134,14 @@ export class ExtensionTransport implements ITransport { options ) } + + parseResponse( + _raw: IRawResponse + ): Promise> { + const result: SDKResponse = { + ok: false, + error: new Error('Should not be called!') as unknown as TError, + } + return Promise.resolve(result) + } } diff --git a/packages/sdk-rtl/src/index.ts b/packages/sdk-rtl/src/index.ts index a655640e9..697d3413e 100644 --- a/packages/sdk-rtl/src/index.ts +++ b/packages/sdk-rtl/src/index.ts @@ -24,8 +24,6 @@ */ -/* Version 21.0.5 */ - export * from './apiMethods' export * from './apiSettings' export * from './authSession' @@ -41,6 +39,7 @@ export * from './delimArray' export * from './extensionSession' export * from './extensionTransport' export * from './oauthSession' -export * from './proxySession' +export * from './paging' export * from './platformServices' +export * from './proxySession' export * from './transport' diff --git a/packages/sdk-rtl/src/paging.spec.ts b/packages/sdk-rtl/src/paging.spec.ts new file mode 100644 index 000000000..36e054bec --- /dev/null +++ b/packages/sdk-rtl/src/paging.spec.ts @@ -0,0 +1,268 @@ +/* + + MIT License + + Copyright (c) 2021 Looker Data Sciences, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + */ + +import { LinkHeader, linkHeaderParser, pager, TotalCountHeader } from './paging' +import { APIMethods } from './apiMethods' +import { AuthSession } from './authSession' +import { ApiSettings, IApiSettings } from './apiSettings' +import { BrowserTransport } from './browserTransport' +import { + IRawResponse, + IRequestProps, + ITransport, + sdkOk, + SDKResponse, +} from './transport' + +const firstUrl = + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=0' +const lastUrl = + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=9' +const prevUrl = + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=3' +const nextUrl = + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=6' + +const firstLink = `<${firstUrl}>; rel="first"` +// Verify extra whitespace characters still get parsed correctly +// The Looker API provides a header without extra whitespace but other Link headers +// may not be formatted that way +const lastLink = `< ${lastUrl} >; rel="\tlast (end of line!)\t"` +const prevLink = `<\t${prevUrl}\n>;\nrel="prev"` +const nextLink = `<${nextUrl}>; rel="next"` +const allLinks = `${firstLink},${lastLink},${prevLink},${nextLink}` +const settings = new ApiSettings({ base_url: 'mocked' }) + +class MockSession extends AuthSession { + constructor( + public readonly activeToken: string, + settings: IApiSettings, + transport: ITransport + ) { + super(settings, transport) + } + + async authenticate(props: IRequestProps) { + props.headers.Authorization = `Bearer ${this.activeToken}` + return props + } + + getToken() { + return Promise.resolve(this.activeToken) + } + + isAuthenticated(): boolean { + return !!this.activeToken + } +} + +const transport = new BrowserTransport(settings) +const session = new MockSession('mocked', settings, transport) +const sdk = new APIMethods(session, '4.0') +const mockedRows = ['one', 'two', 'three', 'four', 'five'] +const totalCount = 10 + +const mockRawResponse = (url?: string, body?: any): IRawResponse => { + const result: IRawResponse = { + ok: true, + body: JSON.stringify(mockedRows), + headers: { [LinkHeader]: allLinks, [TotalCountHeader]: ` ${totalCount}` }, + statusCode: 200, + statusMessage: 'Mocking', + contentType: 'application/json', + url: 'https://mocked', + } + if (url) { + result.url = url + } + if (body) { + result.body = body + } + return result +} + +async function mockSDKSuccess(value: T) { + return Promise.resolve>({ ok: true, value }) +} + +const mockRawResponseSuccess = ( + transport: BrowserTransport, + value: any, + rawResponse: IRawResponse +) => { + if (transport.observer) { + transport.observer(rawResponse) + } + return mockSDKSuccess(value) +} + +// async function mockSDKError(value: T) { +// return Promise.resolve>({ ok: false, error: value }) +// } + +describe('paging', () => { + describe('linkHeaderParser', () => { + it('parses all links', () => { + const actual = linkHeaderParser(allLinks) + const keys = Object.keys(actual) + expect(keys).toEqual(['first', 'last', 'prev', 'next']) + expect(actual.first.rel).toEqual('first') + expect(actual.first.url).toEqual( + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=0' + ) + expect(actual.last.rel).toEqual('last') + expect(actual.last.url).toEqual( + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=9' + ) + expect(actual.last.name).toEqual('last (end of line!)') + expect(actual.next.rel).toEqual('next') + expect(actual.next.url).toEqual( + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=6' + ) + expect(actual.prev.rel).toEqual('prev') + expect(actual.prev.url).toEqual( + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=3' + ) + }) + + it('gets first only', () => { + const actual = linkHeaderParser(firstLink) + const keys = Object.keys(actual) + expect(keys).toEqual(['first']) + expect(actual.first.rel).toEqual('first') + expect(actual.first.url).toEqual( + 'http://localhost/api/4.0/alerts/search?fields=id&limit=3&offset=0' + ) + }) + }) + + describe('pager', () => { + beforeEach(() => { + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue(Promise.resolve(mockRawResponse())) + }) + afterAll(() => { + jest.clearAllMocks() + }) + + it('initializes', async () => { + const actual = await pager( + sdk, + () => + mockRawResponseSuccess( + transport, + mockedRows, + mockRawResponse(firstUrl) + ), + { + timeout: 99, + } + ) + expect(actual).toBeDefined() + expect(actual.limit).toEqual(3) + expect(actual.offset).toEqual(0) + expect(actual.total).toEqual(totalCount) + expect(actual.items).toEqual(mockedRows) + expect(actual.pages).toEqual(4) + expect(actual.page).toEqual(1) + + expect(actual.hasRel('first')).toEqual(true) + expect(actual.hasRel('next')).toEqual(true) + expect(actual.hasRel('prev')).toEqual(true) + expect(actual.hasRel('last')).toEqual(true) + expect(actual.options?.timeout).toEqual(99) + }) + + it('supports paging', async () => { + const paged = await pager(sdk, () => + mockRawResponseSuccess(transport, mockedRows, mockRawResponse()) + ) + expect(paged).toBeDefined() + expect(paged.items).toEqual(mockedRows) + const threeRows = ['one', 'two', 'three'] + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue( + Promise.resolve(mockRawResponse(nextUrl, JSON.stringify(threeRows))) + ) + let items = await sdkOk(paged.nextPage()) + expect(items).toBeDefined() + expect(items).toEqual(threeRows) + expect(paged.offset).toEqual(6) + expect(paged.limit).toEqual(3) + expect(paged.total).toEqual(totalCount) + expect(paged.pages).toEqual(4) + expect(paged.page).toEqual(3) + + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue(Promise.resolve(mockRawResponse(nextUrl, '[]'))) + items = await sdkOk(paged.nextPage()) + expect(items).toBeDefined() + expect(items).toEqual([]) + expect(paged.offset).toEqual(6) + expect(paged.limit).toEqual(3) + expect(paged.total).toEqual(totalCount) + expect(paged.pages).toEqual(4) + expect(paged.page).toEqual(3) + + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue(Promise.resolve(mockRawResponse(prevUrl))) + items = await sdkOk(paged.prevPage()) + expect(items).toBeDefined() + expect(items).toEqual(mockedRows) + expect(paged.offset).toEqual(3) + expect(paged.limit).toEqual(3) + expect(paged.total).toEqual(totalCount) + expect(paged.pages).toEqual(4) + expect(paged.page).toEqual(2) + + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue(Promise.resolve(mockRawResponse(lastUrl))) + items = await sdkOk(paged.lastPage()) + expect(items).toBeDefined() + expect(items).toEqual(mockedRows) + expect(paged.offset).toEqual(9) + expect(paged.limit).toEqual(3) + expect(paged.total).toEqual(totalCount) + expect(paged.pages).toEqual(4) + expect(paged.page).toEqual(4) + + jest + .spyOn(BrowserTransport.prototype, 'rawRequest') + .mockReturnValue(Promise.resolve(mockRawResponse(firstUrl))) + items = await sdkOk(paged.firstPage()) + expect(items).toBeDefined() + expect(items).toEqual(mockedRows) + expect(paged.offset).toEqual(0) + expect(paged.limit).toEqual(3) + expect(paged.total).toEqual(totalCount) + }) + }) +}) diff --git a/packages/sdk-rtl/src/paging.ts b/packages/sdk-rtl/src/paging.ts new file mode 100644 index 000000000..7247251c6 --- /dev/null +++ b/packages/sdk-rtl/src/paging.ts @@ -0,0 +1,403 @@ +/* + + MIT License + + Copyright (c) 2021 Looker Data Sciences, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + */ + +import { + IRawResponse, + ITransportSettings, + sdkOk, + SDKResponse, +} from './transport' +import { IAPIMethods } from './apiMethods' +import { BaseTransport } from './baseTransport' + +export const LinkHeader = 'link' +export const TotalCountHeader = 'X-Total-Count' + +/** + * Types of paging link relative URLs + * based on https://docs.github.com/en/rest/overview/resources-in-the-rest-api#link-header + */ +export type PageLinkRel = 'first' | 'last' | 'next' | 'prev' + +/** Constraints for TSuccess type */ +interface ILength { + length: number +} + +/** Result paging function call */ +export type PagingFunc = () => Promise< + SDKResponse +> + +/** Page link structure */ +export interface IPageLink { + /** Name of link */ + name?: string + /** Type of link */ + rel: PageLinkRel + /** Media type for link */ + mediaType?: string + /** URL for retrieving the link results */ + url: string +} + +/** + * Collection of page links + * + * TODO can this be Record instead and still init to {}? + */ +export type PageLinks = Record + +export interface IPager { + /** Total number of available items being paginated */ + total: number + /** Offset extracted from pager request */ + offset: number + /** Limit extracted from pager request */ + limit: number + /** Paging links extracted from Link header */ + links: PageLinks + /** Latest items returned from response */ + items: TSuccess + /** Captured from the original paging request */ + options?: Partial + /** Total number of pages. -1 if not known. */ + pages: number + /** Current page. -1 if not known. */ + page: number + + /** + * Is the specified link rel defined in the Link header? + * @param link to check + */ + hasRel(link: PageLinkRel): boolean + + /** + * GET the requested relative link + * + * if the requested link is not defined, all calculated values are reset to their defaults, including + * `total`, `items`, `offset`, and `limit` + * + * @param name of Link Relationship + * @param limit optional limit override to replace limit saved in `rel` + * @param offset optional offset override to replace offset saved in `rel` + */ + getRel( + name: PageLinkRel, + limit?: number, + offset?: number + ): Promise> + + /** Get the first page of items. This is the same as offset=0 */ + firstPage(): Promise> + /** + * Get the last page of items + * + * @remarks This link is only provided if `total` is known. + */ + lastPage(): Promise> + /** + * Get the next page of items + * + * @remarks This link is provided if `total` is known, or if the number of items returned == `limit`. In the latter case, this function may return an empty result set. + */ + nextPage(): Promise> + /** + * Get the previous page of items + * + * @remarks This link is provided if the last page was not the first page. + */ + prevPage(): Promise> + + /** `true` if the `next` link is defined */ + more(): boolean +} + +/** + * Parse a link header to extract rels + * @param linkHeader to parse + * + * Several different approaches are discussed at https://stackoverflow.com/questions/8735792/how-to-parse-link-header-from-github-api + * + * Unfortunately, none of them are a good implementation + * + */ +export const linkHeaderParser = (linkHeader: string): PageLinks => { + const re = /<\s*(.*)\s*>;\s*rel="\s*(.*)\s*"\s*/gm + const links = linkHeader.split(',') + const obj: PageLinks = {} + let arrRes + + links.forEach((link) => { + link = link.trim() + while ((arrRes = re.exec(link))) { + const key = arrRes[2].split(' ')[0].trim().toLocaleLowerCase() + obj[key] = { + url: arrRes[1].trim(), + rel: key as PageLinkRel, + name: arrRes[2].trim(), + } + } + }) + return obj +} + +/** Event to observe the paging call */ +export type PageObserver = ( + /** Current retrieved page of results */ + page: TSuccess +) => TSuccess + +/** + * Create an API response pager for an endpoint that returns a Link header + * @param sdk implementation of IAPIMethods. Can be full SDK or functional auth session + * @param pageFunc sdk call that includes a paging header + * @param options transport options override to capture and use in paging requests + * + * @remarks `TSuccess` must be a collection type that supports `length` + */ +export async function pager( + sdk: IAPIMethods, + pageFunc: PagingFunc, + options?: Partial +): Promise> { + return await new Paging(sdk, pageFunc, options).init() +} + +/** + * Create an API response pager and iterate through all pages, calling a page event per page + * @param sdk implementation of IAPIMethods. Can be full SDK object or a functional auth session + * @param pageFunc sdk call that includes a paging header + * @param onPage observer of the latest page of results. Defaults to noop. + * @param options transport options override to capture and use in paging requests + */ +export async function pageAll( + sdk: IAPIMethods, + pageFunc: PagingFunc, + onPage: PageObserver = (page: TSuccess) => page, + options?: Partial +): Promise> { + const paged = await pager(sdk, pageFunc, options) + // Process the first page + onPage(paged.items) + try { + while (paged.more()) { + // Pass the page items to the event + onPage(await sdk.ok(paged.nextPage())) + } + } catch (err) { + return Promise.reject(err) + } + return paged +} + +/** + * Link header pages class + */ +export class Paging + implements IPager +{ + items: TSuccess = [] as unknown as TSuccess + links: PageLinks = {} + total = -1 + offset = -1 + limit = -1 + + private transport: BaseTransport + + /** + * Create an API paginator + * @param sdk functional AuthSession or full SDK implementation + * @param func sdk function to call + * @param options transport overrides to use for subsequent requests + */ + constructor( + public sdk: IAPIMethods, + public func: PagingFunc, + public options?: Partial + ) { + this.transport = sdk.authSession.transport as BaseTransport + } + + private async rawCatch(func: () => any) { + let raw: IRawResponse = {} as IRawResponse + const saved = this.transport.observer + try { + // Capture the raw request for header parsing + this.transport.observer = (response: IRawResponse) => { + raw = response + return response + } + this.items = await sdkOk(func()) + } finally { + // Restore the previous observer (if any) + this.transport.observer = saved + } + if (Object.keys(raw).length === 0 || Object.keys(raw.headers).length === 0) + throw new Error('No paging headers were found') + this.parse(raw) + return this + } + + get page(): number { + if (this.limit < 1 || this.offset < 0) return -1 + const x = this.offset / this.limit + 1 + return Math.ceil(x) + } + + get pages(): number { + if (this.total < 1 || this.limit < 1) return -1 + const x = this.total / this.limit + return Math.ceil(x) + } + + async init() { + return await this.rawCatch(this.func) + } + + hasRel(link: PageLinkRel): boolean { + return !!this.links[link] + } + + more() { + return this.hasRel('next') + } + + /** + * Default string value + * @param value to retrieve or default + * @param defaultValue to apply if string is null + * @param convert function to convert assigned string value + * @private + */ + private static paramDefault( + value: string | null, + defaultValue: any, + convert = (v: string) => parseInt(v, 10) + ) { + if (value === null) return defaultValue + return convert(value) + } + + reset() { + this.links = {} + this.total = this.offset = this.limit = -1 + this.items = [] as unknown as TSuccess + } + + async getRel( + name: PageLinkRel, + limit?: number, + offset?: number + ): Promise> { + const rel = this.links[name] + let result: SDKResponse + this.reset() + if (!rel) { + result = { ok: true, value: this.items } + return result + } + const authenticator = (init: any) => { + return this.sdk.authSession.authenticate(init) + } + + let link = rel.url + if (limit !== undefined) { + if (offset === undefined) { + offset = 0 + } + if (limit < 1 || offset < 0) { + result = { + ok: false, + error: new Error( + 'limit must be > 0 and offset must be >= 0' + ) as unknown as TError, + } + return result + } + const url = new URL(link) + const params = url.searchParams + params.set('limit', limit.toString()) + params.set('offset', offset.toString()) + link = url.toString() + } + const raw = await this.transport.rawRequest( + 'GET', + link, + undefined, + undefined, + authenticator, + this.options + ) + try { + this.parse(raw) + this.items = await sdkOk(this.transport.parseResponse(raw)) + result = { ok: true, value: this.items } + } catch (e) { + result = { ok: false, error: e } + } + return result + } + + parse(raw: IRawResponse): IPager { + const req = new URL(raw.url) + const params = req.searchParams + this.limit = Paging.paramDefault(params.get('limit'), -1) + this.offset = Paging.paramDefault( + params.get('offset'), + this.limit > 0 ? 0 : -1 + ) + const linkHeader = raw.headers.link || raw.headers.Link || raw.headers.LINK + if (linkHeader) { + this.links = linkHeaderParser(linkHeader) + } else { + this.links = {} + } + const totalHeader = raw.headers[TotalCountHeader] + if (totalHeader) { + this.total = parseInt(totalHeader.trim(), 10) + } else { + this.total = -1 + } + return this as unknown as IPager + } + + async firstPage(): Promise> { + return await this.getRel('first') + } + + async lastPage(): Promise> { + return await this.getRel('last') + } + + async nextPage(): Promise> { + return await this.getRel('next') + } + + async prevPage(): Promise> { + return await this.getRel('prev') + } +} diff --git a/packages/sdk-rtl/src/transport.ts b/packages/sdk-rtl/src/transport.ts index e62ff47fc..83c180aa3 100644 --- a/packages/sdk-rtl/src/transport.ts +++ b/packages/sdk-rtl/src/transport.ts @@ -23,6 +23,7 @@ SOFTWARE. */ + import { Agent } from 'https' import { Headers } from 'request' import { Readable } from 'readable-stream' @@ -31,9 +32,7 @@ import { matchCharsetUtf8, matchModeBinary, matchModeString } from './constants' export const agentPrefix = 'TS-SDK' export const LookerAppId = 'x-looker-appid' -/** - * Set to `true` to follow streaming process - */ +/** Set to `true` to follow streaming process */ const tracing = false /** @@ -52,9 +51,7 @@ export function trace(message: string, info?: any) { } } -/** - * ResponseMode for an HTTP request - either binary or "string" - */ +/** ResponseMode for an HTTP request */ export enum ResponseMode { 'binary', // this is a binary response 'string', // this is a "string" response @@ -85,9 +82,7 @@ export const charsetUtf8Pattern = new RegExp(matchCharsetUtf8, 'i') */ export const defaultTimeout = 120 -/** - * Recognized HTTP methods - */ +/** Recognized HTTP methods */ export type HttpMethod = | 'GET' | 'POST' @@ -164,9 +159,7 @@ export enum StatusCode { NetworkAuthRequired, } -/** - * Untyped basic HTTP response type for "raw" HTTP requests - */ +/** Untyped basic HTTP response type for "raw" HTTP requests */ export interface IRawResponse { /** ok is `true` if the response is successful, `false` otherwise */ ok: boolean @@ -182,12 +175,18 @@ export interface IRawResponse { body: any /** Optional performance tracking starting mark name */ startMark?: string + /** Response headers */ + headers: IRequestHeaders } -/** - * Transport plug-in interface - */ +/** IRawResponse observer function type */ +export type RawObserver = (raw: IRawResponse) => IRawResponse + +/** Transport plug-in interface */ export interface ITransport { + /** Observer lambda to process raw responses */ + observer: RawObserver | undefined + /** * HTTP request function for atomic, fully downloaded raw HTTP responses * @@ -229,6 +228,14 @@ export interface ITransport { options?: Partial ): Promise> + /** + * Processes the raw response, converting it into an SDKResponse + * @param raw response result + */ + parseResponse( + raw: IRawResponse + ): Promise> + /** * HTTP request function for a streamable response * @param callback that receives the stream response and pipes it somewhere @@ -260,7 +267,7 @@ export interface ISDKSuccessResponse { value: T } -/** An erroring SDK call. */ +/** An errant SDK call. */ export interface ISDKErrorResponse { /** Whether the SDK call was successful. */ ok: false @@ -278,9 +285,7 @@ export type SDKResponse = | ISDKSuccessResponse | ISDKErrorResponse -/** - * Generic collection - */ +/** Generic collection */ export interface IRequestHeaders { [key: string]: string } @@ -361,9 +366,7 @@ export function isUtf8(contentType: string) { return contentType.match(/;.*\bcharset\b=\butf-8\b/i) } -/** - * Used for name/value pair collections like for QueryParams - */ +/** Used for name/value pair collections like for QueryParams */ export type Values = { [key: string]: any } | null | undefined /** @@ -428,6 +431,7 @@ export function addQueryParams(path: string, obj?: Values) { * @returns a new `Error` object with the failure message */ export function sdkError(response: any) { + const utf8 = 'utf-8' if (typeof response === 'string') { return new Error(response) } @@ -441,11 +445,11 @@ export function sdkError(response: any) { return new Error(error.statusMessage) } if ('error' in error && error.error instanceof Buffer) { - const result = Buffer.from(error.error).toString('utf-8') + const result = Buffer.from(error.error).toString(utf8) return new Error(result) } if (error instanceof Buffer) { - const result = Buffer.from(error).toString('utf-8') + const result = Buffer.from(error).toString(utf8) return new Error(result) } if ('message' in error) {