Experimental pagination feature (sindresorhus#833)

Co-authored-by: Sindre Sorhus <[email protected]>

Author: szmarczak

package.json

@@ -44,7 +44,7 @@
 		"@sindresorhus/is": "^1.0.0",
 		"@szmarczak/http-timer": "^4.0.0",
 		"@types/cacheable-request": "^6.0.1",
-		"cacheable-lookup": "^1.0.0",
+		"cacheable-lookup": "^2.0.0",
 		"cacheable-request": "^7.0.1",
 		"decompress-response": "^5.0.0",
 		"duplexer3": "^0.1.4",
@@ -57,7 +57,7 @@
 		"type-fest": "^0.9.0"
 	},
 	"devDependencies": {
-		"@ava/typescript": "^1.0.0",
+		"@ava/typescript": "^1.1.0",
 		"@sindresorhus/tsconfig": "^0.7.0",
 		"@types/duplexer3": "^0.1.0",
 		"@types/express": "^4.17.2",
@@ -68,7 +68,7 @@
 		"@types/tough-cookie": "^2.3.5",
 		"@typescript-eslint/eslint-plugin": "^2.17.0",
 		"@typescript-eslint/parser": "^2.17.0",
-		"ava": "^3.1.0",
+		"ava": "^3.2.0",
 		"coveralls": "^3.0.4",
 		"create-test-server": "^3.0.1",
 		"del-cli": "^3.0.0",
@@ -101,7 +101,6 @@
 		"files": [
 			"test/*"
 		],
-		"concurrency": 4,
 		"timeout": "1m",
 		"typescript": {
 			"rewritePaths": {
@@ -112,6 +111,9 @@
 	"nyc": {
 		"extension": [
 			".ts"
+		],
+		"exclude": [
+			"**/test/**"
 		]
 	},
 	"xo": {

readme.md

@@ -26,7 +26,9 @@ For browser usage, we recommend [Ky](https://github.com/sindresorhus/ky) by the
 
 ## Highlights
 
-- [Promise & stream API](#api)
+- [Promise API](#api)
+- [Stream API](#streams)
+- [Pagination API (experimental)](#pagination)
 - [Request cancelation](#aborting-the-request)
 - [RFC compliant caching](#cache-adapters)
 - [Follows redirects](#followredirect)
@@ -635,6 +637,49 @@ got('https://api.github.com/some-endpoint', {
 });
 ```
 
+##### \_pagination
+
+Type: `object`
+
+**Note:** This feature is marked as experimental as we're [looking for feedback](https://github.com/sindresorhus/got/issues/1052) on the API and how it works. The feature itself is stable, but the API may change based on feedback. So if you decide to try it out, we suggest locking down the `got` dependency semver range or use a lockfile.
+
+###### \_pagination.transform
+
+Type: `Function`\
+Default: `response => JSON.parse(response.body)`
+
+A function that transform [`Response`](#response) into an array of items. This is where you should do the parsing.
+
+###### \_pagination.paginate
+
+Type: `Function`\
+Default: [`Link` header logic](source/index.ts)
+
+A function that returns an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned.
+
+###### \_pagination.filter
+
+Type: `Function`\
+Default: `(item, allItems) => true`
+
+Checks whether the item should be emitted or not.
+
+###### \_pagination.shouldContinue
+
+Type: `Function`\
+Default: `(item, allItems) => true`
+
+Checks whether the pagination should continue.
+
+For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems) => allItems.some(entry => entry.flag)` instead.
+
+###### \_pagination.countLimit
+
+Type: `number`\
+Default: `Infinity`
+
+The maximum amount of items that should be emitted.
+
 #### Response
 
 The response object will typically be a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage), however, if returned from the cache it will be a [response-like object](https://github.com/lukechilds/responselike) which behaves in the same way.
@@ -785,6 +830,30 @@ If it's not possible to retrieve the body size (can happen when streaming), `tot
 
 The `error` event emitted in case of a protocol error (like `ENOTFOUND` etc.) or status error (4xx or 5xx). The second argument is the body of the server response in case of status error. The third argument is a response object.
 
+#### Pagination
+
+#### got.paginate(url, options?)
+
+Returns an async iterator:
+
+```js
+(async () => {
+	const countLimit = 10;
+
+	const pagination = got.paginate('https://api.github.com/repos/sindresorhus/got/commits', {
+		_pagination: {countLimit}
+	});
+
+	console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`);
+
+	for await (const commitData of pagination) {
+		console.log(commitData.commit.message);
+	}
+})();
+```
+
+See [`options._pagination`](#_pagination) for more pagination options.
+
 #### got.get(url, options?)
 #### got.post(url, options?)
 #### got.put(url, options?)
@@ -1462,6 +1531,7 @@ Some of the Got features may not work properly. See [#899](https://github.com/si
 | Browser support       | :x:                | :x:                | :heavy_check_mark:\* | :heavy_check_mark:       | :heavy_check_mark: | :heavy_check_mark:     |
 | Promise API           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:   | :heavy_check_mark:       | :heavy_check_mark: | :heavy_check_mark:     |
 | Stream API            | :heavy_check_mark: | :heavy_check_mark: | Node.js only         | :x:                      | :x:                | :heavy_check_mark:     |
+| Pagination API        | :sparkle:          | :x:                | :x:                  | :x:                      | :x:                | :x:                    |
 | Request cancelation   | :heavy_check_mark: | :x:                | :heavy_check_mark:   | :heavy_check_mark:       | :heavy_check_mark: | :heavy_check_mark:     |
 | RFC compliant caching | :heavy_check_mark: | :x:                | :x:                  | :x:                      | :x:                | :x:                    |
 | Cookies (out-of-box)  | :heavy_check_mark: | :heavy_check_mark: | :x:                  | :x:                      | :x:                | :x:                    |
@@ -1488,7 +1558,8 @@ Some of the Got features may not work properly. See [#899](https://github.com/si
 \* It's almost API compatible with the browser `fetch` API.\
 \*\* Need to switch the protocol manually. Doesn't accept PUSH streams and doesn't reuse HTTP/2 sessions.\
 \*\*\* Currently, only `DownloadProgress` event is supported, `UploadProgress` event is not supported.\
-:grey_question: Experimental support.
+:sparkle: Almost-stable feature, but the API may change. Don't hestitate to try it out!\
+:grey_question: Feature in early stage of development. Very experimental.
 
 <!-- GITHUB -->
 [k0]: https://github.com/sindresorhus/ky

source/create.ts

@@ -1,4 +1,5 @@
 import {Merge} from 'type-fest';
+import is from '@sindresorhus/is';
 import asPromise, {createRejection} from './as-promise';
 import asStream, {ProxyStream} from './as-stream';
 import * as errors from './errors';
@@ -13,7 +14,8 @@ import {
 	NormalizedOptions,
 	Options,
 	Response,
-	URLOrOptions
+	URLOrOptions,
+	PaginationOptions
 } from './types';
 
 export type HTTPAlias =
@@ -48,17 +50,25 @@ export interface GotRequestMethod {
 	(url: string | OptionsOfTextResponseBody, options?: OptionsOfTextResponseBody): CancelableRequest<Response<string>>;
 	<T>(url: string | OptionsOfJSONResponseBody, options?: OptionsOfJSONResponseBody): CancelableRequest<Response<T>>;
 	(url: string | OptionsOfBufferResponseBody, options?: OptionsOfBufferResponseBody): CancelableRequest<Response<Buffer>>;
+
 	// `resolveBodyOnly` usage
 	<T = string>(url: string | Merge<OptionsOfDefaultResponseBody, ResponseBodyOnly>, options?: Merge<OptionsOfDefaultResponseBody, ResponseBodyOnly>): CancelableRequest<T>;
 	(url: string | Merge<OptionsOfTextResponseBody, ResponseBodyOnly>, options?: Merge<OptionsOfTextResponseBody, ResponseBodyOnly>): CancelableRequest<string>;
 	<T>(url: string | Merge<OptionsOfJSONResponseBody, ResponseBodyOnly>, options?: Merge<OptionsOfJSONResponseBody, ResponseBodyOnly>): CancelableRequest<T>;
 	(url: string | Merge<OptionsOfBufferResponseBody, ResponseBodyOnly>, options?: Merge<OptionsOfBufferResponseBody, ResponseBodyOnly>): CancelableRequest<Buffer>;
+
 	// `asStream` usage
 	<T>(url: string | Merge<Options, {isStream: true}>, options?: Merge<Options, {isStream: true}>): ProxyStream<T>;
 }
 
+export interface GotPaginate {
+	<T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): AsyncIterableIterator<T>;
+	all<T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): Promise<T[]>;
+}
+
 export interface Got extends Record<HTTPAlias, GotRequestMethod>, GotRequestMethod {
 	stream: GotStream;
+	paginate: GotPaginate;
 	defaults: Defaults;
 	GotError: typeof errors.GotError;
 	CacheError: typeof errors.CacheError;
@@ -178,6 +188,64 @@ const create = (defaults: Defaults): Got => {
 		got.stream[method] = (url, options) => got.stream(url, {...options, method});
 	}
 
+	// @ts-ignore The missing property is added below
+	got.paginate = async function * <T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>) {
+		let normalizedOptions = normalizeArguments(url, options, defaults) as NormalizedOptions & PaginationOptions<T>;
+
+		const pagination = normalizedOptions._pagination!;
+
+		if (!is.object(pagination)) {
+			throw new Error('`options._pagination` must be implemented');
+		}
+
+		const all: T[] = [];
+
+		while (true) {
+			// @ts-ignore See https://github.com/sindresorhus/got/issues/954
+			// eslint-disable-next-line no-await-in-loop
+			const result = await got(normalizedOptions);
+
+			// eslint-disable-next-line no-await-in-loop
+			const parsed = await pagination.transform!(result);
+
+			for (const item of parsed) {
+				if (pagination.filter!(item, all)) {
+					if (!pagination.shouldContinue!(item, all)) {
+						return;
+					}
+
+					yield item;
+
+					all.push(item as T);
+
+					if (all.length === pagination.countLimit) {
+						return;
+					}
+				}
+			}
+
+			const optionsToMerge = pagination.paginate!(result);
+
+			if (optionsToMerge === false) {
+				return;
+			}
+
+			if (optionsToMerge !== undefined) {
+				normalizedOptions = normalizeArguments(normalizedOptions, optionsToMerge) as NormalizedOptions & PaginationOptions<T>;
+			}
+		}
+	};
+
+	got.paginate.all = async <T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>) => {
+		const results: T[] = [];
+
+		for await (const item of got.paginate<T>(url, options)) {
+			results.push(item);
+		}
+
+		return results;
+	};
+
 	Object.assign(got, {...errors, mergeOptions});
 	Object.defineProperty(got, 'defaults', {
 		value: defaults.mutableDefaults ? defaults : deepFreeze(defaults),

source/index.ts

@@ -1,5 +1,6 @@
+import {URL} from 'url';
 import create, {defaultHandler} from './create';
-import {Defaults} from './types';
+import {Defaults, Response, GotOptions} from './types';
 
 const defaults: Defaults = {
 	options: {
@@ -64,7 +65,43 @@ const defaults: Defaults = {
 		prefixUrl: '',
 		methodRewriting: true,
 		ignoreInvalidCookies: false,
-		context: {}
+		context: {},
+		_pagination: {
+			transform: (response: Response) => {
+				return JSON.parse(response.body as string);
+			},
+			paginate: response => {
+				if (!Reflect.has(response.headers, 'link')) {
+					return false;
+				}
+
+				const items = (response.headers.link as string).split(',');
+
+				let next: string | undefined;
+				for (const item of items) {
+					const parsed = item.split(';');
+
+					if (parsed[1].includes('next')) {
+						next = parsed[0].trimStart().trim();
+						next = next.slice(1, -1);
+						break;
+					}
+				}
+
+				if (next) {
+					const options: GotOptions = {
+						url: new URL(next)
+					};
+
+					return options;
+				}
+
+				return false;
+			},
+			filter: () => true,
+			shouldContinue: () => true,
+			countLimit: Infinity
+		}
 	},
 	handlers: [defaultHandler],
 	mutableDefaults: false

source/normalize-arguments.ts

@@ -170,7 +170,7 @@ export const preNormalizeArguments = (options: Options, defaults?: NormalizedOpt
 		// Horrible `tough-cookie` check
 		if (setCookie.length === 4 && getCookieString.length === 0) {
 			if (!Reflect.has(setCookie, promisify.custom)) {
-				// @ts-ignore TS is dumb.
+				// @ts-ignore TS is dumb - it says `setCookie` is `never`.
 				setCookie = promisify(setCookie.bind(options.cookieJar));
 				getCookieString = promisify(getCookieString.bind(options.cookieJar));
 			}
@@ -198,6 +198,34 @@ export const preNormalizeArguments = (options: Options, defaults?: NormalizedOpt
 		options = merge({}, defaults, options);
 	}
 
+	// `options._pagination`
+	if (is.object(options._pagination)) {
+		if (defaults && !(Reflect.has(options, 'pagination') && is.undefined(options._pagination))) {
+			options._pagination = {
+				...defaults.pagination,
+				...options._pagination
+			};
+		}
+
+		const pagination = options._pagination!;
+
+		if (!is.function_(pagination.transform)) {
+			throw new Error('`options._pagination.transform` must be implemented');
+		}
+
+		if (!is.function_(pagination.shouldContinue)) {
+			throw new Error('`options._pagination.shouldContinue` must be implemented');
+		}
+
+		if (!is.function_(pagination.filter)) {
+			throw new Error('`options._pagination.filter` must be implemented');
+		}
+
+		if (!is.function_(pagination.paginate)) {
+			throw new Error('`options._pagination.paginate` must be implemented');
+		}
+	}
+
 	// Other values
 	options.decompress = Boolean(options.decompress);
 	options.isStream = Boolean(options.isStream);

source/types.ts

@@ -135,6 +135,7 @@ export type DefaultOptions = Merge<
 			'retry' |
 			'timeout' |
 			'context' |
+			'_pagination' |
 
 			// Should not be present
 			'agent' |
@@ -154,11 +155,22 @@ export type DefaultOptions = Merge<
 		retry: DefaultRetryOptions;
 		timeout: Delays;
 		context: {[key: string]: any};
+		_pagination?: PaginationOptions<unknown>['_pagination'];
 	}
 >;
 /* eslint-enable @typescript-eslint/indent */
 
-export interface GotOptions {
+export interface PaginationOptions<T> {
+	_pagination?: {
+		transform?: (response: Response) => Promise<T[]> | T[];
+		filter?: (item: T, allItems: T[]) => boolean;
+		paginate?: (response: Response) => Options | false;
+		shouldContinue?: (item: T, allItems: T[]) => boolean;
+		countLimit?: number;
+	};
+}
+
+export interface GotOptions extends PaginationOptions<unknown> {
 	[requestSymbol]?: RequestFunction;
 	url?: URL | string;
 	body?: string | Buffer | ReadableStream;
@@ -206,6 +218,7 @@ export interface NormalizedOptions extends Options {
 	cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter;
 	cookieJar?: PromiseCookieJar;
 	maxRedirects: number;
+	pagination?: Required<PaginationOptions<unknown>['_pagination']>;
 	[requestSymbol]: RequestFunction;
 
 	// Other values