feat: preparsed Headers

[PandaWorker]

Let's rewrite the manipulation of request/response headers already?

In undici, [string, string][] for headers are passed almost everywhere, and almost everywhere it is checked and reduced to the same type

Maybe it's worth writing a Headers List and using it everywhere inside the library?

In client-1 client-2, you can immediately set the headersList: new Headers List() and then use this structure everywhere

This structure can be used for all Compose Interceptors and for Handlers

In the current version, only fetch is converted from any type to Record<string, string>, and request just passes this raw data on

My motivation is that I need to make sure that the interceptors and handlers always have the data structure we need and then only transfer it, and not parse Buffer[] in each interceptor or handler, make a modification and throw it further.

1) create request/fetch -> anyHeaders to HeadersList
2) call dispatcher compose interceptors
3) call dispatcher.dispatch

4) llhttp onStatus, create HeadersList()
5) llhttp onHeaderName + onHeaderValue -> HeadersList.append(name, value);
6) call handlers events handler.onHeaders(status, HeadersList, ...)

You can make a separate interceptor that would lead from any type of headers to the Headers List, and would do the same with onHeaders. But do we even need raw headers in the form of Buffer[]?

const kRawHeadersMap = Symbol('raw headers map');

export interface IHeadersList {
	[kRawHeadersMap]: Map<string, string[]>;

	set(name: string, value: string): void;
	append(name: string, value: string): void;

	get(name: string): string | null;
	has(name: string): boolean;
	delete(name: string): boolean;

	keys(name: string): string[];
	values(name: string): string[];

	getCookies(): string[];
	getSetCookies(): string[];

	entries(): IterableIterator<[string, string]>;
	[Symbol.iterator](): IterableIterator<[string, string]>;
}


export class HeadersList implements IHeadersList {
	[kRawHeadersMap]: Map<string, string[]> = new Map();
	
	get size(){
		return this[kRawHeadersMap].size;
	}
	
	set(name: string, value: string): void {
		this[kRawHeadersMap].set(name, [value]);
	}

	append(name: string, value: string): void {
		const values = this[kRawHeadersMap].get(name);

		if (values !== undefined) values.push(value);
		else this[kRawHeadersMap].set(name, [value]);
	}

	get(name: string): string | null {
		const values = this[kRawHeadersMap].get(name);

		if (values !== undefined) return values[0];
		else return null;
	}

	has(name: string): boolean {
		return this[kRawHeadersMap].has(name);
	}

	delete(name: string): boolean {
		return this[kRawHeadersMap].delete(name);
	}

	keys(name: string): string[] {
		if (this[kRawHeadersMap].size === 0) return [];

		name = name.toLowerCase();

		const keys: string[] = [];
		for (const key of this[kRawHeadersMap].keys()) {
			if (name === key.toLowerCase()) keys.push(key);
		}

		return keys;
	}

	values(name: string): string[] {
		return this[kRawHeadersMap].get(name) ?? [];
	}

	getCookies(): string[] {
		const cookies: string[] = [];

		for (const name of this.keys('cookie')) {
			const values = this.values(name);
			cookies.push(...values);
		}

		return cookies;
	}

	getSetCookies(): string[] {
		const setCookies: string[] = [];

		for (const name of this.keys('set-cookie')) {
			const values = this.values(name);
			setCookies.push(...values);
		}

		return setCookies;
	}

	*entries(): IterableIterator<[string, string]> {
		for (const [name, values] of this[kRawHeadersMap].entries()) {
			const lowerCasedName = name.toLowerCase();

			// set-cookie
			if (lowerCasedName === 'set-cookie') {
				for (const value of values) yield [name, value];
			}
			// one value
			else if (values.length === 1) yield [name, values[0]];
			// more values
			else {
				const separator = lowerCasedName === 'cookie' ? '; ' : ', ';
				yield [name, values.join(separator)];
			}
		}
	}

	rawKeys(): IterableIterator<string> {
		return this[kRawHeadersMap].keys();
	}

	rawValues(): IterableIterator<string[]> {
		return this[kRawHeadersMap].values();
	}

	rawEntries(): IterableIterator<[string, string[]]> {
		return this[kRawHeadersMap].entries();
	}

	[Symbol.iterator](): IterableIterator<[string, string]> {
		return this.entries();
	}

	toArray() {
		return Array.from(this.entries());
	}

	toObject() {
		return Object.fromEntries(this.entries());
	}
}

// @ts-expect-error:
const setTimestampInterceptor = dispatch => (opts, handler) => {
	const headers = opts.headers as IHeadersList;

	// request headers list names must be all cased!
	// x-ts-date
	// X-Ts-Date
	// x-TS-date
	headers.set('X-TS-Date', `${(Date.now() / 1000 - 1e3).toFixed(0)}`);
	headers.set('X-ts-Date', `${(Date.now() / 1000 + 1e3).toFixed(0)}`);

	class MyHandler extends DecoratorHandler {
		onRawHeaders(
			status: number,
			headers: Buffer[],
			resume: () => void,
			statusText?: string
		) {}

		onHeaders(
			status: number,
			headersList: IHeadersList,
			resume: () => void,
			statusText?: string
		) {
			// response headers names is all lower cased
			if (headersList.has('content-encoding')) {
				//
			}

			return super.onHeaders(status, headersList, resume, statusText);
		}
	}

	return dispatch(opts, new MyHandler(handler));
};

[KhafraDev]

HeadersList should not be exposed.

[ronag]

WIP #3408

[PandaWorker]

Alternatively, make a wrapper over fetch/request, but this does not solve the problem with the fact that Buffer[] -> Headers are parsed for each request throughout undici and all interceptors in order to already work with them.

I've been doing a little bit here, it may be appropriate to make an implementation of the interceptors, where everything will already be given in the right form

It is simply impossible to write interceptors on the current implementation of undici, since there may be completely different data coming to opts

If there was something like this in undici that was compatible with fetch/request, it would be very convenient.

import undici, { Dispatcher } from 'undici';

type RequestInterceptor = (
	request: Dispatcher.DispatchOptions,
	next: (
		request?: Dispatcher.DispatchOptions
	) => Promise<Dispatcher.ResponseData>
) => Promise<Dispatcher.ResponseData>;

function LoggerInterceptor(prefix: string): RequestInterceptor {
	return async (request, next) => {
		console.log(`[${prefix}] on request:`, request.method);

		const resp = await next();
		console.log(`[${prefix}] on response:`, resp.statusCode);

		return resp;
	};
}

function DecompressInterceptor(): RequestInterceptor {
	return async (request, next) => {
		const resp = await next();
		const { headers } = resp;

		if (resp.body && headers && headers['content-encoding']) {
			// remove headers
			delete headers['content-encoding'];
			delete headers['content-length'];

			resp.body = decompress(resp.body);
		}

		return resp;
	};
}

import { Request, fetch } from 'undici';

type RequestInterceptor = (
	request: Request,
	next: () => Promise<Response>
) => Promise<Response>;

function LoggerInterceptor(): RequestInterceptor {
	return async (request, next) => {
		console.log('Request:', request);

		const resp = await next();

		console.log('Response:', resp);

		return resp;
	};
}

function DecompressInterceptor(): RequestInterceptor {
	return async (request, next) => {
		console.log('Request:', request);

		const resp = await next();

		if (resp.body && resp.headers.has('content-encoding')) {
			const encodings = resp.headers
				.get('content-encoding')!
				.split(',')
				.map(v => v.trim())
				.reverse();

			for (const encoding of encodings) {
				// @ts-expect-error:
				resp.body = resp.body.pipeThrough(new DecompressionStream(encoding));
			}
		}
		return resp;
	};
}

[PandaWorker]

import { scheduler } from 'node:timers/promises';
import undici, { Dispatcher, getGlobalDispatcher } from 'undici';

type RequestInterceptor = (
	request: Dispatcher.DispatchOptions,
	next: () => Promise<Dispatcher.ResponseData>
) => Promise<Dispatcher.ResponseData>;

function composeInterceptors(interceptors: RequestInterceptor[] = []) {
	// Logic for applying interceptors
	return async (
		request: Dispatcher.DispatchOptions,
		next: () => Promise<Dispatcher.ResponseData>
	) => {
		let index = -1;

		const runner = () => {
			index += 1;

			if (index < interceptors.length) {
				// Call the current interceptor and pass the runner as `next`
				return interceptors[index](request, runner);
			} else {
				// No more interceptors, call the original fetch
				return next();
			}
		};

		return runner();
	};
}

function LoggerInterceptor(prefix: string): RequestInterceptor {
	return async (request, next) => {
		console.log(`[${prefix}] on request:`, request.method);

		const resp = await next();

		console.log(
			`[${prefix}] on response:`,
			resp.statusCode,
			`body:`,
			!!resp.body
		);

		return resp;
	};
}

function AsyncInterceptor(): RequestInterceptor {
	return async (request, next) => {
		console.log('wait 1sec');
		await scheduler.wait(1000);

		console.log('wait response');
		const resp = await next();

		console.log('wait 1sec');
		await scheduler.wait(1000);

		console.log('modify response to null');
		// @ts-expect-error:
		resp.body = null;

		console.log('retun resp');
		return resp;
	};
}

type RequestOptions = Partial<Dispatcher.RequestOptions> & {
	dispatcher?: Dispatcher;
} & Partial<Pick<Dispatcher.RequestOptions, 'method' | 'path' | 'origin'>>;

function requestWithInterceptors(interceptors: RequestInterceptor[] = []) {
	const intercept = composeInterceptors([...interceptors]);

	return async (
		url: string | URL,
		options: RequestOptions = {}
	): Promise<Dispatcher.ResponseData> => {
		var waiter = Promise.withResolvers<Dispatcher.ResponseData>();
		var composePromise: Promise<Dispatcher.ResponseData>;

		options.dispatcher ??= getGlobalDispatcher();
		options.dispatcher = options.dispatcher.compose(
			dispatch => (opts, handler) => {
				composePromise = intercept(opts, () => waiter.promise);

				return dispatch(opts, handler);
			}
		);

		undici.request(url, options).then(
			resp => waiter.resolve(resp),
			reason => waiter.reject(reason)
		);


		// @ts-expect-error:
		return composePromise;
	};
}

const request = requestWithInterceptors([
	LoggerInterceptor('1'),
	AsyncInterceptor(),
	LoggerInterceptor('2'),
]);

const response = await request('https://api.ipify.org/');
console.log(response);