keinBot/flumm-fetch
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: enhance fetch function with cookie handling

Browse Source
This commit is contained in:
Flummi 2025-03-18 16:16:08 +01:00
parent 65f1707358
commit fd0eef5cbe
2 changed files with 63 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dist/index.js vendored
View File

@ -3,6 +3,21 @@ import https from "https";
import { URL } from "url";
import querystring from "querystring";
import zlib from "zlib";
const parseCookies = (cookieHeader) => {
const cookies = {};
if (cookieHeader) {
cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("=");
if (key && value)
cookies[key.trim()] = value.trim();
});
}
return cookies;
};
const formatCookies = (cookies) => Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; ");
const addCookiesToRequest = (cookies, headers) => {
headers["Cookie"] = formatCookies(cookies);
};
const decompress = (data, encoding) => {
return encoding === "br" ? zlib.brotliDecompressSync(data) :
encoding === "gzip" ? zlib.gunzipSync(data) :
@ -27,8 +42,9 @@ const readData = (res, mode) => new Promise((resolve, reject) => {
})
.on("error", reject);
});
const fetch = async (urlString, options = {}, redirectCount = 0) => {
const fetch = async (urlString, options = {}, redirectCount = 0, cookies = {}) => {
const { protocol, hostname, pathname, search, port } = new URL(urlString);
options.followRedirects = options.followRedirects ?? true;
if (options.signal?.aborted)
throw new Error("Request aborted");
const body = options.method === "POST" && options.body
@ -47,19 +63,26 @@ const fetch = async (urlString, options = {}, redirectCount = 0) => {
"Accept-Encoding": "br, gzip, deflate"
}
};
addCookiesToRequest(cookies, requestOptions.headers);
return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => {
if (res.headers["set-cookie"])
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
if (redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded"));
return resolve(fetch(new URL(res.headers.location, urlString).toString(), options, redirectCount + 1));
if (!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies));
}
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300))
return reject(new Error(`Request failed with status code ${res.statusCode}`));
resolve({
body: res,
headers: res.headers,
cookies: cookies,
text: () => readData(res, "text"),
json: () => readData(res, "json"),
buffer: () => readData(res, "buffer"),

40
src/index.ts
View File

@ -4,9 +4,12 @@ import { URL } from "url";
import querystring from "querystring";
import zlib from "zlib";
type CookieStorage = Record<string, string>;
type ResponseData = {
body: http.IncomingMessage;
headers: http.IncomingHttpHeaders;
cookies: CookieStorage;
text: () => Promise<string>;
json: () => Promise<JSON>;
buffer: () => Promise<Buffer>;
@ -21,6 +24,25 @@ interface ExtendedRequestOptions extends http.RequestOptions {
signal?: AbortSignal;
}
const parseCookies = (cookieHeader: string | undefined): CookieStorage => {
const cookies: CookieStorage = {};
if(cookieHeader) {
cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("=");
if(key && value)
cookies[key.trim()] = value.trim();
});
}
return cookies;
};
const formatCookies = (cookies: CookieStorage): string =>
Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; ");
const addCookiesToRequest = (cookies: CookieStorage, headers: http.OutgoingHttpHeaders): void => {
headers["Cookie"] = formatCookies(cookies);
};
const decompress = (
data: Buffer,
encoding: string | undefined
@ -59,9 +81,11 @@ const readData = <T>(
const fetch = async (
urlString: string,
options: ExtendedRequestOptions = {},
redirectCount: number = 0
redirectCount: number = 0,
cookies: CookieStorage = {}
): Promise<ResponseData> => {
const { protocol, hostname, pathname, search, port } = new URL(urlString);
options.followRedirects = options.followRedirects ?? true;
if(options.signal?.aborted)
throw new Error("Request aborted");
@ -85,20 +109,32 @@ const fetch = async (
}
};
addCookiesToRequest(cookies, requestOptions.headers!);
return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => {
if(res.headers["set-cookie"])
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
if(redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded"));
return resolve(fetch(new URL(res.headers.location, urlString).toString(), options, redirectCount + 1));
if(!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies));
}
if(res.statusCode && (res.statusCode < 200 || res.statusCode >= 300))
return reject(new Error(`Request failed with status code ${res.statusCode}`));
resolve({
body: res,
headers: res.headers,
cookies: cookies,
text: () => readData<string>(res, "text"),
json: () => readData<JSON>(res, "json"),
buffer: () => readData<Buffer>(res, "buffer"),