keinBot/flumm-fetch
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

class schlass

Browse Source
This commit is contained in:
Flummi 2025-03-18 16:45:00 +01:00
parent fd0eef5cbe
commit e197050aa2
2 changed files with 224 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
dist/index.js vendored
View File

@ -3,50 +3,52 @@ import https from "https";
import { URL } from "url"; import { URL } from "url";
import querystring from "querystring"; import querystring from "querystring";
import zlib from "zlib"; import zlib from "zlib";
const parseCookies = (cookieHeader) => { class fetch {
const cookies = {}; cookies = {};
if (cookieHeader) { constructor() { }
parseCookies(cookieHeader) {
if (!cookieHeader)
return;
cookieHeader.split(";").forEach(cookie => { cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("="); const [key, value] = cookie.split("=");
if (key && value) if (key && value)
cookies[key.trim()] = value.trim(); this.cookies[key.trim()] = value.trim();
}); });
} }
return cookies; formatCookies() {
}; return Object.entries(this.cookies)
const formatCookies = (cookies) => Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; "); .map(([key, value]) => `${key}=${value}`)
const addCookiesToRequest = (cookies, headers) => { .join("; ");
headers["Cookie"] = formatCookies(cookies); }
}; decompress(data, encoding) {
const decompress = (data, encoding) => {
return encoding === "br" ? zlib.brotliDecompressSync(data) : return encoding === "br" ? zlib.brotliDecompressSync(data) :
encoding === "gzip" ? zlib.gunzipSync(data) : encoding === "gzip" ? zlib.gunzipSync(data) :
encoding === "deflate" ? zlib.inflateSync(data) : encoding === "deflate" ? zlib.inflateSync(data) :
data; data;
}; }
const readData = (res, mode) => new Promise((resolve, reject) => { readData(res, mode) {
return new Promise((resolve, reject) => {
const chunks = []; const chunks = [];
res res
.on("data", chunk => chunks.push(chunk)) .on("data", chunk => chunks.push(chunk))
.on("end", () => { .on("end", () => {
try { try {
const data = decompress(Buffer.concat(chunks), res.headers["content-encoding"]); const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
resolve((mode === "json" ? JSON.parse(data.toString("utf8")) : resolve(mode === "json" ? JSON.parse(data.toString("utf8")) :
mode === "buffer" ? data : mode === "buffer" ? data :
mode === "arraybuffer" ? new Uint8Array(data).buffer : mode === "arraybuffer" ? new Uint8Array(data).buffer :
data.toString("utf8"))); data.toString("utf8"));
} }
catch (err) { catch (err) {
reject(err); reject(err);
} }
}) })
.on("error", reject); .on("error", reject);
}); });
const fetch = async (urlString, options = {}, redirectCount = 0, cookies = {}) => { }
const { protocol, hostname, pathname, search, port } = new URL(urlString); async fetch(urlString, options = {}, redirectCount = 0) {
options.followRedirects = options.followRedirects ?? true; options.followRedirects = options.followRedirects ?? true;
if (options.signal?.aborted) const { protocol, hostname, pathname, search, port } = new URL(urlString);
throw new Error("Request aborted");
const body = options.method === "POST" && options.body const body = options.method === "POST" && options.body
? options.headers?.["Content-Type"] === "application/json" ? options.headers?.["Content-Type"] === "application/json"
? JSON.stringify(options.body) ? JSON.stringify(options.body)
@ -60,33 +62,31 @@ const fetch = async (urlString, options = {}, redirectCount = 0, cookies = {}) =
headers: { headers: {
...options.headers, ...options.headers,
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}), ...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
"Accept-Encoding": "br, gzip, deflate" "Cookie": this.formatCookies(),
"Accept-Encoding": "br, gzip, deflate",
} }
}; };
addCookiesToRequest(cookies, requestOptions.headers);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http; const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => { const req = requester.request(requestOptions, res => {
if (res.headers["set-cookie"]) this.parseCookies(res.headers["set-cookie"]?.join("; "));
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
req.destroy();
if (redirectCount >= (options.maxRedirects || 5)) if (redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded")); return reject(new Error("Max redirects exceeded"));
if (!res.headers.location) if (!res.headers.location)
return reject(new Error("Redirect location not provided")); return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString); const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies)); return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
} }
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) return resolve({
return reject(new Error(`Request failed with status code ${res.statusCode}`));
resolve({
body: res, body: res,
headers: res.headers, headers: res.headers,
cookies: cookies, cookies: this.cookies,
text: () => readData(res, "text"), text: () => this.readData(res, "text"),
json: () => readData(res, "json"), json: () => this.readData(res, "json"),
buffer: () => readData(res, "buffer"), buffer: () => this.readData(res, "buffer"),
arrayBuffer: () => readData(res, "arraybuffer") arrayBuffer: () => this.readData(res, "arraybuffer"),
}); });
}); });
req.setTimeout(options.timeout || 5e3, () => { req.setTimeout(options.timeout || 5e3, () => {
@ -104,5 +104,7 @@ const fetch = async (urlString, options = {}, redirectCount = 0, cookies = {}) =
req.write(body); req.write(body);
req.end(); req.end();
}); });
}; }
export default fetch; }
const inst = new fetch();
export default inst.fetch.bind(inst);

126
src/index.ts
View File

@ -4,72 +4,55 @@ import { URL } from "url";
import querystring from "querystring"; import querystring from "querystring";
import zlib from "zlib"; import zlib from "zlib";
type CookieStorage = Record<string, string>; class fetch {
private cookies: Record<string, string> = {};
type ResponseData = { constructor() { }
body: http.IncomingMessage;
headers: http.IncomingHttpHeaders;
cookies: CookieStorage;
text: () => Promise<string>;
json: () => Promise<JSON>;
buffer: () => Promise<Buffer>;
arrayBuffer: () => Promise<ArrayBuffer>;
};
interface ExtendedRequestOptions extends http.RequestOptions { private parseCookies(cookieHeader: string | undefined): void {
body?: any; if(!cookieHeader)
timeout?: number; return;
followRedirects?: boolean;
maxRedirects?: number;
signal?: AbortSignal;
}
const parseCookies = (cookieHeader: string | undefined): CookieStorage => {
const cookies: CookieStorage = {};
if(cookieHeader) {
cookieHeader.split(";").forEach(cookie => { cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("="); const [key, value] = cookie.split("=");
if(key && value) if(key && value)
cookies[key.trim()] = value.trim(); this.cookies[key.trim()] = value.trim();
}); });
} }
return cookies;
};
const formatCookies = (cookies: CookieStorage): string => private formatCookies(): string {
Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; "); return Object.entries(this.cookies)
.map(([key, value]) => `${key}=${value}`)
.join("; ");
}
const addCookiesToRequest = (cookies: CookieStorage, headers: http.OutgoingHttpHeaders): void => { private decompress(
headers["Cookie"] = formatCookies(cookies);
};
const decompress = (
data: Buffer, data: Buffer,
encoding: string | undefined encoding: string | undefined
): Buffer => { ): Buffer {
return encoding === "br" ? zlib.brotliDecompressSync(data) : return encoding === "br" ? zlib.brotliDecompressSync(data) :
encoding === "gzip" ? zlib.gunzipSync(data) : encoding === "gzip" ? zlib.gunzipSync(data) :
encoding === "deflate" ? zlib.inflateSync(data) : encoding === "deflate" ? zlib.inflateSync(data) :
data; data;
}; }
const readData = <T>( private readData<T>(
res: http.IncomingMessage, res: http.IncomingMessage,
mode: "text" | "json" | "buffer" | "arraybuffer" mode: "text" | "json" | "buffer" | "arraybuffer"
): Promise<T> => ): Promise<T> {
new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
res res
.on("data", chunk => chunks.push(chunk)) .on("data", chunk => chunks.push(chunk))
.on("end", () => { .on("end", () => {
try { try {
const data = decompress(Buffer.concat(chunks), res.headers["content-encoding"]); const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
resolve(( resolve(
mode === "json" ? JSON.parse(data.toString("utf8")) : mode === "json" ? JSON.parse(data.toString("utf8")) :
mode === "buffer" ? data : mode === "buffer" ? data :
mode === "arraybuffer" ? new Uint8Array(data).buffer : mode === "arraybuffer" ? new Uint8Array(data).buffer :
data.toString("utf8") data.toString("utf8")
)) as T; );
} }
catch(err: any) { catch(err: any) {
reject(err); reject(err);
@ -77,26 +60,35 @@ const readData = <T>(
}) })
.on("error", reject); .on("error", reject);
}); });
}
const fetch = async ( async fetch(
urlString: string, urlString: string,
options: ExtendedRequestOptions = {}, options: http.RequestOptions & {
redirectCount: number = 0, body?: any;
cookies: CookieStorage = {} timeout?: number;
): Promise<ResponseData> => { followRedirects?: boolean;
const { protocol, hostname, pathname, search, port } = new URL(urlString); maxRedirects?: number;
signal?: AbortSignal;
} = {},
redirectCount = 0
): Promise<{
body: http.IncomingMessage;
headers: http.IncomingHttpHeaders;
cookies: Record<string, string>;
text: () => Promise<string>;
json: () => Promise<JSON>;
buffer: () => Promise<Buffer>;
arrayBuffer: () => Promise<ArrayBuffer>;
}> {
options.followRedirects = options.followRedirects ?? true; options.followRedirects = options.followRedirects ?? true;
if(options.signal?.aborted) const { protocol, hostname, pathname, search, port } = new URL(urlString);
throw new Error("Request aborted"); const body = options.method === "POST" && options.body
const body =
options.method === "POST" && options.body
? options.headers?.["Content-Type"] === "application/json" ? options.headers?.["Content-Type"] === "application/json"
? JSON.stringify(options.body) ? JSON.stringify(options.body)
: querystring.stringify(options.body as querystring.ParsedUrlQueryInput) : querystring.stringify(options.body as querystring.ParsedUrlQueryInput)
: null; : null;
const requestOptions: http.RequestOptions = { const requestOptions: http.RequestOptions = {
hostname, hostname,
port: port || (protocol === "https:" ? 443 : 80), port: port || (protocol === "https:" ? 443 : 80),
@ -105,19 +97,17 @@ const fetch = async (
headers: { headers: {
...options.headers, ...options.headers,
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}), ...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
"Accept-Encoding": "br, gzip, deflate" "Cookie": this.formatCookies(),
"Accept-Encoding": "br, gzip, deflate",
} }
}; };
addCookiesToRequest(cookies, requestOptions.headers!);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http; const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => { const req = requester.request(requestOptions, res => {
if(res.headers["set-cookie"]) this.parseCookies(res.headers["set-cookie"]?.join("; "));
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
req.destroy();
if(redirectCount >= (options.maxRedirects || 5)) if(redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded")); return reject(new Error("Max redirects exceeded"));
@ -125,20 +115,16 @@ const fetch = async (
return reject(new Error("Redirect location not provided")); return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString); const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies)); return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
} }
return resolve({
if(res.statusCode && (res.statusCode < 200 || res.statusCode >= 300))
return reject(new Error(`Request failed with status code ${res.statusCode}`));
resolve({
body: res, body: res,
headers: res.headers, headers: res.headers,
cookies: cookies, cookies: this.cookies,
text: () => readData<string>(res, "text"), text: () => this.readData<string>(res, "text"),
json: () => readData<JSON>(res, "json"), json: () => this.readData<JSON>(res, "json"),
buffer: () => readData<Buffer>(res, "buffer"), buffer: () => this.readData<Buffer>(res, "buffer"),
arrayBuffer: () => readData<ArrayBuffer>(res, "arraybuffer") arrayBuffer: () => this.readData<ArrayBuffer>(res, "arraybuffer"),
}); });
}); });
@ -160,6 +146,8 @@ const fetch = async (
req.write(body); req.write(body);
req.end(); req.end();
}); });
}; }
}
export default fetch; const inst = new fetch();
export default inst.fetch.bind(inst);