keinBot/flumm-fetch
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

class schlass

Browse Source
This commit is contained in:
Flummi 2025-03-18 16:45:00 +01:00
parent fd0eef5cbe
commit e197050aa2
2 changed files with 224 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
dist/index.js vendored
View File

@ -3,106 +3,108 @@ import https from "https";
import { URL } from "url"; import { URL } from "url";
import querystring from "querystring"; import querystring from "querystring";
import zlib from "zlib"; import zlib from "zlib";
const parseCookies = (cookieHeader) => { class fetch {
const cookies = {}; cookies = {};
if (cookieHeader) { constructor() { }
parseCookies(cookieHeader) {
if (!cookieHeader)
return;
cookieHeader.split(";").forEach(cookie => { cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("="); const [key, value] = cookie.split("=");
if (key && value) if (key && value)
cookies[key.trim()] = value.trim(); this.cookies[key.trim()] = value.trim();
}); });
} }
return cookies; formatCookies() {
}; return Object.entries(this.cookies)
const formatCookies = (cookies) => Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; "); .map(([key, value]) => `${key}=${value}`)
const addCookiesToRequest = (cookies, headers) => { .join("; ");
headers["Cookie"] = formatCookies(cookies); }
}; decompress(data, encoding) {
const decompress = (data, encoding) => { return encoding === "br" ? zlib.brotliDecompressSync(data) :
return encoding === "br" ? zlib.brotliDecompressSync(data) : encoding === "gzip" ? zlib.gunzipSync(data) :
encoding === "gzip" ? zlib.gunzipSync(data) : encoding === "deflate" ? zlib.inflateSync(data) :
encoding === "deflate" ? zlib.inflateSync(data) : data;
data; }
}; readData(res, mode) {
const readData = (res, mode) => new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const chunks = []; const chunks = [];
res res
.on("data", chunk => chunks.push(chunk)) .on("data", chunk => chunks.push(chunk))
.on("end", () => { .on("end", () => {
try { try {
const data = decompress(Buffer.concat(chunks), res.headers["content-encoding"]); const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
resolve((mode === "json" ? JSON.parse(data.toString("utf8")) : resolve(mode === "json" ? JSON.parse(data.toString("utf8")) :
mode === "buffer" ? data : mode === "buffer" ? data :
mode === "arraybuffer" ? new Uint8Array(data).buffer : mode === "arraybuffer" ? new Uint8Array(data).buffer :
data.toString("utf8"))); data.toString("utf8"));
} }
catch (err) { catch (err) {
reject(err); reject(err);
} }
}) })
.on("error", reject); .on("error", reject);
}); });
const fetch = async (urlString, options = {}, redirectCount = 0, cookies = {}) => { }
const { protocol, hostname, pathname, search, port } = new URL(urlString); async fetch(urlString, options = {}, redirectCount = 0) {
options.followRedirects = options.followRedirects ?? true; options.followRedirects = options.followRedirects ?? true;
if (options.signal?.aborted) const { protocol, hostname, pathname, search, port } = new URL(urlString);
throw new Error("Request aborted"); const body = options.method === "POST" && options.body
const body = options.method === "POST" && options.body ? options.headers?.["Content-Type"] === "application/json"
? options.headers?.["Content-Type"] === "application/json" ? JSON.stringify(options.body)
? JSON.stringify(options.body) : querystring.stringify(options.body)
: querystring.stringify(options.body) : null;
: null; const requestOptions = {
const requestOptions = { hostname,
hostname, port: port || (protocol === "https:" ? 443 : 80),
port: port || (protocol === "https:" ? 443 : 80), path: pathname + search,
path: pathname + search, method: options.method || "GET",
method: options.method || "GET", headers: {
headers: { ...options.headers,
...options.headers, ...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}), "Cookie": this.formatCookies(),
"Accept-Encoding": "br, gzip, deflate" "Accept-Encoding": "br, gzip, deflate",
}
};
addCookiesToRequest(cookies, requestOptions.headers);
return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => {
if (res.headers["set-cookie"])
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
if (redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded"));
if (!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies));
} }
if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) };
return reject(new Error(`Request failed with status code ${res.statusCode}`)); return new Promise((resolve, reject) => {
resolve({ const requester = protocol === "https:" ? https : http;
body: res, const req = requester.request(requestOptions, res => {
headers: res.headers, this.parseCookies(res.headers["set-cookie"]?.join("; "));
cookies: cookies, if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
text: () => readData(res, "text"), req.destroy();
json: () => readData(res, "json"), if (redirectCount >= (options.maxRedirects || 5))
buffer: () => readData(res, "buffer"), return reject(new Error("Max redirects exceeded"));
arrayBuffer: () => readData(res, "arraybuffer") if (!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
}
return resolve({
body: res,
headers: res.headers,
cookies: this.cookies,
text: () => this.readData(res, "text"),
json: () => this.readData(res, "json"),
buffer: () => this.readData(res, "buffer"),
arrayBuffer: () => this.readData(res, "arraybuffer"),
});
}); });
}); req.setTimeout(options.timeout || 5e3, () => {
req.setTimeout(options.timeout || 5e3, () => { req.destroy();
req.destroy(); reject(new Error("Request timed out"));
reject(new Error("Request timed out"));
});
req.on("error", reject);
if (options.signal) {
options.signal.addEventListener("abort", () => {
req.destroy(new Error("Request aborted"));
reject(new Error("Request aborted"));
}); });
} req.on("error", reject);
if (body) if (options.signal) {
req.write(body); options.signal.addEventListener("abort", () => {
req.end(); req.destroy(new Error("Request aborted"));
}); reject(new Error("Request aborted"));
}; });
export default fetch; }
if (body)
req.write(body);
req.end();
});
}
}
const inst = new fetch();
export default inst.fetch.bind(inst);

266
src/index.ts
View File

@ -4,162 +4,150 @@ import { URL } from "url";
import querystring from "querystring"; import querystring from "querystring";
import zlib from "zlib"; import zlib from "zlib";
type CookieStorage = Record<string, string>; class fetch {
private cookies: Record<string, string> = {};
type ResponseData = { constructor() { }
body: http.IncomingMessage;
headers: http.IncomingHttpHeaders;
cookies: CookieStorage;
text: () => Promise<string>;
json: () => Promise<JSON>;
buffer: () => Promise<Buffer>;
arrayBuffer: () => Promise<ArrayBuffer>;
};
interface ExtendedRequestOptions extends http.RequestOptions { private parseCookies(cookieHeader: string | undefined): void {
body?: any; if(!cookieHeader)
timeout?: number; return;
followRedirects?: boolean;
maxRedirects?: number;
signal?: AbortSignal;
}
const parseCookies = (cookieHeader: string | undefined): CookieStorage => {
const cookies: CookieStorage = {};
if(cookieHeader) {
cookieHeader.split(";").forEach(cookie => { cookieHeader.split(";").forEach(cookie => {
const [key, value] = cookie.split("="); const [key, value] = cookie.split("=");
if(key && value) if(key && value)
cookies[key.trim()] = value.trim(); this.cookies[key.trim()] = value.trim();
}); });
} }
return cookies;
};
const formatCookies = (cookies: CookieStorage): string => private formatCookies(): string {
Object.entries(cookies).map(([key, value]) => `${key}=${value}`).join("; "); return Object.entries(this.cookies)
.map(([key, value]) => `${key}=${value}`)
.join("; ");
}
const addCookiesToRequest = (cookies: CookieStorage, headers: http.OutgoingHttpHeaders): void => { private decompress(
headers["Cookie"] = formatCookies(cookies); data: Buffer,
}; encoding: string | undefined
): Buffer {
return encoding === "br" ? zlib.brotliDecompressSync(data) :
encoding === "gzip" ? zlib.gunzipSync(data) :
encoding === "deflate" ? zlib.inflateSync(data) :
data;
}
const decompress = ( private readData<T>(
data: Buffer, res: http.IncomingMessage,
encoding: string | undefined mode: "text" | "json" | "buffer" | "arraybuffer"
): Buffer => { ): Promise<T> {
return encoding === "br" ? zlib.brotliDecompressSync(data) : return new Promise((resolve, reject) => {
encoding === "gzip" ? zlib.gunzipSync(data) : const chunks: Buffer[] = [];
encoding === "deflate" ? zlib.inflateSync(data) : res
data; .on("data", chunk => chunks.push(chunk))
}; .on("end", () => {
try {
const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
resolve(
mode === "json" ? JSON.parse(data.toString("utf8")) :
mode === "buffer" ? data :
mode === "arraybuffer" ? new Uint8Array(data).buffer :
data.toString("utf8")
);
}
catch(err: any) {
reject(err);
}
})
.on("error", reject);
});
}
const readData = <T>( async fetch(
res: http.IncomingMessage, urlString: string,
mode: "text" | "json" | "buffer" | "arraybuffer" options: http.RequestOptions & {
): Promise<T> => body?: any;
new Promise((resolve, reject) => { timeout?: number;
const chunks: Buffer[] = []; followRedirects?: boolean;
res maxRedirects?: number;
.on("data", chunk => chunks.push(chunk)) signal?: AbortSignal;
.on("end", () => { } = {},
try { redirectCount = 0
const data = decompress(Buffer.concat(chunks), res.headers["content-encoding"]); ): Promise<{
resolve(( body: http.IncomingMessage;
mode === "json" ? JSON.parse(data.toString("utf8")) : headers: http.IncomingHttpHeaders;
mode === "buffer" ? data : cookies: Record<string, string>;
mode === "arraybuffer" ? new Uint8Array(data).buffer : text: () => Promise<string>;
data.toString("utf8") json: () => Promise<JSON>;
)) as T; buffer: () => Promise<Buffer>;
arrayBuffer: () => Promise<ArrayBuffer>;
}> {
options.followRedirects = options.followRedirects ?? true;
const { protocol, hostname, pathname, search, port } = new URL(urlString);
const body = options.method === "POST" && options.body
? options.headers?.["Content-Type"] === "application/json"
? JSON.stringify(options.body)
: querystring.stringify(options.body as querystring.ParsedUrlQueryInput)
: null;
const requestOptions: http.RequestOptions = {
hostname,
port: port || (protocol === "https:" ? 443 : 80),
path: pathname + search,
method: options.method || "GET",
headers: {
...options.headers,
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
"Cookie": this.formatCookies(),
"Accept-Encoding": "br, gzip, deflate",
}
};
return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => {
this.parseCookies(res.headers["set-cookie"]?.join("; "));
if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
req.destroy();
if(redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded"));
if(!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
} }
catch(err: any) { return resolve({
reject(err); body: res,
} headers: res.headers,
}) cookies: this.cookies,
.on("error", reject); text: () => this.readData<string>(res, "text"),
}); json: () => this.readData<JSON>(res, "json"),
buffer: () => this.readData<Buffer>(res, "buffer"),
arrayBuffer: () => this.readData<ArrayBuffer>(res, "arraybuffer"),
});
});
const fetch = async ( req.setTimeout(options.timeout || 5e3, () => {
urlString: string, req.destroy();
options: ExtendedRequestOptions = {}, reject(new Error("Request timed out"));
redirectCount: number = 0, });
cookies: CookieStorage = {}
): Promise<ResponseData> => {
const { protocol, hostname, pathname, search, port } = new URL(urlString);
options.followRedirects = options.followRedirects ?? true;
if(options.signal?.aborted) req.on("error", reject);
throw new Error("Request aborted");
const body = if(options.signal) {
options.method === "POST" && options.body options.signal.addEventListener("abort", () => {
? options.headers?.["Content-Type"] === "application/json" req.destroy(new Error("Request aborted"));
? JSON.stringify(options.body) reject(new Error("Request aborted"));
: querystring.stringify(options.body as querystring.ParsedUrlQueryInput) });
: null;
const requestOptions: http.RequestOptions = {
hostname,
port: port || (protocol === "https:" ? 443 : 80),
path: pathname + search,
method: options.method || "GET",
headers: {
...options.headers,
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
"Accept-Encoding": "br, gzip, deflate"
}
};
addCookiesToRequest(cookies, requestOptions.headers!);
return new Promise((resolve, reject) => {
const requester = protocol === "https:" ? https : http;
const req = requester.request(requestOptions, res => {
if(res.headers["set-cookie"])
cookies = parseCookies(res.headers["set-cookie"].join("; "));
if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
if(redirectCount >= (options.maxRedirects || 5))
return reject(new Error("Max redirects exceeded"));
if(!res.headers.location)
return reject(new Error("Redirect location not provided"));
const nextUrl = new URL(res.headers.location, urlString);
return resolve(fetch(nextUrl.toString(), options, redirectCount + 1, cookies));
} }
if(res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) if(body)
return reject(new Error(`Request failed with status code ${res.statusCode}`)); req.write(body);
req.end();
resolve({
body: res,
headers: res.headers,
cookies: cookies,
text: () => readData<string>(res, "text"),
json: () => readData<JSON>(res, "json"),
buffer: () => readData<Buffer>(res, "buffer"),
arrayBuffer: () => readData<ArrayBuffer>(res, "arraybuffer")
});
}); });
}
}
req.setTimeout(options.timeout || 5e3, () => { const inst = new fetch();
req.destroy(); export default inst.fetch.bind(inst);
reject(new Error("Request timed out"));
});
req.on("error", reject);
if(options.signal) {
options.signal.addEventListener("abort", () => {
req.destroy(new Error("Request aborted"));
reject(new Error("Request aborted"));
});
}
if(body)
req.write(body);
req.end();
});
};
export default fetch;