Compare commits
12 Commits
Author | SHA1 | Date | |
---|---|---|---|
1b75e89ca4 | |||
a8a8701739 | |||
e197050aa2 | |||
fd0eef5cbe | |||
65f1707358 | |||
3f1effbb4a | |||
694c2336b3 | |||
68f345d396 | |||
c395eef497 | |||
cf64ee2e01 | |||
08b74ec61b | |||
6fad46fd95 |
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
node_modules
|
77
dist/cookieManager.js
vendored
Normal file
77
dist/cookieManager.js
vendored
Normal file
@ -0,0 +1,77 @@
|
||||
import fs from "fs";
|
||||
export default class CookieManager {
|
||||
cookies = {};
|
||||
parse(headers, domain) {
|
||||
if (!this.cookies[domain])
|
||||
this.cookies[domain] = {};
|
||||
headers?.forEach(header => {
|
||||
const [cookiePart, ...attributes] = header.split(';').map(part => part.trim());
|
||||
const [name, value] = cookiePart.split('=');
|
||||
const cookie = { value: value || "" };
|
||||
attributes.forEach(attr => {
|
||||
const [key, val] = attr.split('=');
|
||||
const lowerKey = key.toLowerCase();
|
||||
switch (lowerKey) {
|
||||
case 'domain':
|
||||
cookie.domain = val;
|
||||
break;
|
||||
case 'path':
|
||||
cookie.path = val;
|
||||
break;
|
||||
case 'expires':
|
||||
cookie.expires = val ? new Date(val) : undefined;
|
||||
break;
|
||||
case 'max-age':
|
||||
cookie.maxAge = parseInt(val, 10);
|
||||
break;
|
||||
case 'httponly':
|
||||
cookie.httpOnly = true;
|
||||
break;
|
||||
case 'secure':
|
||||
cookie.secure = true;
|
||||
break;
|
||||
case 'samesite':
|
||||
cookie.sameSite = val;
|
||||
break;
|
||||
}
|
||||
});
|
||||
Object.assign(this.cookies[domain], { [name]: cookie });
|
||||
});
|
||||
}
|
||||
format(domain) {
|
||||
this.cleanupExpiredCookies();
|
||||
return Object.entries(this.cookies[domain] || {})
|
||||
.map(([key, value]) => `${key}=${value.value.toString()}`)
|
||||
.join('; ');
|
||||
}
|
||||
getCookies(domain) {
|
||||
this.cleanupExpiredCookies();
|
||||
return this.cookies[domain] || {};
|
||||
}
|
||||
cleanupExpiredCookies() {
|
||||
const now = new Date();
|
||||
Object.keys(this.cookies).forEach(domain => {
|
||||
Object.keys(this.cookies[domain]).forEach(key => {
|
||||
if (this.cookies[domain][key].expires && this.cookies[domain][key].expires < now)
|
||||
delete this.cookies[domain][key];
|
||||
});
|
||||
});
|
||||
}
|
||||
saveToFile(filePath) {
|
||||
this.cleanupExpiredCookies();
|
||||
fs.writeFileSync(filePath, JSON.stringify(this.cookies, null, 2), "utf8");
|
||||
}
|
||||
loadFromFile(filePath) {
|
||||
if (!fs.existsSync(filePath))
|
||||
return console.warn(`The file ${filePath} does not exist.`);
|
||||
const fileContent = fs.readFileSync(filePath, "utf8");
|
||||
const loadedCookies = JSON.parse(fileContent);
|
||||
Object.keys(loadedCookies).forEach(domain => {
|
||||
Object.keys(loadedCookies[domain]).forEach(cookieName => {
|
||||
const cookie = loadedCookies[domain][cookieName];
|
||||
cookie.expires = cookie.expires ? new Date(cookie.expires) : undefined;
|
||||
});
|
||||
});
|
||||
this.cookies = loadedCookies;
|
||||
}
|
||||
}
|
97
dist/index.js
vendored
Normal file
97
dist/index.js
vendored
Normal file
@ -0,0 +1,97 @@
|
||||
import http from "http";
|
||||
import https from "https";
|
||||
import { URL } from "url";
|
||||
import querystring from "querystring";
|
||||
import zlib from "zlib";
|
||||
import CookieManager from './cookieManager.js';
|
||||
export const cookieManager = new CookieManager();
|
||||
class fetch {
|
||||
constructor() { }
|
||||
decompress(data, encoding) {
|
||||
return encoding === "br" ? zlib.brotliDecompressSync(data) :
|
||||
encoding === "gzip" ? zlib.gunzipSync(data) :
|
||||
encoding === "deflate" ? zlib.inflateSync(data) :
|
||||
data;
|
||||
}
|
||||
readData(res, mode) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
res
|
||||
.on("data", chunk => chunks.push(chunk))
|
||||
.on("end", () => {
|
||||
try {
|
||||
const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
|
||||
resolve(mode === "json" ? JSON.parse(data.toString("utf8")) :
|
||||
mode === "buffer" ? data :
|
||||
mode === "arraybuffer" ? new Uint8Array(data).buffer :
|
||||
data.toString("utf8"));
|
||||
}
|
||||
catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
})
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
async fetch(urlString, options = {}, redirectCount = 0) {
|
||||
options.followRedirects = options.followRedirects ?? true;
|
||||
const { protocol, hostname, pathname, search, port } = new URL(urlString);
|
||||
const body = options.method === "POST" && options.body
|
||||
? options.headers?.["Content-Type"] === "application/json"
|
||||
? JSON.stringify(options.body)
|
||||
: querystring.stringify(options.body)
|
||||
: null;
|
||||
const requestOptions = {
|
||||
hostname,
|
||||
port: port || (protocol === "https:" ? 443 : 80),
|
||||
path: pathname + search,
|
||||
method: options.method || "GET",
|
||||
headers: {
|
||||
...options.headers,
|
||||
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
|
||||
"Cookie": cookieManager.format(hostname),
|
||||
"Accept-Encoding": "br, gzip, deflate",
|
||||
}
|
||||
};
|
||||
return new Promise((resolve, reject) => {
|
||||
const requester = protocol === "https:" ? https : http;
|
||||
const req = requester.request(requestOptions, res => {
|
||||
cookieManager.parse(res.headers["set-cookie"], hostname);
|
||||
if (options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
req.destroy();
|
||||
if (redirectCount >= (options.maxRedirects || 5))
|
||||
return reject(new Error("Max redirects exceeded"));
|
||||
if (!res.headers.location)
|
||||
return reject(new Error("Redirect location not provided"));
|
||||
const nextUrl = new URL(res.headers.location, urlString);
|
||||
return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
|
||||
}
|
||||
return resolve({
|
||||
body: res,
|
||||
headers: res.headers,
|
||||
cookies: cookieManager.getCookies(hostname),
|
||||
text: () => this.readData(res, "text"),
|
||||
json: () => this.readData(res, "json"),
|
||||
buffer: () => this.readData(res, "buffer"),
|
||||
arrayBuffer: () => this.readData(res, "arraybuffer"),
|
||||
});
|
||||
});
|
||||
req.setTimeout(options.timeout || 5e3, () => {
|
||||
req.destroy();
|
||||
reject(new Error("Request timed out"));
|
||||
});
|
||||
req.on("error", reject);
|
||||
if (options.signal) {
|
||||
options.signal.addEventListener("abort", () => {
|
||||
req.destroy(new Error("Request aborted"));
|
||||
reject(new Error("Request aborted"));
|
||||
});
|
||||
}
|
||||
if (body)
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
const inst = new fetch();
|
||||
export default inst.fetch.bind(inst);
|
48
package-lock.json
generated
Normal file
48
package-lock.json
generated
Normal file
@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "flumm-fetch",
|
||||
"version": "2.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "flumm-fetch",
|
||||
"version": "2.0.0",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.10",
|
||||
"typescript": "^5.8.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.13.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz",
|
||||
"integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.8.2",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
|
||||
"integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.20.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
|
||||
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
18
package.json
18
package.json
@ -1,22 +1,28 @@
|
||||
{
|
||||
"name": "flumm-fetch",
|
||||
"version": "1.0.2",
|
||||
"version": "2.0.0",
|
||||
"description": "",
|
||||
"main": "src/index.mjs",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
"build": "tsc",
|
||||
"test": "node src/test.mjs"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/kein-Bot/flumm-fetch.git"
|
||||
"url": "git+https://git.lat/keinBot/flumm-fetch.git"
|
||||
},
|
||||
"keywords": [
|
||||
"fetch"
|
||||
],
|
||||
"author": "Flummi",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"bugs": {
|
||||
"url": "https://github.com/kein-Bot/flumm-fetch/issues"
|
||||
"url": "https://git.lat/keinBot/flumm-fetch/issues"
|
||||
},
|
||||
"homepage": "https://github.com/kein-Bot/flumm-fetch#readme"
|
||||
"homepage": "https://git.lat/keinBot/flumm-fetch#readme",
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.10",
|
||||
"typescript": "^5.8.2"
|
||||
}
|
||||
}
|
||||
|
95
src/cookieManager.ts
Normal file
95
src/cookieManager.ts
Normal file
@ -0,0 +1,95 @@
|
||||
import fs from "fs";
|
||||
|
||||
export type Cookie = {
|
||||
value: string;
|
||||
domain?: string;
|
||||
path?: string;
|
||||
expires?: Date;
|
||||
maxAge?: number;
|
||||
httpOnly?: boolean;
|
||||
secure?: boolean;
|
||||
sameSite?: string;
|
||||
}
|
||||
export type Cookies = Record<string, Cookie>;
|
||||
|
||||
export default class CookieManager {
|
||||
private cookies: Record<string, Cookies> = {};
|
||||
|
||||
parse(
|
||||
headers: string[] | undefined,
|
||||
domain: string
|
||||
): void {
|
||||
/* todo
|
||||
* key -> domain | use "domain" only if cookie does not have one
|
||||
* key -> expires | ???
|
||||
*/
|
||||
if(!this.cookies[domain])
|
||||
this.cookies[domain] = {};
|
||||
|
||||
headers?.forEach(header => {
|
||||
const [cookiePart, ...attributes] = header.split(';').map(part => part.trim());
|
||||
const [name, value] = cookiePart.split('=');
|
||||
|
||||
const cookie: Cookie = { value: value || "" };
|
||||
attributes.forEach(attr => {
|
||||
const [key, val] = attr.split('=');
|
||||
const lowerKey = key.toLowerCase();
|
||||
switch(lowerKey) {
|
||||
case 'domain': cookie.domain = val; break;
|
||||
case 'path': cookie.path = val; break;
|
||||
case 'expires': cookie.expires = val ? new Date(val) : undefined; break;
|
||||
case 'max-age': cookie.maxAge = parseInt(val, 10); break;
|
||||
case 'httponly': cookie.httpOnly = true; break;
|
||||
case 'secure': cookie.secure = true; break;
|
||||
case 'samesite': cookie.sameSite = val; break;
|
||||
}
|
||||
});
|
||||
|
||||
Object.assign(this.cookies[domain], { [name]: cookie });
|
||||
});
|
||||
}
|
||||
|
||||
format(domain: string): string {
|
||||
this.cleanupExpiredCookies();
|
||||
return Object.entries(this.cookies[domain] || {})
|
||||
.map(([key, value]) => `${key}=${value.value.toString()}`)
|
||||
.join('; ');
|
||||
}
|
||||
|
||||
getCookies(domain: string): Cookies {
|
||||
this.cleanupExpiredCookies();
|
||||
return this.cookies[domain] || {};
|
||||
}
|
||||
|
||||
cleanupExpiredCookies(): void {
|
||||
const now = new Date();
|
||||
Object.keys(this.cookies).forEach(domain => {
|
||||
Object.keys(this.cookies[domain]).forEach(key => {
|
||||
if(this.cookies[domain][key].expires && this.cookies[domain][key].expires < now)
|
||||
delete this.cookies[domain][key];
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
saveToFile(filePath: string): void {
|
||||
this.cleanupExpiredCookies();
|
||||
fs.writeFileSync(filePath, JSON.stringify(this.cookies, null, 2), "utf8");
|
||||
}
|
||||
|
||||
loadFromFile(filePath: string): void {
|
||||
if(!fs.existsSync(filePath))
|
||||
return console.warn(`The file ${filePath} does not exist.`);
|
||||
|
||||
const fileContent = fs.readFileSync(filePath, "utf8");
|
||||
const loadedCookies: Record<string, Cookies> = JSON.parse(fileContent);
|
||||
|
||||
Object.keys(loadedCookies).forEach(domain => {
|
||||
Object.keys(loadedCookies[domain]).forEach(cookieName => {
|
||||
const cookie = loadedCookies[domain][cookieName];
|
||||
cookie.expires = cookie.expires ? new Date(cookie.expires) : undefined;
|
||||
});
|
||||
});
|
||||
|
||||
this.cookies = loadedCookies;
|
||||
}
|
||||
}
|
@ -1,34 +0,0 @@
|
||||
import http from "http";
|
||||
import https from "https";
|
||||
import url from "url";
|
||||
import querystring from "querystring";
|
||||
|
||||
const readdata = (res, mode, data = "") => new Promise((resolve, reject) => res
|
||||
.setEncoding("utf8")
|
||||
.on("data", chunk => data += chunk)
|
||||
.on("end", () => {
|
||||
switch(mode) {
|
||||
case "text": resolve(data); break;
|
||||
case "json": try { resolve(JSON.parse(data)); } catch(err) { reject(data); } break;
|
||||
case "buffer": resolve(new Buffer.from(data)); break;
|
||||
default: reject("lol no D:"); break;
|
||||
}
|
||||
}));
|
||||
export default (a, options = {}, link = url.parse(a), body = "") => new Promise((resolve, reject) => {
|
||||
options = {...{ hostname: link.hostname, path: link.path, method: "GET" }, ...options};
|
||||
if(options.method === "POST") {
|
||||
body = typeof options.body === "object" ? querystring.stringify(options.body) : options.body;
|
||||
delete options.body;
|
||||
options.headers = {...{
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Content-Length": Buffer.byteLength(body)
|
||||
}, ...options.headers};
|
||||
}
|
||||
(link.protocol === "https:"?https:http).request(options, res => resolve({
|
||||
body: res,
|
||||
headers: res.headers,
|
||||
text: () => readdata(res, "text"),
|
||||
json: () => readdata(res, "json"),
|
||||
buffer: () => readdata(res, "buffer")
|
||||
})).on("error", err => reject(err)).end(body);
|
||||
});
|
137
src/index.ts
Normal file
137
src/index.ts
Normal file
@ -0,0 +1,137 @@
|
||||
import http from "http";
|
||||
import https from "https";
|
||||
import { URL } from "url";
|
||||
import querystring from "querystring";
|
||||
import zlib from "zlib";
|
||||
import CookieManager, { Cookies } from './cookieManager.js';
|
||||
|
||||
export const cookieManager = new CookieManager();
|
||||
|
||||
class fetch {
|
||||
constructor() { }
|
||||
|
||||
private decompress(
|
||||
data: Buffer,
|
||||
encoding: string | undefined
|
||||
): Buffer {
|
||||
return encoding === "br" ? zlib.brotliDecompressSync(data) :
|
||||
encoding === "gzip" ? zlib.gunzipSync(data) :
|
||||
encoding === "deflate" ? zlib.inflateSync(data) :
|
||||
data;
|
||||
}
|
||||
|
||||
private readData<T>(
|
||||
res: http.IncomingMessage,
|
||||
mode: "text" | "json" | "buffer" | "arraybuffer"
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res
|
||||
.on("data", chunk => chunks.push(chunk))
|
||||
.on("end", () => {
|
||||
try {
|
||||
const data = this.decompress(Buffer.concat(chunks), res.headers["content-encoding"]);
|
||||
resolve(
|
||||
mode === "json" ? JSON.parse(data.toString("utf8")) :
|
||||
mode === "buffer" ? data :
|
||||
mode === "arraybuffer" ? new Uint8Array(data).buffer :
|
||||
data.toString("utf8")
|
||||
);
|
||||
}
|
||||
catch(err: any) {
|
||||
reject(err);
|
||||
}
|
||||
})
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async fetch(
|
||||
urlString: string,
|
||||
options: http.RequestOptions & {
|
||||
body?: any;
|
||||
timeout?: number;
|
||||
followRedirects?: boolean;
|
||||
maxRedirects?: number;
|
||||
signal?: AbortSignal;
|
||||
} = {},
|
||||
redirectCount = 0
|
||||
): Promise<{
|
||||
body: http.IncomingMessage;
|
||||
headers: http.IncomingHttpHeaders;
|
||||
cookies: Cookies;
|
||||
text: () => Promise<string>;
|
||||
json: () => Promise<JSON>;
|
||||
buffer: () => Promise<Buffer>;
|
||||
arrayBuffer: () => Promise<ArrayBuffer>;
|
||||
}> {
|
||||
options.followRedirects = options.followRedirects ?? true;
|
||||
|
||||
const { protocol, hostname, pathname, search, port } = new URL(urlString);
|
||||
const body = options.method === "POST" && options.body
|
||||
? options.headers?.["Content-Type"] === "application/json"
|
||||
? JSON.stringify(options.body)
|
||||
: querystring.stringify(options.body as querystring.ParsedUrlQueryInput)
|
||||
: null;
|
||||
const requestOptions: http.RequestOptions = {
|
||||
hostname,
|
||||
port: port || (protocol === "https:" ? 443 : 80),
|
||||
path: pathname + search,
|
||||
method: options.method || "GET",
|
||||
headers: {
|
||||
...options.headers,
|
||||
...(body ? { "Content-Length": Buffer.byteLength(body) } : {}),
|
||||
"Cookie": cookieManager.format(hostname),
|
||||
"Accept-Encoding": "br, gzip, deflate",
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const requester = protocol === "https:" ? https : http;
|
||||
const req = requester.request(requestOptions, res => {
|
||||
cookieManager.parse(res.headers["set-cookie"], hostname);
|
||||
if(options.followRedirects && res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
req.destroy();
|
||||
if(redirectCount >= (options.maxRedirects || 5))
|
||||
return reject(new Error("Max redirects exceeded"));
|
||||
|
||||
if(!res.headers.location)
|
||||
return reject(new Error("Redirect location not provided"));
|
||||
|
||||
const nextUrl = new URL(res.headers.location, urlString);
|
||||
return resolve(this.fetch(nextUrl.toString(), options, redirectCount + 1));
|
||||
}
|
||||
return resolve({
|
||||
body: res,
|
||||
headers: res.headers,
|
||||
cookies: cookieManager.getCookies(hostname),
|
||||
text: () => this.readData<string>(res, "text"),
|
||||
json: () => this.readData<JSON>(res, "json"),
|
||||
buffer: () => this.readData<Buffer>(res, "buffer"),
|
||||
arrayBuffer: () => this.readData<ArrayBuffer>(res, "arraybuffer"),
|
||||
});
|
||||
});
|
||||
|
||||
req.setTimeout(options.timeout || 5e3, () => {
|
||||
req.destroy();
|
||||
reject(new Error("Request timed out"));
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
|
||||
if(options.signal) {
|
||||
options.signal.addEventListener("abort", () => {
|
||||
req.destroy(new Error("Request aborted"));
|
||||
reject(new Error("Request aborted"));
|
||||
});
|
||||
}
|
||||
|
||||
if(body)
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const inst = new fetch();
|
||||
export default inst.fetch.bind(inst);
|
14
tsconfig.json
Normal file
14
tsconfig.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2024",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"removeComments": true
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user