keinBot/flumm-fetch-cookies
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improve performance by using a Set instead of array filtering in fetch and CookieJar.cookiesValidForRequest()

Browse Source 
use fsPromises API instead of the synchronous one
move cookie loading to seperate async CookieJar.load() function
make cookie saving async (CookieJar.save())
add helper function for creating type errors to error.mjs
move CookieParseError to error.mjs
change readme according to changes
bump minor version to 1.3.0
this version requires at least nodejs 11.14.0
This commit is contained in:
jkhsjdhjs
2019-11-25 06:30:12 +01:00
parent d9e3f4154a
commit 5c6ccbd3e8
7 changed files with 97 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/cookie-jar.mjs
View File

@ -1,54 +1,48 @@
import fs from "fs";
import { promises as fs } from "fs";
import url from "url";
import Cookie from "./cookie.mjs";
import { paramError, CookieParseError } from "./errors.mjs";
export default class CookieJar {
constructor(flags, file, cookies) {
this.cookies = new Map();
this.file = file;
constructor(file, flags = "rw", cookies) {
this.flags = flags;
this.file = file;
this.cookies = new Map();
if(typeof this.flags !== "string")
throw new TypeError("First parameter is not a string!");
throw paramError("First", "flags", "new CookieJar()", "string");
if(this.file && typeof this.file !== "string")
throw new TypeError("Second parameter is not a string!");
if(this.file && fs.existsSync(this.file))
this.addFromFile(this.file);
throw paramError("Second", "file", "new CookieJar()", "string");
if(Array.isArray(cookies)) {
if(!cookies.every(c => c instanceof Cookie))
throw new TypeError("Third parameter is not an array of cookies!");
else
cookies.forEach(cookie => this.addCookie(cookie));
throw paramError("Third", "cookies", "new CookieJar()", "[Cookie]");
cookies.forEach(cookie => this.addCookie(cookie));
}
else if(cookies instanceof Cookie)
this.addCookie(cookies);
else if(cookies)
throw new TypeError("Third parameter is neither an array nor a cookie!");
throw paramError("Third", "cookies", "new CookieJar()", ["[Cookie]", "Cookie"]);
}
addCookie(c, fromURL) {
if(typeof c === "string") {
addCookie(cookie, fromURL) {
if(typeof cookie === "string") {
try {
c = new Cookie(c, fromURL);
cookie = new Cookie(cookie, fromURL);
}
catch(error) {
if(error.name === "CookieParseError") {
console.warn("Ignored cookie: " + c);
if(error instanceof CookieParseError) {
console.warn("Ignored cookie: " + cookie);
console.warn("Reason: " + error.message);
return false;
}
else
throw error;
throw error;
}
}
else if(!(c instanceof Cookie))
throw new TypeError("First parameter is neither a string nor a cookie!");
if(!this.cookies.get(c.domain))
this.cookies.set(c.domain, new Map());
this.cookies.get(c.domain).set(c.name, c);
else if(!(cookie instanceof Cookie))
throw paramError("First", "cookie", "CookieJar.addCookie()", ["string", "Cookie"]);
if(!this.cookies.get(cookie.domain))
this.cookies.set(cookie.domain, new Map());
this.cookies.get(cookie.domain).set(cookie.name, cookie);
return true;
}
addFromFile(file) {
JSON.parse(fs.readFileSync(this.file)).forEach(c => this.addCookie(Cookie.fromObject(c)));
}
domains() {
return this.cookies.keys();
}
@ -66,7 +60,7 @@ export default class CookieJar {
yield* this.cookiesDomain(domain);
}
*cookiesValidForRequest(requestURL) {
const namesYielded = [],
const namesYielded = new Set(),
domains = url
.parse(requestURL)
.hostname
@ -76,8 +70,8 @@ export default class CookieJar {
for(const domain of domains) {
for(const cookie of this.cookiesDomain(domain)) {
if(cookie.isValidForRequest(requestURL)
&& namesYielded.every(name => name !== cookie.name)) {
namesYielded.push(cookie.name);
&& !namesYielded.has(cookie.name)) {
namesYielded.add(cookie.name);
yield cookie;
}
}
@ -88,10 +82,15 @@ export default class CookieJar {
this.cookies = new Map();
validCookies.forEach(c => this.addCookie(c));
}
save() {
if(typeof this.file !== "string")
async load(file = this.file) {
if(typeof file !== "string")
throw new Error("No file has been specified for this cookie jar!");
JSON.parse(await fs.readFile(file)).forEach(c => this.addCookie(Cookie.fromObject(c)));
}
async save(file = this.file) {
if(typeof file !== "string")
throw new Error("No file has been specified for this cookie jar!");
// only save cookies that haven't expired
fs.writeFileSync(this.file, JSON.stringify([...this.cookiesValid(false)]));
await fs.writeFile(this.file, JSON.stringify([...this.cookiesValid(false)]));
}
};

10
src/cookie.mjs
View File

@ -1,11 +1,5 @@
import url from "url";
class CookieParseError extends Error {
constructor(...args) {
super(...args);
this.name = "CookieParseError";
}
}
import { paramError, CookieParseError } from "./errors.mjs";
const validateHostname = (cookieHostname, requestHostname, subdomains) => {
cookieHostname = cookieHostname.toLowerCase();
@ -37,7 +31,7 @@ const splitN = (str, sep, n) => {
export default class Cookie {
constructor(str, requestURL) {
if(typeof str !== "string")
throw new TypeError("First parameter is not a string!");
throw paramError("First", "str", "new Cookie()", "string");
const splitted = str.split("; ");
[this.name, this.value] = splitN(splitted[0], "=", 1);

12
src/errors.mjs Normal file
View File

@ -0,0 +1,12 @@
export class CookieParseError extends Error {
constructor(...args) {
super(...args);
this.name = "CookieParseError";
}
}
export function paramError(position, paramName, functionName, validTypes) {
validTypes = [validTypes].flatMap(t => "\"" + t + "\"");
validTypes = validTypes.slice(0, -1).join(", ") + (validTypes.length > 1 ? " or " : "") + validTypes.slice(-1);
return new TypeError(`${position} parameter "${name}" passed to "${functionName}" is not of type ${validTypes}!`);
}

30
src/index.mjs
View File

@ -1,22 +1,28 @@
import fetch from "node-fetch";
import _fetch from "node-fetch";
import CookieJar from "./cookie-jar.mjs";
import Cookie from "./cookie.mjs";
import { paramError } from "./errors.mjs"
async function cookieFetch(cookieJars, url, options) {
async function fetch(cookieJars, url, options) {
let cookies = "";
const addValidFromJars = jars =>
jars
.map(jar => [...jar.cookiesValidForRequest(url)])
.reduce((a, b) => [...a, ...b])
.filter((v, i, a) => a.slice(0, i).every(c => c.name !== v.name)) // filter cookies with duplicate names
.forEach(c => cookies += c.serialize() + "; ");
const addValidFromJars = jars => {
// since multiple cookie jars can be passed, filter duplicates by using a set of cookie names
const set = new Set();
jars.flatMap(jar => [...jar.cookiesValidForRequest(url)])
.forEach(cookie => {
if(set.has(cookie.name))
return;
set.add(cookie.name);
cookies += cookie.serialize() + "; ";
});
};
if(cookieJars) {
if(Array.isArray(cookieJars) && cookieJars.every(c => c instanceof CookieJar))
addValidFromJars(cookieJars.filter(jar => jar.flags.includes("r")));
else if(cookieJars instanceof CookieJar && cookieJars.flags.includes("r"))
addValidFromJars([cookieJars]);
else
throw new TypeError("First paramter is neither a cookie jar nor an array of cookie jars!");
throw paramError("First", "cookieJars", "fetch", ["CookieJar", "[CookieJar]"]);
}
if(cookies) {
if(!options)
@ -25,8 +31,8 @@ async function cookieFetch(cookieJars, url, options) {
options.headers = {};
options.headers.cookie = cookies.slice(0, -2);
}
const result = await fetch(url, options);
// i cannot use headers.get() here because it joins the cookies to a string
const result = await _fetch(url, options);
// I cannot use headers.get() here because it joins the cookies to a string
cookies = result.headers[Object.getOwnPropertySymbols(result.headers)[0]]["set-cookie"];
if(cookies && cookieJars) {
if(Array.isArray(cookieJars)) {
@ -40,4 +46,4 @@ async function cookieFetch(cookieJars, url, options) {
return result;
}
export {cookieFetch as fetch, CookieJar, Cookie};
export {fetch, CookieJar, Cookie};