From 5c6ccbd3e89d65b6b83d86df0f592fa5d8bd0492 Mon Sep 17 00:00:00 2001 From: jkhsjdhjs Date: Mon, 25 Nov 2019 06:30:12 +0100 Subject: [PATCH] improve performance by using a Set instead of array filtering in fetch and CookieJar.cookiesValidForRequest() use fsPromises API instead of the synchronous one move cookie loading to seperate async CookieJar.load() function make cookie saving async (CookieJar.save()) add helper function for creating type errors to error.mjs move CookieParseError to error.mjs change readme according to changes bump minor version to 1.3.0 this version requires at least nodejs 11.14.0 --- README.md | 41 +++++++++++++++++++++-------- package-lock.json | 2 +- package.json | 4 +-- src/cookie-jar.mjs | 65 +++++++++++++++++++++++----------------------- src/cookie.mjs | 10 ++----- src/errors.mjs | 12 +++++++++ src/index.mjs | 30 ++++++++++++--------- 7 files changed, 97 insertions(+), 67 deletions(-) create mode 100644 src/errors.mjs diff --git a/README.md b/README.md index 6b95477..c02f9f5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ # node-fetch-cookies + A [node-fetch](https://github.com/bitinn/node-fetch) wrapper with support for cookies. It supports reading/writing from/to a JSON cookie jar and keeps cookies in memory until you call `CookieJar.save()` to reduce disk I/O. +## For upgrading to version 1.3.0 see [1.3.0 Breaking API Changes](#1.3.0-breaking-api-changes). + + ## Usage Examples ### with file... ```javascript @@ -9,13 +13,16 @@ import {fetch, CookieJar} from "node-fetch-cookies"; (async () => { // creates a CookieJar instance - const cookieJar = new CookieJar("rw", "jar.json"); + const cookieJar = new CookieJar("jar.json"); + + // load cookies from the cookie jar + await cookieJar.load(); // usual fetch usage, except with one or multiple cookie jars as first parameter const response = await fetch(cookieJar, "https://example.com"); // save the received cookies to disk - cookieJar.save(); + await cookieJar.save(); })(); ``` @@ -24,7 +31,7 @@ import {fetch, CookieJar} from "node-fetch-cookies"; import {fetch, CookieJar} from "node-fetch-cookies"; (async () => { - const cookieJar = new CookieJar("rw"); + const cookieJar = new CookieJar(); // log in to some api let response = await fetch(cookieJar, "https://example.com/api/login", { @@ -56,12 +63,12 @@ A class that stores cookies. - `file` The path of the cookie jar on the disk. - `cookies` A [Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map) mapping hostnames to maps, which map cookie names to the respective [Cookie](#class-cookie) instance. -#### new CookieJar(flags[, file, cookies]) -- `flags` A string specifying whether cookies should be read and/or written from/to the jar when passing it as parameter to [fetch](#fetchcookiejar-url-options). +#### new CookieJar([file, flags = `rw` cookies]) +- `file` An optional string containing a relative or absolute path to the file on the disk to use. +- `flags` An optional string specifying whether cookies should be read and/or written from/to the jar when passing it as parameter to [fetch](#fetchcookiejar-url-options). Default: `rw` - `r`: only read from this jar - `w`: only write to this jar - `rw` or `wr`: read/write from/to this jar -- `file` An optional string containing a relative or absolute path to the file on the disk to use. - `cookies` An optional initializer for the cookie jar - either an array of [Cookie](#class-cookie) instances or a single Cookie instance. #### addCookie(cookie[, fromURL]) @@ -74,9 +81,6 @@ In this case `fromURL` must be specified. Returns `true` if the cookie has been added successfully. Returns `false` otherwise. Will log a warning to console if a cookie fails to be parsed. -#### addFromFile(file) -Reads cookies from `file` on the disk and adds the contained cookies. - #### domains() Returns an iterator over all domains currently stored cookies for. @@ -97,9 +101,13 @@ Returns an iterator over all cookies valid for a request to `url`. Removes all expired cookies from the jar. - `sessionEnded`: A boolean. Also removes session cookies if set to `true`. -#### save() -Saves the cookie jar to disk. Only non-expired non-session cookies are saved. +#### async load([file = this.file]) +Reads cookies from `file` on the disk and adds the contained cookies. +- `file`: Path to the file where the cookies should be saved. Default: `this.file`, the file that has been passed to the constructor. +#### async save([file = this.file]) +Saves the cookie jar to `file` on the disk. Only non-expired non-session cookies are saved. +- `file`: Path to the file where the cookies should be saved. Default: `this.file`, the file that has been passed to the constructor. ### Class: Cookie An abstract representation of a cookie. @@ -134,5 +142,16 @@ Returns whether the cookie has expired or not. Returns whether the cookie is valid for a request to `url`. +## 1.3.0 Breaking API Changes +- `new CookieJar(flags, file, cookies)` has been changed to `new CookieJar(file, flags = "rw", cookies)`. +`new CookieJar("rw")` can now be written as `new CookieJar()`, `new CookieJar("rw", "jar.json")` can now be written as `new CookieJar("jar.json")`. +This change has been introduced to simplify the usage of this library, since `rw` is used for `flags` in most cases anyways. +- `CookieJar.addFromFile(file)` has been renamed to the async function `async CookieJar.load([file = this.file])`, which uses the fsPromises API for non-blocking cookie loading. +The default value for `file` is the file passed to the constructor. +- `CookieJar.save(file)` was moved to `async CookieJar.save([file = this.file]) now also uses the fsPromises API. +- `new CookieJar()` now doesn't load cookies from the specified file anymore. To do so, call `await CookieJar.load()` after creating the CookieJar. +**NOTE: `CookieJar.load()` will throw an error if the cookie jar doesn't exist or doesn't contain valid JSON!** + + ## License This project is licensed under the MIT license, see [LICENSE](LICENSE). diff --git a/package-lock.json b/package-lock.json index 4540f0e..f41e0c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "node-fetch-cookies", - "version": "1.2.3", + "version": "1.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 65b06ef..f956e2c 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "node-fetch-cookies", - "version": "1.2.3", + "version": "1.3.0", "description": "node-fetch wrapper that adds support for cookie-jars", "main": "src/index.mjs", "engines": { - "node": ">=10.0.0" + "node": ">=11.14.0" }, "files": [ "src/" diff --git a/src/cookie-jar.mjs b/src/cookie-jar.mjs index fffdcc7..88f4fbe 100644 --- a/src/cookie-jar.mjs +++ b/src/cookie-jar.mjs @@ -1,54 +1,48 @@ -import fs from "fs"; +import { promises as fs } from "fs"; import url from "url"; import Cookie from "./cookie.mjs"; +import { paramError, CookieParseError } from "./errors.mjs"; export default class CookieJar { - constructor(flags, file, cookies) { - this.cookies = new Map(); - this.file = file; + constructor(file, flags = "rw", cookies) { this.flags = flags; + this.file = file; + this.cookies = new Map(); if(typeof this.flags !== "string") - throw new TypeError("First parameter is not a string!"); + throw paramError("First", "flags", "new CookieJar()", "string"); if(this.file && typeof this.file !== "string") - throw new TypeError("Second parameter is not a string!"); - if(this.file && fs.existsSync(this.file)) - this.addFromFile(this.file); + throw paramError("Second", "file", "new CookieJar()", "string"); if(Array.isArray(cookies)) { if(!cookies.every(c => c instanceof Cookie)) - throw new TypeError("Third parameter is not an array of cookies!"); - else - cookies.forEach(cookie => this.addCookie(cookie)); + throw paramError("Third", "cookies", "new CookieJar()", "[Cookie]"); + cookies.forEach(cookie => this.addCookie(cookie)); } else if(cookies instanceof Cookie) this.addCookie(cookies); else if(cookies) - throw new TypeError("Third parameter is neither an array nor a cookie!"); + throw paramError("Third", "cookies", "new CookieJar()", ["[Cookie]", "Cookie"]); } - addCookie(c, fromURL) { - if(typeof c === "string") { + addCookie(cookie, fromURL) { + if(typeof cookie === "string") { try { - c = new Cookie(c, fromURL); + cookie = new Cookie(cookie, fromURL); } catch(error) { - if(error.name === "CookieParseError") { - console.warn("Ignored cookie: " + c); + if(error instanceof CookieParseError) { + console.warn("Ignored cookie: " + cookie); console.warn("Reason: " + error.message); return false; } - else - throw error; + throw error; } } - else if(!(c instanceof Cookie)) - throw new TypeError("First parameter is neither a string nor a cookie!"); - if(!this.cookies.get(c.domain)) - this.cookies.set(c.domain, new Map()); - this.cookies.get(c.domain).set(c.name, c); + else if(!(cookie instanceof Cookie)) + throw paramError("First", "cookie", "CookieJar.addCookie()", ["string", "Cookie"]); + if(!this.cookies.get(cookie.domain)) + this.cookies.set(cookie.domain, new Map()); + this.cookies.get(cookie.domain).set(cookie.name, cookie); return true; } - addFromFile(file) { - JSON.parse(fs.readFileSync(this.file)).forEach(c => this.addCookie(Cookie.fromObject(c))); - } domains() { return this.cookies.keys(); } @@ -66,7 +60,7 @@ export default class CookieJar { yield* this.cookiesDomain(domain); } *cookiesValidForRequest(requestURL) { - const namesYielded = [], + const namesYielded = new Set(), domains = url .parse(requestURL) .hostname @@ -76,8 +70,8 @@ export default class CookieJar { for(const domain of domains) { for(const cookie of this.cookiesDomain(domain)) { if(cookie.isValidForRequest(requestURL) - && namesYielded.every(name => name !== cookie.name)) { - namesYielded.push(cookie.name); + && !namesYielded.has(cookie.name)) { + namesYielded.add(cookie.name); yield cookie; } } @@ -88,10 +82,15 @@ export default class CookieJar { this.cookies = new Map(); validCookies.forEach(c => this.addCookie(c)); } - save() { - if(typeof this.file !== "string") + async load(file = this.file) { + if(typeof file !== "string") + throw new Error("No file has been specified for this cookie jar!"); + JSON.parse(await fs.readFile(file)).forEach(c => this.addCookie(Cookie.fromObject(c))); + } + async save(file = this.file) { + if(typeof file !== "string") throw new Error("No file has been specified for this cookie jar!"); // only save cookies that haven't expired - fs.writeFileSync(this.file, JSON.stringify([...this.cookiesValid(false)])); + await fs.writeFile(this.file, JSON.stringify([...this.cookiesValid(false)])); } }; diff --git a/src/cookie.mjs b/src/cookie.mjs index 85675ac..9ba283a 100644 --- a/src/cookie.mjs +++ b/src/cookie.mjs @@ -1,11 +1,5 @@ import url from "url"; - -class CookieParseError extends Error { - constructor(...args) { - super(...args); - this.name = "CookieParseError"; - } -} +import { paramError, CookieParseError } from "./errors.mjs"; const validateHostname = (cookieHostname, requestHostname, subdomains) => { cookieHostname = cookieHostname.toLowerCase(); @@ -37,7 +31,7 @@ const splitN = (str, sep, n) => { export default class Cookie { constructor(str, requestURL) { if(typeof str !== "string") - throw new TypeError("First parameter is not a string!"); + throw paramError("First", "str", "new Cookie()", "string"); const splitted = str.split("; "); [this.name, this.value] = splitN(splitted[0], "=", 1); diff --git a/src/errors.mjs b/src/errors.mjs new file mode 100644 index 0000000..a5333e9 --- /dev/null +++ b/src/errors.mjs @@ -0,0 +1,12 @@ +export class CookieParseError extends Error { + constructor(...args) { + super(...args); + this.name = "CookieParseError"; + } +} + +export function paramError(position, paramName, functionName, validTypes) { + validTypes = [validTypes].flatMap(t => "\"" + t + "\""); + validTypes = validTypes.slice(0, -1).join(", ") + (validTypes.length > 1 ? " or " : "") + validTypes.slice(-1); + return new TypeError(`${position} parameter "${name}" passed to "${functionName}" is not of type ${validTypes}!`); +} diff --git a/src/index.mjs b/src/index.mjs index 0a22a16..2bc2b98 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -1,22 +1,28 @@ -import fetch from "node-fetch"; +import _fetch from "node-fetch"; import CookieJar from "./cookie-jar.mjs"; import Cookie from "./cookie.mjs"; +import { paramError } from "./errors.mjs" -async function cookieFetch(cookieJars, url, options) { +async function fetch(cookieJars, url, options) { let cookies = ""; - const addValidFromJars = jars => - jars - .map(jar => [...jar.cookiesValidForRequest(url)]) - .reduce((a, b) => [...a, ...b]) - .filter((v, i, a) => a.slice(0, i).every(c => c.name !== v.name)) // filter cookies with duplicate names - .forEach(c => cookies += c.serialize() + "; "); + const addValidFromJars = jars => { + // since multiple cookie jars can be passed, filter duplicates by using a set of cookie names + const set = new Set(); + jars.flatMap(jar => [...jar.cookiesValidForRequest(url)]) + .forEach(cookie => { + if(set.has(cookie.name)) + return; + set.add(cookie.name); + cookies += cookie.serialize() + "; "; + }); + }; if(cookieJars) { if(Array.isArray(cookieJars) && cookieJars.every(c => c instanceof CookieJar)) addValidFromJars(cookieJars.filter(jar => jar.flags.includes("r"))); else if(cookieJars instanceof CookieJar && cookieJars.flags.includes("r")) addValidFromJars([cookieJars]); else - throw new TypeError("First paramter is neither a cookie jar nor an array of cookie jars!"); + throw paramError("First", "cookieJars", "fetch", ["CookieJar", "[CookieJar]"]); } if(cookies) { if(!options) @@ -25,8 +31,8 @@ async function cookieFetch(cookieJars, url, options) { options.headers = {}; options.headers.cookie = cookies.slice(0, -2); } - const result = await fetch(url, options); - // i cannot use headers.get() here because it joins the cookies to a string + const result = await _fetch(url, options); + // I cannot use headers.get() here because it joins the cookies to a string cookies = result.headers[Object.getOwnPropertySymbols(result.headers)[0]]["set-cookie"]; if(cookies && cookieJars) { if(Array.isArray(cookieJars)) { @@ -40,4 +46,4 @@ async function cookieFetch(cookieJars, url, options) { return result; } -export {cookieFetch as fetch, CookieJar, Cookie}; +export {fetch, CookieJar, Cookie};