keinBot/flumm-fetch-cookies
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bugfixes

Browse Source
This commit is contained in:
jkhsjdhjs 2019-08-14 23:35:55 +02:00
parent f613a8c32e
commit 9f498864bb
Signed by: jkhsjdhjs
GPG Key ID: BAC6ADBAB7D576CC
4 changed files with 86 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -50,19 +50,19 @@ Adds a cookie to the jar.
Reads a cookie jar from the disk and adds the contained cookies. Reads a cookie jar from the disk and adds the contained cookies.
#### domains() #### domains()
Returns an array of the domains currently stored cookies for. Returns an iterator over all domains currently stored cookies for.
#### *iterValidForRequest(domain, url) #### *cookiesDomain(domain)
Returns an iterator over all cookies valid for a request to `domain` and `url`. Returns an iterator over all cookies currently stored for `domain`.
#### *iterValid() #### *cookiesValid()
Returns an iterator over all valid (non-expired) cookies. Returns an iterator over all valid (non-expired) cookies.
#### *iterAll() #### *cookiesAll()
Returns an iterator over all cookies currently stored. Returns an iterator over all cookies currently stored.
#### *iter(domain) #### *cookiesValidForRequest(url)
Returns an iterator over all cookies for a specific domain. Returns an iterator over all cookies valid for a request to `url`.
#### deleteExpired() #### deleteExpired()
Removes all expired cookies from the jar. Removes all expired cookies from the jar.

54
src/cookie-jar.mjs
View File

@ -1,4 +1,5 @@
import fs from "fs"; import fs from "fs";
import url from "url";
import Cookie from "./cookie"; import Cookie from "./cookie";
export default class CookieJar { export default class CookieJar {
@ -28,7 +29,7 @@ export default class CookieJar {
addCookie(c, fromURL) { addCookie(c, fromURL) {
if(typeof c === "string") if(typeof c === "string")
c = new Cookie(c, fromURL); c = new Cookie(c, fromURL);
if(!(c instanceof Cookie)) else if(!(c instanceof Cookie))
throw new TypeError("First parameter is neither a string nor a cookie!"); throw new TypeError("First parameter is neither a string nor a cookie!");
if(!this.cookies.get(c.domain)) if(!this.cookies.get(c.domain))
this.cookies.set(c.domain, new Map()); this.cookies.set(c.domain, new Map());
@ -38,35 +39,48 @@ export default class CookieJar {
JSON.parse(fs.readFileSync(this.file)).forEach(c => this.addCookie(Cookie.fromObject(c))); JSON.parse(fs.readFileSync(this.file)).forEach(c => this.addCookie(Cookie.fromObject(c)));
} }
domains() { domains() {
return [...this.cookies.keys()]; return this.cookies.keys();
} }
*iterValidForRequest(domain, url) { *cookiesDomain(domain) {
for(const cookie of this.iter(domain))
if(cookie.isValidForRequest(url))
yield cookie;
}
*iterValid() {
for(const cookie of this.iterAll())
if(!cookie.hasExpired())
yield cookie;
}
*iterAll() {
for(const domain of this.domains())
yield* this.iter(domain);
}
*iter(domain) {
for(const cookie of (this.cookies.get(domain) || []).values()) for(const cookie of (this.cookies.get(domain) || []).values())
yield cookie; yield cookie;
} }
*cookiesValid() {
for(const cookie of this.cookiesAll())
if(!cookie.hasExpired())
yield cookie;
}
*cookiesAll() {
for(const domain of this.domains())
yield* this.cookiesDomain(domain);
}
*cookiesValidForRequest(requestURL) {
const namesYielded = [],
domains = url
.parse(requestURL)
.hostname
.split(".")
.map((_, i, a) => a.slice(i).join("."))
.slice(0, -1);
for(const domain of domains) {
for(const cookie of this.cookiesDomain(domain)) {
if(cookie.isValidForRequest(requestURL)
&& namesYielded.every(name => name !== cookie.name)) {
namesYielded.push(cookie.name);
yield cookie;
}
}
}
}
deleteExpired() { deleteExpired() {
const filteredCookies = [...this.iterValid()]; const validCookies = [...this.cookiesValid()];
this.cookies = new Map(); this.cookies = new Map();
filteredCookies.forEach(c => this.addCookie(c)); validCookies.forEach(c => this.addCookie(c));
} }
save() { save() {
if(typeof this.file !== "string") if(typeof this.file !== "string")
throw new Error("No file has been specified for this cookie jar!"); throw new Error("No file has been specified for this cookie jar!");
// only save cookies that haven't expired // only save cookies that haven't expired
fs.writeFileSync(this.file, JSON.stringify([...this.iterValid()])); fs.writeFileSync(this.file, JSON.stringify([...this.cookiesValid()]));
} }
}; };

52
src/cookie.mjs
View File

@ -1,4 +1,4 @@
import urlParser from "url"; import url from "url";
const validateHostname = (cookieHostname, requestHostname, subdomains) => { const validateHostname = (cookieHostname, requestHostname, subdomains) => {
cookieHostname = cookieHostname.toLowerCase(); cookieHostname = cookieHostname.toLowerCase();
@ -28,12 +28,14 @@ const splitN = (str, sep, n) => {
}; };
export default class Cookie { export default class Cookie {
constructor(str, url) { constructor(str, requestURL) {
if(typeof str !== "string") if(typeof str !== "string")
throw new TypeError("Input not a string"); throw new TypeError("First parameter is not a string!");
const splitted = str.split("; "); const splitted = str.split("; ");
[this.name, this.value] = splitN(splitted[0], "=", 1); [this.name, this.value] = splitN(splitted[0], "=", 1);
if(!this.name)
throw new Error("Invalid cookie name \"" + this.name + "\"");
if(this.value.startsWith("\"") && this.value.endsWith("\"")) if(this.value.startsWith("\"") && this.value.endsWith("\""))
this.value = this.value.slice(1, -1); this.value = this.value.slice(1, -1);
@ -46,12 +48,12 @@ export default class Cookie {
continue; continue;
if(!/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2}[ -](?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[ -]\d{2,4} \d{2}:\d{2}:\d{2} GMT$/.test(v) if(!/^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2}[ -](?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[ -]\d{2,4} \d{2}:\d{2}:\d{2} GMT$/.test(v)
|| (this.expiry = new Date(v)) === "Invalid Date") || (this.expiry = new Date(v)) === "Invalid Date")
throw new TypeError("Invalid value for Expires \"" + v + "\"!"); throw new Error("Invalid value for Expires \"" + v + "\"!");
} }
else if(k === "max-age") { else if(k === "max-age") {
const seconds = ~~+v; const seconds = ~~+v;
if(seconds.toString() !== v) if(seconds.toString() !== v)
throw new TypeError("Invalid value for Max-Age \"" + v + "\"!"); throw new Error("Invalid value for Max-Age \"" + v + "\"!");
this.expiry = new Date(); this.expiry = new Date();
this.expiry.setSeconds(this.expiry.getSeconds() + seconds); this.expiry.setSeconds(this.expiry.getSeconds() + seconds);
} }
@ -67,7 +69,7 @@ export default class Cookie {
else if(k === "samesite") // only relevant for cross site requests, so not for us else if(k === "samesite") // only relevant for cross site requests, so not for us
continue; continue;
else else
throw new TypeError("Invalid key \"" + k + "\" specified!"); throw new Error("Invalid key \"" + k + "\" with value \"" + v + "\" specified!");
} }
else { else {
if(k === "secure") if(k === "secure")
@ -75,23 +77,32 @@ export default class Cookie {
else if(k === "httponly") // only relevant for browsers else if(k === "httponly") // only relevant for browsers
continue; continue;
else else
throw new TypeError("Invalid key \"" + k + "\" specified!"); throw new Error("Invalid key \"" + k + "\" without value specified!");
} }
} }
const parsedURL = url.parse(requestURL);
if(this.name.toLowerCase().startsWith("__secure-") && (!this.secure || parsedURL.protocol !== "https:"))
throw new Error("Cookie has \"__Secure-\" prefix but \"Secure\" isn't set or the cookie is not set via https!");
if(this.name.toLowerCase().startsWith("__host-") && (!this.secure || parsedURL.protocol !== "https:" || this.domain))
throw new Error("Cookie has \"__Host-\" prefix but \"Secure\" isn't set, the cookie is not set via https, \"Domain\" is set or \"Path\" is not equal to \"/\"!");
// assign defaults
if(!this.domain) { if(!this.domain) {
this.domain = urlParser.parse(url).hostname; this.domain = parsedURL.hostname;
this.subdomains = false; this.subdomains = false;
} }
if(!this.path) if(!this.path)
this.path = "/"; this.path = "/";
if(this.name.toLowerCase().startsWith("__secure-") && (!this.secure || !url.toLowerCase().startsWith("https:"))) if(!this.secure)
throw new TypeError("Cookie has \"__Secure-\" prefix but \"Secure\" isn't set or the cookie is not set via https!"); this.secure = false;
if(this.name.toLowerCase().startsWith("__host-") && (!this.secure || !url.toLowerCase().startsWith("https:") || this.domain || this.path !== "/")) if(!this.expiry)
throw new TypeError("Cookie has \"__Host-\" prefix but \"Secure\" isn't set, the cookie is not set via https, \"Domain\" is set or \"Path\" is not equal to \"/\"!"); this.expiry = null;
} }
static fromObject(obj) { static fromObject(obj) {
let c = Object.assign(Object.create(this.prototype), obj); let c = Object.assign(Object.create(this.prototype), obj);
if(c.expiry && typeof c.expiry === "string") if(typeof c.expiry === "string")
c.expiry = new Date(c.expiry); c.expiry = new Date(c.expiry);
return c; return c;
} }
@ -101,17 +112,14 @@ export default class Cookie {
hasExpired() { hasExpired() {
return this.expiry && this.expiry < new Date(); return this.expiry && this.expiry < new Date();
} }
isValidForRequest(url) { isValidForRequest(requestURL) {
if(this.hasExpired()) if(this.hasExpired())
return false; return false;
const parsedURL = urlParser.parse(url); const parsedURL = url.parse(requestURL);
if(parsedURL.protocol !== "http:" && parsedURL.protocol !== "https:") if(parsedURL.protocol !== "http:" && parsedURL.protocol !== "https:"
return false; || this.secure && parsedURL.protocol !== "https:"
if(this.secure && parsedURL.protocol !== "https:") || !validateHostname(this.domain, parsedURL.hostname, this.subdomains)
return false; || !validatePath(this.path, parsedURL.pathname))
if(!validateHostname(this.domain, parsedURL.hostname, this.subdomains))
return false;
if(!validatePath(this.path, parsedURL.pathname))
return false; return false;
return true; return true;
} }

48
src/index.mjs
View File

@ -1,43 +1,27 @@
import fetch from "node-fetch"; import fetch from "node-fetch";
import CookieJar from "./cookie-jar"; import CookieJar from "./cookie-jar";
import Cookie from "./cookie"; import Cookie from "./cookie";
import urlParser from "url";
async function cookieFetch(cookieJars, url, options) { async function cookieFetch(cookieJars, url, options) {
let cookies = ""; let cookies = "";
const domains = const addValidFromJars = jars =>
urlParser jars
.parse(url) .map(jar => [...jar.cookiesValidForRequest(url)])
.hostname
.split(".")
.map((_, i, a) => a.slice(i).join("."))
.slice(0, -1);
const addValidFromJar = jar =>
domains
.map(d => [...jar.iterValidForRequest(d, url)])
.reduce((a, b) => [...a, ...b]) .reduce((a, b) => [...a, ...b])
.filter((v, i, a) => a.slice(0, i).every(c => c.name !== v.name)) //unique .filter((v, i, a) => a.slice(0, i).every(c => c.name !== v.name)) // filter cookies with duplicate names
.forEach(c => cookies += c.serialize() + "; "); .forEach(c => cookies += c.serialize() + "; ");
if(cookieJars) { if(cookieJars) {
if(Array.isArray(cookieJars) && cookieJars.every(c => c instanceof CookieJar)) { if(Array.isArray(cookieJars) && cookieJars.every(c => c instanceof CookieJar))
cookieJars.forEach(jar => { addValidFromJars(cookieJars.filter(jar => jar.flags.includes("r")));
if(!jar.flags.includes("r"))
return;
addValidFromJar(jar);
});
}
else if(cookieJars instanceof CookieJar && cookieJars.flags.includes("r")) else if(cookieJars instanceof CookieJar && cookieJars.flags.includes("r"))
addValidFromJar(cookieJars); addValidFromJars([cookieJars]);
else else
throw new TypeError("First paramter is neither a cookie jar nor an array of cookie jars!"); throw new TypeError("First paramter is neither a cookie jar nor an array of cookie jars!");
} }
if(cookies.length !== 0) { if(cookies) {
if(!options) { if(!options)
options = { options = {};
headers: {} if(!options.headers)
};
}
else if(!options.headers)
options.headers = {}; options.headers = {};
options.headers.cookie = cookies.slice(0, -2); options.headers.cookie = cookies.slice(0, -2);
} }
@ -46,13 +30,11 @@ async function cookieFetch(cookieJars, url, options) {
cookies = result.headers[Object.getOwnPropertySymbols(result.headers)[0]]["set-cookie"]; cookies = result.headers[Object.getOwnPropertySymbols(result.headers)[0]]["set-cookie"];
if(cookies && cookieJars) { if(cookies && cookieJars) {
if(Array.isArray(cookieJars)) { if(Array.isArray(cookieJars)) {
cookieJars.forEach(jar => { cookieJars
if(!jar.flags.includes("w")) .filter(jar => jar.flags.includes("w"))
return; .forEach(jar => cookies.forEach(c => jar.addCookie(c, url)));
cookies.forEach(c => jar.addCookie(c, url));
});
} }
else if(cookieJars.flags.includes("w")) { else if(cookieJars instanceof CookieJar && cookieJars.flags.includes("w")) {
cookies.forEach(c => cookieJars.addCookie(c, url)); cookies.forEach(c => cookieJars.addCookie(c, url));
} }
} }