f0ck/f0ckv2
3
4
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

parser schmarser
All checks were successful
fetch npm modules / f0ck the f0cker (push) Successful in 22s
Details

Browse Source
This commit is contained in:
Flummi 2023-05-01 14:10:11 +02:00
parent d4218ad1e8
commit d2bf6f7afd
2 changed files with 164 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
src/inc/queue.mjs Normal file
View File

@ -0,0 +1,110 @@
import fetch from "flumm-fetch";
import { exec as _exec } from "child_process";
import fs from "fs";
import db from "./sql.mjs";
export default new class queue {
constructor() {
};
addqueue(e, link) {
//this.#queue.push(e, link);
};
exec(cmd) {
return new Promise((resolve, reject) => {
_exec(cmd, { maxBuffer: 5e3 * 1024 }, (err, stdout, stderr) => {
if(err)
return reject(err);
if(stderr)
console.error(stderr);
resolve({ stdout: stdout });
});
});
};
async genuuid() {
return (await db`
select gen_random_uuid() as uuid
`)[0].uuid.substring(0, 8);
};
async checkrepostlink(link) {
const q_repost = await db`
select id
from "items"
where src = ${link}
`;
return q_repost.length > 0 ? q_repost[0].id : false;
};
async checkrepostsum(checksum) {
const q_repost = await db`
select id
from "items"
where checksum = ${checksum}
`;
return q_repost.length > 0 ? q_repost[0].id : false;
};
async getItemID(filename) {
return (await db`
select *
from "items"
where dest = ${filename}
limit 1
`)[0].id;
};
async genThumbnail(filename, mime, itemid, link) {
if(mime.startsWith('video/') || mime == 'image/gif')
await this.exec(`ffmpegthumbnailer -i./public/b/${filename} -s1024 -o./tmp/${itemid}.png`);
else if(mime.startsWith('image/') && mime != 'image/gif')
await this.exec(`convert ./public/b/${filename} ./tmp/${itemid}.png`);
else if(mime.startsWith('audio/')) {
if(link.match(/soundcloud/)) {
let cover = (await this.exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --get-thumbnail "${link}"`)).stdout.trim();
if(!cover.match(/default_avatar/)) {
cover = cover.replace(/-(large|original)\./, '-t500x500.');
try {
await this.exec(`wget "${cover}" -O ./tmp/${itemid}.jpg`);
const size = (await fs.promises.stat(`./tmp/${itemid}.jpg`)).size;
if(size >= 0) {
await this.exec(`convert ./tmp/${itemid}.jpg ./tmp/${itemid}.png`);
await this.exec(`convert ./tmp/${itemid}.jpg ./public/ca/${itemid}.webp`);
}
} catch(err) {}
}
else {
await this.exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
await this.exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
}
}
else {
await this.exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
await this.exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
}
}
await this.exec(`convert "./tmp/${itemid}.png" -resize "128x128^" -gravity center -crop 128x128+0+0 +repage ./public/t/${itemid}.webp`);
await fs.promises.unlink(`./tmp/${itemid}.png`).catch(_=>{});
await fs.promises.unlink(`./tmp/${itemid}.jpg`).catch(_=>{});
return true;
};
// tags
async tagSFW(itemid) {
return await db`
insert into "tags_assign" ${
db({
item_id: itemid,
tag_id: 1,
user_id: 1
})
}
`;
};
};

228
src/inc/trigger/parser.mjs
View File

@ -2,32 +2,26 @@ import cfg from "../config.mjs";
import db from "../sql.mjs";
import lib from "../lib.mjs";
import { getLevel } from "../admin.mjs";
import queue from "../queue.mjs";
import fetch from "flumm-fetch";
import fs from "fs";
import { exec as _exec } from "child_process";
import path from "path";
const exec = cmd => new Promise((resolve, reject) => {
_exec(cmd, { maxBuffer: 5e3 * 1024 }, (err, stdout, stderr) => {
if(err)
return reject(err);
if(stderr)
console.error(stderr);
resolve({ stdout: stdout });
});
});
//const regex = /https?:\/\/[\w\S(\.|:|/)]+/gi;
const regex = /https?:\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?/gi;
const regex = {
all: /https?:\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?/gi,
yt: /(?:youtube\.com\/\S*(?:(?:\/e(?:mbed))?\/|watch\/?\?(?:\S*?&?v\=))|youtu\.be\/)([a-zA-Z0-9_-]{6,11})/gi,
imgur: /^https?:\/\/(\w+\.)?imgur.com\/(\w*\d\w*)+(\.[a-zA-Z]{3,4})?/gi
};
export default async bot => {
return [{
name: "parser",
call: regex,
call: regex.all,
active: true,
f: e => {
const links = e.message.match(regex)?.filter(link => !link.includes("f0ck.me")) || [];
const links = e.message.match(regex.all)?.filter(link => !link.includes("f0ck.me")) || [];
let repost;
if(e.media)
links.push(e.media);
@ -44,56 +38,40 @@ export default async bot => {
console.log(`parsing ${links.length} link${links.length > 1 ? "s" : ""}...`);
links.forEach(async link => {
if(regex.imgur.test(link))
return await e.reply(`imgur is not supported!`);
// check repost (link)
const q_repost = await db`
select id
from "items"
where src = ${link}
`;
if(q_repost.length > 0)
return await e.reply(`repost motherf0cker (link): ${cfg.main.url.full}/${q_repost[0].id}`);
repost = await queue.checkrepostlink(link);
if(repost)
return await e.reply(`repost motherf0cker (link): ${cfg.main.url.full}/${repost}`);
// generate uuid
const uuid = (await db`
select gen_random_uuid() as uuid
`)[0].uuid.substring(0, 8);
const uuid = await queue.genuuid();
const maxfilesize = (getLevel(e.user).level > 50 ? cfg.main.maxfilesize * cfg.main.adminmultiplier : cfg.main.maxfilesize) / 1024;
const maxfilesize = (getLevel(e.user).level > 50 ? cfg.main.maxfilesize * cfg.main.adminmultiplier : cfg.main.maxfilesize);
let meta;
let ext;
// read metadata
try {
meta = JSON.parse((await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --skip-download --dump-json "${link}"`)).stdout);
}
catch(err) {
//e.reply("[error] f0ck has no bock :(");
//console.error(err);
return;
}
if(!Object.values(cfg.mimes).includes(meta.ext.toLowerCase())) {
if(!regex.yt.test(link)) {
const tmphead = (await fetch(link, { method: "HEAD" })).headers["content-type"];
if(!Object.keys(cfg.mimes).includes(tmphead))
return;
meta.ext = cfg.mimes[tmphead];
ext = cfg.mimes[tmphead];
}
let filename = `${uuid}.${meta.ext}`;
const msg = await e.reply(`[charging the f0cker] downloading: ${uuid}`, {
disable_notification: true
});
// download data
const start = new Date();
let source;
if(meta.ext === "mp4") {
source = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize}k --merge-output-format mp4 -o ./tmp/${filename}`)).stdout.trim();
//change 720 to any other available resolution, higher = better quality but bigger filesize
}
else {
source = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize}k -o ./tmp/${filename}`)).stdout.trim();
//change 720 to any other available resolution, higher = better quality but bigger filesize
let source = (await queue.exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize / 1024}k --postprocessor-args "ffmpeg:-bitexact" -o "./tmp/${uuid}.%(ext)s" --print after_move:filepath`)).stdout.trim();
if(!source) {
if(e.type == 'tg')
return await e.editMessageText(msg.result.chat.id, msg.result.message_id, "something went wrong lol");
return await e.reply("something went wrong lol");
}
if(source.match(/larger than/)) {
@ -102,39 +80,40 @@ export default async bot => {
return await e.reply("too large lol");
}
const end = ~~((new Date() - start) / 1e3);
// generate checksum
const checksum = (await exec(`sha256sum ./tmp/${filename}`)).stdout.trim().split(" ")[0];
const size = fs.statSync(`./tmp/${filename}`).size;
// filesize check
const size = fs.statSync(source).size;
if(size > maxfilesize) {
await fs.promises.unlink(source).catch(_=>{});
if(e.type == 'tg')
return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `too large lol. (${lib.formatSize(size)} / ${lib.formatSize(maxfilesize)})`);
return await e.reply(`too large lol. (${lib.formatSize(size)} / ${lib.formatSize(maxfilesize)})`);
}
// mime check
const mime = (await exec(`file --mime-type -b ./tmp/${filename}`)).stdout.trim();
const mime = (await queue.exec(`file --mime-type -b ${source}`)).stdout.trim();
if(!Object.keys(cfg.mimes).includes(mime)) {
if(e.type == 'tg')
return e.editMessageText(msg.result.chat.id, msg.result.message_id, `lol, go f0ck yourself (${mime})`);
return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `lol, go f0ck yourself (${mime})`);
return await e.reply(`lol, go f0ck yourself (${mime})`);
}
if(!Object.values(cfg.mimes).includes(meta.ext.toLowerCase())) {
let tmpext = cfg.mimes[meta.ext.toLowerCase()];
fs.renameSync(`./tmp/${filename}`, `./tmp/${uuid}.${tmpext}`);
filename = `${uuid}.${tmpext}`;
}
// generate checksum
const checksum = (await queue.exec(`sha256sum ${source}`)).stdout.trim().split(" ")[0];
// check repost (checksum)
const q_repostc = await db`
select id
from "items"
where checksum = ${checksum}
`;
if(q_repostc.length > 0) {
repost = await queue.checkrepostsum(checksum);
if(repost) {
await fs.promises.unlink(source).catch(_=>{});
if(e.type == 'tg')
return e.editMessageText(msg.result.chat.id, msg.result.message_id, `repost motherf0cker (checksum): ${cfg.main.url.full}/${q_repostc[0].id}`);
return await e.reply(`repost motherf0cker (checksum): ${cfg.main.url.full}/${q_repostc[0].id}`);
return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `repost motherf0cker (checksum): ${cfg.main.url.full}/${repost}`);
return await e.reply(`repost motherf0cker (checksum): ${cfg.main.url.full}/${repost}`);
}
await fs.promises.copyFile(`./tmp/${filename}`, `./public/b/${filename}`);
await fs.promises.unlink(`./tmp/${filename}`).catch(_=>{});
const filename = path.basename(source);
await fs.promises.copyFile(source, `./public/b/${filename}`);
await fs.promises.unlink(source).catch(_=>{});
await db`
insert into items ${
@ -153,107 +132,20 @@ export default async bot => {
}
`;
const itemid = (await db`
select *
from "items"
where dest = ${filename}
limit 1
`)[0].id;
const itemid = await queue.getItemID(filename);
// generate thumbnail
try {
if(mime.startsWith('video/') || mime == 'image/gif')
await exec(`ffmpegthumbnailer -i./public/b/${filename} -s1024 -o./tmp/${itemid}.png`);
else if(mime.startsWith('image/') && mime != 'image/gif')
await exec(`convert ./public/b/${filename} ./tmp/${itemid}.png`);
else if(mime.startsWith('audio/')) {
if(link.match(/soundcloud/)) {
let cover = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --get-thumbnail "${link}"`)).stdout.trim();
if(!cover.match(/default_avatar/)) {
cover = cover.replace(/-(large|original)\./, '-t500x500.');
try {
await exec(`wget "${cover}" -O ./tmp/${itemid}.jpg`);
const size = (await fs.promises.stat(`./tmp/${itemid}.jpg`)).size;
if(size >= 0) {
await exec(`convert ./tmp/${itemid}.jpg ./tmp/${itemid}.png`);
await exec(`convert ./tmp/${itemid}.jpg ./public/ca/${itemid}.webp`);
}
} catch(err) {}
}
else {
await exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
await exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
}
}
else {
await exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
await exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
}
}
await exec(`convert "./tmp/${itemid}.png" -resize "128x128^" -gravity center -crop 128x128+0+0 +repage ./public/t/${itemid}.webp`);
await fs.promises.unlink(`./tmp/${itemid}.png`).catch(err => {});
await fs.promises.unlink(`./tmp/${itemid}.jpg`).catch(err => {});
await queue.genThumbnail(filename, mime, itemid, link);
} catch(err) {
await exec(`convert ./mugge.png ./public/t/${itemid}.webp`);
await queue.exec(`convert ./mugge.png ./public/t/${itemid}.webp`);
}
let speed = lib.calcSpeed(size, end);
speed = !Number.isFinite(speed) ? "yes" : `${speed.toFixed(2)} Mbit/s`;
// autotagger
/*let tags = [];
let score = 0;
try {
if(mime.startsWith('image')) {
const res = await lib.detectNSFW(filename);
score = res.score;
await db`
insert into "tags_assign" ${
db({
item_id: itemid,
tag_id: res.isNSFW ? 2 : 1,
user_id: 1
})
}
`;
tags.push(res.isNSFW ? 'nsfw' : 'sfw');
if(res.hentai >= .7) {
await db`
insert into "tags_assign" ${
db({
item_id: f.id,
tag_id: 4, // hentai
user_id: 1 // autotagger
})
}
`;
tags.push('hentai');
}
}
else if(mime.startsWith('audio')) {
await db`
insert into "tags_assign" ${
db([{
item_id: itemid,
tag_id: 1,
user_id: 1
}, {
item_id: itemid,
tag_id: 3, // audio
user_id: 1
}])
}
`;
tags.push('sfw', 'audio');
}
} catch(err) {
console.error(err);
}
const outputmsg = `[f0cked] link: ${cfg.main.url.full}/${itemid} | size: ${lib.formatSize(size)} | speed: ${speed}` + (tags.length > 0 ? ` | tags: ${tags.join(', ')} (score: ${score.toFixed(2)})` : '');
*/
await queue.tagSFW(itemid);
const outputmsgirc = `[f0cked] link: ${cfg.main.url.full}/${itemid} | size: ${lib.formatSize(size)} | speed: ${speed}`;
const outputmsgtg = `[f0cked] size: ${lib.formatSize(size)} | speed: ${speed}`;
@ -272,18 +164,6 @@ export default async bot => {
]]
})
});
/*await e.editMessageText(msg.result.chat.id, msg.result.message_id, outputmsgtg, {
reply_markup: JSON.stringify({
inline_keyboard: [[
{ text: 'sfw', callback_data: `b_sfw:${itemid}` },
{ text: 'nsfw', callback_data: `b_nsfw:${itemid}` },
{ text: 'tags', callback_data: `b_tags:${itemid}` },
{ text: '❌ delete', callback_data: `b_delete:${itemid}` }
], [
{ text: `open f0ck #${itemid}`, url: `${cfg.main.url.full}/${itemid}` }
]]
})
});*/
}
else {
await e.reply(outputmsgirc);