parser schmarser

src/inc/queue.mjs

@@ -0,0 +1,110 @@
+import fetch from "flumm-fetch";
+import { exec as _exec } from "child_process";
+import fs from "fs";
+import db from "./sql.mjs";
+
+export default new class queue {
+
+  constructor() {
+
+  };
+
+  addqueue(e, link) {
+    //this.#queue.push(e, link);
+  };
+
+  exec(cmd) {
+    return new Promise((resolve, reject) => {
+      _exec(cmd, { maxBuffer: 5e3 * 1024 }, (err, stdout, stderr) => {
+        if(err)
+          return reject(err);
+        if(stderr)
+          console.error(stderr);
+        resolve({ stdout: stdout });
+      });
+    });
+  };
+
+  async genuuid() {
+    return (await db`
+      select gen_random_uuid() as uuid
+    `)[0].uuid.substring(0, 8);
+  };
+
+  async checkrepostlink(link) {
+    const q_repost = await db`
+      select id
+      from "items"
+      where src = ${link}
+    `;
+    return q_repost.length > 0 ? q_repost[0].id : false;
+  };
+
+  async checkrepostsum(checksum) {
+    const q_repost = await db`
+      select id
+      from "items"
+      where checksum = ${checksum}
+    `;
+    return q_repost.length > 0 ? q_repost[0].id : false;
+  };
+
+  async getItemID(filename) {
+    return (await db`
+      select *
+      from "items"
+      where dest = ${filename}
+      limit 1
+    `)[0].id;
+  };
+
+  async genThumbnail(filename, mime, itemid, link) {
+    if(mime.startsWith('video/') || mime == 'image/gif')
+      await this.exec(`ffmpegthumbnailer -i./public/b/${filename} -s1024 -o./tmp/${itemid}.png`);
+    else if(mime.startsWith('image/') && mime != 'image/gif')
+      await this.exec(`convert ./public/b/${filename} ./tmp/${itemid}.png`);
+    else if(mime.startsWith('audio/')) {
+      if(link.match(/soundcloud/)) {
+        let cover = (await this.exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --get-thumbnail "${link}"`)).stdout.trim();
+        if(!cover.match(/default_avatar/)) {
+          cover = cover.replace(/-(large|original)\./, '-t500x500.');
+          try {
+            await this.exec(`wget "${cover}" -O ./tmp/${itemid}.jpg`);
+            const size = (await fs.promises.stat(`./tmp/${itemid}.jpg`)).size;
+            if(size >= 0) {
+              await this.exec(`convert ./tmp/${itemid}.jpg ./tmp/${itemid}.png`);
+              await this.exec(`convert ./tmp/${itemid}.jpg ./public/ca/${itemid}.webp`);
+            }
+          } catch(err) {}
+        }
+        else {
+          await this.exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
+          await this.exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
+        }
+      }
+      else {
+        await this.exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
+        await this.exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
+      }
+    }
+
+    await this.exec(`convert "./tmp/${itemid}.png" -resize "128x128^" -gravity center -crop 128x128+0+0 +repage ./public/t/${itemid}.webp`);
+    await fs.promises.unlink(`./tmp/${itemid}.png`).catch(_=>{});
+    await fs.promises.unlink(`./tmp/${itemid}.jpg`).catch(_=>{});
+    return true;
+  };
+
+  // tags
+  async tagSFW(itemid) {
+    return await db`
+      insert into "tags_assign" ${
+        db({
+          item_id: itemid,
+          tag_id: 1,
+          user_id: 1
+        })
+      }
+    `;
+  };
+
+};

src/inc/trigger/parser.mjs

@@ -2,32 +2,26 @@ import cfg from "../config.mjs";
 import db from "../sql.mjs";
 import lib from "../lib.mjs";
 import { getLevel } from "../admin.mjs";
+import queue from "../queue.mjs";
 import fetch from "flumm-fetch";
-
 import fs from "fs";
-import { exec as _exec } from "child_process";
+import path from "path";
 
-const exec = cmd => new Promise((resolve, reject) => {
+const regex = {
-  _exec(cmd, { maxBuffer: 5e3 * 1024 }, (err, stdout, stderr) => {
+  all:   /https?:\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?/gi,
-    if(err)
+  yt:    /(?:youtube\.com\/\S*(?:(?:\/e(?:mbed))?\/|watch\/?\?(?:\S*?&?v\=))|youtu\.be\/)([a-zA-Z0-9_-]{6,11})/gi,
-      return reject(err);
+  imgur: /^https?:\/\/(\w+\.)?imgur.com\/(\w*\d\w*)+(\.[a-zA-Z]{3,4})?/gi
-    if(stderr)
+};
-      console.error(stderr);
-    resolve({ stdout: stdout });
-  });
-});
-
-//const regex = /https?:\/\/[\w\S(\.|:|/)]+/gi;
-const regex = /https?:\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?/gi;
 
 export default async bot => {
 
   return [{
     name: "parser",
-    call: regex,
+    call: regex.all,
     active: true,
     f: e => {
-      const links = e.message.match(regex)?.filter(link => !link.includes("f0ck.me")) || [];
+      const links = e.message.match(regex.all)?.filter(link => !link.includes("f0ck.me")) || [];
+      let repost;
 
       if(e.media)
         links.push(e.media);
@@ -44,56 +38,40 @@ export default async bot => {
       console.log(`parsing ${links.length} link${links.length > 1 ? "s" : ""}...`);
 
       links.forEach(async link => {
+        if(regex.imgur.test(link))
+          return await e.reply(`imgur is not supported!`);
+
         // check repost (link)
-        const q_repost = await db`
+        repost = await queue.checkrepostlink(link);
-          select id
+        if(repost)
-          from "items"
+          return await e.reply(`repost motherf0cker (link): ${cfg.main.url.full}/${repost}`);
-          where src = ${link}
-        `;
-        if(q_repost.length > 0)
-          return await e.reply(`repost motherf0cker (link): ${cfg.main.url.full}/${q_repost[0].id}`);
 
         // generate uuid
-        const uuid = (await db`
+        const uuid = await queue.genuuid();
-          select gen_random_uuid() as uuid
-        `)[0].uuid.substring(0, 8);
 
-        const maxfilesize = (getLevel(e.user).level > 50 ? cfg.main.maxfilesize * cfg.main.adminmultiplier : cfg.main.maxfilesize) / 1024;
+        const maxfilesize = (getLevel(e.user).level > 50 ? cfg.main.maxfilesize * cfg.main.adminmultiplier : cfg.main.maxfilesize);
 
-        let meta;
+        let ext;
         // read metadata
-        try {
+        if(!regex.yt.test(link)) {
-          meta = JSON.parse((await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --skip-download --dump-json "${link}"`)).stdout);
-        }
-        catch(err) {
-          //e.reply("[error] f0ck has no bock :(");
-          //console.error(err);
-          return;
-        }
-
-        if(!Object.values(cfg.mimes).includes(meta.ext.toLowerCase())) {
           const tmphead = (await fetch(link, { method: "HEAD" })).headers["content-type"];
           if(!Object.keys(cfg.mimes).includes(tmphead))
             return;
-          meta.ext = cfg.mimes[tmphead];
+          ext = cfg.mimes[tmphead];
         }
 
-        let filename = `${uuid}.${meta.ext}`;
-
         const msg = await e.reply(`[charging the f0cker] downloading: ${uuid}`, {
           disable_notification: true
         });
 
         // download data
         const start = new Date();
-        let source;
+        let source = (await queue.exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize / 1024}k --postprocessor-args "ffmpeg:-bitexact" -o "./tmp/${uuid}.%(ext)s" --print after_move:filepath`)).stdout.trim();
-        if(meta.ext === "mp4") {
+        
-          source = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize}k --merge-output-format mp4 -o ./tmp/${filename}`)).stdout.trim();
+        if(!source) {
-          //change 720 to any other available resolution, higher = better quality but bigger filesize
+          if(e.type == 'tg')
-        }
+            return await e.editMessageText(msg.result.chat.id, msg.result.message_id, "something went wrong lol");
-        else {
+          return await e.reply("something went wrong lol");
-          source = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' "${link}" --max-filesize ${maxfilesize}k -o ./tmp/${filename}`)).stdout.trim();
-          //change 720 to any other available resolution, higher = better quality but bigger filesize
         }
 
         if(source.match(/larger than/)) {
@@ -103,38 +81,39 @@ export default async bot => {
         }
         const end = ~~((new Date() - start) / 1e3);
 
-        // generate checksum
+        // filesize check
-        const checksum = (await exec(`sha256sum ./tmp/${filename}`)).stdout.trim().split(" ")[0];
+        const size = fs.statSync(source).size;
-        const size = fs.statSync(`./tmp/${filename}`).size;
+        if(size > maxfilesize) {
+          await fs.promises.unlink(source).catch(_=>{});
+          if(e.type == 'tg')
+            return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `too large lol. (${lib.formatSize(size)} / ${lib.formatSize(maxfilesize)})`);
+          return await e.reply(`too large lol. (${lib.formatSize(size)} / ${lib.formatSize(maxfilesize)})`);
+        }
         
         // mime check
-        const mime = (await exec(`file --mime-type -b ./tmp/${filename}`)).stdout.trim();
+        const mime = (await queue.exec(`file --mime-type -b ${source}`)).stdout.trim();
         if(!Object.keys(cfg.mimes).includes(mime)) {
           if(e.type == 'tg')
-            return e.editMessageText(msg.result.chat.id, msg.result.message_id, `lol, go f0ck yourself (${mime})`);
+            return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `lol, go f0ck yourself (${mime})`);
           return await e.reply(`lol, go f0ck yourself (${mime})`);
         }
 
-        if(!Object.values(cfg.mimes).includes(meta.ext.toLowerCase())) {
+        // generate checksum
-          let tmpext = cfg.mimes[meta.ext.toLowerCase()];
+        const checksum = (await queue.exec(`sha256sum ${source}`)).stdout.trim().split(" ")[0];
-          fs.renameSync(`./tmp/${filename}`, `./tmp/${uuid}.${tmpext}`);
-          filename = `${uuid}.${tmpext}`;
-        }
 
         // check repost (checksum)
-        const q_repostc = await db`
+        repost = await queue.checkrepostsum(checksum);
-          select id
+        if(repost) {
-          from "items"
+          await fs.promises.unlink(source).catch(_=>{});
-          where checksum = ${checksum}
-        `;
-        if(q_repostc.length > 0) {
           if(e.type == 'tg')
-            return e.editMessageText(msg.result.chat.id, msg.result.message_id, `repost motherf0cker (checksum): ${cfg.main.url.full}/${q_repostc[0].id}`);
+            return await e.editMessageText(msg.result.chat.id, msg.result.message_id, `repost motherf0cker (checksum): ${cfg.main.url.full}/${repost}`);
-          return await e.reply(`repost motherf0cker (checksum): ${cfg.main.url.full}/${q_repostc[0].id}`);
+          return await e.reply(`repost motherf0cker (checksum): ${cfg.main.url.full}/${repost}`);
         }
 
-        await fs.promises.copyFile(`./tmp/${filename}`, `./public/b/${filename}`);
+        const filename = path.basename(source);
-        await fs.promises.unlink(`./tmp/${filename}`).catch(_=>{});
+
+        await fs.promises.copyFile(source, `./public/b/${filename}`);
+        await fs.promises.unlink(source).catch(_=>{});
 
         await db`
           insert into items ${
@@ -153,107 +132,20 @@ export default async bot => {
           }
         `;
 
-        const itemid = (await db`
+        const itemid = await queue.getItemID(filename);
-          select *
-          from "items"
-          where dest = ${filename}
-          limit 1
-        `)[0].id;
 
         // generate thumbnail
         try {
-          if(mime.startsWith('video/') || mime == 'image/gif')
+          await queue.genThumbnail(filename, mime, itemid, link);
-            await exec(`ffmpegthumbnailer -i./public/b/${filename} -s1024 -o./tmp/${itemid}.png`);
-          else if(mime.startsWith('image/') && mime != 'image/gif')
-            await exec(`convert ./public/b/${filename} ./tmp/${itemid}.png`);
-          else if(mime.startsWith('audio/')) {
-            if(link.match(/soundcloud/)) {
-              let cover = (await exec(`yt-dlp -f 'bv*[height<=720]+ba/b[height<=720] / wv*+ba/w' --get-thumbnail "${link}"`)).stdout.trim();
-              if(!cover.match(/default_avatar/)) {
-                cover = cover.replace(/-(large|original)\./, '-t500x500.');
-                try {
-                  await exec(`wget "${cover}" -O ./tmp/${itemid}.jpg`);
-                  const size = (await fs.promises.stat(`./tmp/${itemid}.jpg`)).size;
-                  if(size >= 0) {
-                    await exec(`convert ./tmp/${itemid}.jpg ./tmp/${itemid}.png`);
-                    await exec(`convert ./tmp/${itemid}.jpg ./public/ca/${itemid}.webp`);
-                  }
-                } catch(err) {}
-              }
-              else {
-                await exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
-                await exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
-              }
-            }
-            else {
-              await exec(`ffmpeg -i ./public/b/${filename} -update 1 -map 0:v -map 0:1 -c copy ./tmp/${itemid}.png`);
-              await exec(`convert ./tmp/${itemid}.png ./public/ca/${itemid}.webp`);
-            }
-          }
-      
-          await exec(`convert "./tmp/${itemid}.png" -resize "128x128^" -gravity center -crop 128x128+0+0 +repage ./public/t/${itemid}.webp`);
-          await fs.promises.unlink(`./tmp/${itemid}.png`).catch(err => {});
-          await fs.promises.unlink(`./tmp/${itemid}.jpg`).catch(err => {});
         } catch(err) {
-          await exec(`convert ./mugge.png ./public/t/${itemid}.webp`);
+          await queue.exec(`convert ./mugge.png ./public/t/${itemid}.webp`);
         }
 
         let speed = lib.calcSpeed(size, end);
         speed = !Number.isFinite(speed) ? "yes" : `${speed.toFixed(2)} Mbit/s`;
 
         // autotagger
-        /*let tags = [];
+        await queue.tagSFW(itemid);
-        let score = 0;
-        try {
-          if(mime.startsWith('image')) {
-            const res = await lib.detectNSFW(filename);
-            score = res.score;
-
-            await db`
-              insert into "tags_assign" ${
-                db({
-                  item_id: itemid,
-                  tag_id: res.isNSFW ? 2 : 1,
-                  user_id: 1
-                })
-              }
-            `;
-            tags.push(res.isNSFW ? 'nsfw' : 'sfw');
-
-            if(res.hentai >= .7) {
-              await db`
-                insert into "tags_assign" ${
-                  db({
-                    item_id: f.id,
-                    tag_id: 4, // hentai
-                    user_id: 1 // autotagger
-                  })
-                }
-              `;
-              tags.push('hentai');
-            }
-          }
-          else if(mime.startsWith('audio')) {
-            await db`
-              insert into "tags_assign" ${
-                db([{
-                  item_id: itemid,
-                  tag_id: 1,
-                  user_id: 1
-                }, {
-                  item_id: itemid,
-                  tag_id: 3, // audio
-                  user_id: 1
-                }])
-              }
-            `;
-            tags.push('sfw', 'audio');
-          }
-        } catch(err) {
-          console.error(err);
-        }
-        const outputmsg = `[f0cked] link: ${cfg.main.url.full}/${itemid} | size: ${lib.formatSize(size)} | speed: ${speed}` + (tags.length > 0 ? ` | tags: ${tags.join(', ')} (score: ${score.toFixed(2)})` : '');
-        */
 
         const outputmsgirc = `[f0cked] link: ${cfg.main.url.full}/${itemid} | size: ${lib.formatSize(size)} | speed: ${speed}`;
         const outputmsgtg = `[f0cked] size: ${lib.formatSize(size)} | speed: ${speed}`;
@@ -272,18 +164,6 @@ export default async bot => {
               ]]
             })
           });
-          /*await e.editMessageText(msg.result.chat.id, msg.result.message_id, outputmsgtg, {
-            reply_markup: JSON.stringify({
-              inline_keyboard: [[
-                { text: 'sfw', callback_data: `b_sfw:${itemid}` },
-                { text: 'nsfw', callback_data: `b_nsfw:${itemid}` },
-                { text: 'tags', callback_data: `b_tags:${itemid}` },
-                { text: '❌ delete', callback_data: `b_delete:${itemid}` }
-              ], [
-                { text: `open f0ck #${itemid}`, url: `${cfg.main.url.full}/${itemid}` }
-              ]]
-            })
-          });*/
         }
         else {
           await e.reply(outputmsgirc);