new parser

package.json

@@ -9,10 +9,12 @@
     "cloudscraper": "^1.4.0",
     "coffea": "^0.4.26",
     "du": "^0.1.0",
+    "file-type": "^3.8.0",
     "forever-monitor": "^1.7.0",
     "fs-extra": "^0.30.0",
-    "mime": "^1.3.4",
     "mysql": "^2.11.1",
+    "read-chunk": "^2.0.0",
+    "request": "^2.74.0",
     "swig": "^1.4.2",
     "uuid": "^2.0.2",
     "ytdl-core": "^0.7.17"

src/lib.js

@@ -3,7 +3,7 @@ var http = require('http');
 var https = require('https');
 var exec = require('child_process').exec;
 var crypto = require('crypto');
-var Mime = require('mime');
+var request = require('request');
 
 var bot, sql;
 
@@ -103,6 +103,7 @@ Lib.prototype.generateThumbs = () => {
     rows.forEach((e,i,a) => {
       var thumbnail = outdir+'/'+e.id+'.png';
       if(!fs.existsSync(thumbnail)) {
+        if(e.thumb == '') {
           var cmd;
           switch(e.mime) {
             case "video/mp4":
@@ -132,7 +133,17 @@ Lib.prototype.generateThumbs = () => {
               });
               break;
           }
-        
+        }
+        else {
+          var dat = fs.createWriteStream(thumbnail);
+          request(e.thumb).pipe(dat);
+          dat.on('finish', () => {
+            dat.close();
+            exec('convert '+thumbnail+' -resize "200x200^" -gravity center -crop 128x128+0+0 +repage '+thumbnail, (error) => {
+              Lib.prototype.log((error)?'failed thumbnail for '+e.id+' ('+e.mime+')':'generated thumbnail for '+e.id+' ('+e.mime+')');
+            });
+          });
+        }
       }
     });
   });

src/trigger/parser.js

@@ -2,7 +2,11 @@ var fs = require('fs-extra');
 var uuid = require('uuid');
 var path = require('path');
 var cloudscraper = require('cloudscraper');
-var Mime = require('mime');
+var readChunk = require('read-chunk');
+var fileType = require('file-type');
+var request = require('request');
+var ytdl = require('ytdl-core');
+var Readable = require('stream').Readable;
 var cfg = require('../../cfg/main.json');
 
 module.exports = (bot, lib) => {
@@ -16,71 +20,37 @@ module.exports = (bot, lib) => {
         if(!e.message.match(/\!ignore$/)) {
           var tmp = e.message.match(/https?:\/\/[\w-]+(\.[\w-]+)+\.?(:\d+)?(\/\S*)?/gi); // get links
           tmp.forEach((entry,i,a) => {
-            var tmpdest = uuid.v1().split('-')[0];
             if(!entry.match(/f0ck\.me/i)) {
-              lib.checkRepost(entry, (cbcr) => {
+              getLink(entry, (cb) => {
-                if(cbcr === true) {
+                if(cb.success === true) {
-                  cloudscraper.request({
+                  fs.move(cb.file, cb.file + '.' + cb.info.ext, (err) => {
-                    method: 'GET',
-                    url: entry,
-                    encoding: null,
-                  },
-                  (error, res, data) => {
-                    if(error)
-                      console.log('Error occurred');
-                    else {
-                      var type = res.headers['content-type'];
-                      var length = res.headers['content-length'];
-                      var dest = './b/'+tmpdest+'.'+cfg.allowedMimes[type];
-                      if(cfg.allowedMimes.hasOwnProperty(type)) {
-                        if(length <= cfg.maxFileSize) {
-                          fs.writeFile(dest, data, (err) => {
                     if(!err) {
-                              lib.getCheckSum(dest, (cbcs) => {
-                                lib.checkRepostCheckSum(cbcs, (cbcrcs) => {
-                                  if(cbcrcs === true) {
-                                    var mime = Mime.lookup(dest);
-                                    if(cfg.allowedMimes.hasOwnProperty(mime)) {
                       bot.whois(e.user.getNick(), e.network, (err, cbgu) => {
-                                        lib.sql.query("insert into `f0ck`.`items` (`src`,`dest`,`mime`,`size`,`checksum`,`username`,`userchannel`,`usernetwork`,`stamp`,`active`) values (?,?,?,?,?,?,?,?,?,?)", [
+                        lib.sql.query("insert into `f0ck`.`items` (`src`,`dest`,`mime`,`size`,`checksum`,`username`,`userchannel`,`usernetwork`,`stamp`,`active`,`thumb`) values (?,?,?,?,?,?,?,?,?,?,?)", [
                           entry,
-                                          dest,
+                          cb.file + '.' + cb.info.ext,
-                                          mime,
+                          cb.info.mime,
-                                          length,
+                          cb.size,
-                                          cbcs,
+                          cb.checksum,
                           cbgu['nick'],
                           e.channel.getName(),
                           e.network,
                           Math.floor(new Date() / 1000),
-                                          0
+                          0,
+                          (cb.info.thumb !== null)?cb.info.thumb:''
                         ]).on('result', (result) => {
                           lib.generateThumbs();
-                                          e.reply("https://f0ck.me/"+result.insertId+" - "+path.parse(entry).base+" ("+mime+", ~"+lib.formatSize(length)+") from "+cbgu['nick']+" ("+cbgu['username']+"@"+cbgu['hostname']+")");
+                          e.reply("https://f0ck.me/"+result.insertId+" - "+cb.info.title+" ("+cb.info.mime+", ~"+lib.formatSize(cb.size)+") from "+cbgu['nick']+" ("+cbgu['username']+"@"+cbgu['hostname']+")");
                         }).on('error', (msg) => {
                           e.reply(msg);
                         });
                       });
                     }
-                                    else
-                                      e.reply('lol, go f0ck yourself');
-                                  }
-                                  else
-                                    e.reply("repost motherf0cker: https://f0ck.me/"+cbcrcs);
-                                });
                   });
                 }
                 else
-                              e.reply(err);
+                  if(cb.msg != '')
-                          });
+                    e.reply(cb.msg);
-                        }
-                        else
-                          e.reply('f0ck! your file is too big (~'+lib.formatSize(length)+'), max '+lib.formatSize(cfg.maxFileSize)+' allowed');
-                      }
-                    }
-                  });
-                }
-                else
-                  e.reply('repost motherf0cker: https://f0ck.me/'+cbcr);
               });
             }
           });
@@ -89,4 +59,149 @@ module.exports = (bot, lib) => {
     },
     desc: 'muh'
   });
+  
+  var getLink = (url, cb) => {
+    var yt = /https?:\/\/(www\.)?(yotu\.be\/|youtube\.com\/)((.+\/)?(watch(\?v=|.+&v=))?(v=)?)([\w_-]{11})(&.+)?/gi;
+    var sc = /https?:\/\/(www\.)?(soundcloud\.com|snd\.sc)(\/\S*)(\/\S*)/gi;
+    lib.checkRepost(url, (cbcr) => {
+      if(cbcr === true) {
+        var tmpdest = uuid.v1().split('-')[0];
+        var dat = fs.createWriteStream('./b/' + tmpdest);
+        var info;
+        
+        if(url.match(yt)) { // ytdl
+          ytdl.getInfo(url, (err, inf) => {
+            if(!err) {
+              var title = inf.title;
+              var iurl = inf.iurl;
+              try {
+                ytdl.downloadFromInfo(inf, { filter: (format) => { return format.container === 'webm'; } })
+                  .on('response', (res) => {
+                    if(res.headers['content-length'] > cfg.maxFileSize) {
+                      res.destroy();
+                      dat.close();
+                      fs.unlinkSync('./b/' + tmpdest);
+                      cb({ success: false, msg: 'f0ck! your file is too big (~'+lib.formatSize(res.headers['content-length'])+'), max '+lib.formatSize(cfg.maxFileSize)+' allowed' });
+                    }
+                    else {
+                      info = {
+                        type: 'youtube',
+                        title: title,
+                        mime: 'video/webm',
+                        ext: 'webm',
+                        thumb: iurl
+                      };
+                    }
+                  })
+                  .on('error', (err) => {
+                    dat.close();
+                    fs.unlinkSync('./b/' + tmpdest);
+                    cb({ success: false, msg: ex });
+                  })
+                  .pipe(dat);
+              }
+              catch(ex) {
+                dat.close();
+                fs.unlinkSync('./b/' + tmpdest);
+                cb({ success: false, msg: ex });
+              }
+            }
+          });
+        }
+        else if(url.match(sc)) { // scdl
+          request('https://api.soundcloud.com/resolve.json?client_id=' + cfg.scclientid + '&url=' + url, (err, res, body) => {
+            if(!err && res.statusCode === 200) {
+              var data = JSON.parse(body);
+              request(data.stream_url + ((data.stream_url.indexOf('?') === -1)?'?':'&') + 'client_id=' + cfg.scclientid)
+                .pipe(dat);
+              info = {
+                type: 'soundcloud',
+                title: data.title,
+                mime: 'audio/mpeg',
+                ext: 'mp3',
+                thumb: (data.artwork_url !== null)?data.artwork_url.replace('large.jpg', 't300x300.jpg'):null
+              };
+            }
+            else {
+              dat.close();
+              fs.unlinkSync('./b/' + tmpdest);
+              cb({ success: false, msg: err });
+            }
+          });
+        }
+        else { // various
+          cloudscraper.request({
+            method: 'GET',
+            url: url,
+            encoding: null,
+          },
+          (err, res, data) => {
+            if(!err) {
+              var type = res.headers['content-type'];
+              var length = res.headers['content-length'];
+              if(cfg.allowedMimes.hasOwnProperty(type)) {
+                if(length <= cfg.maxFileSize) {
+                  var s = new Readable
+                  s.push(data);
+                  s.push(null);
+                  s.pipe(dat);
+                  info = {
+                    type: 'other',
+                    title: path.parse(url).base,
+                    mime: type,
+                    ext: cfg.allowedMimes[type],
+                    thumb: null
+                  };
+                }
+                else {
+                  dat.close();
+                  fs.unlinkSync('./b/' + tmpdest);
+                  cb({ success: false, msg: 'f0ck! your file is too big (~'+lib.formatSize(length)+'), max '+lib.formatSize(cfg.maxFileSize)+' allowed' });
+                }
+              }
+              else {
+                dat.close();
+                fs.unlinkSync('./b/' + tmpdest);
+                cb({ success: false, msg: '' });
+              }
+            }
+            else {
+              dat.close();
+              fs.unlinkSync('./b/' + tmpdest);
+              cb({ success: false, msg: err });
+            }
+          });
+        }
+        
+        dat
+          .on('finish', () => {
+            var size = dat.bytesWritten;
+            dat.close();
+            if(size > cfg.maxFileSize)
+              cb({ success: false, msg: 'f0ck! your file is too big (~'+lib.formatSize(size)+'), max '+lib.formatSize(cfg.maxFileSize)+' allowed' });
+            else {
+              lib.getCheckSum('./b/' + tmpdest, (cbcs) => {
+                lib.checkRepostCheckSum(cbcs, (cbcrcs) => {
+                  if(cbcrcs === true) {
+                    var mime = fileType(readChunk.sync('./b/' + tmpdest, 0, 262));
+                    if(cfg.allowedMimes.hasOwnProperty(mime.mime) || info.type === 'soundcloud')
+                      cb({ success: true, info: info, size: size, file: './b/' + tmpdest, checksum: cbcs });
+                    else
+                      cb({ success: false, msg: 'lol, go f0ck yourself ('+mime+')' });
+                  }
+                  else
+                    cb({ success: false, msg: 'repost motherf0cker: https://f0ck.me/'+cbcrcs });
+                });
+              });
+            }
+          })
+          .on('error', (err) => {
+            fs.unlinkSync('./b/' + tmpdest);
+            cb({ success: false, msg: err });
+          });
+      }
+      else
+        cb({ success: false, msg: 'repost motherf0cker: https://f0ck.me/'+cbcr });
+    });
+  };
 };

src/trigger/test.js

@@ -1,12 +1,12 @@
 module.exports = (bot, lib) => {
   lib.trigger.add({
     name: 'test',
-    call: /^\!test/,
+    call: /^!test$/i,
     level: 0,
-    active: 1,
+    active: 0,
     func: (e) => {
-      e.reply('test the best');
+      
     },
-    desc: 'test'
+    desc: 'muh'
   });
 };

src/trigger/ytdl.js

@@ -1,91 +0,0 @@
-var fs = require('fs-extra');
-var ytdl = require('ytdl-core');
-var uuid = require('uuid');
-var cfg = require('../../cfg/main.json');
-
-module.exports = (bot, lib) => {
-  lib.trigger.add({
-    name: 'ytdl',
-    call: /https?:\/\/(www\.)?(yotu\.be\/|youtube\.com\/)((.+\/)?(watch(\?v=|.+&v=))?(v=)?)([\w_-]{11})(&.+)?/gi,
-    level: 0,
-    active: 1,
-    func: (e) => {
-      if(e.channel.getName() == '#f0ck' || e.channel.getName() == '#kbot-dev') {
-        if(!e.message.match(/\!ignore/)) {
-          var tmp = e.message.match(/https?:\/\/(www\.)?(yotu\.be\/|youtube\.com\/)((.+\/)?(watch(\?v=|.+&v=))?(v=)?)([\w_-]{11})(&.+)?/gi); // get links
-          tmp.forEach((entry,i,a) => {
-            var dl = true;
-            var tmpdest = uuid.v1().split('-')[0];
-            lib.checkRepost(entry, (cbcr) => {
-              if(cbcr) {
-                ytdl.getInfo(entry, (err, info) => {
-                  if(!err) {
-                    var title = info.title;
-                    var dat = fs.createWriteStream('./b/'+tmpdest+'.webm');
-                    try {
-                      ytdl.downloadFromInfo(info, { filter: (format) => { return format.container === 'webm'; } })
-                        .on('response', (res) => {
-                          if(res.headers['content-length'] > cfg.maxFileSize) {
-                            res.destroy();
-                            dl = false;
-                            e.reply('f0ck! your file is too big (~'+lib.formatSize(res.headers['content-length'])+'), max '+lib.formatSize(cfg.maxFileSize)+' allowed');
-                          }
-                        })
-                        .on('error', (err) => {
-                          //e.reply(err);
-                        })
-                        .pipe( dat );
-                    }
-                    catch(ex) {
-                      e.reply(ex);
-                    }
-                    dat.on('finish', () => {
-                      if(dl) {
-                        dat.close();
-                          var stat = fs.statSync("./b/"+tmpdest+".webm");
-                          bot.whois(e.user.getNick(), e.network, (err, cbgu) => {
-                            lib.getCheckSum("./b/"+tmpdest+".webm", (cbcs) => {
-                              lib.checkRepostCheckSum(cbcs, (cbcrcs) => {
-                                if(cbcrcs === true) {
-                                  lib.sql.query("insert into `f0ck`.`items` (`src`,`dest`,`mime`,`size`,`checksum`,`username`,`userchannel`,`usernetwork`,`stamp`,`active`) values (?,?,?,?,?,?,?,?,?,?)", [
-                                    entry,
-                                    "./b/"+tmpdest+".webm",
-                                    "video/webm",
-                                    stat.size,
-                                    cbcs,
-                                    cbgu['nick'],
-                                    e.channel.getName(),
-                                    e.network,
-                                    Math.floor(new Date() / 1000),
-                                    0
-                                  ]).on('result', (result) => {
-                                    lib.generateThumbs();
-                                    e.reply("https://f0ck.me/"+result.insertId+" - "+title+" (video/webm, ~"+lib.formatSize(stat.size)+") from "+cbgu['nick']+" ("+cbgu['username']+"@"+cbgu['hostname']+")");
-                                  }).on('error', (msg) => {
-                                    e.reply(msg);
-                                  });
-                                }
-                                else {
-                                  fs.unlink("./b/"+tmpdest+".webm"); // delete repost
-                                  e.reply("repost motherf0cker: https://f0ck.me/"+cbcrcs);
-                                }
-                              });
-                            });
-                          });
-                      }
-                      else
-                        fs.unlink('./b/'+tmpdest+'.webm');
-                    });
-                  }
-                });
-              }
-              else
-                e.reply('repost motherf0cker');
-            });
-          });
-        }
-      }
-    },
-    desc: 'download youtubevideos'
-  });
-};