testing url uplaod via api

src/upload_handler.mjs

@@ -21,6 +21,26 @@ const ARCHIVE_MIMES = new Set(
 );
 const isArchiveMime = (mime) => ARCHIVE_MIMES.has(mime);
 
+const autoTagsFromUrl = (urlString) => {
+    const tags = [];
+    try {
+        const { hostname } = new URL(urlString);
+        const host = hostname.replace(/:\d+$/, '').toLowerCase();
+        const parts = host.split('.');
+        const shortSlds = new Set(['co', 'com', 'net', 'org', 'gov', 'edu', 'ac', 'or', 'ne']);
+        let domain;
+        if (parts.length >= 3 && shortSlds.has(parts[parts.length - 2])) {
+            domain = parts.slice(-3).join('.');
+        } else {
+            domain = parts.slice(-2).join('.');
+        }
+        if (/(?:youtube\.com|youtu\.be)$/i.test(domain) || /(?:youtube\.com|youtu\.be)$/i.test(host)) {
+            tags.push('youtube');
+        }
+    } catch (e) {}
+    return [...new Set(tags)];
+};
+
 // Helper for JSON response
 const sendJson = (res, data, code = 200) => {
     res.writeHead(code, { 'Content-Type': 'application/json' });
@@ -109,15 +129,7 @@ export const handleUpload = async (req, res, self) => {
 
     try {
         const contentType = req.headers['content-type'] || '';
-        
+        let parts = {};
-        // Robust boundary extraction (handles both quoted and unquoted boundaries)
-        const boundaryMatch = contentType.match(/boundary=(?:"([^"]+)"|([^;]+))/);
-
-        if (!contentType || !contentType.includes('multipart/form-data') || !boundaryMatch) {
-            return sendJson(res, { success: false, msg: 'Invalid content type' }, 400);
-        }
-        
-        const boundary = boundaryMatch[1] || boundaryMatch[2];
 
         // Determine max file size early for collectBody
         let effectiveMaxBytes = cfg.main.maxfilesize || (150 * 1024 * 1024);
@@ -132,19 +144,56 @@ export const handleUpload = async (req, res, self) => {
             throw bodyErr;
         }
 
-        const parts = parseMultipart(body, boundary);
+        if (contentType.includes('multipart/form-data')) {
+            const boundaryMatch = contentType.match(/boundary=(?:"([^"]+)"|([^;]+))/);
+            if (!boundaryMatch) {
+                return sendJson(res, { success: false, msg: 'Invalid multipart boundary' }, 400);
+            }
+            const boundary = boundaryMatch[1] || boundaryMatch[2];
+            parts = parseMultipart(body, boundary);
+        } else if (contentType.includes('application/json')) {
+            try {
+                parts = JSON.parse(body.toString());
+            } catch (e) {
+                return sendJson(res, { success: false, msg: 'Invalid JSON body' }, 400);
+            }
+        } else if (contentType.includes('application/x-www-form-urlencoded')) {
+            const params = new URLSearchParams(body.toString());
+            for (const [key, value] of params.entries()) {
+                parts[key] = value;
+            }
+        } else {
+            return sendJson(res, { success: false, msg: 'Invalid content type' }, 400);
+        }
 
         // Validate required fields
         const file = parts.file;
+        let inputUrl = (typeof parts.url === 'string' && parts.url.trim()) ? parts.url.trim() : null;
+        if (inputUrl) {
+            try {
+                const parsed = new URL(inputUrl);
+                if (parsed.searchParams.has('igsh')) {
+                    parsed.searchParams.delete('igsh');
+                    inputUrl = parsed.toString();
+                }
+            } catch (e) {}
+        }
         const rating = parts.rating;
         const tagsRaw = parts.tags;
-        const comment = parts.comment ? parts.comment.trim() : '';
+        const comment = parts.comment ? String(parts.comment).trim() : '';
-        const rawTitle = parts.title ? parts.title.trim() : '';
+        const rawTitle = parts.title ? String(parts.title).trim() : '';
         const title = rawTitle.length > 0 ? rawTitle.substring(0, 500) : null;
 
-        const is_oc = (parts.is_oc === 'true' || parts.is_oc === '1');
+        const is_oc = (parts.is_oc === true || parts.is_oc === 'true' || parts.is_oc === '1');
+        const is_shitpost = (parts.is_shitpost === true || parts.is_shitpost === 'true' || parts.is_shitpost === '1') || cfg.websrv.shitpost_mode === true;
 
-        const is_shitpost = (parts.is_shitpost === 'true' || parts.is_shitpost === '1') || cfg.websrv.shitpost_mode === true;
+        if ((!file || !file.data) && !inputUrl) {
+            return sendJson(res, { success: false, msg: 'No file or URL provided' }, 400);
+        }
+
+        if (inputUrl && cfg.websrv.web_url_upload === false) {
+            return sendJson(res, { success: false, msg: 'URL uploads are disabled' }, 403);
+        }
 
         // Parse visibility: Header 'X-Upload-Visibility' or body field 'visibility' or user default preference
         let targetVisibility = 0;
@@ -180,7 +229,6 @@ export const handleUpload = async (req, res, self) => {
         const nowStamp = ~~(Date.now() / 1000);
         const targetExpiresAt = calculateExpiresAt(rawExpiry, nowStamp);
 
-
         // Always generate a unique item slug for the database
         const itemSlug = lib.generateSlug(11);
 
@@ -189,12 +237,6 @@ export const handleUpload = async (req, res, self) => {
             return sendJson(res, { success: false, msg: `Comment too long (max ${maxLen} characters)` }, 400);
         }
 
-        if (!file || !file.data) {
-            return sendJson(res, { success: false, msg: 'No file provided' }, 400);
-        }
-
-        // In shitpost mode, rating is optional — null means no rating tag is assigned (truly untagged).
-        // If shitpost_require_rating is configured to true, a rating is strictly required.
         const effectiveRating = (rating && ['sfw', 'nsfw', 'nsfl'].includes(rating)) ? rating : null;
 
         if (!is_shitpost && !effectiveRating) {
@@ -209,20 +251,20 @@ export const handleUpload = async (req, res, self) => {
             return sendJson(res, { success: false, msg: 'NSFL mode is currently disabled' }, 400);
         }
 
-        const tags = tagsRaw ? tagsRaw.split(',').map(t => t.trim()).filter(t => t.length > 0 && !['sfw', 'nsfw', 'nsfl'].includes(t.toLowerCase())) : [];
+        const userTags = tagsRaw ? String(tagsRaw).split(',').map(t => t.trim()).filter(t => t.length > 0 && !['sfw', 'nsfw', 'nsfl'].includes(t.toLowerCase())) : [];
+        const autoTags = (inputUrl && !file) ? autoTagsFromUrl(inputUrl) : [];
+        const tags = [...new Set([...userTags, ...autoTags])];
+
         const minTags = getMinTags();
-        // In shitpost mode, tags are optional by default — unless shitpost_min_tags is configured.
         const shitpostMinTags = is_shitpost ? (parseInt(cfg.websrv.shitpost_min_tags) || 0) : 0;
-        if (!is_shitpost && minTags > 0 && tags.length < minTags) {
+        if (!is_shitpost && minTags > 0 && userTags.length < minTags) {
             return sendJson(res, { success: false, msg: `At least ${minTags} tag${minTags !== 1 ? 's' : ''} required` }, 400);
         }
-        if (is_shitpost && shitpostMinTags > 0 && tags.length < shitpostMinTags) {
+        if (is_shitpost && shitpostMinTags > 0 && userTags.length < shitpostMinTags) {
             return sendJson(res, { success: false, msg: `At least ${shitpostMinTags} tag${shitpostMinTags !== 1 ? 's' : ''} required` }, 400);
         }
 
-        // Validate MIME type
+        // Validate MIME type for attached file
-        // cfg.allowedMimes entries can be category prefixes ("image", "video", "audio")
-        // OR exact MIME types ("application/pdf"). Entries with "/" are matched exactly.
         const allowedCats = Array.isArray(cfg.allowedMimes)
             ? cfg.allowedMimes.map(c => c.toLowerCase())
             : null;
@@ -233,27 +275,19 @@ export const handleUpload = async (req, res, self) => {
                 )
               )
             : Object.keys(cfg.mimes);
-        let mime = file.contentType;
 
-        // Browsers often don't know the SWF MIME type and send application/octet-stream or nothing.
+        if (file) {
-        // Normalize it here based on extension so the allowedMimes check doesn't spuriously reject.
+            let mime = file.contentType;
-        // The server-side `file --mime-type` check on line ~248 is the authoritative validation.
+            if ((mime === 'application/octet-stream' || !mime || mime === 'application/x-www-form-urlencoded') &&
-        if ((mime === 'application/octet-stream' || !mime || mime === 'application/x-www-form-urlencoded') &&
+                file.filename && file.filename.toLowerCase().endsWith('.swf')) {
-            file.filename && file.filename.toLowerCase().endsWith('.swf')) {
+                mime = 'application/x-shockwave-flash';
-            mime = 'application/x-shockwave-flash';
+            }
+            if (!allowedMimes.includes(mime)) {
+                return sendJson(res, { success: false, msg: `Invalid file type: ${mime}` }, 400);
+            }
         }
 
-        if (!allowedMimes.includes(mime)) {
-            return sendJson(res, { success: false, msg: `Invalid file type: ${mime}` }, 400);
-        }
-
-        // Size was already validated by collectBody (effectiveMaxBytes)
-        const size = file.data.length;
-
         let manualApproval = getManualApproval();
-
-        // Enforce manual approval for untrusted users (configurable threshold)
-        // Admins and moderators are exempt from this check
         const trustedThreshold = getTrustedUploads();
         if (trustedThreshold > 0 && !req.session.admin && !req.session.is_moderator) {
             try {
@@ -264,18 +298,13 @@ export const handleUpload = async (req, res, self) => {
                     AND is_deleted = false
                 `;
                 if (parseInt(totalUploads[0].count) < trustedThreshold) {
-                    console.log(`[UPLOAD] Forcing manual approval for new user: ${req.session.user} (Upload count: ${totalUploads[0].count}/${trustedThreshold})`);
                     manualApproval = true;
                 }
             } catch (err) {
-                console.error('[UPLOAD] Failed to check total upload count:', err);
-                // Default to manual approval on error for safety if we are unsure
                 manualApproval = true; 
             }
         }
 
-        // Rate Limit Check (if manual approval is disabled)
-        // Admins and moderators are exempt from rate limiting
         if (!manualApproval && !req.session.admin && !req.session.is_moderator) {
             const twelveHoursAgo = ~~(Date.now() / 1000) - (12 * 3600);
             const uploadCount = await db`
@@ -294,6 +323,111 @@ export const handleUpload = async (req, res, self) => {
             }
         }
 
+        // Check for YouTube Embeds when url is provided without a file attachment
+        const ytRegex = /(?:youtube\.com\/\S*(?:(?:\/e(?:mbed))?\/|watch\/?\?(?:\S*?&?v\=))|youtu\.be\/)([a-zA-Z0-9_-]{6,11})/i;
+        const ytMatch = (!file && inputUrl) ? inputUrl.match(ytRegex) : null;
+
+        if (ytMatch && cfg.websrv.enable_youtube_upload !== false) {
+            const videoId = ytMatch[1];
+            const ytUrl = `https://www.youtube.com/watch?v=${videoId}`;
+            const filename = `yt:${videoId}`;
+
+            const [{ id: itemid }] = await db`
+                insert into items ${db({
+                    src: ytUrl,
+                    dest: filename,
+                    mime: 'video/youtube',
+                    size: 0,
+                    checksum: `yt_${videoId}_${Date.now()}`,
+                    phash: null,
+                    username: req.session.user,
+                    userchannel: 'web',
+                    usernetwork: 'web',
+                    stamp: nowStamp,
+                    active: !manualApproval,
+                    is_oc: !!is_oc,
+                    title: title,
+                    visibility: targetVisibility,
+                    slug: itemSlug,
+                    expires_at: targetExpiresAt
+                }, 'src', 'dest', 'mime', 'size', 'checksum', 'phash', 'username', 'userchannel', 'usernetwork', 'stamp', 'active', 'is_oc', 'title', 'visibility', 'slug', 'expires_at')}
+                RETURNING id
+            `;
+
+            try {
+                await db`INSERT INTO comment_subscriptions (user_id, item_id) VALUES (${req.session.id}, ${itemid}) ON CONFLICT DO NOTHING`;
+            } catch (err) {}
+
+            try {
+                await queue.genThumbnail(filename, 'video/youtube', itemid, ytUrl, manualApproval);
+            } catch (err) {
+                const tDir = manualApproval ? path.join(cfg.paths.pending, 't') : cfg.paths.t;
+                await queue.spawn('magick', ['-size', '128x128', 'xc:#1a1a1a', '-gravity', 'center', '-fill', '#666', '-pointsize', '20', '-annotate', '0', 'YouTube', path.join(tDir, `${itemid}.webp`)]).catch(() => {});
+            }
+
+            if (effectiveRating) {
+                const ratingTagId = effectiveRating === 'sfw' ? 1 : (effectiveRating === 'nsfw' ? 2 : (cfg.nsfl_tag_id || 3));
+                await db`insert into tags_assign ${db({ item_id: itemid, tag_id: ratingTagId, user_id: req.session.id })} on conflict do nothing`;
+                await queue.genBlurredThumbnail(itemid, manualApproval).catch(() => {});
+            }
+
+            for (const tagName of tags) {
+                let tagRow = await db`select id from tags where normalized = slugify(${tagName}) limit 1`;
+                let tagId;
+                if (tagRow.length > 0) {
+                    tagId = tagRow[0].id;
+                } else {
+                    const newTag = await db`insert into tags ${db({ name: tagName, normalized: tagName })} returning id`;
+                    tagId = newTag[0].id;
+                }
+                await db`insert into tags_assign ${db({ item_id: itemid, tag_id: tagId, user_id: req.session.id })} on conflict do nothing`;
+            }
+
+            if (comment && comment.length > 0) {
+                try {
+                    const filteredComment = await applyWordFilter(comment);
+                    await db`INSERT INTO comments ${db({ item_id: itemid, user_id: req.session.id, content: filteredComment })}`;
+                } catch (err) {}
+            }
+
+            if (is_oc) {
+                const ocTags = ['oc', 'original content'];
+                for (const ocName of ocTags) {
+                    let tagRow = await db`select id from tags where normalized = slugify(${ocName}) limit 1`;
+                    let tagId = tagRow.length > 0 ? tagRow[0].id : (await db`insert into tags ${db({ name: ocName, normalized: ocName })} returning id`)[0].id;
+                    await db`insert into tags_assign ${db({ item_id: itemid, tag_id: tagId, user_id: req.session.id })} on conflict do nothing`;
+                }
+            }
+
+            const itemRoute = itemSlug ? `/${itemSlug}` : `/${itemid}`;
+            const successMsg = manualApproval ? 'Upload successful! Your upload is pending admin approval.' : 'Upload successful! Your upload is now live.';
+            return sendJson(res, {
+                success: true,
+                msg: successMsg,
+                itemid: itemid,
+                slug: itemSlug,
+                visibility: targetVisibility,
+                manual_approval: manualApproval,
+                redirect: !manualApproval ? itemRoute : null,
+                url: !manualApproval ? `${cfg.main.url.full}${itemRoute}` : `${cfg.main.url.full}/`,
+                file_url: null,
+                dest: filename,
+                mime: 'video/youtube',
+                tag_id: effectiveRating ? (effectiveRating === 'sfw' ? 1 : (effectiveRating === 'nsfw' ? 2 : (cfg.nsfl_tag_id || 3))) : 0,
+                is_oc: !!is_oc,
+                display_name: req.session.display_name || null,
+                username: req.session.user
+            });
+        }
+
+        // Check repost link if uploading via URL (non-YouTube)
+        if (!file && inputUrl && !getBypassDuplicateCheck()) {
+            const repostLink = await queue.checkrepostlink(inputUrl);
+            if (repostLink) {
+                return sendJson(res, { success: false, msg: 'This URL has already been uploaded', repost: repostLink }, 409);
+            }
+        }
+
         // Generate UUID & Base Paths
         const uuid = await queue.genuuid();
         const tmpPath = path.join(cfg.paths.tmp, `${uuid}.tmp`);
@@ -304,8 +438,87 @@ export const handleUpload = async (req, res, self) => {
         await fs.mkdir(path.join(cfg.paths.pending, 't'), { recursive: true });
         await fs.mkdir(path.join(cfg.paths.pending, 'ca'), { recursive: true });
 
-        // Save temporarily to detect actual MIME
+        if (file && file.data) {
-        await fs.writeFile(tmpPath, file.data);
+            await fs.writeFile(tmpPath, file.data);
+        } else if (inputUrl) {
+            const proxyArgs = (cfg.main.socks && cfg.main.socks !== 'undefined') ? ['--proxy', cfg.main.socks] : [];
+            const ytdlpArgs = ['--js-runtimes', 'node', '--geo-bypass', '--extractor-args', 'youtube:player-client=ios,web', '--newline', '--no-colors'];
+            const isInstagram = /instagram\.com/i.test(inputUrl);
+            let downloadedFile = null;
+
+            try {
+                const { stdout } = await queue.spawn('yt-dlp', [
+                    ...proxyArgs, ...ytdlpArgs,
+                    '-f', 'bv*[height<=1080]+ba/b[height<=1080] / wv*+ba/w',
+                    inputUrl,
+                    '--max-filesize', `${effectiveMaxBytes / 1024}k`,
+                    '--postprocessor-args', 'ffmpeg:-bitexact',
+                    '-o', path.join(cfg.paths.tmp, `${uuid}.%(ext)s`),
+                    '--print', 'after_move:filepath',
+                    '--merge-output-format', 'mp4'
+                ], { quiet: true });
+                downloadedFile = stdout.trim().split('\n').map(l => l.trim()).filter(l => l.length > 0).pop();
+            } catch (err1) {
+                if (isInstagram) {
+                    return sendJson(res, { success: false, msg: 'Failed to download Instagram URL' }, 400);
+                }
+                try {
+                    const { stdout } = await queue.spawn('yt-dlp', [
+                        ...proxyArgs, ...ytdlpArgs,
+                        inputUrl,
+                        '--max-filesize', `${effectiveMaxBytes / 1024}k`,
+                        '-o', path.join(cfg.paths.tmp, `${uuid}.%(ext)s`),
+                        '--print', 'after_move:filepath'
+                    ], { quiet: true });
+                    downloadedFile = stdout.trim().split('\n').map(l => l.trim()).filter(l => l.length > 0).pop();
+                } catch (err2) {
+                    const fallbackTmp = path.join(cfg.paths.tmp, `${uuid}.curl_tmp`);
+                    let referer = inputUrl;
+                    try {
+                        const parsedUrl = new URL(inputUrl);
+                        let host = parsedUrl.hostname;
+                        if (host.includes('imgur.com')) host = 'imgur.com';
+                        referer = `${parsedUrl.protocol}//${host}/`;
+                    } catch (e) {}
+
+                    const curlArgs = [
+                        '-s', '-S', '-f', '-L', inputUrl, '-o', fallbackTmp,
+                        '--max-filesize', `${effectiveMaxBytes}`,
+                        '--connect-timeout', '30',
+                        '--max-time', '300',
+                        '--user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                        '--referer', referer,
+                        '-H', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+                        '-H', 'Accept-Language: en-US,en;q=0.9'
+                    ];
+                    if (cfg.main.socks && cfg.main.socks !== 'undefined' && cfg.main.socks !== '') {
+                        const proxyHost = cfg.main.socks.includes('://') ? cfg.main.socks.split('://')[1] : cfg.main.socks;
+                        curlArgs.push('--socks5-hostname', proxyHost);
+                    }
+                    try {
+                        await queue.spawn('curl', curlArgs);
+                        downloadedFile = fallbackTmp;
+                    } catch (curlErr) {
+                        return sendJson(res, { success: false, msg: 'Failed to download file from URL' }, 400);
+                    }
+                }
+            }
+
+            let exists = false;
+            if (downloadedFile) {
+                try {
+                    await fs.access(downloadedFile);
+                    exists = true;
+                } catch (e) {}
+            }
+            if (!exists) {
+                return sendJson(res, { success: false, msg: 'Failed to download file from URL' }, 400);
+            }
+
+            if (downloadedFile !== tmpPath) {
+                await fs.rename(downloadedFile, tmpPath);
+            }
+        }
 
         // Verify actual MIME (second check after file-command detection)
         let actualMime = (await queue.spawn('file', ['--mime-type', '-b', tmpPath])).stdout.trim();
@@ -484,10 +697,18 @@ export const handleUpload = async (req, res, self) => {
 
 
         // Insert
-        const originalFilename = file.filename || null;
+        const size = file?.data ? file.data.length : (await fs.stat(destPath).then(s => s.size).catch(() => 0));
+        let originalFilename = file ? (file.filename || null) : null;
+        if (!originalFilename && inputUrl) {
+            try {
+                const parsedName = path.basename(new URL(inputUrl).pathname);
+                if (parsedName && parsedName !== '/') originalFilename = parsedName;
+            } catch (e) {}
+        }
+
         await db`
           insert into items ${db({
-            src: '',
+            src: inputUrl || '',
             dest: filename,
             mime: actualMime,
             size: size,