reconvert all emojis to webp

config/magick-policy/policy.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE policymap [
+<!ELEMENT policymap (policy)*>
+<!ATTLIST policymap xmlns CDATA #FIXED "">
+<!ELEMENT policy EMPTY>
+<!ATTLIST policy xmlns CDATA #FIXED "">
+<!ATTLIST policy domain NMTOKEN #REQUIRED>
+<!ATTLIST policy name NMTOKEN #IMPLIED>
+<!ATTLIST policy pattern CDATA #IMPLIED>
+<!ATTLIST policy rights NMTOKEN #IMPLIED>
+<!ATTLIST policy stealth NMTOKEN #IMPLIED>
+<!ATTLIST policy value CDATA #IMPLIED>
+]>
+<policymap>
+  <policy domain="resource" name="thread" value="2"/>
+  <policy domain="resource" name="time" value="120"/>
+  <policy domain="resource" name="file" value="768"/>
+  <policy domain="resource" name="memory" value="768MiB"/>
+  <policy domain="resource" name="map" value="2GiB"/>
+  <policy domain="resource" name="area" value="8KP"/>
+  <policy domain="resource" name="disk" value="2GiB"/>
+  <policy domain="resource" name="list-length" value="65536"/>
+  <policy domain="resource" name="width" value="8KP"/>
+  <policy domain="resource" name="height" value="8KP"/>
+  <policy domain="cache" name="memory-map" value="anonymous"/>
+  <policy domain="filter" rights="none" pattern="*"/>
+  <policy domain="path" rights="none" pattern="fd:*"/>
+  <policy domain="path" rights="none" pattern="/etc/*"/>
+  <policy domain="path" rights="read" pattern="/etc/IM*"/>
+  <policy domain="path" rights="none" pattern="*../*"/>
+  <policy domain="path" rights="none" pattern="@*"/>
+  <policy domain="module" rights="write" pattern="{MSL,MVG,PS,SVG,TXT,URL,XPS}"/>
+  <policy domain="coder" rights="none" pattern="EPHEMERAL" />
+  <policy domain="coder" rights="none" pattern="URL" />
+  <policy domain="coder" rights="none" pattern="HTTPS" />
+  <policy domain="coder" rights="none" pattern="MVG" />
+  <policy domain="coder" rights="none" pattern="MSL" />
+  <policy domain="coder" rights="none" pattern="TEXT" />
+  <policy domain="coder" rights="none" pattern="SHOW" />
+  <policy domain="coder" rights="none" pattern="WIN" />
+  <policy domain="coder" rights="none" pattern="PLT" />
+  <policy domain="coder" rights="write" pattern="PS" />
+  <policy domain="coder" rights="write" pattern="PS2" />
+  <policy domain="coder" rights="write" pattern="PS3" />
+  <policy domain="coder" rights="write" pattern="PDF" />
+  <policy domain="coder" rights="write" pattern="XPS" />
+  <policy domain="coder" rights="write" pattern="PCL" />
+</policymap>

scripts/reconvert_emojis.mjs

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+/**
+ * Reconvert all existing emoji images to WebP.
+ * - Converts non-WebP files (PNG, GIF, JPG, etc.) to lossless WebP via ImageMagick.
+ * - Keeps transparency. Handles animated GIFs via -coalesce.
+ * - Updates the `url` column in `custom_emojis` to the new .webp path.
+ * - Deletes the original file after successful conversion.
+ *
+ * Usage:
+ *   STORAGE_DIR=f0ckm-data DB_HOST=localhost DB_PORT=5454 node scripts/reconvert_emojis.mjs
+ */
+
+import db from "../src/inc/sql.mjs";
+import cfg from "../src/inc/config.mjs";
+import { promises as fs } from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { execFile as _execFile } from "child_process";
+import { promisify } from "util";
+
+const execFile = promisify(_execFile);
+
+// Custom IM policy that raises list-length from 128 → 65536 to handle large animated GIFs.
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const MAGICK_POLICY_PATH = path.resolve(__dirname, '../config/magick-policy');
+const magickEnv = { ...process.env, MAGICK_CONFIGURE_PATH: MAGICK_POLICY_PATH };
+
+const emojisDir = cfg.paths.emojis;
+console.log(`[reconvert_emojis] Emoji directory: ${emojisDir}`);
+
+let converted = 0;
+let skipped = 0;
+let errors = 0;
+
+try {
+    const emojis = await db`SELECT id, name, url FROM custom_emojis ORDER BY id`;
+    console.log(`[reconvert_emojis] Found ${emojis.length} emojis in DB.\n`);
+
+    for (const emoji of emojis) {
+        const { id, name, url } = emoji;
+
+        // Only handle locally hosted emojis (not external URLs)
+        if (!url || !url.startsWith('/s/emojis/')) {
+            console.log(`[${id}] "${name}" — skipping (external URL: ${url})`);
+            skipped++;
+            continue;
+        }
+
+        const filename = path.basename(url);
+        const ext = path.extname(filename).toLowerCase().replace('.', '');
+
+        if (ext === 'webp') {
+            // Verify the file actually exists, then skip
+            const fullPath = path.join(emojisDir, filename);
+            try {
+                await fs.access(fullPath);
+                console.log(`[${id}] "${name}" — already WebP, skipping.`);
+            } catch {
+                console.warn(`[${id}] "${name}" — already WebP in DB but file missing: ${fullPath}`);
+            }
+            skipped++;
+            continue;
+        }
+
+        const originalPath = path.join(emojisDir, filename);
+        const webpFilename = filename.replace(/\.[^.]+$/, '.webp');
+        const webpPath = path.join(emojisDir, webpFilename);
+        const newUrl = `/s/emojis/${webpFilename}`;
+
+        try {
+            await fs.access(originalPath);
+        } catch {
+            console.warn(`[${id}] "${name}" — original file missing: ${originalPath}, skipping.`);
+            skipped++;
+            continue;
+        }
+
+        try {
+            console.log(`[${id}] "${name}" — converting ${ext.toUpperCase()} → WebP …`);
+            // -coalesce: handles animated GIFs properly
+            // -quality 80: lossy WebP — ~50-70% smaller than GIF, transparency preserved
+            await execFile('magick', [originalPath, '-coalesce', '-quality', '80', webpPath], { env: magickEnv });
+
+            // Verify output exists and is non-empty
+            const stat = await fs.stat(webpPath);
+            if (!stat || stat.size === 0) throw new Error('Output file is empty');
+
+            // Update DB
+            await db`UPDATE custom_emojis SET url = ${newUrl} WHERE id = ${id}`;
+
+            // Remove original
+            await fs.unlink(originalPath);
+
+            console.log(`[${id}] "${name}" — ✓ converted (${(stat.size / 1024).toFixed(1)} KB) → ${newUrl}`);
+            converted++;
+        } catch (err) {
+            console.error(`[${id}] "${name}" — ✗ FAILED: ${err.message}`);
+            // Clean up partial output if it exists
+            await fs.unlink(webpPath).catch(() => {});
+            errors++;
+        }
+    }
+
+    // Notify connected clients of updated emoji list
+    if (converted > 0) {
+        await db`NOTIFY emojis_updated, '{}'`;
+        console.log(`\n[reconvert_emojis] Notified clients of emoji update.`);
+    }
+
+    console.log(`\n[reconvert_emojis] Done.`);
+    console.log(`  Converted : ${converted}`);
+    console.log(`  Skipped   : ${skipped}`);
+    console.log(`  Errors    : ${errors}`);
+
+    process.exit(errors > 0 ? 1 : 0);
+} catch (err) {
+    console.error('[reconvert_emojis] Fatal error:', err);
+    process.exit(1);
+}

src/emoji_upload_handler.mjs

@@ -4,6 +4,16 @@ import lib from "./inc/lib.mjs";
 import cfg from "./inc/config.mjs";
 import { parseMultipart, collectBody } from "./inc/multipart.mjs";
 import path from "path";
+import { fileURLToPath } from "url";
+import { execFile as _execFile } from "child_process";
+import { promisify } from "util";
+
+const execFile = promisify(_execFile);
+
+// Custom IM policy that raises list-length from 128 → 65536 to handle large animated GIFs.
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const MAGICK_POLICY_PATH = path.resolve(__dirname, '../config/magick-policy');
+const magickEnv = { ...process.env, MAGICK_CONFIGURE_PATH: MAGICK_POLICY_PATH };
 
 const sendJson = (res, data, code = 200) => {
     res.writeHead(code, { 'Content-Type': 'application/json' });
@@ -70,20 +80,37 @@ export const handleEmojiUpload = async (req, res) => {
 
         const file = parts.file;
         if (file && file.data && file.data.length > 0) {
+            const randSuffix = Math.random().toString(36).substring(7);
             const extMatch = file.filename.match(/\.([a-z0-9]+)$/i);
-            const ext = extMatch ? extMatch[1].toLowerCase() : 'png';
+            const originalExt = extMatch ? extMatch[1].toLowerCase() : 'png';
-            const filename = `${name}_${Math.random().toString(36).substring(7)}.${ext}`;
 
-            // Emojis go to public/s/emojis
+            const webpFilename = `${name}_${randSuffix}.webp`;
-            const filePath = path.join(cfg.paths.emojis, filename);
+            const webpPath = path.join(cfg.paths.emojis, webpFilename);
-            console.error(`[BOOT] [EMOJI HANDLER] Writing file to: ${filePath} (Size: ${file.data.length})`);
+
-            await fs.writeFile(filePath, file.data);
+            if (originalExt === 'webp') {
+                // Already WebP — write directly
+                console.error(`[BOOT] [EMOJI HANDLER] Writing WebP directly to: ${webpPath} (Size: ${file.data.length})`);
+                await fs.writeFile(webpPath, file.data);
+            } else {
+                // Write original to a temp file, then convert to WebP via magick
+                const tmpFilename = `${name}_${randSuffix}_tmp.${originalExt}`;
+                const tmpPath = path.join(cfg.paths.emojis, tmpFilename);
+                console.error(`[BOOT] [EMOJI HANDLER] Writing temp file: ${tmpPath} (Size: ${file.data.length})`);
+                await fs.writeFile(tmpPath, file.data);
+                try {
+                    // -coalesce handles animated GIFs; quality 80 = lossy WebP (good visual quality, ~50-70% smaller than GIF)
+                    await execFile('magick', [tmpPath, '-coalesce', '-quality', '80', webpPath], { env: magickEnv });
+                    console.error(`[BOOT] [EMOJI HANDLER] Converted to WebP: ${webpPath}`);
+                } finally {
+                    await fs.unlink(tmpPath).catch(() => {});
+                }
+            }
 
             // Verify write
-            const exists = (await fs.stat(filePath)).size > 0;
+            const stat = await fs.stat(webpPath);
-            if (!exists) throw new Error("File write verification failed");
+            if (!stat || stat.size === 0) throw new Error("File write/conversion verification failed");
 
-            url = `/s/emojis/${filename}`;
+            url = `/s/emojis/${webpFilename}`;
         }
 
         if (!url) {

src/inc/routes/emojis.mjs

@@ -1,6 +1,18 @@
 import db from "../sql.mjs";
-
 import lib from "../lib.mjs";
+import cfg from "../config.mjs";
+import { promises as fs } from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { execFile as _execFile } from "child_process";
+import { promisify } from "util";
+
+const execFile = promisify(_execFile);
+
+// Custom IM policy that raises list-length from 128 → 65536 to handle large animated GIFs.
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const MAGICK_POLICY_PATH = path.resolve(__dirname, '../../../config/magick-policy');
+const magickEnv = { ...process.env, MAGICK_CONFIGURE_PATH: MAGICK_POLICY_PATH };
 
 export default (router, tpl) => {
 
@@ -25,7 +37,75 @@ export default (router, tpl) => {
         }
     });
 
+    // Reconvert all existing emoji files to WebP (Admin only)
+    router.post('/api/v2/admin/emojis/reconvert', async (req, res) => {
+        if (!req.session || !req.session.admin) {
+            return res.reply({ code: 403, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ success: false, message: "Forbidden" }) });
+        }
+        const csrfToken = req.headers['x-csrf-token'];
+        if (!req.session.csrf_token || !csrfToken || csrfToken !== req.session.csrf_token) {
+            return res.reply({ code: 403, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ success: false, message: "Invalid CSRF token" }) });
+        }
 
+        try {
+            const emojis = await db`SELECT id, name, url FROM custom_emojis ORDER BY id`;
+            const emojisDir = cfg.paths.emojis;
+            let converted = 0, skipped = 0, errors = 0;
+            const errorList = [];
+
+            for (const emoji of emojis) {
+                const { id, name, url } = emoji;
+
+                // Only handle locally hosted emojis
+                if (!url || !url.startsWith('/s/emojis/')) { skipped++; continue; }
+
+                const filename = path.basename(url);
+                const ext = path.extname(filename).toLowerCase().replace('.', '');
+
+                if (ext === 'webp') { skipped++; continue; }
+
+                const originalPath = path.join(emojisDir, filename);
+                const webpFilename = filename.replace(/\.[^.]+$/, '.webp');
+                const webpPath = path.join(emojisDir, webpFilename);
+                const newUrl = `/s/emojis/${webpFilename}`;
+
+                try {
+                    await fs.access(originalPath);
+                } catch {
+                    console.warn(`[EMOJI RECONVERT] Missing file for emoji "${name}": ${originalPath}`);
+                    skipped++;
+                    continue;
+                }
+
+                try {
+                    await execFile('magick', [originalPath, '-coalesce', '-quality', '80', webpPath], { env: magickEnv });
+                    const stat = await fs.stat(webpPath);
+                    if (!stat || stat.size === 0) throw new Error('Output file is empty');
+                    await db`UPDATE custom_emojis SET url = ${newUrl} WHERE id = ${id}`;
+                    await fs.unlink(originalPath);
+                    console.log(`[EMOJI RECONVERT] Converted "${name}": ${filename} → ${webpFilename}`);
+                    converted++;
+                } catch (err) {
+                    console.error(`[EMOJI RECONVERT] Failed "${name}":`, err.message);
+                    await fs.unlink(webpPath).catch(() => {});
+                    errorList.push(`${name}: ${err.message}`);
+                    errors++;
+                }
+            }
+
+            if (converted > 0) {
+                await db`NOTIFY emojis_updated, '{}'`;
+            }
+
+            return res.reply({
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ success: true, converted, skipped, errors, errorList })
+            });
+        } catch (e) {
+            console.error('[EMOJI RECONVERT] Fatal error:', e);
+            return res.reply({ code: 500, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ success: false, message: e.message }) });
+        }
+    });
 
     // Delete emoji (Admin only)
     router.post(/\/api\/v2\/admin\/emojis\/(?<id>\d+)\/delete/, async (req, res) => {

views/admin/emojis.html

@@ -24,6 +24,12 @@
             </div>
         </div>
 
+        <div style="margin-bottom: 20px; display: flex; align-items: center; gap: 12px; flex-wrap: wrap;">
+            <button id="reconvert-webp" class="btn-upload" style="width: auto; padding: 7px 18px; border: 1px solid var(--nav-border-color); background: var(--bg); color: var(--white); cursor: pointer;">
+                Reconvert All to WebP
+            </button>
+            <span id="reconvert-status" style="font-size: 0.85em; opacity: 0.8;"></span>
+        </div>
 
         <div id="emoji-list" class="emoji-grid">
             <!-- Populated by JS -->
@@ -124,6 +130,39 @@
             const btnAddEmoji = document.getElementById('add-emoji');
             if (btnAddEmoji) btnAddEmoji.addEventListener('click', addEmoji);
 
+            // Reconvert all emojis to WebP
+            const reconvertEmojis = async () => {
+                const btn = document.getElementById('reconvert-webp');
+                const status = document.getElementById('reconvert-status');
+                if (!btn || !status) return;
+                if (!confirm('Reconvert all non-WebP emojis to WebP? This will delete the originals.')) return;
+
+                btn.disabled = true;
+                status.textContent = '\u23F3 Converting\u2026';
+
+                try {
+                    const csrf = '{{ csrf_token }}';
+                    const res = await fetch('/api/v2/admin/emojis/reconvert', {
+                        method: 'POST',
+                        headers: { 'X-CSRF-Token': csrf, 'X-Requested-With': 'XMLHttpRequest' }
+                    });
+                    const result = await res.json();
+                    if (result.success) {
+                        status.textContent = '\u2705 Done \u2014 converted: ' + result.converted + ', skipped: ' + result.skipped + ', errors: ' + result.errors;
+                        if (result.converted > 0) loadEmojis();
+                    } else {
+                        status.textContent = '\u274C Failed: ' + (result.message || 'Unknown error');
+                    }
+                } catch (err) {
+                    status.textContent = '\u274C Error: ' + err.message;
+                } finally {
+                    btn.disabled = false;
+                }
+            };
+
+            const btnReconvert = document.getElementById('reconvert-webp');
+            if (btnReconvert) btnReconvert.addEventListener('click', reconvertEmojis);
+            
             // Live Update Listener (SSE dispatched via f0ckm.js)
             document.addEventListener('f0ck:emojis_updated', loadEmojis);