From a0ef6586847776ed7b21a1d3b204e6b59d3fa625 Mon Sep 17 00:00:00 2001 From: Kibi Kelburton Date: Tue, 2 Jun 2026 16:35:34 +0200 Subject: [PATCH] new filenames + backfill --- scripts/backfill_uuid_filenames.mjs | 243 ++++++++++++++++++++++++++++ src/avatar_handler.mjs | 3 +- src/inc/queue.mjs | 7 +- src/rethumb_handler.mjs | 3 +- 4 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 scripts/backfill_uuid_filenames.mjs diff --git a/scripts/backfill_uuid_filenames.mjs b/scripts/backfill_uuid_filenames.mjs new file mode 100644 index 0000000..1256ac1 --- /dev/null +++ b/scripts/backfill_uuid_filenames.mjs @@ -0,0 +1,243 @@ +import db from '../src/inc/sql.mjs'; +import cfg from '../src/inc/config.mjs'; +import path from 'path'; +import fs from 'fs/promises'; +import crypto from 'crypto'; + +const isDryRun = process.argv.includes('--dry-run'); +const limitArg = process.argv.indexOf('--limit'); +const limit = limitArg !== -1 ? parseInt(process.argv[limitArg + 1], 10) : null; +const BATCH_SIZE = 1000; + +async function run() { + console.log(`[BACKFILL] Starting long UUID migration & backfill script${isDryRun ? ' (DRY RUN)' : ''}${limit ? ` (Limit: ${limit})` : ''}`); + + if (!isDryRun) { + console.log('[MIGRATION] Applying database schema migrations...'); + // Drop views first because they depend on columns we want to alter + await db`DROP VIEW IF EXISTS public.items_sfw CASCADE`; + await db`DROP VIEW IF EXISTS public.items_li CASCADE`; + + // Alter dest columns in items and comment_files + await db`ALTER TABLE public.items ALTER COLUMN dest TYPE character varying(255)`; + await db`ALTER TABLE public.comment_files ALTER COLUMN dest TYPE character varying(255)`; + + // Recreate public.items_li view + await db` + CREATE OR REPLACE VIEW public.items_li AS + SELECT items.id, + items.src, + items.dest, + items.mime, + items.size, + items.checksum, + items.username, + items.userchannel, + items.usernetwork, + items.stamp + FROM ((public.items + JOIN public.tags_assign ta1 ON (((ta1.tag_id = 1) AND (ta1.item_id = items.id)))) + JOIN public.tags_assign ta2 ON (((NOT (ta2.tag_id IN ( SELECT tags_nsfp.id + FROM public.tags_nsfp))) AND (ta2.item_id = items.id)))) + WHERE items.active + GROUP BY items.id; + `; + + // Recreate public.items_sfw view + await db` + CREATE OR REPLACE VIEW public.items_sfw AS + SELECT ( SELECT + CASE + WHEN (tags_assign.tag_id > 0) THEN tags_assign.tag_id + ELSE 0 + END AS "case" + FROM public.tags_assign + WHERE ((tags_assign.tag_id = ANY (ARRAY[1, 2])) AND (tags_assign.item_id = items.id))) AS sfw, + ( SELECT + CASE + WHEN (tags_assign.tag_id > 0) THEN 1 + ELSE 0 + END AS "case" + FROM public.tags_assign + WHERE ((tags_assign.tag_id IN ( SELECT tags_nsfp.id + FROM public.tags_nsfp)) AND (tags_assign.item_id = items.id)) + LIMIT 1) AS nsfp, + id, + src, + dest, + mime, + size, + checksum, + username, + userchannel, + usernetwork, + stamp, + active + FROM public.items; + `; + console.log('[MIGRATION] Schema changes applied and views successfully recreated.'); + } + + // Map to keep track of all renamed files: old_filename -> new_filename + const renameMap = new Map(); + + // 1. Process items + console.log('[BACKFILL] Fetching items to process...'); + const items = await db`SELECT id, dest FROM items`; + console.log(`[BACKFILL] Found ${items.length} items in DB.`); + + let itemsProcessed = 0; + let itemsRenamedCount = 0; + + for (const item of items) { + if (limit && itemsProcessed >= limit) break; + itemsProcessed++; + + const ext = path.extname(item.dest); + const base = path.basename(item.dest, ext); + + // If name is already 48 characters long, skip + if (base.length === 48) { + continue; + } + + const newUuid = crypto.randomBytes(24).toString('hex'); + const newDest = `${newUuid}${ext}`; + renameMap.set(item.dest, newDest); + + // Physical files could be in active, pending or deleted folders + const pathsToCheck = [ + path.join(cfg.paths.b, item.dest), + path.join(cfg.paths.pending, 'b', item.dest), + path.join(cfg.paths.deleted, 'b', item.dest) + ]; + + let foundPath = null; + for (const p of pathsToCheck) { + try { + const stat = await fs.lstat(p); + if (!stat.isSymbolicLink()) { + foundPath = p; + break; + } + } catch (e) {} + } + + if (foundPath) { + const newPath = path.join(path.dirname(foundPath), newDest); + console.log(`[BACKFILL] Renaming item file: ${foundPath} -> ${newPath}`); + if (!isDryRun) { + await fs.rename(foundPath, newPath); + } + } else { + console.log(`[BACKFILL] [WARNING] Item physical file not found/is symlink for: ${item.dest}`); + } + + if (!isDryRun) { + await db`UPDATE items SET dest = ${newDest} WHERE id = ${item.id}`; + } + itemsRenamedCount++; + + if (itemsRenamedCount % BATCH_SIZE === 0) { + console.log(`[BACKFILL] Processed ${itemsRenamedCount} item renames...`); + } + } + + // 2. Process comment_files + console.log('[BACKFILL] Fetching comment files to process...'); + const commentFiles = await db`SELECT id, dest FROM comment_files`; + console.log(`[BACKFILL] Found ${commentFiles.length} comment files in DB.`); + + let commentsProcessed = 0; + let commentsRenamedCount = 0; + + for (const cf of commentFiles) { + if (limit && commentsProcessed >= limit) break; + commentsProcessed++; + + const ext = path.extname(cf.dest); + const base = path.basename(cf.dest, ext); + + if (base.length === 48) { + continue; + } + + const newUuid = crypto.randomBytes(24).toString('hex'); + const newDest = `${newUuid}${ext}`; + renameMap.set(cf.dest, newDest); + + const oldPath = path.join(cfg.paths.c, cf.dest); + const newPath = path.join(cfg.paths.c, newDest); + + try { + const stat = await fs.lstat(oldPath); + if (!stat.isSymbolicLink()) { + console.log(`[BACKFILL] Renaming comment file: ${oldPath} -> ${newPath}`); + if (!isDryRun) { + await fs.rename(oldPath, newPath); + } + } + } catch (e) {} + + // Rename corresponding thumbnail in t/ (cf_${old_uuid}.webp -> cf_${new_uuid}.webp) + const oldThumbPath = path.join(cfg.paths.t, `cf_${base}.webp`); + const newThumbPath = path.join(cfg.paths.t, `cf_${newUuid}.webp`); + try { + await fs.access(oldThumbPath); + console.log(`[BACKFILL] Renaming comment thumbnail: ${oldThumbPath} -> ${newThumbPath}`); + if (!isDryRun) { + await fs.rename(oldThumbPath, newThumbPath); + } + } catch (e) {} + + if (!isDryRun) { + await db`UPDATE comment_files SET dest = ${newDest} WHERE id = ${cf.id}`; + } + commentsRenamedCount++; + + if (commentsRenamedCount % BATCH_SIZE === 0) { + console.log(`[BACKFILL] Processed ${commentsRenamedCount} comment file renames...`); + } + } + + // 3. Update Symlinks in c/ + console.log('[BACKFILL] Scanning /c/ directory for symlinks to update...'); + try { + const filesInC = await fs.readdir(cfg.paths.c); + for (const f of filesInC) { + const filePath = path.join(cfg.paths.c, f); + try { + const stat = await fs.lstat(filePath); + if (stat.isSymbolicLink()) { + const target = await fs.readlink(filePath); + const targetBasename = path.basename(target); + if (renameMap.has(targetBasename)) { + const newTargetBasename = renameMap.get(targetBasename); + // Re-construct the symlink target path, pointing to public/b or public/c + const resolvedTargetAbs = path.resolve(path.dirname(filePath), target); + const resolvedTargetNewAbs = path.join(path.dirname(resolvedTargetAbs), newTargetBasename); + const relativeNewTarget = path.relative(path.dirname(filePath), resolvedTargetNewAbs); + + console.log(`[BACKFILL] Updating symlink: ${filePath} -> ${relativeNewTarget}`); + if (!isDryRun) { + await fs.unlink(filePath); + await fs.symlink(relativeNewTarget, filePath); + } + } + } + } catch (e) { + console.error(`[BACKFILL] [ERROR] Failed processing file/symlink ${filePath}:`, e.message); + } + } + } catch (e) { + console.error('[BACKFILL] Error reading /c/ directory:', e.message); + } + + console.log(`[BACKFILL] Completed. Items renamed: ${itemsRenamedCount}. Comment files renamed: ${commentsRenamedCount}.`); + process.exit(0); +} + +run().catch(err => { + console.error('[BACKFILL] Fatal error:', err); + process.exit(1); +}); diff --git a/src/avatar_handler.mjs b/src/avatar_handler.mjs index 76855ee..3068d3a 100644 --- a/src/avatar_handler.mjs +++ b/src/avatar_handler.mjs @@ -19,7 +19,8 @@ const sendJson = (res, data, code = 200) => { // Generate UUID using the same method as video uploads const genuuid = async () => { - return (await db`select gen_random_uuid() as uuid`)[0].uuid.substring(0, 8); + const raw = (await db`select replace(gen_random_uuid()::text, '-', '') || replace(gen_random_uuid()::text, '-', '') as uuid`)[0].uuid; + return raw.substring(0, 48); }; export const handleAvatarUpload = async (req, res) => { diff --git a/src/inc/queue.mjs b/src/inc/queue.mjs index daa3cee..ed32870 100644 --- a/src/inc/queue.mjs +++ b/src/inc/queue.mjs @@ -277,9 +277,10 @@ export default new class queue { }; async genuuid() { - return (await db` - select gen_random_uuid() as uuid - `)[0].uuid.substring(0, 8); + const raw = (await db` + select replace(gen_random_uuid()::text, '-', '') || replace(gen_random_uuid()::text, '-', '') as uuid + `)[0].uuid; + return raw.substring(0, 48); }; async checkrepostlink(link) { diff --git a/src/rethumb_handler.mjs b/src/rethumb_handler.mjs index 7b12b7a..04dcae8 100644 --- a/src/rethumb_handler.mjs +++ b/src/rethumb_handler.mjs @@ -104,7 +104,8 @@ export const handleRethumbUpload = async (req, res, itemId) => { } // Save to tmp for verification - const uuid = (await db`select gen_random_uuid() as uuid`)[0].uuid.substring(0, 8); + const raw = (await db`select replace(gen_random_uuid()::text, '-', '') || replace(gen_random_uuid()::text, '-', '') as uuid`)[0].uuid; + const uuid = raw.substring(0, 48); const tmpPath = path.join(cfg.paths.tmp, `rethumb_${uuid}_${itemId}`); await fs.mkdir(cfg.paths.tmp, { recursive: true });