From 8cdd4fa69a2b90613c2511eb1f138eb4af336384 Mon Sep 17 00:00:00 2001 From: Kibi Kelburton Date: Mon, 10 Aug 2026 17:36:10 +0200 Subject: [PATCH] add import script --- docker-compose.yml | 1 + package.json | 3 +- scripts/import.mjs | 529 +++++++++++++++++++++++++++++++++++++++++++++ src/inc/config.mjs | 15 +- 4 files changed, 546 insertions(+), 2 deletions(-) create mode 100644 scripts/import.mjs diff --git a/docker-compose.yml b/docker-compose.yml index 02acf22..d62a57b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,7 @@ services: - ./f0ckm-data/hall_cache/:/opt/f0ckm/public/hall_cache/:Z - ./f0ckm-data/hall_custom/:/opt/f0ckm/public/hall_custom/:Z - ./f0ckm-data/koepfe/:/opt/f0ckm/public/s/koepfe/:Z + - ./f0ckm-data/import/:/opt/f0ckm/f0ckm-data/import/:Z - ./f0ckm-data/manifest.json:/opt/f0ckm/public/manifest.json:Z environment: diff --git a/package.json b/package.json index 6be573d..22df826 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "build": "node scripts/build-css.mjs", "copy-vendor": "node scripts/copy-vendor.mjs", "seed": "node scripts/seed.mjs", - "create-admin": "node scripts/create-admin.mjs" + "create-admin": "node scripts/create-admin.mjs", + "import": "STORAGE_DIR=f0ckm-data DB_HOST=localhost DB_PORT=5454 node scripts/import.mjs" }, "author": "Kibi Kelburton", "license": "MIT", diff --git a/scripts/import.mjs b/scripts/import.mjs new file mode 100644 index 0000000..2a9e942 --- /dev/null +++ b/scripts/import.mjs @@ -0,0 +1,529 @@ +import fs from "fs"; +import path from "path"; +import db from "../src/inc/sql.mjs"; +import lib from "../src/inc/lib.mjs"; +import cfg from "../src/inc/config.mjs"; +import queue from "../src/inc/queue.mjs"; +import f0cklib from "../src/inc/routeinc/f0cklib.mjs"; +import { getBypassDuplicateCheck } from "../src/inc/settings.mjs"; + +// Helper for parsing CLI flags +function parseArgs() { + const args = process.argv.slice(2); + const options = { + user: null, + rating: null, + tags: [], + dir: cfg.paths.import || path.resolve("f0ckm-data/import"), + keep: false, + bypass: null, // null means use config default, true = force bypass, false = force check + is_oc: false, + title: null, + help: false + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--help' || arg === '-h') { + options.help = true; + } else if (arg === '--user' || arg === '-u') { + options.user = args[++i]; + } else if (arg === '--rating' || arg === '-r') { + options.rating = args[++i]?.toLowerCase(); + } else if (arg === '--tags' || arg === '-t') { + const rawTags = args[++i]; + if (rawTags) { + options.tags = rawTags.split(',').map(t => t.trim()).filter(Boolean); + } + } else if (arg === '--dir' || arg === '-d') { + options.dir = path.resolve(args[++i]); + } else if (arg === '--keep' || arg === '-k') { + options.keep = true; + } else if (arg === '--bypass' || arg === '-b') { + options.bypass = true; + } else if (arg === '--check') { + options.bypass = false; + } else if (arg === '--oc') { + options.is_oc = true; + } else if (arg === '--title') { + options.title = args[++i]; + } + } + + return options; +} + +function showHelp() { + console.log(` +f0ckm Bulk Import Script +======================== +Imports all media files in f0ckm-data/import (or custom directory) into f0ckm. + +Usage: + node scripts/import.mjs [options] + +Options: + --user, -u User to attribute uploads to (defaults to first admin) + --rating, -r Default rating for imported items + --tags, -t Global tags to assign to imported items + --dir, -d Directory to scan (default: f0ckm-data/import) + --keep, -k Keep source files in import dir after import (default: delete) + --bypass, -b Force bypass of duplicate checksum/pHash checks + --check Force duplicate checking even if config enables bypass + --oc Mark all imported files as Original Content (OC) + --title Set title for imported files + --help, -h Show this help screen + +Folder & Sidecar JSON Features: + - Rating Subfolders: Files in 'import/sfw/', 'import/nsfw/', or 'import/nsfl/' automatically receive that rating. + - Sidecar Metadata: Place a file.jpg.json or file.json alongside file.jpg to supply metadata: + { "title": "My Post", "rating": "sfw", "tags": ["funny", "cat"], "comment": "First comment", "is_oc": true } +`); +} + +// Derive archive MIME types from cfg.mimes +const ARCHIVE_MIMES = new Set( + Object.entries(cfg.mimes || {}) + .filter(([mime, ext]) => mime.startsWith('application/') && !['swf', 'pdf'].includes(ext)) + .map(([mime]) => mime) +); +const isArchiveMime = (mime) => ARCHIVE_MIMES.has(mime); + +// Recursively collect all target files in directory (follows symlinked dirs and files) +async function collectFiles(dir, visitedDirs = new Set()) { + const results = []; + + let canonicalDir; + try { + canonicalDir = await fs.promises.realpath(dir); + } catch (_) { + canonicalDir = dir; + } + + if (visitedDirs.has(canonicalDir)) { + return results; // Avoid infinite loops on circular symlink references + } + visitedDirs.add(canonicalDir); + + let entries; + try { + entries = await fs.promises.readdir(dir); + } catch (e) { + console.warn(`[IMPORT] Warning: unable to read directory '${dir}':`, e.message); + return results; + } + + for (const name of entries) { + if (name.startsWith('.')) continue; // skip hidden files & .gitkeep + const fullPath = path.join(dir, name); + + try { + // fs.promises.stat follows symlinks to inspect the underlying file/dir + const st = await fs.promises.stat(fullPath); + + if (st.isDirectory()) { + const subResults = await collectFiles(fullPath, visitedDirs); + results.push(...subResults); + } else if (st.isFile()) { + if (name.endsWith('.json')) continue; // skip sidecar metadata files + results.push(fullPath); + } + } catch (err) { + console.warn(`[IMPORT] Warning: unable to stat '${fullPath}' (broken symlink?):`, err.message); + } + } + return results; +} + +// Read optional sidecar JSON metadata +async function readSidecarMetadata(filePath) { + const candidates = [ + `${filePath}.json`, + path.join(path.dirname(filePath), `${path.parse(filePath).name}.json`) + ]; + + for (const cand of candidates) { + try { + if (fs.existsSync(cand)) { + const raw = await fs.promises.readFile(cand, 'utf-8'); + return { meta: JSON.parse(raw), sidecarPath: cand }; + } + } catch (e) { + console.warn(`[IMPORT] Warning: failed to parse sidecar JSON at ${cand}:`, e.message); + } + } + return { meta: {}, sidecarPath: null }; +} + +async function runImport() { + const opts = parseArgs(); + + if (opts.help) { + showHelp(); + process.exit(0); + } + + console.log(`--- f0ckm Bulk Import Starting ---`); + console.log(`Scan Directory: ${opts.dir}`); + + if (!fs.existsSync(opts.dir)) { + console.error(`[IMPORT ERROR] Directory does not exist: ${opts.dir}`); + process.exit(1); + } + + // 1. Resolve Target User + let targetUser = null; + if (opts.user) { + const userRows = await db` + SELECT id, "user", login, admin, is_moderator + FROM "user" + WHERE LOWER(login) = ${opts.user.toLowerCase()} OR LOWER("user") = ${opts.user.toLowerCase()} + LIMIT 1 + `; + if (userRows.length > 0) { + targetUser = userRows[0]; + } else { + console.error(`[IMPORT ERROR] Specified user '${opts.user}' not found in database.`); + process.exit(1); + } + } else { + const adminRows = await db` + SELECT id, "user", login, admin, is_moderator + FROM "user" + WHERE admin = true + ORDER BY id ASC + LIMIT 1 + `; + if (adminRows.length > 0) { + targetUser = adminRows[0]; + } else { + const anyUser = await db`SELECT id, "user", login, admin, is_moderator FROM "user" ORDER BY id ASC LIMIT 1`; + if (anyUser.length > 0) { + targetUser = anyUser[0]; + } else { + console.error(`[IMPORT ERROR] No users found in database. Please run npm run create-admin first.`); + process.exit(1); + } + } + } + + console.log(`Attributing uploads to user: ${targetUser.user} (ID: ${targetUser.id})`); + + // Ensure target directories exist + await fs.promises.mkdir(cfg.paths.b, { recursive: true }); + await fs.promises.mkdir(cfg.paths.t, { recursive: true }); + await fs.promises.mkdir(cfg.paths.ca, { recursive: true }); + + // 2. Discover files + const fileList = await collectFiles(opts.dir); + + if (fileList.length === 0) { + console.log(`[IMPORT] No files found in ${opts.dir}. Place files in '${opts.dir}' and re-run.`); + process.exit(0); + } + + console.log(`Found ${fileList.length} file(s) to process.\n`); + + const allowedCats = Array.isArray(cfg.allowedMimes) ? cfg.allowedMimes.map(c => c.toLowerCase()) : null; + const allowedMimes = allowedCats + ? Object.keys(cfg.mimes).filter(m => allowedCats.some(cat => cat.includes('/') ? m === cat : m.startsWith(`${cat}/`))) + : Object.keys(cfg.mimes); + + let successCount = 0; + let skippedCount = 0; + let failedCount = 0; + + // 3. Process each file sequentially + for (let index = 0; index < fileList.length; index++) { + const filePath = fileList[index]; + const baseName = path.basename(filePath); + const relPath = path.relative(opts.dir, filePath); + const prefix = `[${index + 1}/${fileList.length}]`; + + try { + // Check sidecar JSON metadata + const { meta: sidecarMeta, sidecarPath } = await readSidecarMetadata(filePath); + + // Determine Rating: Sidecar -> CLI option -> Folder name (import/sfw/ etc.) -> Default + let effectiveRating = sidecarMeta.rating || opts.rating; + if (!effectiveRating) { + const parentDirName = path.basename(path.dirname(filePath)).toLowerCase(); + if (['sfw', 'nsfw', 'nsfl', 'untagged'].includes(parentDirName)) { + effectiveRating = parentDirName; + } + } + if (effectiveRating === 'untagged') effectiveRating = null; + if (!effectiveRating && !cfg.websrv.shitpost_mode) { + effectiveRating = 'sfw'; // Default fallback if rating required + } + + // Determine Title & Comments & OC + const effectiveTitle = sidecarMeta.title || opts.title || null; + const effectiveComment = sidecarMeta.comment || null; + const effectiveIsOc = (sidecarMeta.is_oc !== undefined) ? !!sidecarMeta.is_oc : opts.is_oc; + const effectiveVisibility = (typeof sidecarMeta.visibility === 'number') ? sidecarMeta.visibility : 0; + + // Collect tags + let mergedTags = [...opts.tags]; + if (Array.isArray(sidecarMeta.tags)) { + mergedTags.push(...sidecarMeta.tags); + } else if (typeof sidecarMeta.tags === 'string') { + mergedTags.push(...sidecarMeta.tags.split(',').map(t => t.trim())); + } + mergedTags = mergedTags.filter(t => t && !['sfw', 'nsfw', 'nsfl'].includes(t.toLowerCase())); + + // Verify file stats & MIME + const stat = await fs.promises.stat(filePath); + if (stat.size === 0) { + console.warn(`${prefix} [SKIP] ${relPath} (0 bytes)`); + skippedCount++; + continue; + } + + // Detect actual MIME via file command (with -L to follow symlinks) + let actualMime = (await queue.spawn('file', ['-L', '--mime-type', '-b', filePath])).stdout.trim(); + + if ((actualMime === 'application/octet-stream' || !actualMime) && baseName.toLowerCase().endsWith('.swf')) { + actualMime = 'application/x-shockwave-flash'; + } + + if (!allowedMimes.includes(actualMime)) { + console.warn(`${prefix} [SKIP] ${relPath} - Unsupported MIME type: '${actualMime}'`); + skippedCount++; + continue; + } + + // Phase A: SHA-256, PHash, Dimensions, Audio-MP4 probe + const needsPHash = !isArchiveMime(actualMime); + const needsDims = !isArchiveMime(actualMime); + const needsAudioMP4 = (actualMime === 'video/mp4' || actualMime === 'video/quicktime'); + + const [checksum, phash, dimResult, audioMp4Result] = await Promise.all([ + queue.spawn('sha256sum', [filePath]).then(r => r.stdout.trim().split(' ')[0]), + needsPHash ? queue.generatePHash(filePath).catch(e => { console.error(`[IMPORT] PHash error:`, e.message); return null; }) : Promise.resolve(null), + needsDims ? (async () => { + try { + if (actualMime.startsWith('image/')) { + const { stdout: magickOut } = await queue.spawn('magick', ['identify', '-format', '%wx%h\n', filePath + '[0]'], { quiet: true, ignoreExitCode: true }); + const match = magickOut.trim().split('\n')[0].match(/^(\d+)x(\d+)$/); + if (match) return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) }; + } else if (actualMime.startsWith('video/') && actualMime !== 'video/youtube') { + const { stdout: probeOut } = await queue.spawn('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height', '-of', 'csv=p=0', filePath], { quiet: true, ignoreExitCode: true }); + const dimParts = probeOut.trim().split(','); + if (dimParts.length >= 2) { + const w = parseInt(dimParts[0], 10); + const h = parseInt(dimParts[1], 10); + if (!isNaN(w) && !isNaN(h) && w > 0 && h > 0) return { width: w, height: h }; + } + } + } catch (_) {} + return null; + })() : Promise.resolve(null), + needsAudioMP4 ? (async () => { + const ext = baseName.split('.').pop().toLowerCase(); + if (['m4a', 'aac'].includes(ext)) return 'audio/mp4'; + try { + const probe = await queue.spawn('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=codec_type', '-of', 'csv=p=0', filePath]); + if (!probe.stdout.trim()) return 'audio/mp4'; + } catch (_) {} + return null; + })() : Promise.resolve(null) + ]); + + if (audioMp4Result) { + actualMime = audioMp4Result; + } + + // Phase B: Duplicate check + const bypassDupes = (opts.bypass !== null) ? opts.bypass : (getBypassDuplicateCheck() || cfg.websrv.bypass_duplicate_check === true); + if (!bypassDupes) { + const [repostBySum, repostByPhash] = await Promise.all([ + queue.checkrepostsum(checksum), + phash ? queue.checkrepostphash(phash) : Promise.resolve(null) + ]); + + if (repostBySum) { + console.warn(`${prefix} [SKIP] ${relPath} - Duplicate file (matches existing Item #${repostBySum})`); + skippedCount++; + continue; + } + if (repostByPhash) { + console.warn(`${prefix} [SKIP] ${relPath} - Visual duplicate (matches existing Item #${repostByPhash})`); + skippedCount++; + continue; + } + } + + // Generate target filename & UUID + const uuid = await queue.genuuid(); + const ext = cfg.mimes[actualMime] || 'bin'; + const filename = `${uuid}.${ext}`; + const destPath = path.join(cfg.paths.b, filename); + const nowStamp = ~~(Date.now() / 1000); + const itemSlug = lib.generateSlug(11); + + // Copy file to public/b + await fs.promises.copyFile(filePath, destPath); + + const insertChecksum = bypassDupes ? `${checksum}_bypass_${Date.now()}` : checksum; + + // Insert into items table + await db` + INSERT INTO items ${db({ + src: '', + dest: filename, + mime: actualMime, + size: stat.size, + checksum: insertChecksum, + phash: phash, + username: targetUser.user, + userchannel: 'import_script', + usernetwork: 'local', + stamp: nowStamp, + active: true, + is_oc: effectiveIsOc, + original_filename: baseName, + title: effectiveTitle, + width: dimResult?.width ?? null, + height: dimResult?.height ?? null, + visibility: effectiveVisibility, + slug: itemSlug, + expires_at: null + }, 'src', 'dest', 'mime', 'size', 'checksum', 'phash', 'username', 'userchannel', 'usernetwork', 'stamp', 'active', 'is_oc', 'original_filename', 'title', 'width', 'height', 'visibility', 'slug', 'expires_at')} + `; + + const itemid = await queue.getItemID(filename); + + // Subscribe uploader to comments + try { + await db` + INSERT INTO comment_subscriptions (user_id, item_id) + VALUES (${targetUser.id}, ${itemid}) + ON CONFLICT DO NOTHING + `; + } catch (_) {} + + // Generate thumbnails + try { + await queue.genThumbnail(filename, actualMime, itemid, '', false, 512); + if (actualMime.startsWith('audio/') && queue._lastCoverExtracted) { + await db`UPDATE items SET has_coverart = TRUE WHERE id = ${itemid}`; + } + } catch (err) { + console.warn(`${prefix} [WARN] Thumbnail generation warning for #${itemid}:`, err.message); + } + + // Blurred thumbnail + await queue.genBlurredThumbnail(itemid, false); + + // Assign Rating Tag + if (effectiveRating) { + const ratingTagId = effectiveRating === 'sfw' ? 1 : (effectiveRating === 'nsfw' ? 2 : (cfg.nsfl_tag_id || 3)); + await db` + INSERT INTO tags_assign ${db({ item_id: itemid, tag_id: ratingTagId, user_id: targetUser.id })} + `; + } + + // Assign Custom Tags + for (const tagName of mergedTags) { + let tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagName}) LIMIT 1`; + if (tagRow.length === 0) { + await db`INSERT INTO tags ${db({ tag: tagName }, 'tag')}`; + tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagName}) LIMIT 1`; + } + const tagId = tagRow[0].id; + await db` + INSERT INTO tags_assign ${db({ item_id: itemid, tag_id: tagId, user_id: targetUser.id })} + ON CONFLICT DO NOTHING + `; + } + + // Auto OC tags + if (effectiveIsOc) { + for (const tagname of ['oc', 'original content']) { + let tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagname}) LIMIT 1`; + if (tagRow.length === 0) { + await db`INSERT INTO tags ${db({ tag: tagname }, 'tag')}`; + tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagname}) LIMIT 1`; + } + await db` + INSERT INTO tags_assign ${db({ item_id: itemid, tag_id: tagRow[0].id, user_id: targetUser.id })} + ON CONFLICT DO NOTHING + `; + } + } + + // Auto Flash tags + if (actualMime === 'application/x-shockwave-flash' || actualMime === 'application/vnd.adobe.flash.movie') { + for (const tagname of ['Flash', 'SWF']) { + let tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagname}) LIMIT 1`; + if (tagRow.length === 0) { + await db`INSERT INTO tags ${db({ tag: tagname }, 'tag')}`; + tagRow = await db`SELECT id FROM tags WHERE normalized = slugify(${tagname}) LIMIT 1`; + } + await db` + INSERT INTO tags_assign ${db({ item_id: itemid, tag_id: tagRow[0].id, user_id: targetUser.id })} + ON CONFLICT DO NOTHING + `; + } + } + + // Optional first comment + if (effectiveComment && effectiveComment.trim().length > 0) { + await db` + INSERT INTO comments ${db({ + item_id: itemid, + user_id: targetUser.id, + content: effectiveComment.trim() + })} + `; + } + + // PostgreSQL Live Notification + try { + await db`SELECT pg_notify('new_item', ${JSON.stringify({ + id: itemid, + dest: filename, + mime: actualMime, + username: targetUser.user, + display_name: targetUser.display_name || null, + tag_id: effectiveRating ? (effectiveRating === 'sfw' ? 1 : (effectiveRating === 'nsfw' ? 2 : (cfg.nsfl_tag_id || 3))) : 0, + is_oc: !!effectiveIsOc + })})`; + } catch (_) {} + + // Clear cache + f0cklib.clearCountCache(); + + // Cleanup imported source file unless --keep was passed + if (!opts.keep) { + await fs.promises.unlink(filePath).catch(() => {}); + if (sidecarPath) { + await fs.promises.unlink(sidecarPath).catch(() => {}); + } + } + + console.log(`${prefix} [SUCCESS] Imported '${relPath}' -> Item #${itemid} (Rating: ${effectiveRating || 'untagged'}, MIME: ${actualMime})`); + successCount++; + + } catch (err) { + console.error(`${prefix} [ERROR] Failed to import '${relPath}':`, err.message); + failedCount++; + } + } + + console.log(`\n========================================`); + console.log(`Import Complete!`); + console.log(`Total Scanned : ${fileList.length}`); + console.log(`Successful : ${successCount}`); + console.log(`Skipped : ${skippedCount}`); + console.log(`Failed : ${failedCount}`); + console.log(`========================================`); + + process.exit(0); +} + +runImport().catch(err => { + console.error(`[IMPORT FATAL ERROR]`, err); + process.exit(1); +}); diff --git a/src/inc/config.mjs b/src/inc/config.mjs index 6662acd..0284ba3 100644 --- a/src/inc/config.mjs +++ b/src/inc/config.mjs @@ -16,6 +16,18 @@ if (process.env.NODE_ENV === 'production') { config.main.development = false; } +if (config.main.development) { + if (!process.env.DB_HOST && (config.sql.host === 'f0ckm-db' || !config.sql.host)) { + config.sql.host = 'localhost'; + } + if (!process.env.DB_PORT && config.sql.port === 5432) { + config.sql.port = 5454; + } + if (!process.env.STORAGE_DIR) { + process.env.STORAGE_DIR = 'f0ckm-data'; + } +} + // Set timezone from config if not already set via environment variable if (!process.env.TZ && config.main.timezone) { process.env.TZ = config.main.timezone; @@ -57,7 +69,8 @@ config.paths = { pending: resolvePath('pending'), deleted: resolvePath('deleted'), logs: resolvePath('logs'), - tmp: resolvePath('tmp') + tmp: resolvePath('tmp'), + import: storage ? path.resolve(path.join(path.resolve(storage), 'import')) : path.resolve(path.join(base, 'f0ckm-data/import')) }; export default config;