add dimensions
This commit is contained in:
@@ -906,7 +906,9 @@ CREATE TABLE public.items (
|
|||||||
is_oc boolean DEFAULT false,
|
is_oc boolean DEFAULT false,
|
||||||
xd_score integer DEFAULT 0 NOT NULL,
|
xd_score integer DEFAULT 0 NOT NULL,
|
||||||
original_filename text,
|
original_filename text,
|
||||||
title text
|
title text,
|
||||||
|
width integer,
|
||||||
|
height integer
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
154
scripts/backfill_dimensions.mjs
Normal file
154
scripts/backfill_dimensions.mjs
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
/**
|
||||||
|
* Backfill script: populate width/height for existing image and video items.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node scripts/backfill_dimensions.mjs
|
||||||
|
* node scripts/backfill_dimensions.mjs --dry-run (print without writing)
|
||||||
|
* node scripts/backfill_dimensions.mjs --limit 500 (process first N items)
|
||||||
|
*
|
||||||
|
* Skips: audio, flash, PDF, YouTube items (width/height left as NULL).
|
||||||
|
* Skips: items where the physical file is missing.
|
||||||
|
* Safe to run multiple times — only processes rows where width IS NULL.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { spawn as _spawn } from 'child_process';
|
||||||
|
import db from '../src/inc/sql.mjs';
|
||||||
|
import cfg from '../src/inc/config.mjs';
|
||||||
|
import path from 'path';
|
||||||
|
import fs from 'fs/promises';
|
||||||
|
|
||||||
|
const isDryRun = process.argv.includes('--dry-run');
|
||||||
|
const limitArg = process.argv.indexOf('--limit');
|
||||||
|
const limit = limitArg !== -1 ? parseInt(process.argv[limitArg + 1], 10) : null;
|
||||||
|
const BATCH = 50;
|
||||||
|
|
||||||
|
// ---- tiny spawn helper (no shell) ----
|
||||||
|
const spawn = (cmd, args, opts = {}) =>
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
|
const child = _spawn(cmd, args, opts);
|
||||||
|
let out = '';
|
||||||
|
let err = '';
|
||||||
|
child.stdout?.on('data', d => { out += d; });
|
||||||
|
child.stderr?.on('data', d => { err += d; });
|
||||||
|
child.on('close', code => {
|
||||||
|
if (code !== 0 && !opts.ignoreExitCode) {
|
||||||
|
const e = new Error(`${cmd} exited ${code}`);
|
||||||
|
e.stderr = err;
|
||||||
|
return reject(e);
|
||||||
|
}
|
||||||
|
resolve(out);
|
||||||
|
});
|
||||||
|
child.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- probe helpers ----
|
||||||
|
async function getDimsImage(filePath) {
|
||||||
|
try {
|
||||||
|
// magick identify -format "%wx%h" reports raw pixel dimensions
|
||||||
|
const out = await spawn('magick', ['identify', '-format', '%wx%h\n', filePath + '[0]'], { ignoreExitCode: true });
|
||||||
|
// Take the first line (multi-frame GIF etc. may have many)
|
||||||
|
const line = out.trim().split('\n')[0];
|
||||||
|
const match = line.match(/^(\d+)x(\d+)$/);
|
||||||
|
if (match) return { w: parseInt(match[1], 10), h: parseInt(match[2], 10) };
|
||||||
|
} catch (_) {}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getDimsVideo(filePath) {
|
||||||
|
try {
|
||||||
|
const out = await spawn('ffprobe', [
|
||||||
|
'-v', 'error', '-select_streams', 'v:0',
|
||||||
|
'-show_entries', 'stream=width,height',
|
||||||
|
'-of', 'csv=p=0', filePath
|
||||||
|
], { ignoreExitCode: true });
|
||||||
|
const parts = out.trim().split(',');
|
||||||
|
if (parts.length < 2) return null;
|
||||||
|
const w = parseInt(parts[0], 10);
|
||||||
|
const h = parseInt(parts[1], 10);
|
||||||
|
if (w > 0 && h > 0) return { w, h };
|
||||||
|
} catch (_) {}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- main ----
|
||||||
|
async function run() {
|
||||||
|
console.log(`[BACKFILL] Starting dimension backfill${isDryRun ? ' (DRY RUN)' : ''}${limit ? ` (limit: ${limit})` : ''}`);
|
||||||
|
|
||||||
|
// Count pending
|
||||||
|
const [{ count }] = await db`
|
||||||
|
SELECT count(*) FROM items
|
||||||
|
WHERE width IS NULL
|
||||||
|
AND active = true
|
||||||
|
AND (mime LIKE 'image/%' OR (mime LIKE 'video/%' AND mime != 'video/youtube'))
|
||||||
|
`;
|
||||||
|
const total = parseInt(count, 10);
|
||||||
|
const toProcess = limit ? Math.min(total, limit) : total;
|
||||||
|
console.log(`[BACKFILL] ${total} items need backfill${limit ? `, processing up to ${toProcess}` : ''}`);
|
||||||
|
|
||||||
|
let offset = 0;
|
||||||
|
let updated = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
while (offset < toProcess) {
|
||||||
|
const rows = await db`
|
||||||
|
SELECT id, dest, mime FROM items
|
||||||
|
WHERE width IS NULL
|
||||||
|
AND active = true
|
||||||
|
AND (mime LIKE 'image/%' OR (mime LIKE 'video/%' AND mime != 'video/youtube'))
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT ${BATCH} OFFSET ${offset}
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (rows.length === 0) break;
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const filePath = path.join(cfg.paths.b, row.dest);
|
||||||
|
|
||||||
|
// Check file exists (may be deleted/purged)
|
||||||
|
try {
|
||||||
|
await fs.access(filePath);
|
||||||
|
} catch {
|
||||||
|
console.log(`[BACKFILL] SKIP #${row.id} — file missing: ${row.dest}`);
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let dims = null;
|
||||||
|
try {
|
||||||
|
if (row.mime.startsWith('image/')) {
|
||||||
|
dims = await getDimsImage(filePath);
|
||||||
|
} else if (row.mime.startsWith('video/')) {
|
||||||
|
dims = await getDimsVideo(filePath);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`[BACKFILL] PROBE ERROR #${row.id}: ${e.message}`);
|
||||||
|
failed++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dims) {
|
||||||
|
console.log(`[BACKFILL] SKIP #${row.id} — no dimensions found (${row.mime})`);
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[BACKFILL] ${isDryRun ? '[DRY]' : 'UPDATE'} #${row.id} → ${dims.w}×${dims.h}`);
|
||||||
|
if (!isDryRun) {
|
||||||
|
await db`UPDATE items SET width = ${dims.w}, height = ${dims.h} WHERE id = ${row.id}`;
|
||||||
|
}
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += rows.length;
|
||||||
|
console.log(`[BACKFILL] Progress: ${Math.min(offset, toProcess)} / ${toProcess} (updated=${updated}, skipped=${skipped}, failed=${failed})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[BACKFILL] Done. updated=${updated}, skipped=${skipped}, failed=${failed}`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch(err => {
|
||||||
|
console.error('[BACKFILL] Fatal error:', err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -550,7 +550,8 @@
|
|||||||
"direct_url": "Direkt-Link",
|
"direct_url": "Direkt-Link",
|
||||||
"view_file": "Datei anzeigen",
|
"view_file": "Datei anzeigen",
|
||||||
"metadata": "Metadaten",
|
"metadata": "Metadaten",
|
||||||
"sha256": "SHA-256-Hash"
|
"sha256": "SHA-256-Hash",
|
||||||
|
"dimensions": "Abmessungen"
|
||||||
},
|
},
|
||||||
"meme": {
|
"meme": {
|
||||||
"add_text_layer": "Textebene hinzufügen",
|
"add_text_layer": "Textebene hinzufügen",
|
||||||
|
|||||||
@@ -554,7 +554,8 @@
|
|||||||
"direct_url": "Direct URL",
|
"direct_url": "Direct URL",
|
||||||
"view_file": "View File",
|
"view_file": "View File",
|
||||||
"metadata": "Metadata",
|
"metadata": "Metadata",
|
||||||
"sha256": "SHA-256 Hash"
|
"sha256": "SHA-256 Hash",
|
||||||
|
"dimensions": "Dimensions"
|
||||||
},
|
},
|
||||||
"meme": {
|
"meme": {
|
||||||
"add_text_layer": "Add Text Layer",
|
"add_text_layer": "Add Text Layer",
|
||||||
|
|||||||
@@ -550,7 +550,8 @@
|
|||||||
"direct_url": "Directe URL",
|
"direct_url": "Directe URL",
|
||||||
"view_file": "Bestand bekijken",
|
"view_file": "Bestand bekijken",
|
||||||
"metadata": "Metadata",
|
"metadata": "Metadata",
|
||||||
"sha256": "SHA-256 Hash"
|
"sha256": "SHA-256 Hash",
|
||||||
|
"dimensions": "Afmetingen"
|
||||||
},
|
},
|
||||||
"meme": {
|
"meme": {
|
||||||
"add_text_layer": "Tekstlaag Toevoegen",
|
"add_text_layer": "Tekstlaag Toevoegen",
|
||||||
|
|||||||
@@ -551,7 +551,8 @@
|
|||||||
"direct_url": "Direktelfe",
|
"direct_url": "Direktelfe",
|
||||||
"view_file": "Datei betrachten",
|
"view_file": "Datei betrachten",
|
||||||
"metadata": "Metadaten",
|
"metadata": "Metadaten",
|
||||||
"sha256": "SHA-256-Streuwert"
|
"sha256": "SHA-256-Streuwert",
|
||||||
|
"dimensions": "Abmessungen"
|
||||||
},
|
},
|
||||||
"meme": {
|
"meme": {
|
||||||
"add_text_layer": "Textebene hinzufügen",
|
"add_text_layer": "Textebene hinzufügen",
|
||||||
|
|||||||
@@ -726,7 +726,9 @@ export default {
|
|||||||
is_comments_locked: actitem.is_comments_locked || false,
|
is_comments_locked: actitem.is_comments_locked || false,
|
||||||
is_oc: actitem.is_oc || false,
|
is_oc: actitem.is_oc || false,
|
||||||
is_repost: actitem.checksum ? actitem.checksum.includes('_bypass_') : false,
|
is_repost: actitem.checksum ? actitem.checksum.includes('_bypass_') : false,
|
||||||
reposts: repostItems
|
reposts: repostItems,
|
||||||
|
width: actitem.width || null,
|
||||||
|
height: actitem.height || null
|
||||||
},
|
},
|
||||||
title: `${actitem.id} - ${cfg.websrv.domain}`,
|
title: `${actitem.id} - ${cfg.websrv.domain}`,
|
||||||
pagination: {
|
pagination: {
|
||||||
|
|||||||
@@ -21,6 +21,10 @@ db`ALTER TABLE items ADD COLUMN IF NOT EXISTS original_filename text`.catch(() =
|
|||||||
// One-time migration: restore title column for backwards compatibility with old databases
|
// One-time migration: restore title column for backwards compatibility with old databases
|
||||||
db`ALTER TABLE items ADD COLUMN IF NOT EXISTS title text`.catch(() => {});
|
db`ALTER TABLE items ADD COLUMN IF NOT EXISTS title text`.catch(() => {});
|
||||||
|
|
||||||
|
// One-time migration: add width/height columns for image and video dimension storage
|
||||||
|
db`ALTER TABLE items ADD COLUMN IF NOT EXISTS width integer`.catch(() => {});
|
||||||
|
db`ALTER TABLE items ADD COLUMN IF NOT EXISTS height integer`.catch(() => {});
|
||||||
|
|
||||||
export const handleUpload = async (req, res, self) => {
|
export const handleUpload = async (req, res, self) => {
|
||||||
// Manual session lookup is required here because this handler is called from a
|
// Manual session lookup is required here because this handler is called from a
|
||||||
// bypass middleware that runs in parallel with the main session middleware.
|
// bypass middleware that runs in parallel with the main session middleware.
|
||||||
@@ -343,6 +347,44 @@ export const handleUpload = async (req, res, self) => {
|
|||||||
// Suffix it so the INSERT can proceed — the file is genuinely a new item entry.
|
// Suffix it so the INSERT can proceed — the file is genuinely a new item entry.
|
||||||
const insertChecksum = getBypassDuplicateCheck() ? `${checksum}_bypass_${Date.now()}` : checksum;
|
const insertChecksum = getBypassDuplicateCheck() ? `${checksum}_bypass_${Date.now()}` : checksum;
|
||||||
|
|
||||||
|
// Probe pixel dimensions for images and videos (null for audio/flash/pdf/youtube)
|
||||||
|
let itemWidth = null;
|
||||||
|
let itemHeight = null;
|
||||||
|
try {
|
||||||
|
if (actualMime.startsWith('image/')) {
|
||||||
|
// Use magick identify — handles all image formats, already present for thumbnailing
|
||||||
|
const { stdout: magickOut } = await queue.spawn('magick', [
|
||||||
|
'identify', '-format', '%wx%h\n', destPath + '[0]'
|
||||||
|
], { quiet: true, ignoreExitCode: true });
|
||||||
|
const line = magickOut.trim().split('\n')[0];
|
||||||
|
const match = line.match(/^(\d+)x(\d+)$/);
|
||||||
|
if (match) {
|
||||||
|
itemWidth = parseInt(match[1], 10);
|
||||||
|
itemHeight = parseInt(match[2], 10);
|
||||||
|
}
|
||||||
|
} else if (actualMime.startsWith('video/') && actualMime !== 'video/youtube') {
|
||||||
|
// Use ffprobe for videos — reads first video stream dimensions
|
||||||
|
const { stdout: probeOut } = await queue.spawn('ffprobe', [
|
||||||
|
'-v', 'error',
|
||||||
|
'-select_streams', 'v:0',
|
||||||
|
'-show_entries', 'stream=width,height',
|
||||||
|
'-of', 'csv=p=0',
|
||||||
|
destPath
|
||||||
|
], { quiet: true, ignoreExitCode: true });
|
||||||
|
const parts = probeOut.trim().split(',');
|
||||||
|
if (parts.length >= 2) {
|
||||||
|
const w = parseInt(parts[0], 10);
|
||||||
|
const h = parseInt(parts[1], 10);
|
||||||
|
if (!isNaN(w) && !isNaN(h) && w > 0 && h > 0) {
|
||||||
|
itemWidth = w;
|
||||||
|
itemHeight = h;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (dimErr) {
|
||||||
|
console.warn(`[UPLOAD] Dimension probe failed for ${actualMime} (non-fatal):`, dimErr.message);
|
||||||
|
}
|
||||||
|
|
||||||
// Insert
|
// Insert
|
||||||
const originalFilename = file.filename || null;
|
const originalFilename = file.filename || null;
|
||||||
await db`
|
await db`
|
||||||
@@ -360,9 +402,10 @@ export const handleUpload = async (req, res, self) => {
|
|||||||
active: !manualApproval,
|
active: !manualApproval,
|
||||||
is_oc: is_oc,
|
is_oc: is_oc,
|
||||||
original_filename: originalFilename,
|
original_filename: originalFilename,
|
||||||
title: title
|
title: title,
|
||||||
}, 'src', 'dest', 'mime', 'size', 'checksum', 'phash', 'username', 'userchannel', 'usernetwork', 'stamp', 'active', 'is_oc', 'original_filename', 'title')
|
width: itemWidth,
|
||||||
}
|
height: itemHeight
|
||||||
|
}, 'src', 'dest', 'mime', 'size', 'checksum', 'phash', 'username', 'userchannel', 'usernetwork', 'stamp', 'active', 'is_oc', 'original_filename', 'title', 'width', 'height')}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const itemid = await queue.getItemID(filename);
|
const itemid = await queue.getItemID(filename);
|
||||||
|
|||||||
@@ -222,6 +222,12 @@
|
|||||||
<th>{{ t('info_modal.file_size') || 'File Size' }}</th>
|
<th>{{ t('info_modal.file_size') || 'File Size' }}</th>
|
||||||
<td>{{ item.size }}</td>
|
<td>{{ item.size }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
@if(item.width && item.height)
|
||||||
|
<tr>
|
||||||
|
<th>{{ t('info_modal.dimensions') || 'Dimensions' }}</th>
|
||||||
|
<td>{{ item.width }} × {{ item.height }}</td>
|
||||||
|
</tr>
|
||||||
|
@endif
|
||||||
<tr>
|
<tr>
|
||||||
<th>{{ t('info_modal.mime_type') || 'MIME Type' }}</th>
|
<th>{{ t('info_modal.mime_type') || 'MIME Type' }}</th>
|
||||||
<td><code>{{ item.mime }}</code></td>
|
<td><code>{{ item.mime }}</code></td>
|
||||||
|
|||||||
@@ -196,6 +196,12 @@
|
|||||||
<th>{{ t('info_modal.file_size') || 'File Size' }}</th>
|
<th>{{ t('info_modal.file_size') || 'File Size' }}</th>
|
||||||
<td>{{ item.size }}</td>
|
<td>{{ item.size }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
@if(item.width && item.height)
|
||||||
|
<tr>
|
||||||
|
<th>{{ t('info_modal.dimensions') || 'Dimensions' }}</th>
|
||||||
|
<td>{{ item.width }} × {{ item.height }}</td>
|
||||||
|
</tr>
|
||||||
|
@endif
|
||||||
<tr>
|
<tr>
|
||||||
<th>{{ t('info_modal.mime_type') || 'MIME Type' }}</th>
|
<th>{{ t('info_modal.mime_type') || 'MIME Type' }}</th>
|
||||||
<td><code>{{ item.mime }}</code></td>
|
<td><code>{{ item.mime }}</code></td>
|
||||||
|
|||||||
Reference in New Issue
Block a user