diff --git a/server/render.js b/server/render.js new file mode 100644 index 0000000..84b3f72 --- /dev/null +++ b/server/render.js @@ -0,0 +1,304 @@ +import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { TARGET_SECONDS } from './settings.js'; + +export const OUTPUT = { width: 1920, height: 1080, fps: 30, sampleRate: 48000 }; + +// Erhöhen, wenn sich die Filterkette ändert: dann verfallen zwischengespeicherte Segmente. +const RENDER_VERSION = 1; +const MUSIC_FADE_SECONDS = 2; + +const AUDIO_FORMAT = `aformat=sample_fmts=fltp:sample_rates=${OUTPUT.sampleRate}:channel_layouts=stereo`; +// Zwischenstände tragen unkomprimierten Ton; erst das Endergebnis bekommt AAC. +// So summieren sich über viele Übergänge keine Verschiebungen auf. +const SEGMENT_CODECS = [ + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '20', '-pix_fmt', 'yuv420p', + '-r', String(OUTPUT.fps), '-video_track_timescale', '15360', + '-c:a', 'pcm_s16le', +]; + +const hash = (value) => + crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 32); +const factor = (volume) => Math.round(volume) / 100; +const seconds = (value) => value.toFixed(3); + +function readyAsset(project, slot) { + const asset = project.assets?.[slot]; + return asset?.status === 'ready' ? asset : null; +} + +// Beschreibt, was gerendert wird – ohne Pfade, damit der Plan vergleichbar bleibt. +export function planRender(project) { + const { settings } = project; + + const bookend = (slot) => { + const asset = readyAsset(project, slot); + if (!settings[slot].enabled || !asset) return null; + return { + kind: slot, + target: { type: 'asset', slot }, + source: `${slot}:${asset.revision}`, + speed: 1, + outDuration: asset.media.duration, + audio: asset.media.hasAudio ? { volume: factor(settings[slot].volume), tempo: 1 } : null, + voice: null, + }; + }; + + const clips = project.clips + .filter((clip) => clip.status === 'ready' && !settings.excluded.includes(clip.index)) + .map((clip) => { + const { duration, hasAudio } = clip.media; + const speed = settings.speedUp && duration > TARGET_SECONDS ? duration / TARGET_SECONDS : 1; + const voice = readyAsset(project, `voice-${clip.index}`); + return { + kind: 'clip', + index: clip.index, + target: { type: 'clip', index: clip.index }, + source: `clip:${clip.index}:${clip.file}:${clip.size}`, + speed, + outDuration: duration / speed, + audio: !settings.muteClips && hasAudio ? { volume: factor(settings.clipVolume), tempo: speed } : null, + voice: voice && { + target: { type: 'asset', slot: `voice-${clip.index}` }, + source: `voice:${voice.revision}`, + volume: factor(settings.voiceVolume) * factor(settings.voiceVolumes[clip.index] ?? 100), + }, + }; + }); + + const intro = bookend('intro'); + const outro = bookend('outro'); + const segments = [intro, ...clips, outro].filter(Boolean); + for (const segment of segments) { + const { source, speed, outDuration, audio, voice } = segment; + segment.key = hash({ + version: RENDER_VERSION, + source, + speed, + outDuration, + audio, + voice: voice && { source: voice.source, volume: voice.volume }, + }); + } + + const clipsDuration = clips.reduce((sum, clip) => sum + clip.outDuration, 0); + const musicAsset = readyAsset(project, 'music'); + const music = + settings.music.enabled && musicAsset && clips.length > 0 + ? { + target: { type: 'asset', slot: 'music' }, + source: `music:${musicAsset.revision}`, + volume: factor(settings.music.volume), + duration: clipsDuration, + } + : null; + + return { + intro, + clips, + outro, + music, + clipsDuration, + duration: segments.reduce((sum, segment) => sum + segment.outDuration, 0), + key: hash({ + segments: segments.map((segment) => segment.key), + music: music && { source: music.source, volume: music.volume }, + }), + }; +} + +// atempo schafft je Stufe höchstens Faktor 2, also verketten. +function tempoFilters(tempo) { + const filters = []; + let rest = tempo; + while (rest > 2) { + filters.push('atempo=2'); + rest /= 2; + } + if (rest !== 1) filters.push(`atempo=${rest.toFixed(6)}`); + return filters; +} + +// Ein Segment: auf 1920×1080 bei 30 fps, Hochkant mit unscharfem Hintergrund, +// Ton aus Clip (oder Stille) plus Voice-over, exakt auf Segmentlänge. +export function segmentArgs(segment, inputs, output) { + const { width, height, fps } = OUTPUT; + const duration = seconds(segment.outDuration); + const speed = segment.speed === 1 ? '' : `setpts=PTS/${segment.speed.toFixed(6)},`; + + const filters = [ + `[0:v]scale=iw*sar:ih,setsar=1,${speed}fps=${fps},split=2[bgsrc][fgsrc]`, + `[bgsrc]scale=${width / 4}:${height / 4}:force_original_aspect_ratio=increase,` + + `crop=${width / 4}:${height / 4},boxblur=8:2,scale=${width}:${height}[bg]`, + `[fgsrc]scale=${width}:${height}:force_original_aspect_ratio=decrease[fg]`, + '[bg][fg]overlay=(W-w)/2:(H-h)/2,setsar=1,format=yuv420p[v]', + segment.audio + ? `[0:a]${[...tempoFilters(segment.audio.tempo), `volume=${segment.audio.volume}`, AUDIO_FORMAT].join(',')}[base]` + : `anullsrc=r=${OUTPUT.sampleRate}:cl=stereo,${AUDIO_FORMAT}[base]`, + ]; + if (segment.voice) { + filters.push(`[1:a]volume=${segment.voice.volume},${AUDIO_FORMAT}[voice]`); + filters.push('[base][voice]amix=inputs=2:duration=first:normalize=0[mixed]'); + } + filters.push(`[${segment.voice ? 'mixed' : 'base'}]apad,atrim=0:${duration}[a]`); + + return [ + '-i', inputs.video, + ...(segment.voice ? ['-i', inputs.voice] : []), + '-filter_complex', filters.join(';'), + '-map', '[v]', '-map', '[a]', '-t', duration, + ...SEGMENT_CODECS, + output, + ]; +} + +// Musik unter den Clip-Teil: wiederholt bis zum Ende, dort gekürzt und ausgeblendet. +export function musicArgs(music, inputs, output) { + const duration = seconds(music.duration); + const fade = Math.min(MUSIC_FADE_SECONDS, music.duration); + const fadeStart = seconds(Math.max(0, music.duration - fade)); + return [ + '-i', inputs.middle, + '-stream_loop', '-1', '-i', inputs.music, + '-filter_complex', + `[1:a]${AUDIO_FORMAT},volume=${music.volume},atrim=0:${duration},` + + `afade=t=out:st=${fadeStart}:d=${seconds(fade)}[m];` + + '[0:a][m]amix=inputs=2:duration=first:normalize=0[a]', + '-map', '0:v', '-map', '[a]', '-c:v', 'copy', '-c:a', 'pcm_s16le', + output, + ]; +} + +export function concatArgs(listFile, output, { final }) { + return [ + '-f', 'concat', '-safe', '0', '-i', listFile, + '-map', '0:v', '-map', '0:a', '-c:v', 'copy', + ...(final ? ['-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart'] : ['-c:a', 'copy']), + output, + ]; +} + +// Startet ffmpeg und meldet die bereits erzeugte Ausgabezeit in Sekunden. +export function runFfmpeg(args, { onTime, signal } = {}) { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(new Error('Abgebrochen.')); + const child = spawn( + 'ffmpeg', + ['-hide_banner', '-nostdin', '-y', '-v', 'error', '-progress', 'pipe:1', ...args], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + let buffer = ''; + const abort = () => child.kill('SIGKILL'); + signal?.addEventListener('abort', abort, { once: true }); + + child.stdout.on('data', (data) => { + buffer += data; + const lines = buffer.split('\n'); + buffer = lines.pop(); + for (const line of lines) { + const match = /^out_time_us=(\d+)/.exec(line); + if (match && onTime) onTime(Number(match[1]) / 1e6); + } + }); + child.stderr.on('data', (data) => { + stderr = (stderr + data).slice(-4000); + }); + child.on('error', reject); + child.on('close', (code) => { + signal?.removeEventListener('abort', abort); + if (signal?.aborted) reject(new Error('Abgebrochen.')); + else if (code === 0) resolve(); + else reject(new Error(`ffmpeg ist fehlgeschlagen: ${stderr.trim().split('\n').slice(-3).join(' | ')}`)); + }); + }); +} + +async function exists(file) { + try { + await fsp.access(file); + return true; + } catch { + return false; + } +} + +async function writeConcatList(file, entries) { + const quote = (entry) => `file '${entry.replace(/'/g, "'\\''")}'`; + await fsp.writeFile(file, `${entries.map(quote).join('\n')}\n`); +} + +// Rendert den Plan nach output. resolve(target) liefert den Pfad einer Eingabedatei. +export async function renderPlan({ plan, resolve, workDir, output, onProgress = () => {}, signal }) { + if (plan.clips.length === 0) throw new Error('Keine Clips für das Video ausgewählt.'); + + const cacheDir = path.join(workDir, 'cache'); + const tempDir = path.join(workDir, 'tmp'); + await fsp.mkdir(cacheDir, { recursive: true }); + await fsp.rm(tempDir, { recursive: true, force: true }); + await fsp.mkdir(tempDir, { recursive: true }); + + // Gewichte für die Fortschrittsanzeige: Segmente kosten fast die ganze Zeit. + const musicWeight = plan.music ? plan.clipsDuration * 0.1 : 0; + const finalWeight = plan.duration * 0.05; + const total = plan.duration + musicWeight + finalWeight; + let done = 0; + const report = (current = 0) => onProgress(Math.min(1, (done + current) / total)); + + const segmentFile = (segment) => path.join(cacheDir, `${segment.key}.mov`); + const segments = [plan.intro, ...plan.clips, plan.outro].filter(Boolean); + for (const segment of segments) { + const file = segmentFile(segment); + if (!(await exists(file))) { + const temp = path.join(tempDir, `${segment.key}.mov`); + const inputs = { + video: resolve(segment.target), + voice: segment.voice ? resolve(segment.voice.target) : null, + }; + await runFfmpeg(segmentArgs(segment, inputs, temp), { + signal, + onTime: (time) => report(Math.min(time, segment.outDuration)), + }); + await fsp.rename(temp, file); + } + done += segment.outDuration; + report(); + } + + const middleList = path.join(tempDir, 'middle.txt'); + let middle = path.join(tempDir, 'middle.mov'); + await writeConcatList(middleList, plan.clips.map(segmentFile)); + await runFfmpeg(concatArgs(middleList, middle, { final: false }), { signal }); + + if (plan.music) { + const withMusic = path.join(tempDir, 'middle-music.mov'); + await runFfmpeg(musicArgs(plan.music, { middle, music: resolve(plan.music.target) }, withMusic), { + signal, + onTime: (time) => report((Math.min(time, plan.clipsDuration) / plan.clipsDuration) * musicWeight), + }); + middle = withMusic; + done += musicWeight; + report(); + } + + const finalList = path.join(tempDir, 'final.txt'); + const parts = [plan.intro && segmentFile(plan.intro), middle, plan.outro && segmentFile(plan.outro)]; + await writeConcatList(finalList, parts.filter(Boolean)); + const tempOutput = path.join(tempDir, 'output.mp4'); + await runFfmpeg(concatArgs(finalList, tempOutput, { final: true }), { + signal, + onTime: (time) => report((Math.min(time, plan.duration) / plan.duration) * finalWeight), + }); + await fsp.rename(tempOutput, output); + onProgress(1); + + // Aufräumen: Zwischenstände und Segmente, die der aktuelle Plan nicht mehr braucht. + await fsp.rm(tempDir, { recursive: true, force: true }); + const keep = new Set(segments.map((segment) => `${segment.key}.mov`)); + for (const entry of await fsp.readdir(cacheDir)) { + if (!keep.has(entry)) await fsp.rm(path.join(cacheDir, entry), { force: true }); + } +} diff --git a/test/render.test.js b/test/render.test.js new file mode 100644 index 0000000..e8fc68b --- /dev/null +++ b/test/render.test.js @@ -0,0 +1,200 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, before, test } from 'node:test'; +import { probe } from '../server/media.js'; +import { planRender, renderPlan, segmentArgs } from '../server/render.js'; +import { defaultSettings } from '../server/settings.js'; + +const media = (duration, hasAudio = true) => ({ duration, hasVideo: true, hasAudio, width: 320, height: 180 }); + +function fakeProject(settings = {}) { + return { + clips: [ + { index: 0, file: 'a.mp4', size: 10, status: 'ready', media: media(95) }, + { index: 1, file: 'b.mp4', size: 20, status: 'ready', media: media(4, false) }, + { index: 2, file: 'c.mp4', size: 30, status: 'ready', media: media(30) }, + { index: 3, file: 'd.mp4', size: 40, status: 'pending' }, + ], + assets: { + intro: { status: 'ready', revision: 'i1', media: media(3) }, + music: { status: 'ready', revision: 'm1', media: { duration: 60, hasAudio: true } }, + 'voice-2': { status: 'ready', revision: 'v1', media: { duration: 8, hasAudio: true } }, + }, + settings: { ...defaultSettings(), ...settings }, + }; +} + +test('Lange Clips werden auf 10 Sekunden beschleunigt, kurze bleiben', () => { + const plan = planRender(fakeProject()); + assert.deepEqual(plan.clips.map((clip) => clip.index), [0, 1, 2]); + assert.equal(plan.clips[0].speed, 9.5); + assert.equal(plan.clips[0].outDuration, 10); + assert.equal(plan.clips[1].speed, 1); + assert.equal(plan.clips[1].outDuration, 4); + assert.equal(plan.clipsDuration, 24); +}); + +test('Ohne Beschleunigen behalten Clips ihre Länge', () => { + const plan = planRender(fakeProject({ speedUp: false })); + assert.equal(plan.clips[0].outDuration, 95); +}); + +test('Stummgeschaltete Clips haben keinen eigenen Ton, Intro und Outro schon', () => { + const plan = planRender(fakeProject({ intro: { enabled: true, volume: 80 } })); + assert.ok(plan.clips.every((clip) => clip.audio === null)); + assert.deepEqual(plan.intro.audio, { volume: 0.8, tempo: 1 }); +}); + +test('Mit Ton wird der Clip-Ton im selben Tempo beschleunigt', () => { + const plan = planRender(fakeProject({ muteClips: false, clipVolume: 50 })); + assert.deepEqual(plan.clips[0].audio, { volume: 0.5, tempo: 9.5 }); + assert.equal(plan.clips[1].audio, null, 'Clip ohne Tonspur'); + const args = segmentArgs(plan.clips[0], { video: 'in.mp4' }, 'out.mov').join(' '); + assert.match(args, /atempo=2,atempo=2,atempo=2,atempo=1\.187500/); +}); + +test('Voice-over-Lautstärke ist Gesamtregler mal Einzelregler', () => { + const plan = planRender(fakeProject({ voiceVolume: 50, voiceVolumes: { 2: 80 } })); + assert.equal(plan.clips[2].voice.volume, 0.4); + assert.equal(plan.clips[0].voice, null); +}); + +test('Ausgeschlossene Clips und fehlende Dateien fallen raus', () => { + const plan = planRender(fakeProject({ excluded: [0], intro: { enabled: true, volume: 100 } })); + assert.deepEqual(plan.clips.map((clip) => clip.index), [1, 2]); + assert.equal(plan.duration, 3 + 4 + 10); + const withoutAsset = fakeProject({ outro: { enabled: true, volume: 100 } }); + assert.equal(planRender(withoutAsset).outro, null); +}); + +test('Musik läuft nur unter den Clips', () => { + const plan = planRender(fakeProject({ music: { enabled: true, volume: 30 }, intro: { enabled: true, volume: 100 } })); + assert.equal(plan.music.duration, 24); + assert.equal(plan.music.volume, 0.3); +}); + +test('Musiklautstärke ändert keine Clip-Segmente, Clip-Lautstärke im Stumm-Modus auch nicht', () => { + const base = planRender(fakeProject({ music: { enabled: true, volume: 30 } })); + const louder = planRender(fakeProject({ music: { enabled: true, volume: 90 }, clipVolume: 20 })); + assert.deepEqual(louder.clips.map((c) => c.key), base.clips.map((c) => c.key)); + assert.notEqual(louder.key, base.key); + const unmuted = planRender(fakeProject({ muteClips: false })); + assert.notEqual(unmuted.clips[0].key, base.clips[0].key); +}); + +// --- Echter Schnitt mit kleinen Testvideos --- + +const hasFfmpeg = (() => { + try { + execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +})(); + +let dir; +const files = {}; +const ffmpeg = (...args) => execFileSync('ffmpeg', ['-v', 'error', '-y', ...args]); + +// Mittlere Lautstärke eines Ausschnitts in dB (volumedetect schreibt auf stderr) +function meanVolumeOf(file, start, duration) { + const { stderr } = spawnSync( + 'ffmpeg', + ['-v', 'info', '-ss', String(start), '-t', String(duration), '-i', file, '-af', 'volumedetect', '-f', 'null', '-'], + { encoding: 'utf8' }, + ); + const match = /mean_volume: (-?[\d.]+|-inf) dB/.exec(stderr); + if (!match) throw new Error(`Keine Lautstärke gemessen: ${stderr.slice(-300)}`); + return match[1] === '-inf' ? -Infinity : Number(match[1]); +} + +before(() => { + if (!hasFfmpeg) return; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fdc-render-')); + const file = (name) => (files[name] = path.join(dir, name)); + // 12 s Hochkant wie vom Handy (quer gespeichert, gedreht), mit Ton + ffmpeg('-f', 'lavfi', '-i', 'testsrc=size=320x180:rate=30', '-f', 'lavfi', '-i', 'sine=frequency=500', + '-t', '12', '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'aac', '-shortest', file('raw.mp4')); + ffmpeg('-i', files['raw.mp4'], '-c', 'copy', '-metadata:s:v:0', 'rotate=90', file('clip0.mp4')); + // 4 s quer, ohne Ton + ffmpeg('-f', 'lavfi', '-i', 'testsrc2=size=320x180:rate=25', '-t', '4', '-c:v', 'libx264', '-preset', 'ultrafast', file('clip1.mp4')); + // 2 s Intro mit Ton + ffmpeg('-f', 'lavfi', '-i', 'color=c=blue:size=320x180:rate=30', '-f', 'lavfi', '-i', 'sine=frequency=800', + '-t', '2', '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'aac', '-shortest', file('intro.mp4')); + // 3 s Musik (muss wiederholt werden) und 3 s Voice-over + ffmpeg('-f', 'lavfi', '-i', 'sine=frequency=300', '-t', '3', file('music.mp3')); + ffmpeg('-f', 'lavfi', '-i', 'sine=frequency=1000', '-t', '3', '-c:a', 'libopus', file('voice.webm')); +}); + +after(() => { + if (dir) fs.rmSync(dir, { recursive: true, force: true }); +}); + +async function realProject(settings) { + return { + clips: [ + { index: 0, file: 'clip0.mp4', size: 1, status: 'ready', media: await probe(files['clip0.mp4']) }, + { index: 1, file: 'clip1.mp4', size: 1, status: 'ready', media: await probe(files['clip1.mp4']) }, + ], + assets: { + intro: { status: 'ready', revision: 'i', media: await probe(files['intro.mp4']) }, + music: { status: 'ready', revision: 'm', media: await probe(files['music.mp3']) }, + 'voice-1': { status: 'ready', revision: 'v', media: await probe(files['voice.webm']) }, + }, + settings: { ...defaultSettings(), ...settings }, + }; +} + +const resolve = (target) => + target.type === 'clip' + ? files[`clip${target.index}.mp4`] + : { intro: files['intro.mp4'], music: files['music.mp3'], 'voice-1': files['voice.webm'] }[target.slot]; + +test('Schnitt: Intro, beschleunigter Hochkant-Clip, Voice-over und Musik', { skip: !hasFfmpeg && 'ffmpeg fehlt', timeout: 180000 }, async () => { + const workDir = path.join(dir, 'work'); + const output = path.join(dir, 'out.mp4'); + const project = await realProject({ + intro: { enabled: true, volume: 100 }, + music: { enabled: true, volume: 60 }, + }); + const progress = []; + await renderPlan({ plan: planRender(project), resolve, workDir, output, onProgress: (p) => progress.push(p) }); + + const result = await probe(output); + // 2 s Intro + 12 s auf 10 s beschleunigt + 4 s + assert.ok(Math.abs(result.duration - 16) < 0.2, `Länge ${result.duration}`); + assert.equal(result.width, 1920); + assert.equal(result.height, 1080); + assert.equal(result.hasAudio, true); + assert.equal(progress.at(-1), 1); + assert.ok(progress.every((value, i) => i === 0 || value >= progress[i - 1]), 'Fortschritt steigt'); + + // Intro hat Ton, stummgeschalteter Clip trägt nur Musik bzw. Voice-over + assert.ok(meanVolumeOf(output, 0.5, 1) > -40, 'Intro hörbar'); + assert.ok(meanVolumeOf(output, 3, 2) > -40, 'Musik unter dem stummen Clip hörbar'); + assert.ok(meanVolumeOf(output, 12.5, 1.5) > -40, 'Voice-over hörbar'); + + // Zweiter Lauf mit anderer Musiklautstärke rechnet die Segmente nicht neu + const cacheDir = path.join(workDir, 'cache'); + const before = new Map(fs.readdirSync(cacheDir).map((f) => [f, fs.statSync(path.join(cacheDir, f)).mtimeMs])); + assert.equal(before.size, 3); + project.settings.music.volume = 20; + await renderPlan({ plan: planRender(project), resolve, workDir, output }); + for (const [name, mtime] of before) { + assert.equal(fs.statSync(path.join(cacheDir, name)).mtimeMs, mtime, `${name} neu gerechnet`); + } +}); + +test('Schnitt: ohne Musik und Voice-over sind stummgeschaltete Clips still', { skip: !hasFfmpeg && 'ffmpeg fehlt', timeout: 180000 }, async () => { + const output = path.join(dir, 'still.mp4'); + const project = await realProject({}); + delete project.assets['voice-1']; + await renderPlan({ plan: planRender(project), resolve, workDir: path.join(dir, 'work2'), output }); + const result = await probe(output); + assert.ok(Math.abs(result.duration - 14) < 0.2, `Länge ${result.duration}`); + assert.ok(meanVolumeOf(output, 1, 10) < -80, 'still'); +});