diff --git a/backend/cli.py b/backend/cli.py index 27dd014..360d2b5 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -550,6 +550,14 @@ def cmd_process(args): print(f"Error: Video not found: {video_path}", file=sys.stderr) sys.exit(1) + # An episode with no picture is cut the same way and drawn differently. Said + # here rather than discovered later: this used to run all the way through + # Whisper and then die inside get_dimensions on "No video stream found". + from services.audiogram import is_audio_only + if is_audio_only(video_path): + print(" Audio-only episode: clips will be rendered as audiograms " + "(waveform + captions).") + # Resolve transcript from preset if not given on CLI if not args.transcript and config.get("transcript_path"): args.transcript = config["transcript_path"] diff --git a/backend/services/audiogram.py b/backend/services/audiogram.py new file mode 100644 index 0000000..7740606 --- /dev/null +++ b/backend/services/audiogram.py @@ -0,0 +1,242 @@ +"""Episodes that were never filmed. + +A podcast recorded as audio has no frame to crop, no face to follow and nothing +to burn captions onto. Everything downstream of the transcript assumes +otherwise: `get_dimensions` raises "No video stream found". It raises it after +transcription, so an hour of audio pays for its full Whisper run to be told that +it was never going to produce a clip, and the message it gets reads like a bug +in the tool rather than a fact about the file. + +The header knew all along. This is the part that reads it. +""" + +import os +from typing import Optional + +import numpy as np + +from services.media_probe import get_video_info +from utils.proc import run as proc_run + + +def _moving_picture(streams: list) -> bool: + """Whether any stream is something you could actually crop.""" + for stream in streams or []: + if stream.get("codec_type") != "video": + continue + # Cover art in an mp3 is a video stream by ffprobe's reckoning, and a + # single still is not something to crop or follow a face around. + if (stream.get("disposition") or {}).get("attached_pic"): + continue + if str(stream.get("avg_frame_rate") or "0/0").split("/")[0] in ("0", ""): + continue + return True + return False + + +def is_audio_only(path: str) -> bool: + """Positively sound, positively no picture. + + Deliberately stricter than "no video found", because this one is allowed to + stop a run. A file ffprobe cannot read at all is not evidence of anything: + it might be a stub, a truncated download, or a container this build has no + demuxer for. Refusing those with a message about audio would swap one + unhelpful failure for a differently unhelpful one, so anything short of "I + read this, it has sound, and it has no moving picture" is left to the path + it has always taken. + """ + try: + info = get_video_info(path) + except Exception: + return False + streams = info.get("streams") or [] + if not streams: + return False + if not any(stream.get("codec_type") == "audio" for stream in streams): + return False + return not _moving_picture(streams) + + +def cover_art(path: str) -> bool: + """Whether the file carries embedded artwork worth drawing behind the bars.""" + try: + info = get_video_info(path) + except Exception: + return False + return any( + (stream.get("disposition") or {}).get("attached_pic") + for stream in info.get("streams", []) or [] + ) + + +def extract_cover(path: str, out_path: str) -> Optional[str]: + """Pull the embedded artwork out, or return None if there is none.""" + if not cover_art(path): + return None + result = proc_run( + [os.environ.get("PODCLI_FFMPEG", "ffmpeg"), "-y", "-i", path, + "-an", "-vcodec", "copy", out_path, "-loglevel", "error"], + timeout=60, check=False, + ) + return out_path if result.returncode == 0 and os.path.exists(out_path) else None + + +def envelope( + audio_path: str, + start: float, + end: float, + fps: int = 30, + bars: int = 48, + wav_path: Optional[str] = None, +) -> list[list[float]]: + """Loudness per bar per frame, for the window between start and end. + + One row per rendered frame, each row `bars` values in 0..1. Computed here + rather than in the browser because these samples are already being read on + this side for moment detection, and shipping an hour of PCM into a Remotion + render to have it averaged there would be the same arithmetic somewhere + slower and harder to test. + + Root mean square rather than peak: peaks make every bar full height on any + voice that clips, and the whole point of the bars is that they move. + """ + from services.audio_events import _read_waveform_16k_mono + + if end <= start or fps <= 0 or bars <= 0: + return [] + + samples = _read_waveform_16k_mono(audio_path, wav_path=wav_path) + if samples is None or samples.size == 0: + return [] + + rate = 16_000 + first = max(0, int(start * rate)) + last = min(samples.size, int(end * rate)) + if last <= first: + return [] + + window = samples[first:last] + frames = max(1, int(round((end - start) * fps))) + per_frame = max(1, window.size // frames) + + # Trimmed to a whole number of frames and bars so the reshape is exact; the + # remainder is a fraction of one frame at the tail. + per_bar = max(1, per_frame // bars) + usable = frames * bars * per_bar + if usable > window.size: + frames = max(1, window.size // (bars * per_bar)) + usable = frames * bars * per_bar + + block = window[:usable].reshape(frames, bars, per_bar) + rms = np.sqrt(np.mean(np.square(block, dtype=np.float64), axis=2)) + + # Scaled against the loudest bar in this window rather than against full + # scale, so a quietly recorded episode still moves. + peak = float(rms.max()) + if peak <= 0: + return [[0.0] * bars for _ in range(frames)] + scaled = np.clip(rms / peak, 0.0, 1.0) + return [[round(float(v), 4) for v in row] for row in scaled] + + +def render_audiogram( + audio_path: str, + start_second: float, + end_second: float, + caption_style: str, + spec, + transcript_words: Optional[list] = None, + title: str = "clip", + output_dir: Optional[str] = None, + fps: int = 30, + bars: int = 48, + progress_callback=None, +) -> dict: + """One clip from an episode with no picture. + + Returns what generate_clip returns, because it is called in its place and + nothing above it should have to know which road the file took. + """ + import json + import shutil + import tempfile + + def say(percent, message): + if progress_callback: + progress_callback(percent, message) + + out_dir = output_dir or os.path.join(os.getcwd(), "output") + os.makedirs(out_dir, exist_ok=True) + safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in title)[:60] or "clip" + final_path = os.path.join(out_dir, f"{safe}.mp4") + + say(10, "Reading the waveform") + levels = envelope(audio_path, start_second, end_second, fps=fps, bars=bars) + if not levels: + raise ValueError(f"No audio to draw between {start_second}s and {end_second}s") + + # Word timings are absolute in the episode; the render starts at zero. + words = [ + { + "word": w.get("word", ""), + "start": max(0.0, float(w.get("start", 0)) - start_second), + "end": max(0.0, float(w.get("end", 0)) - start_second), + } + for w in (transcript_words or []) + if start_second <= float(w.get("start", 0)) < end_second + ] + + work = tempfile.mkdtemp(prefix="audiogram_") + try: + cover = extract_cover(audio_path, os.path.join(work, "cover.jpg")) + props_path = os.path.join(work, "props.json") + with open(props_path, "w", encoding="utf-8") as fh: + json.dump({ + "words": words, + "levels": levels, + "styleName": caption_style, + "bg": "#0B0B0F", + "accent": "#FFE000", + "coverSrc": cover, + "title": title if title != "clip" else None, + }, fh) + + say(35, "Drawing the waveform") + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + script = os.path.join(os.path.dirname(root), "remotion", "render-audiogram.mjs") + result = proc_run( + [os.environ.get("PODCLI_NODE", "node"), script, + "--props", props_path, "--audio", audio_path, + "--start", str(start_second), "--end", str(end_second), + "--output", final_path, + "--fps", str(fps), "--width", str(spec.width), "--height", str(spec.height)], + timeout=1800, check=False, + ) + if result.returncode != 0 or not os.path.exists(final_path): + tail = (getattr(result, "stderr", "") or "")[-600:] + raise RuntimeError(f"Audiogram render failed:\n{tail}") + finally: + shutil.rmtree(work, ignore_errors=True) + + say(100, "Done") + size_mb = round(os.path.getsize(final_path) / (1024 * 1024), 2) + return { + "output_path": final_path, + "duration": round(end_second - start_second, 2), + "file_size_mb": size_mb, + "title": title, + "start_second": start_second, + "end_second": end_second, + "caption_style": caption_style, + "crop_strategy": "audiogram", + "format": spec.name, + } + + +def audio_only_message(path: str) -> str: + """What to tell somebody whose file has no picture in it.""" + return ( + f"{os.path.basename(path)} has no video track, so there is nothing to crop " + "or burn captions onto. Clips are made from video; pass the recording that " + "has the picture in it." + ) diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py index b9118be..f978eaa 100644 --- a/backend/services/clip_generator.py +++ b/backend/services/clip_generator.py @@ -710,6 +710,25 @@ def generate_clip( spec = get_format(format) + # An episode that was never filmed takes the other road entirely. Branching + # here rather than at the five call sites means every caller of this + # function gets it, and none of them has to know the difference: what comes + # back is the same dict describing the same window. + from services.audiogram import is_audio_only + if is_audio_only(video_path): + from services.audiogram import render_audiogram + return render_audiogram( + audio_path=video_path, + start_second=start_second, + end_second=end_second, + caption_style=caption_style, + spec=spec, + transcript_words=transcript_words, + title=title, + output_dir=output_dir, + progress_callback=progress_callback, + ) + if trim_opening is None: trim_opening = not (keep_segments and len(keep_segments) > 0) diff --git a/remotion/render-audiogram.mjs b/remotion/render-audiogram.mjs new file mode 100644 index 0000000..8e6094f --- /dev/null +++ b/remotion/render-audiogram.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +/** + * Render an audiogram: captions and a moving waveform over the show's artwork, + * for an episode that has no picture of its own. + * + * The video is rendered silent and the window's audio muxed on afterwards, the + * same way render-bookend.mjs does it. Remotion could take the audio itself, + * but that means decoding the whole source inside the render for a window a + * single ffmpeg copy can cut out. + * + * Usage: + * node remotion/render-audiogram.mjs \ + * --props /path/to/props.json \ + * --audio /path/to/episode.mp3 \ + * --start 12.5 --end 45.0 \ + * --output /path/to/clip.mp4 \ + * [--fps 30] [--width 1080] [--height 1920] + * + * The props file carries words, style, levels, colours, cover and title. It is + * a file rather than a flag because the level data is one row per frame and + * would not survive an argv limit. + */ + +import { renderMedia, selectComposition } from "@remotion/renderer"; +import { getCachedBundle } from "./bundle-cache.mjs"; +import path from "path"; +import fs from "fs"; +import os from "os"; +import crypto from "crypto"; +import { spawnSync } from "child_process"; + +function parseArgs() { + const args = process.argv.slice(2); + const opts = {}; + for (let i = 0; i < args.length; i++) { + if (args[i].startsWith("--") && i + 1 < args.length) { + opts[args[i].replace(/^--/, "")] = args[i + 1]; + i++; + } + } + return opts; +} + +async function main() { + const opts = parseArgs(); + if (!opts.props || !opts.output || !opts.audio) { + console.error( + "Usage: render-audiogram.mjs --props --audio --start --end --output ", + ); + process.exit(1); + } + + const props = JSON.parse(fs.readFileSync(opts.props, "utf-8")); + const fps = parseInt(opts.fps || "30", 10); + const width = parseInt(opts.width || "1080", 10); + const height = parseInt(opts.height || "1920", 10); + const start = parseFloat(opts.start || "0"); + const end = parseFloat(opts.end || "0"); + const seconds = Math.max(0.1, end - start); + + // The levels decide the length: they were computed for this window at this + // frame rate, so trusting them keeps the bars in step with the audio rather + // than drifting a frame at a time. + const durationInFrames = props.levels?.length + ? props.levels.length + : Math.round(seconds * fps); + + const inputProps = { + words: props.words || [], + levels: props.levels || [], + audiogramBg: props.bg || "#0B0B0F", + audiogramAccent: props.accent || "#FFE000", + coverSrc: props.coverSrc, + audiogramTitle: props.title, + styleName: props.styleName || "hormozi", + singleLine: props.singleLine === true, + }; + + const bundleLocation = await getCachedBundle({ + onBundle: () => console.log(" Remotion: bundling (first run, or src/config changed)..."), + }); + const composition = await selectComposition({ + serveUrl: bundleLocation, + id: "Audiogram", + inputProps, + }); + + const seed = `${path.resolve(opts.output)}:${process.pid}`; + const id = crypto.createHash("md5").update(seed).digest("hex").slice(0, 12); + const silentVideo = path.join(os.tmpdir(), `audiogram_${id}.mp4`); + + console.log( + `Audiogram: ${durationInFrames}f @ ${fps}fps, ${width}x${height}, ` + + `${inputProps.levels.length ? inputProps.levels[0].length : 0} bars`, + ); + + await renderMedia({ + composition: { ...composition, durationInFrames, fps, width, height }, + serveUrl: bundleLocation, + codec: "h264", + outputLocation: silentVideo, + inputProps, + crf: 18, + concurrency: Math.max(2, Math.min(os.cpus().length, 8)), + }); + + const ffmpeg = process.env.PODCLI_FFMPEG || "ffmpeg"; + const mux = spawnSync( + ffmpeg, + [ + "-y", + "-i", silentVideo, + "-ss", String(start), + "-t", String(seconds), + "-i", opts.audio, + "-map", "0:v:0", + "-map", "1:a:0", + "-c:v", "copy", + "-c:a", "aac", + "-b:a", "192k", + "-ar", "44100", + "-ac", "2", + "-shortest", + "-movflags", "+faststart", + opts.output, + ], + { encoding: "utf-8" }, + ); + try { + fs.unlinkSync(silentVideo); + } catch { + // A leftover temp file is not worth failing a finished render over. + } + if (mux.status !== 0 || !fs.existsSync(opts.output)) { + console.error(`Audio mux failed:\n${(mux.stderr || "").slice(-800)}`); + process.exit(1); + } + + console.log(` ✓ ${opts.output}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/remotion/src/Audiogram.tsx b/remotion/src/Audiogram.tsx new file mode 100644 index 0000000..1749a82 --- /dev/null +++ b/remotion/src/Audiogram.tsx @@ -0,0 +1,144 @@ +import React from "react"; +import { AbsoluteFill, Img, useCurrentFrame, useVideoConfig } from "remotion"; +import { HormoziCaptions } from "./components/HormoziCaptions"; +import { KaraokeCaptions } from "./components/KaraokeCaptions"; +import { SubtleCaptions } from "./components/SubtleCaptions"; +import { BrandedCaptions } from "./components/BrandedCaptions"; +import type { Word, CaptionStyle } from "./types"; + +/** + * What an episode that was never filmed looks like. + * + * There is no frame to crop and no face to follow, so the picture has to be + * made rather than found: the show's artwork behind, the voice drawn as bars, + * and the same captions every other clip gets. + * + * The bars are handed in already reduced, one row of levels per frame, because + * the samples are read on the Python side for moment detection anyway. Shipping + * an hour of PCM into a browser to average it here would be the same arithmetic + * somewhere slower and harder to test. + */ +export interface AudiogramProps { + words: Word[]; + style: CaptionStyle; + /** One row per frame, each row a level per bar in 0..1. */ + levels: number[][]; + bg: string; + accent: string; + /** The show's artwork, when the file carried any. */ + coverSrc?: string; + title?: string; + singleLine?: boolean; +} + +/** Bars fall faster than they rise, which is what makes them read as a voice. */ +const smooth = (levels: number[][], frame: number, bar: number): number => { + const now = levels[Math.min(frame, levels.length - 1)]?.[bar] ?? 0; + const before = levels[Math.max(0, Math.min(frame, levels.length - 1) - 1)]?.[bar] ?? 0; + return now >= before ? now : before * 0.6 + now * 0.4; +}; + +export const Audiogram: React.FC = ({ + words, + style, + levels, + bg, + accent, + coverSrc, + title, + singleLine = false, +}) => { + const frame = useCurrentFrame(); + const { width, height } = useVideoConfig(); + + const CaptionComponent = { + hormozi: HormoziCaptions, + karaoke: KaraokeCaptions, + subtle: SubtleCaptions, + branded: BrandedCaptions, + }[style.name] ?? HormoziCaptions; + + const bars = levels[0]?.length ?? 0; + // The bars sit above the captions rather than behind them: a waveform under + // moving text is two things competing for the same pixels. + const bandHeight = Math.round(height * 0.16); + const barWidth = bars > 0 ? Math.max(2, Math.floor((width * 0.82) / bars / 1.6)) : 0; + const gap = bars > 0 ? Math.max(2, Math.floor((width * 0.82 - barWidth * bars) / Math.max(1, bars - 1))) : 0; + + return ( + + {coverSrc && ( + + + + )} + + {title && ( +
+ {title} +
+ )} + + {bars > 0 && ( +
+ {Array.from({ length: bars }, (_, bar) => { + const level = smooth(levels, frame, bar); + return ( +
+ ); + })} +
+ )} + + {style.name === "branded" ? ( + + ) : ( + + )} + + ); +}; diff --git a/remotion/src/Root.tsx b/remotion/src/Root.tsx index ca1545e..f04dd50 100644 --- a/remotion/src/Root.tsx +++ b/remotion/src/Root.tsx @@ -2,6 +2,7 @@ import React from "react"; import { Composition, continueRender, delayRender, getInputProps } from "remotion"; import { CaptionedClip } from "./CaptionedClip"; import { Bookend } from "./Bookend"; +import { Audiogram } from "./Audiogram"; import { STYLES } from "./types"; import type { Word } from "./types"; import type { CaptionPosition, LogoPosition } from "./types"; @@ -38,6 +39,11 @@ const inputProps = getInputProps() as { captionFontScale?: number; logoPosition?: LogoPosition; singleLine?: boolean; + levels?: number[][]; + coverSrc?: string; + audiogramBg?: string; + audiogramAccent?: string; + audiogramTitle?: string; durationInFrames?: number; fps?: number; bookendMode?: "intro" | "outro"; @@ -85,6 +91,24 @@ export const RemotionRoot: React.FC = () => { singleLine: inputProps.singleLine === true, }} /> +