#!/usr/bin/env python3 """Build Apify input for shephouse/youtube-transcript-scraper-bulk, then write .srt/.vtt files. prepare: read a list of YouTube URLs or IDs (one per line, or a CSV with a url / link / video column) and write input.json with startUrls plus outputFormats including srt and vtt. The Actor's default formats are only text and segments, so SRT/VTT have to be requested. split: read the Actor dataset JSON (an array, a single object, or JSONL) and write .srt / .vtt from the srt and vtt fields. The script does not invent cues from plain text or segments. Standard library only. No token or login is read or stored. Fields written into the input file are limited to ones in the Actor input schema. """ import csv, json, os, re, sys ID_RE = re.compile(r"^[A-Za-z0-9_-]{11}$") SAFE_RE = re.compile(r"^[A-Za-z0-9_-]+$") YOUTUBEISH = re.compile( r"(youtube\.com|youtu\.be|youtube-nocookie\.com|^[A-Za-z0-9_-]{11}$)", re.I, ) URL_HEADERS = { "url", "urls", "link", "links", "video", "video_url", "video url", "youtube", "youtube_url", "youtube url", "starturl", "start url", "starturls", } def video_id(u): """Return an 11-character video ID when the line is clearly one video, else None.""" u = (u or "").strip() if ID_RE.match(u): return u m = re.search( r"(?:youtube(?:-nocookie)?\.com/(?:watch\?(?:[^#]*&)?v=|shorts/|embed/|live/)|youtu\.be/)([A-Za-z0-9_-]{11})", u, re.I, ) return m.group(1) if m else None def usage(): print( "usage: python3 youtube_srt_vtt_export.py prepare URLS_FILE [INPUT_JSON]\n" " python3 youtube_srt_vtt_export.py split DATASET_JSON [OUT_DIR]\n" "prepare options: --languages en,de --max-transcripts 100 --no-auto", file=sys.stderr, ) sys.exit(2) def opt(flag, default=None): if flag in sys.argv: i = sys.argv.index(flag) if default is False: return True if i + 1 >= len(sys.argv): sys.exit(f"missing value for {flag}") return sys.argv[i + 1] return default def positional(after, n, default=None): """nth positional arg after the subcommand, skipping flags and their values.""" args = sys.argv[1:] pos = [] skip = 0 for a in args: if skip: skip -= 1 continue if a in ("--languages", "--max-transcripts"): skip = 1 continue if a.startswith("--"): continue pos.append(a) # pos[0] is subcommand if len(pos) > n: return pos[n] return default def read_urls(path): raw = open(path, encoding="utf-8-sig").read() # CSV if the first line looks like a header with a URL column first = raw.splitlines()[0] if raw.strip() else "" headers = [h.strip().strip('"').lower() for h in first.split(",")] url_col = next((h for h in headers if h in URL_HEADERS), None) urls, notes = [], [] if url_col and "," in first: with open(path, newline="", encoding="utf-8-sig") as f: for row in csv.DictReader(f): key = next(k for k in row if k.strip().lower() == url_col) u = (row.get(key) or "").strip() if u: urls.append(u) if not urls: notes.append("CSV had a URL column but no values") return urls, notes for i, line in enumerate(raw.splitlines(), 1): s = line.strip() if not s or s.startswith("#"): continue urls.append(s) return urls, notes def prepare(path, out="input.json"): urls, notes = read_urls(path) seen_lines, seen_ids, start, skipped = set(), set(), [], [] for u in urls: if u in seen_lines: skipped.append((u, "duplicate")) continue seen_lines.add(u) vid = video_id(u) if vid and vid in seen_ids: skipped.append((u, "duplicate of video " + vid)) continue if vid: seen_ids.add(vid) if not YOUTUBEISH.search(u): notes.append( f"{u}: no YouTube URL or 11-character ID; " "the Actor treats plain text as a search" ) start.append(u) if not start: sys.exit("no URLs or IDs found in " + path) langs = opt("--languages", "en") languages = [x.strip() for x in langs.split(",") if x.strip()] or ["en"] max_t = opt("--max-transcripts", "100") try: max_t = int(max_t) except ValueError: sys.exit("--max-transcripts must be an integer") if max_t < 1: sys.exit("--max-transcripts must be >= 1") payload = { "startUrls": start, "languages": languages, "outputFormats": ["text", "segments", "srt", "vtt"], "maxTranscripts": max_t, } if opt("--no-auto", False): payload["allowAutoGenerated"] = False with open(out, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) f.write("\n") print(f"{len(start)} URLs -> {out}") for u, why in skipped: print(f" skipped {u}: {why}") for n in notes: print(f" note: {n}") def load_dataset(path): text = open(path, encoding="utf-8-sig").read().strip() if not text: sys.exit(path + " is empty") if text[0] in "[{": data = json.loads(text) if isinstance(data, dict): # Apify sometimes wraps items if isinstance(data.get("items"), list): return data["items"] return [data] if isinstance(data, list): return data sys.exit("dataset JSON must be an array or an object") # JSONL items = [] for i, line in enumerate(text.splitlines(), 1): s = line.strip() if not s: continue try: items.append(json.loads(s)) except json.JSONDecodeError as e: sys.exit(f"{path} line {i}: {e}") return items def file_stem(item, index, used): vid = str(item.get("videoId") or "").strip() if vid and SAFE_RE.match(vid): stem = vid else: stem = f"row-{index:03d}" if vid: print(f" note: row {index} videoId {vid!r} is not a safe filename; wrote {stem}") else: print(f" note: row {index} has no videoId; wrote {stem}") n, candidate = 0, stem while candidate in used: n += 1 candidate = f"{stem}-{n}" if candidate != stem: print(f" note: duplicate filename {stem}; wrote {candidate}") used.add(candidate) return candidate def split(path, outdir="subtitles"): items = load_dataset(path) os.makedirs(outdir, exist_ok=True) written, skipped, used = 0, [], set() for i, item in enumerate(items, 1): if not isinstance(item, dict): skipped.append((f"row {i}", "not an object")) continue status = item.get("status") or "" srt = item.get("srt") vtt = item.get("vtt") srt_s = srt.strip() if isinstance(srt, str) else "" vtt_s = vtt.strip() if isinstance(vtt, str) else "" label = item.get("videoId") or f"row {i}" if not srt_s and not vtt_s: why = f"no srt/vtt (status={status or 'missing'}" err = item.get("errorCode") if err: why += f", {err}" why += ")" skipped.append((label, why)) continue stem = file_stem(item, i, used) if srt_s: open(os.path.join(outdir, stem + ".srt"), "w", encoding="utf-8").write( srt if srt.endswith("\n") else srt + "\n" ) written += 1 if vtt_s: open(os.path.join(outdir, stem + ".vtt"), "w", encoding="utf-8").write( vtt if vtt.endswith("\n") else vtt + "\n" ) written += 1 print(f"{written} files -> {outdir}/") for label, why in skipped: print(f" skipped {label}: {why}") def main(): if len(sys.argv) < 3: usage() cmd = sys.argv[1] if cmd == "prepare": prepare(positional("prepare", 1), positional("prepare", 2, "input.json")) elif cmd == "split": split(positional("split", 1), positional("split", 2, "subtitles")) else: usage() if __name__ == "__main__": main()