#!/usr/bin/env python3 """ myrient-dl.py — Download US/English ROMs from Myrient and sort into a RomM-style library. Usage: python3 myrient-dl.py # download all platforms (Phase 1 first) python3 myrient-dl.py gb gba nes # download specific platforms python3 myrient-dl.py --list # show platforms and estimated sizes python3 myrient-dl.py --dry-run gb # show what would be downloaded """ import sys import os import re import logging import argparse import urllib.request import urllib.error import urllib.parse from html.parser import HTMLParser from pathlib import Path # ── Configuration ───────────────────────────────────────────────────────────── # Where ROMs land (one subfolder per platform key) and where the log goes. LIBRARY_DIR = "./roms" LOG_FILE = "./myrient-dl.log" MYRIENT_BASE = "https://myrient.erista.me/files" # key: (collection, directory name on Myrient, estimated size, apply region filter) PLATFORMS = { # Phase 1 — Small/cartridge systems "gb": ("No-Intro", "Nintendo - Game Boy", "~2 GB", True), "gbc": ("No-Intro", "Nintendo - Game Boy Color", "~2 GB", True), "gba": ("No-Intro", "Nintendo - Game Boy Advance", "~5 GB", True), "nes": ("No-Intro", "Nintendo - Nintendo Entertainment System (Headered)", "~1 GB", True), "snes": ("No-Intro", "Nintendo - Super Nintendo Entertainment System", "~2 GB", True), "n64": ("No-Intro", "Nintendo - Nintendo 64 (BigEndian)", "~3 GB", True), "genesis": ("No-Intro", "Sega - Mega Drive - Genesis", "~1 GB", True), # Phase 2 — Medium "nds": ("No-Intro", "Nintendo - Nintendo DS (Decrypted)", "~50 GB", True), # Phase 3 — Large/disc "ps": ("Redump", "Sony - PlayStation", "~500 GB", True), "psp": ("Redump", "Sony - PlayStation Portable", "~500 GB", True), "dreamcast": ("Redump", "Sega - Dreamcast", "~100 GB", True), "gc": ("Redump", "Nintendo - GameCube - NKit RVZ [zstd-19-128k]", "~200 GB", True), # Phase 4 — Very large / curated "ps2": ("Redump", "Sony - PlayStation 2", "~500 GB-1 TB (curated 250)", True), "wii": ("Redump", "Nintendo - Wii - NKit RVZ [zstd-19-128k]", "~150 GB (curated 35)", True), "wiiu": ("Redump", "Nintendo - Wii U - WUX", "~400 GB (curated 50)", True), # Phase 5 — Handhelds (large) "3ds": ("No-Intro", "Nintendo - Nintendo 3DS (Decrypted)", "~300 GB", True), "saturn": ("Redump", "Sega - Saturn", "~120 GB", True), "xbox": ("Redump", "Microsoft - Xbox", "~70 GB (curated 10)", True), # Arcade "arcade": ("FinalBurn Neo", "arcade", "~30 GB", False), } PHASE_ORDER = ["gb", "gbc", "gba", "nes", "snes", "n64", "genesis", "nds", "ps", "psp", "dreamcast", "gc", "ps2", "wii", "wiiu", "3ds", "arcade"] # ── Curated lists ───────────────────────────────────────────────────────────── # Each entry is a substring matched case-insensitively against Redump filenames. PS2_CURATED = [ "Grand Theft Auto - San Andreas", "Grand Theft Auto - Vice City", "Grand Theft Auto III", "Grand Theft Auto - Liberty City Stories", "God of War II", "God of War (USA)", # anchored to avoid matching II "Devil May Cry 3", "Devil May Cry 2", "Devil May Cry (USA)", "Okami", "Ookami", "Shadow of the Colossus", "Ico (", "Prince of Persia - The Sands of Time", "Prince of Persia - Warrior Within", "Prince of Persia - The Two Thrones", "Bully", "Psychonauts", "Beyond Good & Evil", "Beyond Good and Evil", "Onimusha - Warlords", "Onimusha 2", "Onimusha 3", "Onimusha - Dawn of Dreams", "Viewtiful Joe (USA)", "Viewtiful Joe 2", "Katamari Damacy", "We Love Katamari", "God Hand", "Killer7", "Simpsons, The - Hit & Run", "Simpsons, The - Hit and Run", "Spider-Man 2 (USA)", "Manhunt (USA)", "Tomb Raider - Legend", "Tomb Raider - Anniversary", "Maximo - Ghosts to Glory", "Maximo vs. Army of Zin", "Freedom Fighters", "Psi-Ops", "Warriors, The", "Mark of Kri", "Indigo Prophecy", "Odin Sphere", "Jak and Daxter - The Precursor Legacy", "Jak II", "Jak 3", "Jak X - Combat Racing", "Ratchet & Clank (USA)", "Ratchet & Clank - Going Commando", "Ratchet & Clank - Up Your Arsenal", "Ratchet - Deadlocked", "Sly Cooper and the Thievius Raccoonus", "Sly 2 - Band of Thieves", "Sly 3 - Honor Among Thieves", "Klonoa 2", "Ape Escape 2", "Ape Escape 3", "Rayman 2 - Revolution", "Final Fantasy X (", "Final Fantasy X-2", "Final Fantasy XII", "Kingdom Hearts (USA)", "Kingdom Hearts II", "Kingdom Hearts - Re-Chain of Memories", "Dragon Quest VIII", "Persona 3 FES", "Persona 4", "Shin Megami Tensei - Nocturne", "Shin Megami Tensei - Digital Devil Saga", "Shin Megami Tensei - Digital Devil Saga 2", "Dark Cloud (USA)", "Dark Cloud 2", "Rogue Galaxy", "Disgaea - Hour of Darkness", "Disgaea 2", "Xenosaga Episode I", "Xenosaga Episode II", "Xenosaga Episode III", "Tales of the Abyss", "Tales of Legendia", "Suikoden III", "Suikoden IV", "Suikoden V", "Shadow Hearts (USA)", "Shadow Hearts - Covenant", "Shadow Hearts - From the New World", "Star Ocean - Till the End of Time", "Valkyrie Profile 2", "Wild Arms 3", "Wild Arms - Alter Code", "Baldur's Gate - Dark Alliance", "Baldur's Gate - Dark Alliance II", "Champions of Norrath", "Champions - Return to Arms", "Ar tonelico - Melody of Elemia", "Ar tonelico II", "Grandia II", "Grandia III", "Breath of Fire - Dragon Quarter", "Radiata Stories", "Atelier Iris", "Metal Gear Solid 2 - Substance", "Metal Gear Solid 3 - Subsistence", "Tom Clancy's Splinter Cell (USA)", "Tom Clancy's Splinter Cell - Pandora Tomorrow", "Tom Clancy's Splinter Cell - Chaos Theory", "Hitman 2 - Silent Assassin", "Hitman - Blood Money", "Resident Evil 4", "Resident Evil - Code - Veronica X", "Silent Hill 2", "Silent Hill 3", "Silent Hill 4", "Fatal Frame (USA)", "Fatal Frame II", "Fatal Frame III", "Haunting Ground", "Clock Tower 3", "Siren (USA)", "Thing, The", "TimeSplitters 2", "TimeSplitters - Future Perfect", "Black (USA)", "SOCOM - U.S. Navy SEALs (USA)", "SOCOM II", "Star Wars - Battlefront (USA)", "Star Wars - Battlefront II", "Medal of Honor - Frontline", "Medal of Honor - Rising Sun", "Max Payne (USA)", "Max Payne 2", "Killzone (USA)", "Half-Life (USA)", "Darkwatch", "Red Faction (USA)", "Red Faction II", "Mercenaries - Playground of Destruction", "Tekken 5", "Tekken Tag Tournament", "SoulCalibur II", "SoulCalibur III", "Virtua Fighter 4 - Evolution", "Virtua Fighter 4 (USA)", "Guilty Gear X2", "Guilty Gear XX Accent Core Plus", "Marvel vs. Capcom 2", "Capcom vs. SNK 2", "Street Fighter Alpha Anthology", "King of Fighters '98 Ultimate Match", "King of Fighters, The - '98", "Dragon Ball Z - Budokai 3", "Dragon Ball Z - Budokai Tenkaichi 3", "Def Jam - Fight for NY", "WWE SmackDown! - Here Comes the Pain", "WWE SmackDown vs. Raw 2006", "Gran Turismo 3", "Gran Turismo 4", "Burnout 2", "Burnout 3 - Takedown", "Burnout Revenge", "Need for Speed - Underground (", "Need for Speed - Underground 2", "Need for Speed - Most Wanted", "Need for Speed - Hot Pursuit 2", "Midnight Club 3", "SSX (", "SSX Tricky", "SSX 3", "ATV Offroad Fury 2", "Twisted Metal - Black", "Wipeout Fusion", "Tony Hawk's Pro Skater 3", "Tony Hawk's Pro Skater 4", "Tony Hawk's Underground", "Tony Hawk's Underground 2", "Madden NFL 2004", "Madden NFL 2005", "ESPN NFL 2K5", "NBA Street (USA)", "NBA Street Vol. 2", "NBA Street V3", "Fight Night Round 2", "Fight Night Round 3", "MLB 06", "FIFA Soccer 06", "NHL Hitz 2003", "Tiger Woods PGA Tour 2004", "Guitar Hero (USA)", "Guitar Hero II", "Guitar Hero III", "Amplitude", "Frequency", "Gitaroo Man", "Rez (USA)", "Metal Slug Anthology", "Mega Man Anniversary Collection", "Mega Man X Collection", "Gradius V", "R-Type Final", "Destroy All Humans! (USA)", "Destroy All Humans! 2", "Scarface - The World Is Yours", "Ace Combat 04", "Ace Combat 5", "Ace Combat Zero", "War of the Monsters", "Front Mission 4", "Lord of the Rings, The - The Two Towers", "Lord of the Rings, The - The Return of the King", "LEGO Star Wars - The Video Game", "LEGO Star Wars II", "Marvel - Ultimate Alliance", "X-Men Legends (USA)", "X-Men Legends II", "Star Wars - Jedi Starfighter", "Dot Hack Part 1 - Infection", "Dot Hack Part 2 - Mutation", "Dot Hack Part 3 - Outbreak", "Dot Hack Part 4 - Quarantine", "Dot Hack G.U. Vol. 1", "Dot Hack G.U. Vol. 2", "Dot Hack G.U. Vol. 3", "Yakuza (USA)", "Yakuza 2", "Zone of the Enders (USA)", "Zone of the Enders - The 2nd Runner", "Okage - Shadow King", "Growlanser - Heritage of War", "Growlanser Generations", "SkyGunner", "Fire Pro Wrestling Returns", "OutRun 2006", "Castlevania - Lament of Innocence", "Castlevania - Curse of Darkness", "Contra - Shattered Soldier", "007 - Everything or Nothing", "007 - Nightfire", "Aggressive Inline", "NFL Street (USA)", "Smuggler's Run (USA)", "Smuggler's Run 2", "Escape from Monkey Island", ] # Classic Controller / standard gamepad games only (no motion-required games). WII_CURATED = [ "Xenoblade Chronicles", "Last Story, The", "Pandora's Tower", "Arc Rise Fantasia", "Tales of Symphonia - Dawn of the New World", "Fire Emblem - Radiant Dawn", "New Super Mario Bros. Wii", "Donkey Kong Country Returns", "Kirby's Return to Dream Land", "Kirby's Epic Yarn", "Wario Land - Shake It", "Rayman Origins", "Boy and His Blob, A", "Super Smash Bros. Brawl", "Tatsunoko vs. Capcom", "Dragon Ball Z - Budokai Tenkaichi 3", "Mario Kart Wii", "Need for Speed - Carbon", "Monster Hunter Tri", "MadWorld", "Muramasa - The Demon Blade", "No More Heroes (USA)", "No More Heroes 2", "Ookami", "Guitar Hero III", "Rock Band 2", "Call of Duty - Modern Warfare - Reflex", "Call of Duty - World at War", "Call of Duty - Black Ops", "GoldenEye 007", "Boom Blox (USA)", "Zack & Wiki", "Metal Slug Anthology", "SNK Arcade Classics Vol. 1", ] WIIU_CURATED = [ "Legend of Zelda, The - Breath of the Wild", "Legend of Zelda, The - Twilight Princess HD", "Legend of Zelda, The - The Wind Waker HD", "Hyrule Warriors", "Super Mario 3D World", "Super Mario Maker", "New Super Mario Bros. U + New Super Luigi U", "Mario Kart 8", "Captain Toad - Treasure Tracker", "Paper Mario - Color Splash", "Mario Party 10", "Super Smash Bros. for Wii U", "Tekken Tag Tournament 2", "Injustice - Gods Among Us", "Donkey Kong Country - Tropical Freeze", "Rayman Legends", "Kirby and the Rainbow Curse", "Yoshi's Woolly World", "Shovel Knight", "Shantae - Half-Genie Hero", "Bayonetta (USA)", "Bayonetta 2", "Wonderful 101, The", "Ninja Gaiden 3 - Razor's Edge", "Devil's Third", "Darksiders - Warmastered Edition", "Xenoblade Chronicles X", "Tokyo Mirage Sessions FE", "Splatoon", "Call of Duty - Black Ops II", "Deus Ex - Human Revolution - Director's Cut", "Tom Clancy's Splinter Cell - Blacklist", "Mass Effect 3 - Special Edition", "ZombiU", "Pikmin 3 (USA)", "Monster Hunter 3 Ultimate", "Game & Wario", "NES Remix Pack", "LEGO City Undercover", "Minecraft - Wii U Edition", "Terraria", "Axiom Verge", "Runbow - Deluxe Edition", "SteamWorld Collection", "Sonic Lost World", "Sonic & All-Stars Racing Transformed", "Batman - Arkham City - Armored Edition", "Watch_Dogs", "Assassin's Creed IV - Black Flag", "Warriors Orochi 3 Hyper", "Star Fox Zero", "Star Fox Guard", "Nintendo Land", "Shakedown - Hawaii", "Pokken Tournament", ] CURATED_LISTS = {"ps2": PS2_CURATED, "wii": WII_CURATED, "wiiu": WIIU_CURATED} # ── Filters ──────────────────────────────────────────────────────────────────── INCLUDE_RE = re.compile(r"\((USA|USA, Canada|USA, Asia|World)\)") INCLUDE_EU_EN_RE = re.compile(r"\(Europe\).*\(En[,\)]") EXCLUDE_RE = re.compile(r"\((Beta|Proto|Sample|Sampler|Program|BIOS|Unl)[^)]*\)|\bDemo\b", re.IGNORECASE) REVISION_RE = re.compile(r"\(Rev ([0-9A-Za-z]+)\)") VERSION_RE = re.compile(r"\(v(\d+\.\d+)\)") EDITION_RE = re.compile(r"\((Greatest Hits|Collector's Edition|Special Edition|Black Edition)\)", re.IGNORECASE) class LinkExtractor(HTMLParser): """Extract href values from tags in an HTML page.""" def __init__(self): super().__init__() self.links = [] def handle_starttag(self, tag, attrs): if tag == "a": for name, value in attrs: if name == "href" and value and not value.startswith("?") and not value.startswith("/"): self.links.append(value) def fetch_file_list(url): """Fetch a Myrient directory listing and return decoded filenames.""" logging.info("Fetching listing: %s", url) req = urllib.request.Request(url, headers={"User-Agent": "myrient-dl/1.0"}) with urllib.request.urlopen(req, timeout=60) as resp: html = resp.read().decode("utf-8", errors="replace") parser = LinkExtractor() parser.feed(html) filenames = [] for link in parser.links: if link in ("../", "./"): continue decoded = urllib.parse.unquote(link) filenames.append(decoded) return filenames def matches_us_english(filename): if INCLUDE_RE.search(filename): return True if INCLUDE_EU_EN_RE.search(filename): return True return False def should_exclude(filename): return bool(EXCLUDE_RE.search(filename)) def base_name_for_dedup(filename): """Strip revision/version/region/language tags and extension for deduplication.""" name = filename name = re.sub(r"\.[^.]+$", "", name) name = REVISION_RE.sub("", name) name = VERSION_RE.sub("", name) name = EDITION_RE.sub("", name) name = re.sub(r"\((?:USA|USA, Canada|USA, Asia|World|Europe)[^)]*\)", "", name) name = re.sub(r"\((?:En|Ja|Fr|De|Es|It|Ko|Pt|Ru|Nl|Sv|No|Da|Pl|El|Hu)(?:,[A-Z][a-z])*\)", "", name) name = re.sub(r"\s+", " ", name).strip() return name def revision_key(filename): """Extract revision/version number for sorting. Higher = newer.""" m = REVISION_RE.search(filename) if m: val = m.group(1) try: return 1000 + int(val) except ValueError: return 1000 + ord(val.upper()) - ord("A") + 1 m = VERSION_RE.search(filename) if m: try: return int(float(m.group(1)) * 100) except ValueError: return 0 return 0 def deduplicate(filenames): """Keep only the highest revision of each title.""" groups = {} for fn in filenames: key = base_name_for_dedup(fn) if key not in groups or revision_key(fn) > revision_key(groups[key]): groups[key] = fn return sorted(groups.values()) def filter_files(filenames): """Apply US/English include, exclude, and dedup filters.""" included = [f for f in filenames if matches_us_english(f)] filtered = [f for f in included if not should_exclude(f)] return deduplicate(filtered) def filter_curated(filenames, curated_titles): """Filter by curated title list + USA region + exclusions.""" patterns = [] for title in curated_titles: escaped = re.escape(title) patterns.append(re.compile(r"(?:^|[\s\-])" + escaped, re.IGNORECASE)) usa = [f for f in filenames if INCLUDE_RE.search(f)] matched = [] for f in usa: if any(p.search(f) for p in patterns): matched.append(f) filtered = [f for f in matched if not should_exclude(f)] return deduplicate(filtered) def get_remote_size(url): """HEAD request for Content-Length. Returns None if unavailable.""" req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "myrient-dl/1.0"}) try: with urllib.request.urlopen(req, timeout=30) as resp: cl = resp.getheader("Content-Length") return int(cl) if cl else None except (urllib.error.URLError, ValueError): return None def download_file(url, dest_path): """Download a file with resume support. Returns True if downloaded, False if skipped/failed.""" dest = Path(dest_path) dest.parent.mkdir(parents=True, exist_ok=True) remote_size = get_remote_size(url) local_size = dest.stat().st_size if dest.exists() else 0 if remote_size is not None and local_size == remote_size: logging.info(" SKIP (complete): %s", dest.name) return False if remote_size is not None and local_size > remote_size: logging.warning(" Local file larger than remote, re-downloading: %s", dest.name) local_size = 0 headers = {"User-Agent": "myrient-dl/1.0"} mode = "wb" if local_size > 0 and remote_size is not None and local_size < remote_size: headers["Range"] = f"bytes={local_size}-" mode = "ab" logging.info(" RESUME from %d bytes: %s", local_size, dest.name) else: local_size = 0 logging.info(" DOWNLOAD: %s", dest.name) req = urllib.request.Request(url, headers=headers) try: with urllib.request.urlopen(req, timeout=300) as resp: with open(dest, mode) as f: downloaded = local_size while True: chunk = resp.read(1024 * 1024) if not chunk: break f.write(chunk) downloaded += len(chunk) if remote_size: pct = downloaded * 100 // remote_size if pct % 10 == 0 and (downloaded - len(chunk)) * 100 // remote_size < pct: logging.info(" %d%% (%d / %d MB)", pct, downloaded // (1024*1024), remote_size // (1024*1024)) except (urllib.error.URLError, OSError) as e: logging.error(" FAILED: %s — %s", dest.name, e) return False logging.info(" DONE: %s", dest.name) return True def build_url(collection, directory, filename): encoded_col = urllib.parse.quote(collection, safe="") encoded_dir = urllib.parse.quote(directory, safe="") encoded_file = urllib.parse.quote(filename, safe="") return f"{MYRIENT_BASE}/{encoded_col}/{encoded_dir}/{encoded_file}" def build_listing_url(collection, directory): encoded_col = urllib.parse.quote(collection, safe="") encoded_dir = urllib.parse.quote(directory, safe="") return f"{MYRIENT_BASE}/{encoded_col}/{encoded_dir}/" def process_platform(platform, dry_run=False): """Download all matching ROMs for a single platform.""" if platform not in PLATFORMS: logging.error("Unknown platform: %s", platform) return collection, directory, est_size, use_region_filter = PLATFORMS[platform] dest_dir = os.path.join(LIBRARY_DIR, platform) logging.info("=" * 70) logging.info("Platform: %s | Collection: %s | Est: %s", platform, collection, est_size) logging.info("Directory: %s", directory) logging.info("Destination: %s", dest_dir) logging.info("=" * 70) listing_url = build_listing_url(collection, directory) try: all_files = fetch_file_list(listing_url) except (urllib.error.URLError, OSError) as e: logging.error("Failed to fetch listing for %s: %s", platform, e) return logging.info("Total files in directory: %d", len(all_files)) if platform in CURATED_LISTS: matched = filter_curated(all_files, CURATED_LISTS[platform]) logging.info("Files after curated list filter: %d", len(matched)) elif use_region_filter: matched = filter_files(all_files) logging.info("Files after US/English filter + dedup: %d", len(matched)) else: matched = sorted(all_files) logging.info("No region filter (downloading all %d files)", len(matched)) if dry_run: print(f"\n[DRY RUN] {platform}: {len(matched)} files would be downloaded to {dest_dir}/") for fn in matched: print(f" {fn}") return os.makedirs(dest_dir, exist_ok=True) downloaded = 0 skipped = 0 failed = 0 for i, fn in enumerate(matched, 1): url = build_url(collection, directory, fn) dest_path = os.path.join(dest_dir, fn) logging.info("[%d/%d] %s", i, len(matched), fn) try: result = download_file(url, dest_path) if result: downloaded += 1 else: skipped += 1 except Exception as e: logging.error(" ERROR: %s — %s", fn, e) failed += 1 logging.info("-" * 70) logging.info("Platform %s complete: %d downloaded, %d skipped, %d failed", platform, downloaded, skipped, failed) logging.info("-" * 70) def list_platforms(): phases = [ ("Phase 1 — Small/cartridge", ["gb", "gbc", "gba", "nes", "snes", "n64", "genesis"]), ("Phase 2 — Medium", ["nds"]), ("Phase 3 — Large/disc", ["ps", "psp", "dreamcast", "gc"]), ("Phase 4 — Very large", ["ps2", "wii", "wiiu"]), ("Phase 5 — Handhelds", ["3ds"]), ("Arcade", ["arcade"]), ] for phase_name, keys in phases: print(f"\n{phase_name}:") for k in keys: collection, directory, est, _ = PLATFORMS[k] print(f" {k:<12} {directory:<55} {est}") def main(): parser = argparse.ArgumentParser(description="Download US/English ROMs from Myrient") parser.add_argument("platforms", nargs="*", help="Platforms to download (default: all)") parser.add_argument("--list", action="store_true", help="List available platforms") parser.add_argument("--dry-run", action="store_true", help="Show what would be downloaded") args = parser.parse_args() if args.list: list_platforms() return log_handlers = [logging.StreamHandler(sys.stdout)] if not args.dry_run: log_dir = os.path.dirname(LOG_FILE) if log_dir: os.makedirs(log_dir, exist_ok=True) log_handlers.append(logging.FileHandler(LOG_FILE, mode="a")) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", handlers=log_handlers, ) targets = args.platforms if args.platforms else PHASE_ORDER for p in targets: if p not in PLATFORMS: print(f"Unknown platform: {p}") print("Use --list to see available platforms.") sys.exit(1) logging.info("Myrient ROM downloader starting") logging.info("Targets: %s", ", ".join(targets)) logging.info("Dry run: %s", args.dry_run) for platform in targets: process_platform(platform, dry_run=args.dry_run) logging.info("All done.") if __name__ == "__main__": main() # src: ezra, contact: ezkru69@mail.com