flatten_object_store.py
python
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago
| 1 | #!/usr/bin/env python3 |
| 2 | """Migrate LocalBackend objects to the flat global content-addressed layout. |
| 3 | |
| 4 | Old layouts (both broken): |
| 5 | <root>/<repo_id>/<object_id> — legacy per-repo paths |
| 6 | <root>/objects/<object_id> — double-objects bug (root was set to |
| 7 | /data/musehub/objects but code appended |
| 8 | "objects/" again) |
| 9 | |
| 10 | Correct layout: |
| 11 | <root>/<object_id> — flat, globally content-addressed |
| 12 | (root = musehub_objects_dir, default |
| 13 | /data/musehub/objects) |
| 14 | |
| 15 | This script: |
| 16 | 1. Walks the objects root directory. |
| 17 | 2. Moves any file not at the top level into <root>/<filename>. |
| 18 | 3. Updates storage_uri in musehub_objects for every moved file. |
| 19 | 4. Prints a summary. |
| 20 | |
| 21 | Run inside the container or on the EC2 instance where /data is mounted: |
| 22 | python3 deploy/flatten_object_store.py [--root /data/musehub/objects] [--db-url ...] |
| 23 | |
| 24 | Dry-run mode (no writes): |
| 25 | python3 deploy/flatten_object_store.py --dry-run |
| 26 | """ |
| 27 | from __future__ import annotations |
| 28 | |
| 29 | import argparse |
| 30 | import os |
| 31 | import shutil |
| 32 | import sys |
| 33 | from pathlib import Path |
| 34 | |
| 35 | |
| 36 | def parse_args() -> argparse.Namespace: |
| 37 | p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| 38 | p.add_argument("--root", default=os.environ.get("MUSEHUB_OBJECTS_DIR", "/data/musehub/objects"), |
| 39 | help="Objects root directory (musehub_objects_dir)") |
| 40 | p.add_argument("--db-url", default=os.environ.get("DATABASE_URL"), |
| 41 | help="PostgreSQL connection URL for updating storage_uri") |
| 42 | p.add_argument("--dry-run", action="store_true", help="Print what would happen, make no changes") |
| 43 | return p.parse_args() |
| 44 | |
| 45 | |
| 46 | def flatten(root: Path, dry_run: bool) -> dict[str, str]: |
| 47 | """Move all non-top-level object files to root. |
| 48 | |
| 49 | Returns {old_uri: new_uri} for every file that was moved. |
| 50 | """ |
| 51 | moved: dict[str, str] = {} |
| 52 | # Walk every file under root |
| 53 | for dirpath, dirnames, filenames in os.walk(root): |
| 54 | dp = Path(dirpath) |
| 55 | if dp == root: |
| 56 | # Top-level files are already correct — skip |
| 57 | continue |
| 58 | for fname in filenames: |
| 59 | src = dp / fname |
| 60 | dst = root / fname |
| 61 | old_uri = "local://" + str(src) |
| 62 | new_uri = "local://" + str(dst) |
| 63 | if dst.exists(): |
| 64 | # Already migrated or duplicate — verify content matches then remove src |
| 65 | if src.read_bytes() == dst.read_bytes(): |
| 66 | print(f" dup {src.relative_to(root)} (already at top level)") |
| 67 | if not dry_run: |
| 68 | src.unlink() |
| 69 | else: |
| 70 | print(f" CONFLICT {src.relative_to(root)} — content differs, skipping", file=sys.stderr) |
| 71 | continue |
| 72 | print(f" move {src.relative_to(root)} → {fname}") |
| 73 | if not dry_run: |
| 74 | shutil.move(str(src), str(dst)) |
| 75 | moved[old_uri] = new_uri |
| 76 | |
| 77 | # Remove now-empty subdirectories |
| 78 | if not dry_run: |
| 79 | for dirpath, dirnames, filenames in os.walk(root, topdown=False): |
| 80 | dp = Path(dirpath) |
| 81 | if dp == root: |
| 82 | continue |
| 83 | try: |
| 84 | dp.rmdir() # only removes empty dirs |
| 85 | print(f" rmdir {dp.relative_to(root)}") |
| 86 | except OSError: |
| 87 | pass # not empty — leave it |
| 88 | |
| 89 | return moved |
| 90 | |
| 91 | |
| 92 | def update_db(moved: dict[str, str], db_url: str, dry_run: bool) -> None: |
| 93 | """Bulk-update storage_uri in musehub_objects for all moved files.""" |
| 94 | if not moved: |
| 95 | print("No DB updates needed.") |
| 96 | return |
| 97 | |
| 98 | try: |
| 99 | import psycopg2 # type: ignore[import] |
| 100 | except ImportError: |
| 101 | print("psycopg2 not available — skipping DB update. Run manually:", file=sys.stderr) |
| 102 | for old, new in moved.items(): |
| 103 | print(f" UPDATE musehub_objects SET storage_uri = '{new}' WHERE storage_uri = '{old}';") |
| 104 | return |
| 105 | |
| 106 | conn = psycopg2.connect(db_url.replace("+asyncpg", "").replace("+psycopg2", "")) |
| 107 | cur = conn.cursor() |
| 108 | updated = 0 |
| 109 | for old_uri, new_uri in moved.items(): |
| 110 | if dry_run: |
| 111 | print(f" [dry] UPDATE storage_uri: {old_uri[:60]}... → {new_uri[:60]}...") |
| 112 | else: |
| 113 | cur.execute( |
| 114 | "UPDATE musehub_objects SET storage_uri = %s WHERE storage_uri = %s", |
| 115 | (new_uri, old_uri), |
| 116 | ) |
| 117 | updated += cur.rowcount |
| 118 | if not dry_run: |
| 119 | conn.commit() |
| 120 | print(f" Updated {updated} DB row(s).") |
| 121 | conn.close() |
| 122 | |
| 123 | |
| 124 | def main() -> None: |
| 125 | args = parse_args() |
| 126 | root = Path(args.root) |
| 127 | |
| 128 | if not root.exists(): |
| 129 | print(f"Root does not exist: {root}", file=sys.stderr) |
| 130 | sys.exit(1) |
| 131 | |
| 132 | print(f"{'[DRY RUN] ' if args.dry_run else ''}Flattening object store: {root}") |
| 133 | moved = flatten(root, args.dry_run) |
| 134 | print(f"\n{'Would move' if args.dry_run else 'Moved'} {len(moved)} file(s).") |
| 135 | |
| 136 | if args.db_url: |
| 137 | print("\nUpdating DB storage_uri...") |
| 138 | update_db(moved, args.db_url, args.dry_run) |
| 139 | else: |
| 140 | print("\nNo --db-url provided — skipping DB update.") |
| 141 | if moved: |
| 142 | print("Run with --db-url to update storage_uri in musehub_objects.") |
| 143 | |
| 144 | print("\nDone.") |
| 145 | |
| 146 | |
| 147 | if __name__ == "__main__": |
| 148 | main() |
File History
1 commit
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago