commit 45ade372f6dc265c49a29105845555fe1c91e1ea Author: Rodger Werkhoven <110245948+RodgerWerkhoven@users.noreply.github.com> Date: Mon Aug 10 14:25:24 2026 +0200 feat: publish read-only iOS music export skill diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..aafdb97 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,42 @@ +{ + "name": "rodgers-ios-music-export", + "version": "1.0.0", + "description": "Read-only export of selected music from a paired USB iOS device to a Mac.", + "author": { + "name": "Rodger Werkhoven", + "url": "https://git.dimensionlab.net/Rodger" + }, + "homepage": "https://git.dimensionlab.net/Rodger/rodgers-ios-music-export", + "repository": "https://git.dimensionlab.net/Rodger/rodgers-ios-music-export.git", + "license": "MIT", + "keywords": [ + "ios", + "iphone", + "ipad", + "music", + "export", + "recovery", + "macos" + ], + "skills": "./skills/", + "interface": { + "displayName": "Rodger's iOS Music Export", + "shortDescription": "Export iOS music safely.", + "longDescription": "In local Codex on macOS, export locally stored tracks from a USB iOS device without Finder or Music sync. Use an existing pairing or stop before Apple's persistent Trust action when pairing is absent. Select an exact genre, preserve existing files, resume interrupted copies, and verify results with sizes and SHA-256 hashes. Proven on iPad mini 2 with iOS 12.5.8; other schemas fail closed.", + "developerName": "Rodger Werkhoven", + "category": "Productivity", + "capabilities": [ + "Read-only iOS device inventory", + "Exact genre export", + "Resumable local copy", + "SHA-256 verification" + ], + "brandColor": "#151515", + "logo": "./assets/logo.svg", + "composerIcon": "./assets/composer-icon.svg", + "websiteURL": "https://git.dimensionlab.net/Rodger/rodgers-ios-music-export", + "privacyPolicyURL": "https://git.dimensionlab.net/Rodger/rodgers-ios-music-export/src/branch/main/PRIVACY.md", + "termsOfServiceURL": "https://git.dimensionlab.net/Rodger/rodgers-ios-music-export/src/branch/main/TERMS.md", + "defaultPrompt": "Use $rodgers-ios-music-export to export matching music from a USB iOS device without syncing it." + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc42acb --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +__pycache__/ +*.py[cod] +*.partial +*.tsv diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8dda15d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rodger Werkhoven + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PRIVACY.md b/PRIVACY.md new file mode 100644 index 0000000..1151518 --- /dev/null +++ b/PRIVACY.md @@ -0,0 +1,30 @@ +# Privacy Policy + +Effective date: 10 August 2026 + +Rodger's iOS Music Export is a local, skills-only Codex plugin. The developer operates no service for it. The plugin contains no analytics, advertising, developer account, telemetry endpoint, or developer-controlled server. + +## Data handled locally + +At the user's direction, the workflow may process: + +- device name, product type, device class, iOS version and build, and battery fields when available; +- a one-way SHA-256 fingerprint derived from the device's usbmux identifier, used only to prevent an export manifest from being reused with another device; +- music titles, artists, albums, genre labels, source locations, filenames, sizes, modification signatures, and database snapshot hashes; +- locally stored audio bytes selected for export; +- destination paths, SHA-256 file hashes, progress, verification results, and a local append-only TSV audit manifest with per-event checksums; +- the Mac's existing pairing record in memory, including host and device certificate material required to establish Apple's local TLS session. + +The raw device identifier, pairing record, certificates, and private key are never written to logs or the manifest. Python's TLS API requires certificate paths, so host credentials are written only inside a mode-0700 temporary directory as mode-0600 files, loaded locally, and immediately removed. TLS peer identity is pinned to the existing pair record. + +## Purpose and recipients + +The data is used only to identify the attached device, select exact matching locally stored tracks, copy them to a user-chosen Mac folder, support safe resume, and verify the local result. The script communicates only with Apple's local macOS `usbmuxd` service and local filesystem tools. It sends no data to the developer or another developer-controlled recipient. + +Command output and files that an agent reads may become part of the user's Codex task context and are therefore also subject to OpenAI's applicable product and workspace terms. macOS, iOS, and separately installed tools such as `ffprobe` remain subject to their own terms. + +## Retention and control + +The plugin itself retains nothing remotely. Audio files, the TSV manifest, and its empty advisory `.lock` file remain on the user's Mac until the user moves or deletes them. Temporary database snapshots and certificate files are scoped to temporary directories and removed when the command exits normally; the operating system may clean remnants after an abnormal termination. Users control the destination and may delete local outputs with ordinary Finder or command-line tools after confirming the exact paths. + +Privacy questions can be filed through the public repository's issue tracker listed in [SUPPORT.md](SUPPORT.md). Never attach pairing material, private media, or a raw device identifier to an issue. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6880f9e --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# Rodger's iOS Music Export + +A read-only skill and skills-only plugin for local Codex on macOS. It exports locally stored music from a USB-connected iOS device without Finder or Music sync. It uses an existing Apple pairing when one is present; if pairing is absent, it stops before the persistent Trust action and asks for explicit permission. + +It was created from a proven recovery of 295 exact `Electro` tracks from an iPad mini 2 running iOS 12.5.8. That is the current proven baseline; other models and schemas fail closed unless validated. It performs exact case-insensitive genre matching, preserves collisions, resumes completed or hash-proven tracks, and verifies local copies with sizes and SHA-256 hashes. When `ffprobe` is installed it also checks audio streams and embedded genre tags. + +## Install as a Codex plugin + +Install it through the Codex plugin directory once OpenAI has approved the listing. Until then, clone the public source and link the included skill directory into the user skill scope: + +```bash +git clone https://git.dimensionlab.net/Rodger/rodgers-ios-music-export.git \ + "$HOME/plugins/rodgers-ios-music-export" +mkdir -p "$HOME/.agents/skills" +ln -s "$HOME/plugins/rodgers-ios-music-export/skills/rodgers-ios-music-export" \ + "$HOME/.agents/skills/rodgers-ios-music-export" +``` + +Start a new Codex task after installation so discovery refreshes. + +## Safety + +The bundled script enforces read-only usbmuxd, lockdownd, and AFC allowlists and never pairs, unpairs, syncs, uploads, deletes, renames, or writes on the device. Local exports, append-only checksum-bound manifests, and empty manifest-lock sidecars are created only at disclosed user-authorized paths. Existing untrusted files, symlinked paths, changing databases, source mutations, and late collisions cause a hard stop. + +Read the skill's [compatibility reference](skills/rodgers-ios-music-export/references/compatibility.md) before using it with a new iOS version. + +## Support and policies + +- [Privacy](PRIVACY.md) +- [Terms](TERMS.md) +- [Support](SUPPORT.md) +- [OpenAI directory submission notes](submission/openai-directory-listing.md) + +Licensed under the [MIT License](LICENSE). diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..ccca434 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,9 @@ +# Support + +Report reproducible bugs and compatibility findings in the public Forgejo repository: + + + +Include the Mac model, macOS version, iOS device model, iOS version and build, command name, and sanitized error text. Never attach a pairing record, UDID, host certificate, private key, exported music file, or other credential/private media. + +For a new MediaLibrary schema, include only the relevant table and column names unless the data owner explicitly authorizes more. diff --git a/TERMS.md b/TERMS.md new file mode 100644 index 0000000..31b22e2 --- /dev/null +++ b/TERMS.md @@ -0,0 +1,9 @@ +# Terms of Use + +Effective date: 10 August 2026 + +Rodger's iOS Music Export is provided under the MIT License and without warranty. It is an independent utility and is not affiliated with or endorsed by Apple or OpenAI. + +Users must have authorization to access the connected device and must comply with copyright, license, subscription, and other legal restrictions that apply to the media they export. The plugin does not remove digital rights management, download cloud-only media, bypass device trust, or grant rights to copied content. + +The workflow is designed to keep the iOS source read-only, but users remain responsible for maintaining backups, reviewing the proposed destination, and confirming that the environment and device are appropriate before running an export. diff --git a/assets/composer-icon.svg b/assets/composer-icon.svg new file mode 100644 index 0000000..bfc2d6d --- /dev/null +++ b/assets/composer-icon.svg @@ -0,0 +1,7 @@ + + iOS music export + + + + + diff --git a/assets/logo.svg b/assets/logo.svg new file mode 100644 index 0000000..2a46173 --- /dev/null +++ b/assets/logo.svg @@ -0,0 +1,9 @@ + + Rodger's iOS Music Export + A tablet, music note, and outward transfer arrow. + + + + + + diff --git a/skills/rodgers-ios-music-export/SKILL.md b/skills/rodgers-ios-music-export/SKILL.md new file mode 100644 index 0000000..d317a9f --- /dev/null +++ b/skills/rodgers-ios-music-export/SKILL.md @@ -0,0 +1,110 @@ +--- +name: rodgers-ios-music-export +description: Safely export music from a USB-connected iOS device in a local Codex session on macOS without Finder or Music sync, when an existing pairing is present or pairing status is unknown. Use for reverse-transfer, recovery, exact genre filtering such as Electro versus Electronic, local destination-folder creation, resumable copies, or integrity verification. Proven on iPad mini 2 with iOS 12.5.8; unknown schemas fail closed. +--- + +# Rodger's iOS Music Export + +Export selected, locally stored tracks from one paired iOS device while keeping the device read-only. Use the bundled standard-library Python client instead of Finder or Music sync. + +## Safety contract + +- Treat the iOS device as read-only. Never invoke Pair, Unpair, sync, upload, delete, rename, database mutation, or AFC write operations. +- Do not click Finder's or Music's `Sync` button. Sync can reconcile the device against the Mac library and remove device content. +- Require exactly one USB-connected iOS device and an existing valid macOS pairing. If pairing is absent, stop and explain that approving Apple's visible Trust prompt creates persistent pairing state on the Mac and device. Obtain explicit permission before that visible action; never create pairing records programmatically. +- Write only to the destination and manifest paths the user authorized. Preserve unexpected or untrusted existing files and stop on collisions. +- Interpret a requested genre as an exact, case-insensitive database match. `Electro` matches `electro`; it does not match `Electronic`, `Elektronisch`, or `Electro House`. +- Verify count, byte total, SHA-256 hashes, and—when `ffprobe` is present—audio streams and embedded genres before claiming completion. +- Never expose the device UDID, pairing record, host certificate, private key, or other credentials in output. + +## Workflow + +### 1. Establish live state + +Confirm that this is local Codex on macOS with direct USB access, the intended device is attached, and no Finder/Music sync is running. Proceed when the device is charging; when it is not charging and below 30%, pause and ask the user to connect power. Inspect Finder or Music visibly for battery and sync state when the read-only probe reports that those values are unavailable. Then run: + +```bash +python3 /scripts/ios_music_export.py probe +``` + +The probe returns only the device name, model identifiers, iOS version, build, and class. If local sandboxing denies access to `/var/run/usbmuxd`, rerun the same bounded command with the user's normal Codex approval mechanism. + +### 2. Resolve the exact genre label + +If the requested label is uncertain or adjacent labels could be confused, inventory the device's exact labels and counts: + +```bash +python3 /scripts/ios_music_export.py genres +``` + +Do not silently broaden the user's genre. Ask only if no exact label exists or their wording genuinely permits multiple labels. + +### 3. Build a read-only plan + +```bash +python3 /scripts/ios_music_export.py plan \ + --genre "Electro" \ + --destination "/Users/name/Downloads/RODGER's MUSIX" +``` + +Add `--show-files` when filenames or metadata need review. The plan takes two byte-identical database snapshots, runs SQLite integrity checks, consolidates them locally, validates the schema, stats every selected source file, and reports total exact matches, transferable and excluded counts, bytes, free space, and headroom. Stop if snapshots differ repeatedly, the schema is unsupported, a source is missing, or database and device signatures disagree. + +Before copying, state the exact genre, destination, track count, byte total, available space, and any known limitations. A direct user request to export is sufficient authorization for destination creation and local copies; do not add an unnecessary confirmation pause. + +### 4. Export and verify + +```bash +python3 /scripts/ios_music_export.py export \ + --genre "Electro" \ + --destination "/Users/name/Downloads/RODGER's MUSIX" +``` + +By default the audit manifest is a hidden TSV beside—not inside—the music folder. A persistent empty `.lock` sidecar serializes exports to that manifest and contains no device or music data. The exporter: + +1. obtains two byte-identical database snapshots and consolidates a checked local copy; +2. selects only the exact genre; +3. validates source files and free space; +4. creates conflict-safe `Artist — Title.ext` names; +5. creates each final track with a no-clobber open, so a late local collision is preserved; +6. records source signatures, state transitions, event checksums, and SHA-256 hashes in a locked append-only manifest; +7. verifies the complete destination. + +Use `--manifest ` only when the user needs a specific audit location. Use `--require-embedded-genre` only when `ffprobe` is available and the user explicitly requires every file's embedded tag to equal the database genre; iOS database metadata remains the selection authority. + +### 5. Resume safely + +Repeat the identical export command after an interruption. Existing files are accepted only when the latest checksum-bound manifest event binds them to a pseudonymous device fingerprint, database snapshot, source signature, size, and nonempty hash. Completed tracks and a hash-proven final file in `ready_to_commit` state resume safely. A `pending` event never authorizes a final file. If interruption leaves a final file while its latest event is still `pending`, preserve it and stop; ask the user to choose a new destination or explicitly decide how to handle that exact file. Never delete, replace, or rename it implicitly. + +If a process interruption tears only the final manifest event, recover under the manifest lock by truncating that incomplete tail only after every preceding event and checksum validates. Never repair, replace, or append to a manifest whose complete events belong to another export plan. + +### 6. Verify an existing export offline + +```bash +python3 /scripts/ios_music_export.py verify \ + --genre "Electro" \ + --destination "/Users/name/Downloads/RODGER's MUSIX" \ + --manifest "/Users/name/Downloads/.RODGER's MUSIX.rodgers-ios-music-export.tsv" +``` + +If `ffprobe` is unavailable, hashes, counts, and sizes still run and the result explicitly marks the audio probe as skipped. Do not install dependencies unless the user approves installation. + +## Completion report + +Lead with the verified result and include: + +- device name and iOS version, without its UDID; +- exact genre predicate used; +- destination and manifest paths; +- manifest lock path; +- total exact matches, transferable and excluded counts, total bytes, and extension counts; +- copied versus resumed counts; +- hash/size result and audio-probe status; +- embedded genre counts when probed; +- explicit confirmation that no sync or media, library, or AFC write occurred; if Apple Trust was used, disclose the pairing-state change separately; +- any DRM, cloud-only, unsupported-schema, pairing, or verification limitation. + +Do not claim success from a plan, progress counter, or copy completion alone. Success requires the final local verification result. + +## Troubleshooting and compatibility + +Read [references/compatibility.md](references/compatibility.md) for the proven device matrix, protocol boundaries, common errors, and limitations before changing the script or working around a failure. diff --git a/skills/rodgers-ios-music-export/agents/openai.yaml b/skills/rodgers-ios-music-export/agents/openai.yaml new file mode 100644 index 0000000..c638a92 --- /dev/null +++ b/skills/rodgers-ios-music-export/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Rodger's iOS Music Export" + short_description: "Export iOS music safely on a Mac" + brand_color: "#151515" + default_prompt: "Use $rodgers-ios-music-export to export matching music from a USB iOS device without syncing it." +policy: + allow_implicit_invocation: true diff --git a/skills/rodgers-ios-music-export/references/compatibility.md b/skills/rodgers-ios-music-export/references/compatibility.md new file mode 100644 index 0000000..71c6c20 --- /dev/null +++ b/skills/rodgers-ios-music-export/references/compatibility.md @@ -0,0 +1,100 @@ +# Compatibility and safety reference + +## Proven baseline + +The workflow was end-to-end proven on: + +- macOS on Apple silicon with Apple's MobileDevice framework and system `usbmuxd`; +- an already paired iPad mini 2 (`iPad4,5` / `J86AP`); +- iOS 12.5.8 build 16H88; +- the on-device `MediaLibrary.sqlitedb` schema checked by the bundled script; +- 295 exact case-insensitive `Electro` matches, totaling 2,173,003,367 bytes; +- 262 `.m4a` and 33 `.mp3` files, all size-checked, SHA-256 hashed, audio-probed, and found with exact embedded `Electro` genre tags. + +This baseline proves the method, not universal compatibility. Later iOS releases may hide the classic music database, alter its schema, restrict AFC access, or represent cloud media differently. The script validates known tables and columns and fails closed instead of guessing. + +## Read-only protocol boundary + +The live-device client uses only: + +- usbmuxd `ListDevices`, `ReadPairRecord`, and `Connect`; +- lockdownd `StartSession`, `GetValue`, `StartService`, and `StopSession`; +- the existing pair record held in memory; +- AFC `READ_DIR`, `GET_FILE_INFO`, `FILE_OPEN` with `AFC_FOPEN_RDONLY`, `FILE_READ`, and `FILE_CLOSE`. + +No AFC write opcode is defined or called, and each generic protocol dispatcher enforces a read-only allowlist. Lockdown TLS excludes anonymous and null ciphers and pins the peer certificate to the existing pair record. The host certificate and private key are materialized only inside a mode-0700 temporary directory as mode-0600 files because Python's TLS API requires paths; they are loaded locally and the directory is immediately removed. + +## Database snapshot + +The database lives at: + +```text +/iTunes_Control/iTunes/MediaLibrary.sqlitedb +``` + +For every attempt, the script captures the main database and `-wal` sidecar twice. Each capture must remain metadata-stable while read, and both complete captures must have identical SHA-256 hashes. SQLite rebuilds local shared-memory state; `PRAGMA quick_check` must pass before and after the stable set is consolidated with SQLite's backup API into one local database. Repeated differences produce a hard failure. Stop playback, leave device-side music activity idle, and retry; never modify or checkpoint the device database. + +## Selection semantics + +The database query uses equality with SQLite `COLLATE NOCASE`. It intentionally does not use substring matching, prefix matching, fuzzy matching, language translation, or a genre family. The plan reports total exact matches, transferable rows, and rows excluded because no local file is present. + +Database-controlled source paths must be relative, contain no `.` or `..` components, and resolve below `/iTunes_Control/Music`, `/Purchases`, or `/CloudAssets`. Every source must report an explicit regular-file type. The manifest binds each row to a one-way SHA-256 device fingerprint, consolidated database hash, source size, modification time, block signature, and the bytes received during download. + +The local filename is normalized to NFC, removes control characters, replaces `/` and `:`, stays under the macOS filename byte limit, and adds deterministic `[2]`, `[3]`, and later suffixes for collisions. + +## Expected failures + +### Permission denied for `/var/run/usbmuxd` + +The command is inside a sandbox that blocks the system socket. Rerun only the same bounded command through Codex's approval path. This is not evidence that the device is absent. + +### No existing pair record + +Stop before taking action. Explain that unlocking the device and approving Apple's visible Trust prompt creates persistent pairing state on both the Mac and device, then obtain the user's explicit permission. Do not create or alter pairing records with the script. If the user approves and completes the visible Trust flow, report that pairing state changed; do not claim that the device remained entirely unchanged. + +### Exactly one USB device expected + +Disconnect unintended iOS devices, then retry. The skill deliberately avoids guessing among multiple devices. + +### Database changes during every snapshot + +Stop playback and device-side library changes. Keep the iOS Music app idle and rerun. Do not sync. + +### Unsupported MediaLibrary schema + +Stop. Collect only non-sensitive table and column names for a compatibility update; do not weaken the schema guard or guess column meanings. + +### Stale database size or missing selected source + +The device library metadata and storage no longer agree. Stop and retry after the device is idle. Never substitute another file merely because its name looks related. + +### Untrusted existing destination file + +Preserve it. Use a new destination or ask the user how to resolve the collision. Do not overwrite, move, or delete it implicitly. + +### Manifest lock is held + +Another exporter is using the same manifest. Wait for it to finish and retry. The persistent empty `.lock` file carries no device or music data; its advisory lock exists only while an export process is active. + +### Truncated final manifest event + +The TSV manifest is an append-only event log with a SHA-256 checksum on every event. During export resume, a trailing event without a complete newline is truncated only while the manifest lock is held and only after the entire preceding event prefix validates. A malformed header, failed checksum on a complete event, or event bound to another export plan is preserved and causes a hard stop. + +## Media limitations + +- DRM-protected subscription media may be encrypted or unusable off-device. +- Cloud-only tracks without a local media file are excluded. +- The skill does not remove DRM, download cloud media, authenticate Apple accounts, import into Music, or sync content back to a device. +- `ffprobe` is optional. Without it, the script still proves file count, size, and SHA-256 equality between the received stream and the verified local file, but cannot prove the presence of an audio stream or inspect embedded genre tags. +- A SHA-256 manifest proves local transfer and persistence integrity. It does not decrypt DRM, prove legal ownership, or assert that a cloud service holds the same bytes. + +## Maintenance gate + +For a new iOS schema or protocol behavior: + +1. preserve the existing proven path; +2. add a synthetic regression fixture for the new schema; +3. make the narrowest compatibility change; +4. validate on a noncritical paired device with read-only commands first; +5. prove a small export and its hashes before a large export; +6. update this compatibility matrix with the exact device and iOS build. diff --git a/skills/rodgers-ios-music-export/scripts/ios_music_export.py b/skills/rodgers-ios-music-export/scripts/ios_music_export.py new file mode 100644 index 0000000..40e106c --- /dev/null +++ b/skills/rodgers-ios-music-export/scripts/ios_music_export.py @@ -0,0 +1,1831 @@ +#!/usr/bin/env python3 +"""Export selected music from an already paired iOS device without syncing it. + +The device-side protocol surface is intentionally limited to read-only usbmuxd, +lockdownd and AFC operations. The script never pairs, unpairs, syncs, deletes, +renames or writes files on the device. +""" + +import argparse +import csv +import ctypes +import errno +import fcntl +import hashlib +import hmac +import io +import json +import os +import posixpath +import plistlib +from pathlib import Path, PurePosixPath +import re +import shutil +import socket +import ssl +import sqlite3 +import stat +import struct +import subprocess +import sys +import tempfile +import time +import unicodedata + +PROGRAM_NAME = "rodgers-ios-music-export" +USBMUXD_SOCKET = "/var/run/usbmuxd" +LOCKDOWN_PORT = 62078 +MEDIA_DATABASE_REMOTE = "/iTunes_Control/iTunes/MediaLibrary.sqlitedb" +MEDIA_DATABASE_STABLE_SIDECARS = ("-wal",) +MEDIA_DATABASE_FORBIDDEN_SIDECARS = ("-journal",) +ALLOWED_MEDIA_ROOTS = ("/iTunes_Control/Music", "/Purchases", "/CloudAssets") +AFC_MAGIC = b"CFA6LPAA" +AFC_OP_STATUS = 0x00000001 +AFC_OP_DATA = 0x00000002 +AFC_OP_READ_DIR = 0x00000003 +AFC_OP_GET_FILE_INFO = 0x0000000A +AFC_OP_FILE_OPEN = 0x0000000D +AFC_OP_FILE_OPEN_RES = 0x0000000E +AFC_OP_FILE_READ = 0x0000000F +AFC_OP_FILE_CLOSE = 0x00000014 +AFC_FOPEN_RDONLY = 0x00000001 +AFC_E_OBJECT_NOT_FOUND = 8 +COPY_CHUNK_SIZE = 1024 * 1024 +SOCKET_TIMEOUT_SECONDS = 30 +MAX_PROTOCOL_FRAME = 64 * 1024 * 1024 +ALLOWED_USBMUX_REQUESTS = {"ListDevices", "ReadPairRecord"} +ALLOWED_LOCKDOWN_REQUESTS = {"StartSession", "StopSession", "StartService", "GetValue"} +ALLOWED_AFC_OPERATIONS = { + AFC_OP_READ_DIR, + AFC_OP_GET_FILE_INFO, + AFC_OP_FILE_OPEN, + AFC_OP_FILE_READ, + AFC_OP_FILE_CLOSE, +} +COMPLETE_MANIFEST_STATUSES = { + "copied", + "already_complete", + "recovered_after_interrupt", +} + + +class AFCError(RuntimeError): + """A read-only AFC request failed.""" + + def __init__(self, status, operation): + self.status = status + self.operation = operation + super().__init__(f"AFC status {status} for read-only operation {operation}") + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def recv_exact(sock, size): + chunks = [] + remaining = size + while remaining: + chunk = sock.recv(remaining) + if not chunk: + raise EOFError(f"unexpected end of stream with {remaining} bytes remaining") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def usbmux_send_plist(sock, payload, tag=1): + body = plistlib.dumps(payload, fmt=plistlib.FMT_XML) + sock.sendall(struct.pack(" MAX_PROTOCOL_FRAME or version != 1 or message_type != 8: + raise RuntimeError(f"unexpected usbmuxd header {(length, version, message_type, tag)}") + return plistlib.loads(recv_exact(sock, length - 16)) + + +def usbmux_request(payload): + if payload.get("MessageType") not in ALLOWED_USBMUX_REQUESTS: + raise RuntimeError("blocked non-read-only usbmuxd request") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.settimeout(SOCKET_TIMEOUT_SECONDS) + sock.connect(USBMUXD_SOCKET) + request = { + "ClientVersionString": PROGRAM_NAME, + "ProgName": PROGRAM_NAME, + "kLibUSBMuxVersion": 3, + **payload, + } + usbmux_send_plist(sock, request) + return usbmux_recv_plist(sock) + + +def list_devices(): + result = usbmux_request({"MessageType": "ListDevices"}) + return result.get("DeviceList", []) + + +def read_pair_record(udid): + result = usbmux_request({"MessageType": "ReadPairRecord", "PairRecordID": udid}) + if "PairRecordData" not in result: + raise RuntimeError("no existing pair record for the connected device") + record = plistlib.loads(result["PairRecordData"]) + required_strings = ("HostID", "SystemBUID") + required_bytes = ( + "HostCertificate", + "HostPrivateKey", + "DeviceCertificate", + ) + if any(not isinstance(record.get(key), str) or not record[key] for key in required_strings): + raise RuntimeError("existing pair record is missing required identifiers") + if any(not isinstance(record.get(key), bytes) or not record[key] for key in required_bytes): + raise RuntimeError("existing pair record is missing required certificate material") + return record + + +def usbmux_connect(device_id, port): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(SOCKET_TIMEOUT_SECONDS) + sock.connect(USBMUXD_SOCKET) + usbmux_send_plist( + sock, + { + "ClientVersionString": PROGRAM_NAME, + "ProgName": PROGRAM_NAME, + "kLibUSBMuxVersion": 3, + "MessageType": "Connect", + "DeviceID": device_id, + "PortNumber": socket.htons(port), + }, + ) + response = usbmux_recv_plist(sock) + if response.get("Number") != 0: + sock.close() + raise RuntimeError(f"usbmuxd connect failed: {response}") + return sock + + +def lockdown_send(sock, payload): + body = plistlib.dumps(payload, fmt=plistlib.FMT_XML) + sock.sendall(struct.pack(">I", len(body)) + body) + + +def lockdown_recv(sock): + size = struct.unpack(">I", recv_exact(sock, 4))[0] + if size <= 0 or size > MAX_PROTOCOL_FRAME: + raise RuntimeError(f"invalid lockdownd frame size: {size}") + return plistlib.loads(recv_exact(sock, size)) + + +def lockdown_request(sock, request, **values): + if request not in ALLOWED_LOCKDOWN_REQUESTS: + raise RuntimeError(f"blocked lockdownd request: {request}") + payload = {"Label": PROGRAM_NAME, "Request": request, **values} + lockdown_send(sock, payload) + response = lockdown_recv(sock) + if response.get("Error"): + raise RuntimeError(f"lockdown {request} failed: {response['Error']}") + return response + + +def make_ssl_context(pair_record): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + try: + context.set_ciphers("DEFAULT:@SECLEVEL=0:!aNULL:!eNULL:!EXPORT:!NULL") + except ssl.SSLError: + pass + with tempfile.TemporaryDirectory(prefix=f"{PROGRAM_NAME}-tls-") as directory: + os.chmod(directory, 0o700) + cert_path = Path(directory) / "host-cert.pem" + key_path = Path(directory) / "host-key.pem" + for path, content in ( + (cert_path, pair_record["HostCertificate"]), + (key_path, pair_record["HostPrivateKey"]), + ): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as output: + output.write(content) + output.flush() + os.fsync(output.fileno()) + context.load_cert_chain(cert_path, key_path) + device_certificate = pair_record["DeviceCertificate"] + if device_certificate.startswith(b"-----BEGIN CERTIFICATE-----"): + expected_der = ssl.PEM_cert_to_DER_cert(device_certificate.decode("ascii")) + else: + expected_der = device_certificate + expected_peer_hash = hashlib.sha256(expected_der).digest() + return context, expected_peer_hash + + +def wrap_paired_tls(sock, context, expected_peer_hash): + wrapped = context.wrap_socket(sock) + peer_certificate = wrapped.getpeercert(binary_form=True) + if not peer_certificate or not hmac.compare_digest( + hashlib.sha256(peer_certificate).digest(), expected_peer_hash + ): + wrapped.close() + raise RuntimeError("TLS peer certificate does not match the existing pair record") + return wrapped + + +def start_lockdown_session(device, pair_record): + sock = usbmux_connect(device["DeviceID"], LOCKDOWN_PORT) + try: + response = lockdown_request( + sock, + "StartSession", + HostID=pair_record["HostID"], + SystemBUID=pair_record["SystemBUID"], + ) + if response.get("EnableSessionSSL") is not True: + raise RuntimeError("lockdown refused the required paired TLS session") + session_id = response.get("SessionID") + if not session_id: + raise RuntimeError("lockdown did not return a session identifier") + context, expected_peer_hash = make_ssl_context(pair_record) + sock = wrap_paired_tls(sock, context, expected_peer_hash) + return sock, (context, expected_peer_hash), session_id + except Exception: + sock.close() + raise + + +def start_service(device, pair_record, name): + if name != "com.apple.afc": + raise RuntimeError(f"blocked device service: {name}") + lockdown, tls_state, session_id = start_lockdown_session(device, pair_record) + service = None + try: + response = lockdown_request(lockdown, "StartService", Service=name) + service = usbmux_connect(device["DeviceID"], response["Port"]) + if response.get("EnableServiceSSL"): + service = wrap_paired_tls(service, *tls_state) + except Exception: + if service is not None: + service.close() + if session_id: + try: + lockdown_request(lockdown, "StopSession", SessionID=session_id) + except (EOFError, OSError, RuntimeError): + pass + lockdown.close() + raise + return service, lockdown, session_id + + +class AFCClient: + def __init__(self, sock, lockdown=None, session_id=None): + self.sock = sock + self.lockdown = lockdown + self.session_id = session_id + self.packet_number = 0 + + def close(self): + try: + self.sock.close() + finally: + if self.lockdown is not None: + if self.session_id: + try: + lockdown_request( + self.lockdown, "StopSession", SessionID=self.session_id + ) + except (EOFError, OSError, RuntimeError) as error: + log(f"WARNING could not confirm StopSession: {error}") + self.lockdown.close() + self.lockdown = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def request(self, operation, payload=b""): + if operation not in ALLOWED_AFC_OPERATIONS: + raise RuntimeError(f"blocked AFC operation: {operation}") + if operation == AFC_OP_FILE_OPEN: + if len(payload) < 9 or struct.unpack(" MAX_PROTOCOL_FRAME + or this_length < 40 + or this_length > entire_length + or reply_packet != packet + ): + raise RuntimeError("invalid AFC response header") + body = recv_exact(self.sock, entire_length - 40) + if reply_operation == AFC_OP_STATUS: + status = struct.unpack("= 8 else -1 + if status != 0: + raise AFCError(status, operation) + return reply_operation, body + + def read_directory(self, path): + operation, body = self.request(AFC_OP_READ_DIR, path.encode() + b"\0") + if operation != AFC_OP_DATA: + raise RuntimeError(f"unexpected AFC directory response {operation}") + return [name.decode("utf-8", "replace") for name in body.rstrip(b"\0").split(b"\0")] + + def get_file_info(self, path): + operation, body = self.request(AFC_OP_GET_FILE_INFO, path.encode() + b"\0") + if operation != AFC_OP_DATA: + raise RuntimeError(f"unexpected AFC file-info response {operation}") + values = body.rstrip(b"\0").split(b"\0") + return { + values[index].decode("utf-8", "replace"): values[index + 1].decode("utf-8", "replace") + for index in range(0, len(values) - 1, 2) + } + + def download(self, remote_path, local_path, destination_fd=None): + operation, body = self.request( + AFC_OP_FILE_OPEN, + struct.pack(" maximum_bytes: + value = value[:-1] + return value.rstrip() + + +REQUIRED_SCHEMA = { + "item": {"item_pid", "genre_id", "item_artist_pid", "album_pid", "base_location_id"}, + "item_extra": {"item_pid", "title", "location", "file_size"}, + "genre": {"genre_id", "genre"}, + "item_artist": {"item_artist_pid", "item_artist"}, + "album": {"album_pid", "album"}, + "base_location": {"base_location_id", "path"}, +} + + +MANIFEST_FIELDS = [ + "item_pid", + "genre", + "source_device_fingerprint", + "database_snapshot_sha256", + "title", + "artist", + "album", + "file_size", + "source_mtime", + "source_blocks", + "remote_path", + "filename", + "partial_name", + "actual_size", + "sha256", + "status", + "event_sha256", +] + + +def open_media_database(database_path): + connection = sqlite3.connect(f"file:{Path(database_path).resolve()}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + validate_database_schema(connection) + return connection + + +def validate_database_schema(connection): + present_tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + missing_tables = sorted(set(REQUIRED_SCHEMA) - present_tables) + if missing_tables: + raise RuntimeError( + "unsupported MediaLibrary schema; missing tables: " + ", ".join(missing_tables) + ) + missing_columns = [] + for table, required_columns in REQUIRED_SCHEMA.items(): + present_columns = { + row[1] for row in connection.execute(f'PRAGMA table_info("{table}")') + } + for column in sorted(required_columns - present_columns): + missing_columns.append(f"{table}.{column}") + if missing_columns: + raise RuntimeError( + "unsupported MediaLibrary schema; missing columns: " + ", ".join(missing_columns) + ) + + +def list_genres(database_path): + connection = open_media_database(database_path) + try: + rows = connection.execute( + """ + SELECT + g.genre, + COUNT(*) AS total_tracks, + SUM( + CASE WHEN x.location IS NOT NULL AND x.location != '' THEN 1 ELSE 0 END + ) AS transferable_tracks + FROM item AS i + JOIN genre AS g USING (genre_id) + LEFT JOIN item_extra AS x USING (item_pid) + GROUP BY g.genre + ORDER BY g.genre COLLATE NOCASE + """ + ).fetchall() + finally: + connection.close() + return [ + { + "genre": row["genre"], + "total_tracks": int(row["total_tracks"]), + "transferable_tracks": int(row["transferable_tracks"] or 0), + "excluded_without_local_file": int(row["total_tracks"]) + - int(row["transferable_tracks"] or 0), + } + for row in rows + ] + + +def get_genre_counts(database_path, genre_name): + connection = open_media_database(database_path) + try: + row = connection.execute( + """ + SELECT + COUNT(*) AS total_tracks, + SUM( + CASE WHEN x.location IS NOT NULL AND x.location != '' THEN 1 ELSE 0 END + ) AS transferable_tracks + FROM item AS i + JOIN genre AS g USING (genre_id) + LEFT JOIN item_extra AS x USING (item_pid) + WHERE g.genre = ? COLLATE NOCASE + """, + (genre_name,), + ).fetchone() + finally: + connection.close() + total = int(row["total_tracks"] or 0) + transferable = int(row["transferable_tracks"] or 0) + return { + "total_exact_matches": total, + "transferable_tracks": transferable, + "excluded_without_local_file": total - transferable, + } + + +def make_export_plan(database_path, genre_name): + connection = open_media_database(database_path) + try: + rows = connection.execute( + """ + SELECT + i.item_pid, + x.title, + COALESCE(ar.item_artist, '') AS artist, + COALESCE(al.album, '') AS album, + x.location, + b.path AS base_path, + x.file_size + FROM item AS i + JOIN item_extra AS x USING (item_pid) + JOIN genre AS g USING (genre_id) + LEFT JOIN item_artist AS ar USING (item_artist_pid) + LEFT JOIN album AS al USING (album_pid) + JOIN base_location AS b USING (base_location_id) + WHERE g.genre = ? COLLATE NOCASE + AND x.location IS NOT NULL + AND x.location != '' + ORDER BY ar.item_artist COLLATE NOCASE, x.title COLLATE NOCASE, i.item_pid + """, + (genre_name,), + ).fetchall() + finally: + connection.close() + + plan = [] + used_names = set() + for row in rows: + remote_path = make_safe_remote_media_path(row["base_path"], row["location"]) + raw_extension = PurePosixPath(row["location"]).suffix.lower() + extension = raw_extension if re.fullmatch(r"\.[a-z0-9]{1,10}", raw_extension) else "" + artist = normalize_filename_part(row["artist"], "Unknown Artist") + title = normalize_filename_part(row["title"], f"Track {row['item_pid']}") + stem = truncate_utf8(f"{artist} — {title}", 235 - len(extension.encode("utf-8"))) + candidate = f"{stem}{extension}" + suffix_number = 2 + collision_key = unicodedata.normalize("NFD", candidate).casefold() + while collision_key in used_names: + suffix = f" [{suffix_number}]" + short_stem = truncate_utf8(stem, 235 - len((suffix + extension).encode("utf-8"))) + candidate = f"{short_stem}{suffix}{extension}" + collision_key = unicodedata.normalize("NFD", candidate).casefold() + suffix_number += 1 + used_names.add(collision_key) + plan.append( + { + "item_pid": str(row["item_pid"]), + "genre": genre_name, + "source_device_fingerprint": "", + "database_snapshot_sha256": "", + "title": row["title"] or "", + "artist": row["artist"] or "", + "album": row["album"] or "", + "file_size": int(row["file_size"] or 0), + "source_mtime": "", + "source_blocks": "", + "remote_path": remote_path, + "filename": candidate, + } + ) + return plan + + +def make_safe_remote_media_path(base_path, location): + values = [str(base_path or ""), str(location or "")] + for value in values: + path = PurePosixPath(value) + if not value or "\0" in value or path.is_absolute() or any( + part in {"", ".", ".."} for part in path.parts + ): + raise RuntimeError(f"unsafe media database path component: {value!r}") + remote_path = "/" + str(PurePosixPath(values[0]) / PurePosixPath(values[1])) + if not any( + remote_path == root or remote_path.startswith(root + "/") + for root in ALLOWED_MEDIA_ROOTS + ): + raise RuntimeError(f"media path is outside approved roots: {remote_path}") + return remote_path + + +def try_get_file_info(client, remote_path): + try: + return client.get_file_info(remote_path) + except AFCError as error: + if error.status == AFC_E_OBJECT_NOT_FOUND: + return None + raise + + +def file_signature(info): + if info is None: + return None + return tuple(info.get(key) for key in ("st_size", "st_mtime", "st_blocks")) + + +def capture_database_set(client, capture_directory): + capture_directory = Path(capture_directory) + capture_directory.mkdir(parents=True, exist_ok=False) + remote_paths = [MEDIA_DATABASE_REMOTE] + [ + MEDIA_DATABASE_REMOTE + suffix for suffix in MEDIA_DATABASE_STABLE_SIDECARS + ] + forbidden_paths = [ + MEDIA_DATABASE_REMOTE + suffix for suffix in MEDIA_DATABASE_FORBIDDEN_SIDECARS + ] + observed_paths = remote_paths + forbidden_paths + before = {path: try_get_file_info(client, path) for path in observed_paths} + if before[MEDIA_DATABASE_REMOTE] is None: + raise RuntimeError(f"media database not found at {MEDIA_DATABASE_REMOTE}") + if any(before[path] is not None for path in forbidden_paths): + return None + + local_paths = {} + for remote_path in remote_paths: + info = before[remote_path] + if info is None: + continue + local_path = capture_directory / posixpath.basename(remote_path) + partial_path = local_path.with_name(local_path.name + ".partial") + try: + actual_size, _ = client.download(remote_path, partial_path) + if actual_size != int(info.get("st_size", 0)): + return None + os.chmod(partial_path, 0o600) + os.replace(partial_path, local_path) + local_paths[remote_path] = local_path + finally: + partial_path.unlink(missing_ok=True) + + after = {path: try_get_file_info(client, path) for path in observed_paths} + if any(after[path] is not None for path in forbidden_paths): + return None + if any( + file_signature(before[path]) != file_signature(after[path]) + for path in observed_paths + ): + return None + return { + "main": local_paths[MEDIA_DATABASE_REMOTE], + "hashes": {path: sha256_file(local_path) for path, local_path in local_paths.items()}, + } + + +def consolidate_database_snapshot(snapshot, output_path): + source_path = Path(snapshot["main"]) + local_shm = Path(str(source_path) + "-shm") + local_shm.unlink(missing_ok=True) + source = sqlite3.connect(f"file:{source_path.resolve()}?mode=ro", uri=True) + destination = None + try: + quick_check = [row[0] for row in source.execute("PRAGMA quick_check").fetchall()] + if quick_check != ["ok"]: + raise RuntimeError("SQLite quick_check failed for the stable media snapshot") + output_path = Path(output_path) + output_path.unlink(missing_ok=True) + destination = sqlite3.connect(output_path) + source.backup(destination) + destination.commit() + consolidated_check = [ + row[0] for row in destination.execute("PRAGMA quick_check").fetchall() + ] + if consolidated_check != ["ok"]: + raise RuntimeError("SQLite quick_check failed after snapshot consolidation") + finally: + if destination is not None: + destination.close() + source.close() + os.chmod(output_path, 0o600) + return output_path + + +def snapshot_media_database(client, work_directory, attempts=5): + work_directory = Path(work_directory) + work_directory.mkdir(parents=True, exist_ok=True) + for attempt in range(1, attempts + 1): + first = capture_database_set(client, work_directory / f"attempt-{attempt}-a") + second = capture_database_set(client, work_directory / f"attempt-{attempt}-b") + if first is not None and second is not None and first["hashes"] == second["hashes"]: + try: + return consolidate_database_snapshot( + second, work_directory / "MediaLibrary.consolidated.sqlitedb" + ) + except (RuntimeError, sqlite3.Error) as error: + log(f"DATABASE_CONSOLIDATION_RETRY attempt={attempt}/{attempts} error={error}") + else: + log(f"DATABASE_SNAPSHOT_RETRY attempt={attempt}/{attempts}") + time.sleep(0.2) + + raise RuntimeError( + "the media database did not produce two identical valid snapshots; stop playback, " + "leave the Music app idle, and retry" + ) + + +def validate_source_plan( + client, plan, source_device_fingerprint="offline-test", database_snapshot_sha256="offline-test" +): + for index, row in enumerate(plan, 1): + info = try_get_file_info(client, row["remote_path"]) + if info is None: + raise RuntimeError(f"selected source file is missing: {row['remote_path']}") + if info.get("st_ifmt") != "S_IFREG": + raise RuntimeError(f"selected source is not a regular file: {row['remote_path']}") + actual_size = int(info.get("st_size", 0)) + source_mtime = info.get("st_mtime") + if source_mtime is None: + raise RuntimeError(f"selected source has no modification signature: {row['remote_path']}") + database_size = int(row["file_size"]) + if database_size and database_size != actual_size: + raise RuntimeError( + f"stale media database size for {row['remote_path']}: " + f"database={database_size} device={actual_size}" + ) + row["file_size"] = actual_size + row["source_mtime"] = str(source_mtime) + row["source_blocks"] = str(info.get("st_blocks", "")) + row["source_device_fingerprint"] = source_device_fingerprint + row["database_snapshot_sha256"] = database_snapshot_sha256 + if index % 100 == 0 or index == len(plan): + log(f"SOURCE_PREFLIGHT {index}/{len(plan)}") + return plan + + +def ensure_source_signature(client, row): + info = try_get_file_info(client, row["remote_path"]) + if info is None or info.get("st_ifmt") != "S_IFREG": + raise RuntimeError(f"source is no longer a regular file: {row['remote_path']}") + current = ( + str(info.get("st_size", "")), + str(info.get("st_mtime", "")), + str(info.get("st_blocks", "")), + ) + expected = ( + str(row["file_size"]), + str(row["source_mtime"]), + str(row["source_blocks"]), + ) + if current != expected: + raise RuntimeError(f"source changed after planning: {row['remote_path']}") + + +def sha256_file(path, dir_fd=None): + digest = hashlib.sha256() + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, dir_fd=dir_fd) + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode): + os.close(descriptor) + raise RuntimeError(f"refusing to hash a non-regular file: {path}") + with os.fdopen(descriptor, "rb") as source: + while True: + chunk = source.read(COPY_CHUNK_SIZE) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def lexical_absolute_path(path): + return Path(os.path.abspath(os.path.expanduser(os.fspath(path)))) + + +def reject_symlink_components(path): + path = lexical_absolute_path(path) + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + try: + current_stat = os.lstat(current) + except FileNotFoundError: + continue + if stat.S_ISLNK(current_stat.st_mode): + raise RuntimeError(f"refusing path with symlink component: {current}") + return path + + +def open_destination_directory(destination): + flags = os.O_RDONLY | os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(destination, flags) + descriptor_stat = os.fstat(descriptor) + if not stat.S_ISDIR(descriptor_stat.st_mode): + os.close(descriptor) + raise RuntimeError(f"destination is not a regular directory: {destination}") + assert_destination_identity(destination, descriptor) + return descriptor + + +def assert_destination_identity(destination, descriptor): + try: + path_stat = os.lstat(destination) + except FileNotFoundError as error: + raise RuntimeError(f"destination path changed concurrently: {destination}") from error + descriptor_stat = os.fstat(descriptor) + if not stat.S_ISDIR(path_stat.st_mode) or ( + path_stat.st_dev, + path_stat.st_ino, + ) != (descriptor_stat.st_dev, descriptor_stat.st_ino): + raise RuntimeError(f"destination path changed concurrently: {destination}") + + +def full_sync(descriptor): + if sys.platform == "darwin" and hasattr(fcntl, "F_FULLFSYNC"): + fcntl.fcntl(descriptor, fcntl.F_FULLFSYNC) + else: + os.fsync(descriptor) + + +def manifest_event(row): + event = {field: str(row.get(field, "")) for field in MANIFEST_FIELDS[:-1]} + canonical = json.dumps(event, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + event["event_sha256"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return event + + +def encode_manifest_event(row): + event = manifest_event(row) + encoded = { + field: json.dumps(event[field], ensure_ascii=False, separators=(",", ":")) + for field in MANIFEST_FIELDS[:-1] + } + encoded["event_sha256"] = event["event_sha256"] + return encoded + + +def serialize_manifest_events(rows, include_header=False): + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=MANIFEST_FIELDS, delimiter="\t") + if include_header: + writer.writeheader() + writer.writerows(encode_manifest_event(row) for row in rows) + return output.getvalue().encode("utf-8") + + +def parse_manifest_bytes(data, manifest_path, require_nonempty=False): + if not data.endswith(b"\n"): + raise RuntimeError(f"existing manifest has a truncated final event: {manifest_path}") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + raise RuntimeError(f"existing manifest is not valid UTF-8: {manifest_path}") from error + reader = csv.DictReader(io.StringIO(text, newline=""), delimiter="\t") + if reader.fieldnames != MANIFEST_FIELDS: + raise RuntimeError(f"existing manifest has an incompatible format: {manifest_path}") + rows = list(reader) + latest = {} + for stored_row in rows: + try: + row = { + field: json.loads(stored_row[field]) + for field in MANIFEST_FIELDS[:-1] + } + except (KeyError, TypeError, json.JSONDecodeError) as error: + raise RuntimeError(f"existing manifest event encoding failed: {manifest_path}") from error + if any(not isinstance(value, str) for value in row.values()): + raise RuntimeError(f"existing manifest event encoding failed: {manifest_path}") + recorded_checksum = stored_row.get("event_sha256", "") + expected = manifest_event(row)["event_sha256"] + if not isinstance(recorded_checksum, str) or not hmac.compare_digest( + recorded_checksum, expected + ): + raise RuntimeError(f"existing manifest event checksum failed: {manifest_path}") + row["event_sha256"] = recorded_checksum + latest[row["filename"]] = row + if require_nonempty and not latest: + raise RuntimeError(f"manifest is empty: {manifest_path}") + return latest + + +def parse_manifest_prefix(data, manifest_path, require_nonempty=False): + if data.endswith(b"\n"): + return ( + parse_manifest_bytes(data, manifest_path, require_nonempty=require_nonempty), + len(data), + False, + ) + final_newline = data.rfind(b"\n") + if final_newline < 0: + raise RuntimeError(f"existing manifest has a truncated header: {manifest_path}") + complete_prefix = data[: final_newline + 1] + records = parse_manifest_bytes( + complete_prefix, manifest_path, require_nonempty=require_nonempty + ) + return records, len(complete_prefix), True + + +def read_all(descriptor): + os.lseek(descriptor, 0, os.SEEK_SET) + chunks = [] + while True: + chunk = os.read(descriptor, COPY_CHUNK_SIZE) + if not chunk: + return b"".join(chunks) + chunks.append(chunk) + + +def load_manifest( + manifest_path, + require_present=False, + require_nonempty=False, + tolerate_truncated_tail=False, +): + manifest_path = reject_symlink_components(manifest_path) + if not os.path.lexists(manifest_path): + if require_present: + raise RuntimeError(f"manifest does not exist: {manifest_path}") + return {} + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(manifest_path, flags) + try: + manifest_stat = os.fstat(descriptor) + if not stat.S_ISREG(manifest_stat.st_mode): + raise RuntimeError(f"manifest is not a regular file: {manifest_path}") + if tolerate_truncated_tail: + records, _, _ = parse_manifest_prefix( + read_all(descriptor), manifest_path, require_nonempty=require_nonempty + ) + return records + return parse_manifest_bytes(read_all(descriptor), manifest_path, require_nonempty) + finally: + os.close(descriptor) + + +def rename_no_clobber(parent_descriptor, source_name, target_name): + if sys.platform == "darwin": + libc = ctypes.CDLL(None, use_errno=True) + renameatx = libc.renameatx_np + renameatx.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameatx.restype = ctypes.c_int + result = renameatx( + parent_descriptor, + os.fsencode(source_name), + parent_descriptor, + os.fsencode(target_name), + 0x00000004, + ) + if result != 0: + error_number = ctypes.get_errno() + if error_number == errno.EEXIST: + raise FileExistsError(error_number, os.strerror(error_number), target_name) + raise OSError(error_number, os.strerror(error_number), target_name) + return + os.link( + source_name, + target_name, + src_dir_fd=parent_descriptor, + dst_dir_fd=parent_descriptor, + follow_symlinks=False, + ) + os.unlink(source_name, dir_fd=parent_descriptor) + + +def create_manifest_no_clobber(parent_descriptor, manifest_path, rows): + descriptor, temporary_path = tempfile.mkstemp( + prefix=f".{manifest_path.name}.", suffix=".partial", dir=manifest_path.parent + ) + temporary_name = Path(temporary_path).name + installed = False + try: + os.chmod(temporary_path, 0o600) + payload = serialize_manifest_events(rows, include_header=True) + while payload: + written = os.write(descriptor, payload) + if written <= 0: + raise IOError(f"could not create manifest: {manifest_path}") + payload = payload[written:] + full_sync(descriptor) + rename_no_clobber(parent_descriptor, temporary_name, manifest_path.name) + installed = True + full_sync(parent_descriptor) + finally: + os.close(descriptor) + if not installed: + try: + os.unlink(temporary_name, dir_fd=parent_descriptor) + except FileNotFoundError: + pass + + +def write_manifest_atomic(manifest_path, rows, expected_plan=None): + """Append checksum-bound state events without replacing an existing pathname.""" + manifest_path = reject_symlink_components(manifest_path) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + reject_symlink_components(manifest_path.parent) + parent_flags = os.O_RDONLY | os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + parent_flags |= os.O_NOFOLLOW + parent_descriptor = os.open(manifest_path.parent, parent_flags) + flags = os.O_RDWR | os.O_APPEND + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = None + try: + try: + descriptor = os.open(manifest_path.name, flags, dir_fd=parent_descriptor) + except FileNotFoundError: + if expected_plan is not None: + validate_manifest_records_for_plan( + {row["filename"]: row for row in rows}, expected_plan, manifest_path + ) + try: + create_manifest_no_clobber(parent_descriptor, manifest_path, rows) + return + except FileExistsError: + descriptor = os.open(manifest_path.name, flags, dir_fd=parent_descriptor) + descriptor_stat = os.fstat(descriptor) + if not stat.S_ISREG(descriptor_stat.st_mode): + raise RuntimeError(f"manifest is not a regular file: {manifest_path}") + fcntl.flock(descriptor, fcntl.LOCK_EX) + existing_records, complete_length, had_torn_tail = parse_manifest_prefix( + read_all(descriptor), manifest_path + ) + if expected_plan is not None: + validate_manifest_records_for_plan(existing_records, expected_plan, manifest_path) + if had_torn_tail: + os.ftruncate(descriptor, complete_length) + full_sync(descriptor) + payload = serialize_manifest_events(rows) + while payload: + written = os.write(descriptor, payload) + if written <= 0: + raise IOError(f"could not append manifest event: {manifest_path}") + payload = payload[written:] + full_sync(descriptor) + current_stat = os.stat( + manifest_path.name, dir_fd=parent_descriptor, follow_symlinks=False + ) + if (current_stat.st_dev, current_stat.st_ino) != ( + descriptor_stat.st_dev, + descriptor_stat.st_ino, + ): + raise RuntimeError(f"manifest changed concurrently; preserving it: {manifest_path}") + full_sync(parent_descriptor) + finally: + if descriptor is not None: + os.close(descriptor) + os.close(parent_descriptor) + + +def manifest_record_matches(record, row): + return all( + str(record.get(key, "")) == str(row.get(key, "")) + for key in ( + "item_pid", + "genre", + "source_device_fingerprint", + "database_snapshot_sha256", + "file_size", + "source_mtime", + "source_blocks", + "remote_path", + "filename", + ) + ) + + +def validate_manifest_records_for_plan(records, plan, manifest_path): + plan_by_filename = {row["filename"]: row for row in plan} + for filename, record in records.items(): + planned_row = plan_by_filename.get(filename) + if planned_row is None or not manifest_record_matches(record, planned_row): + raise RuntimeError( + f"existing manifest belongs to another export plan; preserving it: {manifest_path}" + ) + + +def inspect_existing_destination( + destination, plan, previous_records, destination_descriptor=None +): + destination = reject_symlink_components(destination) + owns_descriptor = destination_descriptor is None + if destination_descriptor is None: + if not os.path.lexists(destination): + return { + "trusted_files": {}, + "ready_finals": {}, + "required_bytes": sum(int(row["file_size"]) for row in plan), + } + destination_descriptor = open_destination_directory(destination) + try: + assert_destination_identity(destination, destination_descriptor) + planned_names = {row["filename"] for row in plan} + actual_names = { + name for name in os.listdir(destination_descriptor) if name != ".DS_Store" + } + unexpected = sorted(actual_names - planned_names) + if unexpected: + raise RuntimeError( + "destination contains entries outside this export plan; preserving them: " + + ", ".join(unexpected[:10]) + ) + + trusted = {} + ready_finals = {} + required_bytes = 0 + for row in plan: + filename = row["filename"] + final_path = destination / filename + record = previous_records.get(filename) + if filename in actual_names: + final_stat = os.stat( + filename, dir_fd=destination_descriptor, follow_symlinks=False + ) + if not stat.S_ISREG(final_stat.st_mode): + raise RuntimeError(f"refusing non-regular destination entry: {final_path}") + if not record or not manifest_record_matches(record, row): + raise FileExistsError( + f"preserving untrusted existing file without matching manifest: {final_path}" + ) + recorded_digest = record.get("sha256", "") + if not recorded_digest: + raise RuntimeError(f"existing file has no trusted source hash: {final_path}") + status_value = record.get("status") + if record.get("partial_name") or status_value not in ( + COMPLETE_MANIFEST_STATUSES | {"ready_to_commit"} + ): + raise RuntimeError( + f"existing file manifest status is not complete: {final_path}" + ) + if final_stat.st_size != int(row["file_size"]): + raise IOError(f"existing file size mismatch: {final_path}") + digest = sha256_file(filename, dir_fd=destination_descriptor) + if digest != recorded_digest: + raise IOError(f"existing file hash mismatch: {final_path}") + trusted[filename] = digest + if status_value == "ready_to_commit": + ready_finals[filename] = digest + continue + + required_bytes += int(row["file_size"]) + assert_destination_identity(destination, destination_descriptor) + return { + "trusted_files": trusted, + "ready_finals": ready_finals, + "required_bytes": required_bytes, + } + finally: + if owns_descriptor: + os.close(destination_descriptor) + + +def nearest_existing_directory(path): + current = reject_symlink_components(path) + while not os.path.lexists(current): + if current.parent == current: + raise RuntimeError(f"cannot resolve an existing parent for {path}") + current = current.parent + current_stat = os.lstat(current) + if not stat.S_ISDIR(current_stat.st_mode): + current = current.parent + return current + + +def check_free_space(destination, required_bytes): + parent = nearest_existing_directory(Path(destination).parent) + free_bytes = shutil.disk_usage(parent).free + headroom = min(1024**3, max(100 * 1024**2, int(required_bytes * 0.02))) + needed = required_bytes + headroom + if free_bytes < needed: + raise RuntimeError( + f"not enough free space: required_with_headroom={needed} available={free_bytes}" + ) + return {"required_bytes": required_bytes, "headroom_bytes": headroom, "free_bytes": free_bytes} + + +def default_manifest_path(destination): + destination = Path(destination) + return destination.parent / f".{destination.name}.rodgers-ios-music-export.tsv" + + +class ManifestLock: + def __init__(self, manifest_path): + self.path = Path(str(reject_symlink_components(manifest_path)) + ".lock") + self.descriptor = None + + def __enter__(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + reject_symlink_components(self.path.parent) + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + self.descriptor = os.open(self.path, flags, 0o600) + lock_stat = os.fstat(self.descriptor) + if not stat.S_ISREG(lock_stat.st_mode): + os.close(self.descriptor) + self.descriptor = None + raise RuntimeError(f"manifest lock is not a regular file: {self.path}") + try: + fcntl.flock(self.descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + os.close(self.descriptor) + self.descriptor = None + raise RuntimeError(f"another export holds the manifest lock: {self.path}") + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.descriptor is not None: + fcntl.flock(self.descriptor, fcntl.LOCK_UN) + os.close(self.descriptor) + self.descriptor = None + + +def _export_plan_locked(client, plan, destination, manifest_path): + if not plan: + raise RuntimeError("the exact requested genre has no transferable tracks") + destination = reject_symlink_components(destination) + manifest_path = reject_symlink_components(manifest_path) + if manifest_path == destination or manifest_path.is_relative_to(destination): + raise RuntimeError("store the manifest outside the destination music folder") + + previous_records = load_manifest(manifest_path, tolerate_truncated_tail=True) + validate_manifest_records_for_plan(previous_records, plan, manifest_path) + destination.mkdir(parents=True, exist_ok=True) + reject_symlink_components(destination) + destination_descriptor = open_destination_directory(destination) + try: + return _export_to_open_destination( + client, + plan, + destination, + manifest_path, + previous_records, + destination_descriptor, + ) + finally: + os.close(destination_descriptor) + + +def _export_to_open_destination( + client, + plan, + destination, + manifest_path, + previous_records, + destination_descriptor, +): + existing = inspect_existing_destination( + destination, + plan, + previous_records, + destination_descriptor=destination_descriptor, + ) + space = check_free_space(destination, existing["required_bytes"]) + + completed_bytes = 0 + copied_tracks = 0 + resumed_tracks = 0 + recovered_tracks = 0 + expected_bytes = sum(int(row["file_size"]) for row in plan) + for index, row in enumerate(plan, 1): + filename = row["filename"] + final_path = destination / filename + assert_destination_identity(destination, destination_descriptor) + digest = existing["trusted_files"].get(filename) + if digest: + actual_size = os.stat( + filename, dir_fd=destination_descriptor, follow_symlinks=False + ).st_size + if filename in existing["ready_finals"]: + status = "recovered_after_interrupt" + recovered_tracks += 1 + else: + status = "already_complete" + resumed_tracks += 1 + else: + pending_record = { + **row, + "partial_name": "", + "actual_size": "0", + "sha256": "", + "status": "pending", + } + write_manifest_atomic(manifest_path, [pending_record], expected_plan=plan) + ensure_source_signature(client, row) + try: + actual_size, digest = client.download( + row["remote_path"], filename, destination_fd=destination_descriptor + ) + except FileExistsError as error: + raise RuntimeError(f"refusing to replace destination entry: {final_path}") from error + ensure_source_signature(client, row) + assert_destination_identity(destination, destination_descriptor) + if actual_size != int(row["file_size"]): + raise IOError( + f"size mismatch for {row['remote_path']}: " + f"{actual_size} != {row['file_size']}" + ) + ready_record = { + **row, + "partial_name": "", + "actual_size": str(actual_size), + "sha256": digest, + "status": "ready_to_commit", + } + write_manifest_atomic(manifest_path, [ready_record], expected_plan=plan) + assert_destination_identity(destination, destination_descriptor) + status = "copied" + copied_tracks += 1 + + completed_bytes += actual_size + completed_record = { + **row, + "partial_name": "", + "actual_size": str(actual_size), + "sha256": digest, + "status": status, + } + assert_destination_identity(destination, destination_descriptor) + write_manifest_atomic(manifest_path, [completed_record], expected_plan=plan) + assert_destination_identity(destination, destination_descriptor) + log( + f"EXPORT_PROGRESS {index}/{len(plan)} bytes={completed_bytes}/{expected_bytes} " + f"status={status} file={row['filename']}" + ) + + return { + "tracks": len(plan), + "bytes": completed_bytes, + "copied_tracks": copied_tracks, + "resumed_tracks": resumed_tracks, + "recovered_tracks": recovered_tracks, + "destination": str(destination), + "manifest": str(manifest_path), + "space_preflight": space, + } + + +def export_plan(client, plan, destination, manifest_path): + if not plan: + raise RuntimeError("the exact requested genre has no transferable tracks") + destination = reject_symlink_components(destination) + manifest_path = reject_symlink_components(manifest_path) + if manifest_path == destination or manifest_path.is_relative_to(destination): + raise RuntimeError("store the manifest outside the destination music folder") + with ManifestLock(manifest_path): + return _export_plan_locked(client, plan, destination, manifest_path) + + +def probe_audio_file(ffprobe_path, path): + try: + probe = subprocess.run( + [ + ffprobe_path, + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=codec_name:format=format_name,duration,size:format_tags=genre", + "-of", + "json", + str(path), + ], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + except subprocess.TimeoutExpired: + return None, "ffprobe_timeout" + if probe.returncode != 0: + return None, "ffprobe_failed" + try: + return json.loads(probe.stdout), None + except json.JSONDecodeError: + return None, "ffprobe_invalid_json" + + +def verify_export( + manifest_path, + destination, + expected_genre, + probe_audio=True, + require_embedded_genre=False, +): + manifest_path = reject_symlink_components(manifest_path) + destination = reject_symlink_components(destination) + if not os.path.lexists(destination): + raise RuntimeError(f"destination is not a regular directory: {destination}") + destination_descriptor = open_destination_directory(destination) + try: + return _verify_export_open_destination( + manifest_path, + destination, + expected_genre, + probe_audio, + require_embedded_genre, + destination_descriptor, + ) + finally: + os.close(destination_descriptor) + + +def _verify_export_open_destination( + manifest_path, + destination, + expected_genre, + probe_audio, + require_embedded_genre, + destination_descriptor, +): + assert_destination_identity(destination, destination_descriptor) + records = load_manifest(manifest_path, require_present=True, require_nonempty=True) + rows = list(records.values()) + expected_names = set(records) + actual_names = { + name for name in os.listdir(destination_descriptor) if name != ".DS_Store" + } + missing = sorted(expected_names - actual_names) + unexpected = sorted(actual_names - expected_names) + non_regular = [] + size_errors = [] + hash_errors = [] + manifest_errors = [] + probe_errors = [] + no_audio_stream = [] + genre_other_files = [] + genre_missing_files = [] + genre_exact = 0 + actual_bytes = 0 + manifest_expected_bytes = 0 + extension_counts = {} + ffprobe_path = shutil.which("ffprobe") if probe_audio else None + + for index, row in enumerate(rows, 1): + filename = row.get("filename", "") + row_errors = [] + if not filename or filename != os.path.basename(filename) or filename in {".", ".."}: + row_errors.append("unsafe_filename") + if row.get("genre", "").casefold() != expected_genre.casefold(): + row_errors.append("wrong_genre") + if re.fullmatch(r"[0-9a-f]{64}", row.get("source_device_fingerprint", "")) is None: + row_errors.append("invalid_device_fingerprint") + if re.fullmatch(r"[0-9a-f]{64}", row.get("database_snapshot_sha256", "")) is None: + row_errors.append("invalid_database_snapshot") + remote_path = row.get("remote_path", "") + remote_parts = PurePosixPath(remote_path) + if ( + not remote_path + or "\0" in remote_path + or not remote_parts.is_absolute() + or any(part in {".", ".."} for part in remote_parts.parts) + or not any( + remote_path == root or remote_path.startswith(root + "/") + for root in ALLOWED_MEDIA_ROOTS + ) + ): + row_errors.append("unsafe_remote_path") + if not row.get("source_mtime"): + row_errors.append("missing_source_signature") + if row.get("status") not in COMPLETE_MANIFEST_STATUSES: + row_errors.append("incomplete_status") + if row.get("partial_name"): + row_errors.append("staging_name_present") + recorded_digest = row.get("sha256", "") + if re.fullmatch(r"[0-9a-f]{64}", recorded_digest) is None: + row_errors.append("invalid_sha256") + try: + expected_size = int(row["file_size"]) + recorded_size = int(row["actual_size"]) + if expected_size < 0 or recorded_size != expected_size: + row_errors.append("manifest_size_mismatch") + elif expected_size >= 0: + manifest_expected_bytes += expected_size + except (KeyError, TypeError, ValueError): + expected_size = -1 + row_errors.append("invalid_size") + if row_errors: + manifest_errors.append({"filename": filename, "errors": row_errors}) + if "unsafe_filename" in row_errors: + continue + + path = destination / row["filename"] + if row["filename"] not in actual_names: + continue + path_stat = os.stat( + row["filename"], + dir_fd=destination_descriptor, + follow_symlinks=False, + ) + if not stat.S_ISREG(path_stat.st_mode): + non_regular.append(row["filename"]) + continue + actual_size = path_stat.st_size + actual_bytes += actual_size + extension = path.suffix.lower() + extension_counts[extension] = extension_counts.get(extension, 0) + 1 + if actual_size != expected_size: + size_errors.append(row["filename"]) + if re.fullmatch(r"[0-9a-f]{64}", recorded_digest) and sha256_file( + row["filename"], dir_fd=destination_descriptor + ) != recorded_digest: + hash_errors.append(row["filename"]) + + if ffprobe_path: + assert_destination_identity(destination, destination_descriptor) + data, error = probe_audio_file(ffprobe_path, path) + if error: + probe_errors.append(row["filename"]) + else: + if not data.get("streams"): + no_audio_stream.append(row["filename"]) + tags = data.get("format", {}).get("tags", {}) + genre = next( + (value for key, value in tags.items() if key.casefold() == "genre"), None + ) + if genre is None: + genre_missing_files.append(row["filename"]) + elif genre.casefold() == expected_genre.casefold(): + genre_exact += 1 + else: + genre_other_files.append(row["filename"]) + if index % 50 == 0 or index == len(rows): + log(f"VERIFY_PROGRESS {index}/{len(rows)}") + + expected_bytes = manifest_expected_bytes + summary = { + "manifest_tracks": len(rows), + "destination_tracks": len(actual_names), + "expected_bytes": expected_bytes, + "actual_bytes": actual_bytes, + "extension_counts": extension_counts, + "missing": missing, + "unexpected": unexpected, + "non_regular": non_regular, + "size_errors": size_errors, + "hash_errors": hash_errors, + "manifest_errors": manifest_errors, + "audio_probe": "completed" if ffprobe_path else "skipped_ffprobe_unavailable_or_disabled", + "probe_errors": probe_errors, + "no_audio_stream": no_audio_stream, + "embedded_genre_exact": genre_exact, + "embedded_genre_other": genre_other_files, + "embedded_genre_missing": genre_missing_files, + } + fatal_lists = [ + missing, + unexpected, + non_regular, + size_errors, + hash_errors, + manifest_errors, + probe_errors, + no_audio_stream, + ] + if require_embedded_genre: + if not ffprobe_path: + raise RuntimeError("strict embedded-genre verification requires ffprobe") + fatal_lists.extend([genre_other_files, genre_missing_files]) + assert_destination_identity(destination, destination_descriptor) + if actual_bytes != expected_bytes or any(fatal_lists): + raise RuntimeError("export verification failed: " + json.dumps(summary, ensure_ascii=False)) + return summary + + +def select_device(): + devices = [ + item + for item in list_devices() + if item.get("Properties", {}).get("ConnectionType") == "USB" + ] + if len(devices) != 1: + raise RuntimeError(f"expected exactly one USB iOS device, found {len(devices)}") + return devices[0] + + +def require_macos(): + if sys.platform != "darwin": + raise RuntimeError("live iOS device access is supported only on macOS") + + +def get_device_and_pair_record(): + require_macos() + device = select_device() + properties = device.get("Properties", {}) + udid = properties.get("SerialNumber") + if not udid: + raise RuntimeError("the connected USB device has no usbmuxd serial identifier") + return device, read_pair_record(udid) + + +def probe_device(): + device, pair_record = get_device_and_pair_record() + lockdown, _, session_id = start_lockdown_session(device, pair_record) + try: + response = lockdown_request(lockdown, "GetValue") + try: + battery_response = lockdown_request( + lockdown, "GetValue", Domain="com.apple.mobile.battery" + ) + except RuntimeError: + battery_response = {} + finally: + if session_id: + try: + lockdown_request(lockdown, "StopSession", SessionID=session_id) + except (EOFError, OSError, RuntimeError) as error: + log(f"WARNING could not confirm StopSession: {error}") + lockdown.close() + safe_keys = [ + "DeviceName", + "ProductType", + "ProductVersion", + "BuildVersion", + "DeviceClass", + "BatteryCurrentCapacity", + "BatteryIsCharging", + "ExternalConnected", + ] + values = response.get("Value", {}) + battery_values = battery_response.get("Value", {}) + if not isinstance(battery_values, dict): + battery_values = {} + result = {key: battery_values.get(key, values.get(key)) for key in safe_keys} + if result.get("BatteryCurrentCapacity") is None: + result["BatteryCheck"] = "requires a visible device or Finder check" + result["SyncState"] = "requires a visible Finder or Music check" + return result + + +def with_media_database(action): + device, pair_record = get_device_and_pair_record() + with AFCClient(*start_service(device, pair_record, "com.apple.afc")) as client: + with tempfile.TemporaryDirectory(prefix=f"{PROGRAM_NAME}-") as work_directory: + database_path = snapshot_media_database(client, work_directory) + serial = str(device.get("Properties", {}).get("SerialNumber", "")) + source_device_fingerprint = hashlib.sha256( + f"{PROGRAM_NAME}\0{serial}".encode("utf-8") + ).hexdigest() + context = { + "source_device_fingerprint": source_device_fingerprint, + "database_snapshot_sha256": sha256_file(database_path), + } + return action(client, database_path, context) + + +def build_parser(): + parser = argparse.ArgumentParser( + description="Read-only export of selected music from one paired USB iOS device." + ) + parser.add_argument("--version", action="version", version=f"%(prog)s 1.0.0") + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("probe", help="show non-sensitive identity for the paired USB device") + subparsers.add_parser("genres", help="list exact genre labels and transferable track counts") + + plan_parser = subparsers.add_parser("plan", help="build a read-only exact-genre export plan") + plan_parser.add_argument("--genre", required=True) + plan_parser.add_argument("--destination") + plan_parser.add_argument("--manifest") + plan_parser.add_argument("--show-files", action="store_true") + + export_parser = subparsers.add_parser("export", help="copy and verify an exact genre") + export_parser.add_argument("--genre", required=True) + export_parser.add_argument("--destination", required=True) + export_parser.add_argument("--manifest") + export_parser.add_argument("--no-audio-probe", action="store_true") + export_parser.add_argument("--require-embedded-genre", action="store_true") + + verify_parser = subparsers.add_parser("verify", help="verify a completed local export") + verify_parser.add_argument("--manifest", required=True) + verify_parser.add_argument("--destination", required=True) + verify_parser.add_argument("--genre", required=True) + verify_parser.add_argument("--no-audio-probe", action="store_true") + verify_parser.add_argument("--require-embedded-genre", action="store_true") + return parser + + +def run(args): + if args.command == "probe": + return probe_device() + if args.command == "verify": + return verify_export( + args.manifest, + args.destination, + args.genre, + probe_audio=not args.no_audio_probe, + require_embedded_genre=args.require_embedded_genre, + ) + if args.command == "genres": + return with_media_database( + lambda _client, database, _context: {"genres": list_genres(database)} + ) + if args.command == "plan": + def plan_action(client, database, context): + plan = validate_source_plan( + client, + make_export_plan(database, args.genre), + **context, + ) + counts = get_genre_counts(database, args.genre) + result = { + "genre": args.genre, + "tracks": len(plan), + "bytes": sum(int(row["file_size"]) for row in plan), + **counts, + } + if args.destination: + destination = reject_symlink_components(args.destination) + manifest = reject_symlink_components( + args.manifest or default_manifest_path(destination) + ) + previous = load_manifest(manifest) + existing = inspect_existing_destination(destination, plan, previous) + result["destination"] = str(destination) + result["manifest"] = str(manifest) + result["space_preflight"] = check_free_space( + destination, existing["required_bytes"] + ) + if args.show_files: + result["files"] = [ + { + "artist": row["artist"], + "title": row["title"], + "album": row["album"], + "bytes": row["file_size"], + "filename": row["filename"], + } + for row in plan + ] + return result + + return with_media_database(plan_action) + if args.command == "export": + destination = reject_symlink_components(args.destination) + manifest = ( + reject_symlink_components(args.manifest) + if args.manifest + else reject_symlink_components(default_manifest_path(destination)) + ) + + def export_action(client, database, context): + plan = validate_source_plan( + client, + make_export_plan(database, args.genre), + **context, + ) + counts = get_genre_counts(database, args.genre) + export_summary = export_plan(client, plan, destination, manifest) + verification = verify_export( + manifest, + destination, + args.genre, + probe_audio=not args.no_audio_probe, + require_embedded_genre=args.require_embedded_genre, + ) + return {"selection": counts, "export": export_summary, "verification": verification} + + return with_media_database(export_action) + raise RuntimeError(f"unhandled command: {args.command}") + + +def main(): + args = build_parser().parse_args() + try: + result = run(args) + except (AFCError, EOFError, OSError, RuntimeError, sqlite3.Error, ValueError) as error: + log(f"ERROR {error}") + return 1 + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/submission/openai-directory-listing.md b/submission/openai-directory-listing.md new file mode 100644 index 0000000..f13d7b3 --- /dev/null +++ b/submission/openai-directory-listing.md @@ -0,0 +1,66 @@ +# OpenAI plugin directory submission dossier + +## Listing copy + +- **Name:** Rodger's iOS Music Export +- **Slug:** `rodgers-ios-music-export` +- **Developer:** Rodger Werkhoven +- **Category:** Productivity +- **Short description:** Export iOS music safely. +- **Long description:** In local Codex on macOS, export locally stored tracks from a USB iOS device without Finder or Music sync. Use an existing pairing or stop before Apple's persistent Trust action when pairing is absent. Select an exact genre, preserve existing files, resume interrupted copies, and verify results with sizes and SHA-256 hashes. Proven on iPad mini 2 with iOS 12.5.8; other schemas fail closed. +- **Availability requested:** Global, subject to OpenAI review and local macOS/USB capability. +- **Repository:** +- **Privacy:** +- **Terms:** +- **Support:** +- **License:** MIT +- **Logo:** `assets/logo.svg` +- **Composer icon:** `assets/composer-icon.svg` +- **Screenshots:** None; this is a skills-only plugin with no plugin UI. + +## Positive test cases + +1. **Exact export:** `Mijn oude iPad hangt via USB aan mijn Mac. Exporteer alle tracks met exact genre Electro naar een nieuwe map in Downloads, zonder te synchroniseren.` + - Invoke the skill, probe read-only, plan exact case-insensitive matches, report transferable and excluded counts plus space, export without sync, and verify the complete local result. +2. **Inventory only:** `Welke genres en aantallen staan op mijn gekoppelde iPhone? Alleen inventariseren; niets kopiëren.` + - Run `probe` and `genres`; create no destination, manifest, or device mutation. +3. **Offline verification:** `Controleer of mijn eerdere iPad-muziekexport compleet en intact is aan de hand van het manifest.` + - Run `verify` without contacting the device; reject absent, empty, incomplete, wrong-genre, or invalid-hash manifests. +4. **Interrupted export:** `Ga verder met de eerder onderbroken Jazz-export zonder bestaande bestanden te overschrijven.` + - Accept only manifest-bound completed hashes or a hash-proven `ready_to_commit` final file; never trust or delete a hashless `pending` final file. +5. **Legacy recovery in English:** `Recover all locally stored Ambient tracks from this paired USB iOS device to a new Mac folder without modifying the device.` + - Trigger the skill, qualify unsupported schemas and cloud/DRM limits, and finish only after local verification. + +## Negative test cases + +1. **Sync request:** `Synchroniseer mijn hele Mac Music-bibliotheek naar de iPad.` + - Do not invoke this reverse-export workflow; it forbids Finder/Music sync. +2. **DRM bypass:** `Haal de DRM van mijn Apple Music-downloads en exporteer ze.` + - Refuse DRM removal; explain that protected or cloud-only media may be unusable or excluded. +3. **No local USB runtime:** `Export these iPhone tracks from Codex cloud where the phone is not attached.` + - Explain that the skill requires local Codex on macOS with direct USB/usbmuxd access and take no action. + +## Release notes for 1.0.0 + +- Exact case-insensitive genre selection with adjacent-label protection. +- Two byte-identical database captures, SHA-256 comparison, SQLite `quick_check`, and local backup consolidation. +- Read-only protocol allowlists, bounded frames/timeouts, paired-certificate pinning, and credential-safe logging. +- Source-root confinement, pseudonymous device/database binding, source signatures, free-space headroom, and deterministic filenames. +- Locked append-only checksum-bound manifest with torn-tail recovery and late-plan collision protection, hash-proven final recovery, symlink rejection, and no-clobber final-file creation. +- Offline count, size, SHA-256, status, genre, regular-file, and optional `ffprobe` verification. + +## Evidence checklist + +- Official skill validator: pass. +- Official plugin validator: pass. +- Python compilation: pass. +- Unit and fault-injection tests: pass; exact count recorded in the release verification report. +- Live read-only probe on iPad mini 2 / iOS 12.5.8: pass. +- Live exact `Electro` plan: 295 transferable of 295 exact matches, 2,173,003,367 bytes, with destination free-space preflight; no destination or manifest created. +- Static opcode review: no device-side write, Pair, Unpair, or sync operation defined or reachable. +- Public repository and policy URLs: verify anonymously after publication. +- Publisher security scan: attach portal result when available. + +## Publisher and review gate + +The package can be built, validated, published, and locally installed automatically. Final submission requires Rodger Werkhoven to sign in to OpenAI's plugin submission portal with a verified Platform identity, confirm the correct organization and Apps Management role, select global availability, review public metadata and policies, accept all submission attestations, and send the package for review. Directory eligibility for a local-only macOS USB skill must be confirmed by OpenAI. Do not claim worldwide directory availability until OpenAI approves and publishes the listing. diff --git a/tests/test_ios_music_export.py b/tests/test_ios_music_export.py new file mode 100644 index 0000000..055f54c --- /dev/null +++ b/tests/test_ios_music_export.py @@ -0,0 +1,592 @@ +import hashlib +import importlib.util +import os +from pathlib import Path +import sqlite3 +import struct +import tempfile +import unittest +from unittest import mock + + +PLUGIN_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ( + PLUGIN_ROOT + / "skills" + / "rodgers-ios-music-export" + / "scripts" + / "ios_music_export.py" +) +SPEC = importlib.util.spec_from_file_location("ios_music_export", SCRIPT_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def create_media_database(path): + connection = sqlite3.connect(path) + connection.executescript( + """ + CREATE TABLE item ( + item_pid INTEGER PRIMARY KEY, + genre_id INTEGER, + item_artist_pid INTEGER, + album_pid INTEGER, + base_location_id INTEGER + ); + CREATE TABLE item_extra ( + item_pid INTEGER PRIMARY KEY, + title TEXT, + location TEXT, + file_size INTEGER + ); + CREATE TABLE genre (genre_id INTEGER PRIMARY KEY, genre TEXT); + CREATE TABLE item_artist (item_artist_pid INTEGER PRIMARY KEY, item_artist TEXT); + CREATE TABLE album (album_pid INTEGER PRIMARY KEY, album TEXT); + CREATE TABLE base_location (base_location_id INTEGER PRIMARY KEY, path TEXT); + """ + ) + connection.executemany( + "INSERT INTO genre VALUES (?, ?)", + [(1, "Electro"), (2, "electro"), (3, "Elektronisch"), (4, "Electro House")], + ) + connection.execute("INSERT INTO item_artist VALUES (10, 'Test/Artist')") + connection.execute("INSERT INTO album VALUES (20, 'Test Album')") + connection.execute("INSERT INTO base_location VALUES (30, 'iTunes_Control/Music')") + rows = [ + (101, 1, "Same: Track", "F00/AAAA.m4a", 5), + (102, 2, "Same: Track", "F01/BBBB.M4A", 6), + (103, 3, "Andere track", "F02/CCCC.m4a", 7), + (104, 4, "House track", "F03/DDDD.m4a", 8), + ] + for item_pid, genre_id, title, location, size in rows: + connection.execute("INSERT INTO item VALUES (?, ?, 10, 20, 30)", (item_pid, genre_id)) + connection.execute( + "INSERT INTO item_extra VALUES (?, ?, ?, ?)", + (item_pid, title, location, size), + ) + connection.commit() + connection.close() + + +class FakeAFCClient: + def __init__(self, content): + self.content = content + self.download_calls = [] + + def download(self, remote_path, local_path, destination_fd=None): + self.download_calls.append(remote_path) + data = self.content[remote_path] + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(local_path, flags, 0o600, dir_fd=destination_fd) + with os.fdopen(descriptor, "wb") as output: + output.write(data) + return len(data), hashlib.sha256(data).hexdigest() + + def get_file_info(self, remote_path): + if remote_path not in self.content: + raise MODULE.AFCError(MODULE.AFC_E_OBJECT_NOT_FOUND, MODULE.AFC_OP_GET_FILE_INFO) + data = self.content[remote_path] + return { + "st_ifmt": "S_IFREG", + "st_size": str(len(data)), + "st_mtime": "1", + "st_blocks": "1", + } + + +def sample_plan(): + return [ + { + "item_pid": "1", + "genre": "Electro", + "source_device_fingerprint": "a" * 64, + "database_snapshot_sha256": "b" * 64, + "title": "One", + "artist": "Artist", + "album": "Album", + "file_size": 5, + "source_mtime": "1", + "source_blocks": "1", + "remote_path": "/iTunes_Control/Music/F00/one.m4a", + "filename": "Artist — One.m4a", + }, + { + "item_pid": "2", + "genre": "Electro", + "source_device_fingerprint": "a" * 64, + "database_snapshot_sha256": "b" * 64, + "title": "Two", + "artist": "Artist", + "album": "Album", + "file_size": 6, + "source_mtime": "1", + "source_blocks": "1", + "remote_path": "/iTunes_Control/Music/F01/two.mp3", + "filename": "Artist — Two.mp3", + }, + ] + + +class IOSMusicExportTests(unittest.TestCase): + def test_exact_case_insensitive_genre_and_collision_names(self): + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / "MediaLibrary.sqlitedb" + create_media_database(database) + plan = MODULE.make_export_plan(database, "ELECTRO") + + self.assertEqual([row["item_pid"] for row in plan], ["101", "102"]) + self.assertEqual(plan[0]["filename"], "Test⁄Artist — Same - Track.m4a") + self.assertEqual(plan[1]["filename"], "Test⁄Artist — Same - Track [2].m4a") + self.assertTrue(plan[0]["remote_path"].endswith("/F00/AAAA.m4a")) + + def test_adjacent_genres_are_not_selected(self): + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / "MediaLibrary.sqlitedb" + create_media_database(database) + plan = MODULE.make_export_plan(database, "Elektronisch") + + self.assertEqual(len(plan), 1) + self.assertEqual(plan[0]["item_pid"], "103") + + def test_unknown_schema_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / "MediaLibrary.sqlitedb" + sqlite3.connect(database).close() + with self.assertRaisesRegex(RuntimeError, "unsupported MediaLibrary schema"): + MODULE.make_export_plan(database, "Electro") + + def test_two_identical_database_captures_are_consolidated_and_checked(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source.sqlitedb" + create_media_database(source) + client = FakeAFCClient({MODULE.MEDIA_DATABASE_REMOTE: source.read_bytes()}) + snapshot = MODULE.snapshot_media_database(client, root / "snapshot-work") + plan = MODULE.make_export_plan(snapshot, "Electro") + + self.assertEqual(len(plan), 2) + self.assertEqual(client.download_calls.count(MODULE.MEDIA_DATABASE_REMOTE), 2) + + def test_export_hashes_verifies_and_resumes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + plan = sample_plan() + client = FakeAFCClient({ + "/iTunes_Control/Music/F00/one.m4a": b"12345", + "/iTunes_Control/Music/F01/two.mp3": b"abcdef", + }) + first = MODULE.export_plan(client, plan, destination, manifest) + verification = MODULE.verify_export( + manifest, destination, "Electro", probe_audio=False + ) + second_client = FakeAFCClient({}) + second = MODULE.export_plan(second_client, plan, destination, manifest) + + self.assertEqual(first["copied_tracks"], 2) + self.assertEqual(verification["manifest_tracks"], 2) + self.assertEqual(verification["hash_errors"], []) + self.assertEqual(second["resumed_tracks"], 2) + self.assertEqual(second_client.download_calls, []) + + def test_untrusted_existing_file_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory).resolve() / "music" + destination.mkdir() + (destination / "Artist — One.m4a").write_bytes(b"12345") + plan = [sample_plan()[0]] + with self.assertRaisesRegex(FileExistsError, "preserving untrusted"): + MODULE.inspect_existing_destination(destination, plan, {}) + + def test_verify_requires_present_nonempty_manifest_and_correct_genre(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + destination.mkdir() + manifest = root / "manifest.tsv" + with self.assertRaisesRegex(RuntimeError, "manifest does not exist"): + MODULE.verify_export(manifest, destination, "Electro", probe_audio=False) + MODULE.write_manifest_atomic(manifest, []) + with self.assertRaisesRegex(RuntimeError, "manifest is empty"): + MODULE.verify_export(manifest, destination, "Electro", probe_audio=False) + + client = FakeAFCClient({ + "/iTunes_Control/Music/F00/one.m4a": b"12345", + "/iTunes_Control/Music/F01/two.mp3": b"abcdef", + }) + MODULE.export_plan(client, sample_plan(), destination, manifest) + with self.assertRaisesRegex(RuntimeError, "wrong_genre"): + MODULE.verify_export(manifest, destination, "Jazz", probe_audio=False) + + def test_pending_manifest_never_trusts_same_size_final_file(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + destination.mkdir() + manifest = root / "manifest.tsv" + row = sample_plan()[0] + (destination / row["filename"]).write_bytes(b"WRONG") + pending = { + **row, + "partial_name": "", + "actual_size": "0", + "sha256": "", + "status": "pending", + } + MODULE.write_manifest_atomic(manifest, [pending]) + with self.assertRaisesRegex(RuntimeError, "no trusted source hash"): + MODULE.inspect_existing_destination(destination, [row], MODULE.load_manifest(manifest)) + + pending["sha256"] = hashlib.sha256(b"WRONG").hexdigest() + MODULE.write_manifest_atomic(manifest, [pending]) + with self.assertRaisesRegex(RuntimeError, "status is not complete"): + MODULE.inspect_existing_destination(destination, [row], MODULE.load_manifest(manifest)) + + def test_late_collision_is_not_overwritten(self): + class RacingClient(FakeAFCClient): + def __init__(self, content, final_path): + super().__init__(content) + self.final_path = final_path + + def download(self, remote_path, local_path, destination_fd=None): + self.final_path.write_bytes(b"RIVAL") + return super().download( + remote_path, local_path, destination_fd=destination_fd + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + row = sample_plan()[0] + final_path = destination / row["filename"] + client = RacingClient({row["remote_path"]: b"12345"}, final_path) + with self.assertRaisesRegex(RuntimeError, "refusing to replace"): + MODULE.export_plan(client, [row], destination, manifest) + self.assertEqual(final_path.read_bytes(), b"RIVAL") + + def test_destination_symlink_swap_cannot_redirect_track_write(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + moved_destination = root / "music-original" + escape = root / "escape" + escape.mkdir() + manifest = root / "manifest.tsv" + row = sample_plan()[0] + + class SwappingClient(FakeAFCClient): + def download(self, remote_path, local_path, destination_fd=None): + destination.rename(moved_destination) + destination.symlink_to(escape, target_is_directory=True) + return super().download( + remote_path, local_path, destination_fd=destination_fd + ) + + client = SwappingClient({row["remote_path"]: b"12345"}) + with self.assertRaisesRegex(RuntimeError, "destination path changed concurrently"): + MODULE.export_plan(client, [row], destination, manifest) + self.assertFalse((escape / row["filename"]).exists()) + self.assertEqual((moved_destination / row["filename"]).read_bytes(), b"12345") + self.assertEqual(MODULE.load_manifest(manifest)[row["filename"]]["status"], "pending") + + def test_download_failure_preserves_later_resume_records(self): + class FailingClient(FakeAFCClient): + def download(self, remote_path, local_path, destination_fd=None): + raise IOError("injected download failure") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + plan = sample_plan() + content = { + "/iTunes_Control/Music/F00/one.m4a": b"12345", + "/iTunes_Control/Music/F01/two.mp3": b"abcdef", + } + MODULE.export_plan(FakeAFCClient(content), plan, destination, manifest) + (destination / plan[0]["filename"]).unlink() + + with self.assertRaisesRegex(IOError, "injected"): + MODULE.export_plan(FailingClient(content), plan, destination, manifest) + records = MODULE.load_manifest(manifest) + self.assertEqual(set(records), {plan[0]["filename"], plan[1]["filename"]}) + self.assertTrue(records[plan[1]["filename"]]["sha256"]) + + summary = MODULE.export_plan(FakeAFCClient(content), plan, destination, manifest) + self.assertEqual(summary["copied_tracks"], 1) + self.assertEqual(summary["resumed_tracks"], 1) + + def test_interrupted_direct_final_is_preserved_and_not_trusted(self): + class PartialFailureClient(FakeAFCClient): + def download(self, remote_path, local_path, destination_fd=None): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open( + local_path, flags, 0o600, dir_fd=destination_fd + ) + with os.fdopen(descriptor, "wb") as output: + output.write(b"12") + raise IOError("injected mid-track interruption") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + row = sample_plan()[0] + client = PartialFailureClient({row["remote_path"]: b"12345"}) + with self.assertRaisesRegex(IOError, "mid-track interruption"): + MODULE.export_plan(client, [row], destination, manifest) + final_path = destination / row["filename"] + self.assertEqual(final_path.read_bytes(), b"12") + with self.assertRaisesRegex(RuntimeError, "no trusted source hash"): + MODULE.export_plan(FakeAFCClient({row["remote_path"]: b"12345"}), [row], destination, manifest) + self.assertEqual(final_path.read_bytes(), b"12") + + def test_ready_final_is_hash_verified_and_recovered(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + destination.mkdir() + manifest = root / "manifest.tsv" + row = sample_plan()[0] + final_path = destination / row["filename"] + final_path.write_bytes(b"12345") + ready = { + **row, + "partial_name": "", + "actual_size": "5", + "sha256": hashlib.sha256(b"12345").hexdigest(), + "status": "ready_to_commit", + } + MODULE.write_manifest_atomic(manifest, [ready]) + + summary = MODULE.export_plan(FakeAFCClient({}), [row], destination, manifest) + verification = MODULE.verify_export( + manifest, destination, "Electro", probe_audio=False + ) + self.assertEqual(summary["recovered_tracks"], 1) + self.assertEqual(verification["hash_errors"], []) + self.assertEqual(final_path.read_bytes(), b"12345") + + def test_unrelated_manifest_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + unrelated = { + **sample_plan()[0], + "filename": "Unrelated.m4a", + "partial_name": "", + "actual_size": "5", + "sha256": "c" * 64, + "status": "copied", + } + MODULE.write_manifest_atomic(manifest, [unrelated]) + original = manifest.read_bytes() + with self.assertRaisesRegex(RuntimeError, "another export plan"): + MODULE.export_plan( + FakeAFCClient({sample_plan()[0]["remote_path"]: b"12345"}), + [sample_plan()[0]], + destination, + manifest, + ) + self.assertEqual(manifest.read_bytes(), original) + self.assertFalse(destination.exists()) + + def test_late_unrelated_manifest_collision_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + row = sample_plan()[0] + unrelated = { + **row, + "filename": "Late unrelated.m4a", + "partial_name": "", + "actual_size": "5", + "sha256": "c" * 64, + "status": "copied", + } + original_inspect = MODULE.inspect_existing_destination + collided_bytes = [] + + def inject_manifest_collision( + target, plan, previous_records, destination_descriptor=None + ): + result = original_inspect( + target, + plan, + previous_records, + destination_descriptor=destination_descriptor, + ) + MODULE.write_manifest_atomic(manifest, [unrelated]) + with manifest.open("ab") as output: + output.write(b"torn-unrelated-tail") + collided_bytes.append(manifest.read_bytes()) + return result + + with mock.patch.object( + MODULE, "inspect_existing_destination", side_effect=inject_manifest_collision + ): + with self.assertRaisesRegex(RuntimeError, "another export plan"): + MODULE.export_plan( + FakeAFCClient({row["remote_path"]: b"12345"}), + [row], + destination, + manifest, + ) + self.assertEqual(manifest.read_bytes(), collided_bytes[0]) + with self.assertRaisesRegex(RuntimeError, "truncated final event"): + MODULE.load_manifest(manifest) + + def test_fault_during_initial_manifest_write_installs_no_manifest(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + manifest = root / "manifest.tsv" + row = { + **sample_plan()[0], + "partial_name": "", + "actual_size": "0", + "sha256": "", + "status": "pending", + } + real_write = os.write + + def fail_before_first_byte(descriptor, payload): + raise OSError(28, "injected no space") + + with mock.patch.object(MODULE.os, "write", side_effect=fail_before_first_byte): + with self.assertRaisesRegex(OSError, "injected no space"): + MODULE.write_manifest_atomic(manifest, [row], expected_plan=[row]) + self.assertFalse(manifest.exists()) + + def fail_after_partial_header(descriptor, payload): + real_write(descriptor, payload[:7]) + raise OSError(28, "injected partial header") + + with mock.patch.object(MODULE.os, "write", side_effect=fail_after_partial_header): + with self.assertRaisesRegex(OSError, "injected partial header"): + MODULE.write_manifest_atomic(manifest, [row], expected_plan=[row]) + self.assertFalse(manifest.exists()) + self.assertEqual(list(root.glob("*.partial")), []) + + MODULE.write_manifest_atomic(manifest, [row], expected_plan=[row]) + self.assertEqual(MODULE.load_manifest(manifest)[row["filename"]]["status"], "pending") + + def test_torn_trailing_manifest_event_is_recovered_under_export_lock(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + destination = root / "music" + manifest = root / "manifest.tsv" + row = sample_plan()[0] + row["title"] = "Line one\nLine two" + MODULE.export_plan( + FakeAFCClient({row["remote_path"]: b"12345"}), + [row], + destination, + manifest, + ) + with manifest.open("ab") as output: + output.write(b"torn-final-event") + with self.assertRaisesRegex(RuntimeError, "truncated final event"): + MODULE.load_manifest(manifest) + + summary = MODULE.export_plan(FakeAFCClient({}), [row], destination, manifest) + verification = MODULE.verify_export( + manifest, destination, "Electro", probe_audio=False + ) + self.assertEqual(summary["resumed_tracks"], 1) + self.assertEqual(verification["hash_errors"], []) + self.assertTrue(manifest.read_bytes().endswith(b"\n")) + self.assertNotIn(b"Line one\nLine two", manifest.read_bytes()) + + def test_symlink_destination_and_unexpected_directory_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + real_destination = root / "real" + real_destination.mkdir() + linked_destination = root / "linked" + linked_destination.symlink_to(real_destination, target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, "symlink component"): + MODULE.inspect_existing_destination(linked_destination, sample_plan(), {}) + + (real_destination / "unexpected-directory").mkdir() + with self.assertRaisesRegex(RuntimeError, "entries outside"): + MODULE.inspect_existing_destination(real_destination, sample_plan(), {}) + + def test_media_paths_are_confined_to_approved_roots(self): + self.assertEqual( + MODULE.make_safe_remote_media_path("iTunes_Control/Music/F00", "ABCD.m4a"), + "/iTunes_Control/Music/F00/ABCD.m4a", + ) + with self.assertRaisesRegex(RuntimeError, "unsafe media database path"): + MODULE.make_safe_remote_media_path("iTunes_Control/Music/F00", "../secret") + with self.assertRaisesRegex(RuntimeError, "outside approved roots"): + MODULE.make_safe_remote_media_path("Safari", "history.db") + + def test_protocol_dispatchers_block_unknown_operations(self): + with self.assertRaisesRegex(RuntimeError, "blocked AFC operation"): + MODULE.AFCClient(None).request(0xFFFF) + writable_open = struct.pack("