#!/usr/bin/env python3
"""Leyr vs other self-hosted music servers: one server per invocation.

    bench.py SERVER [--phase main|scan] [--run N] [--idle SECONDS] [--out DIR]

SERVER is one of leyr, navidrome, gonic, jellyfin. Only the standard library
is used. Everything runs in throwaway rootless podman containers named
bench-*, each with the same CPU and memory limits, the same read-only music
mount, and a fresh named volume. See bench/README.md.
"""
import argparse, hashlib, http.client, json, os, re, statistics, subprocess
import threading, time, urllib.parse
import rss_breakdown

MUSIC = os.environ["BENCH_MUSIC"]  # the subset copy, a host path
CPUS = os.environ.get("BENCH_CPUS", "4")
MEM = os.environ.get("BENCH_MEM", "4g")
CPUSET = os.environ.get("BENCH_CPUSET", "")  # optional: pin every container to these cores (e.g. 16-19)
NET = os.environ.get("BENCH_NETWORK", "bench-noegress")  # an internal podman network: no route out, published ports still work
LIBRARY = os.environ.get("BENCH_LIBRARY", "")  # a label written into every result file ("public" or "real")
BREAKDOWN = bool(os.environ.get("BENCH_BREAKDOWN"))  # also record RSS by owner (rss_breakdown.py) at the peak and at idle
LONG_FLAC_HINT = os.environ.get("BENCH_LONG_FLAC", "Long Form Drone")  # part of the path of one FLAC over 25 minutes


def sh(*a, check=True):
    r = subprocess.run(a, capture_output=True, text=True)
    if check and r.returncode:
        raise RuntimeError(f"{a}: {r.stderr[:500]}")
    return r.stdout.strip()


def pct(xs, p):
    xs = sorted(xs)
    if not xs:
        return None
    return xs[min(len(xs) - 1, int(p / 100 * len(xs)))]


def summ(xs):
    if not xs:
        return {"n": 0}
    return {"n": len(xs), "median": statistics.median(xs), "p95": pct(xs, 95),
            "min": min(xs), "max": max(xs)}


# ---------------------------------------------------------------- process stats
def tree_pids(root):
    kids = {}
    for d in os.listdir("/proc"):
        if d.isdigit():
            try:
                st = open(f"/proc/{d}/stat").read()
                ppid = int(st.rsplit(")", 1)[1].split()[1])
                kids.setdefault(ppid, []).append(int(d))
            except Exception:
                pass
    out, todo = [], [root]
    while todo:
        p = todo.pop()
        out.append(p)
        todo += kids.get(p, [])
    return out


def proc_stats(root):
    """Sum RSS, PSS, private (anonymous) memory and threads over the container's process tree. Anonymous is read from
    smaps_rollup: pages owned by the process and not backed by a file, so the executable, libraries and mapped
    database files (which the kernel can reclaim at will) are left out."""
    rss = pss = thr = procs = anon = 0
    for p in tree_pids(root):
        try:
            for ln in open(f"/proc/{p}/status"):
                if ln.startswith("VmRSS:"):
                    rss += int(ln.split()[1])
                elif ln.startswith("Threads:"):
                    thr += int(ln.split()[1])
            for ln in open(f"/proc/{p}/smaps_rollup"):
                if ln.startswith("Pss:"):
                    pss += int(ln.split()[1])
                elif ln.startswith("Anonymous:"):
                    anon += int(ln.split()[1])
            procs += 1
        except Exception:
            pass
    return {"rss_mb": rss / 1024, "pss_mb": pss / 1024, "anon_mb": anon / 1024,
            "threads": thr, "procs": procs}


def cpu_seconds(root):
    """User+system CPU seconds used so far by the container's process tree (live processes)."""
    tck = os.sysconf("SC_CLK_TCK")
    tot = 0
    for p in tree_pids(root):
        try:
            f = open(f"/proc/{p}/stat").read().rsplit(")", 1)[1].split()
            tot += int(f[11]) + int(f[12])
        except Exception:
            pass
    return tot / tck


def cgroup_mem_mb(name):
    try:
        out = sh("podman", "stats", "--no-stream", "--format", "{{.MemUsage}}", name)
        m = re.match(r"([\d.]+)\s*([kKMG]i?B)", out)
        return float(m.group(1)) * {"kB": 1 / 1024, "KiB": 1 / 1024, "MB": 1, "MiB": 1, "GB": 1024, "GiB": 1024}[m.group(2)]
    except Exception:
        return None


class Sampler(threading.Thread):
    def __init__(self, pid, every=0.5):
        super().__init__(daemon=True)
        self.pid, self.every, self.samples, self.ev = pid, every, [], threading.Event()
        self.peak = None

    def run(self):
        while not self.ev.is_set():
            st = proc_stats(self.pid)
            self.samples.append(st)
            if BREAKDOWN and (self.peak is None or st["rss_mb"] > self.peak[0]):
                self.peak = (st["rss_mb"], rss_breakdown.snapshot(self.pid))  # the owners at the highest RSS seen
            self.ev.wait(self.every)

    def stop(self):
        self.ev.set()
        self.join()
        rs = [s["rss_mb"] for s in self.samples]
        an = [s["anon_mb"] for s in self.samples]  # private (anonymous) memory, see proc_stats
        out = {"samples": len(rs), "rss_peak_mb": max(rs, default=None),
               "rss_median_mb": statistics.median(rs) if rs else None,
               "anon_peak_mb": max(an, default=None),
               "anon_median_mb": statistics.median(an) if an else None,
               "threads_peak": max((s["threads"] for s in self.samples), default=None)}
        if self.peak:
            out["peak_breakdown"] = self.peak[1]
        return out


# ---------------------------------------------------------------- HTTP helpers
class Conn:
    def __init__(self, port):
        self.port, self.c = port, None

    def req(self, path, headers=None, method="GET", body=None):
        for attempt in range(2):
            try:
                if self.c is None:
                    self.c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=120)
                self.c.request(method, path, body=body, headers=headers or {})
                r = self.c.getresponse()
                return r.status, r.read(), r
            except (http.client.HTTPException, OSError):
                self.c = None
                if attempt:
                    raise


def get_json(port, path, headers=None, method="GET", body=None):
    st, data, _ = Conn(port).req(path, headers, method, body)
    try:
        return st, json.loads(data)
    except Exception:
        return st, data[:300]


def wait_http(port, path, ok=lambda st: st == 200, timeout=180, t0=None):
    t0 = t0 or time.time()
    while time.time() - t0 < timeout:
        try:
            st, _, _ = Conn(port).req(path)
            if ok(st):
                return time.time() - t0
        except Exception:
            pass
        time.sleep(0.01)
    raise TimeoutError(path)


def subsonic_qs(user, pw, extra=None):
    salt = "benchsalt"
    q = {"u": user, "t": hashlib.md5((pw + salt).encode()).hexdigest(), "s": salt,
         "v": "1.16.1", "c": "leyr-bench", "f": "json"}
    q.update(extra or {})
    return urllib.parse.urlencode(q)


# ---------------------------------------------------------------- servers
class Server:
    name = ""
    image = ""
    inner_port = 0
    host_port = 0
    ping_path = ""
    env = {}
    volumes = []   # (volume suffix, container path)
    music_mount = "/music"
    extra_args = []
    user_name = "bench"
    user_pass = "benchpass-123"

    def cname(self):
        return f"bench-{self.name}"

    def vols(self):
        return [(f"bench-{self.name}-{s}", p) for s, p in self.volumes]

    def create(self):
        for v, _ in self.vols():
            sh("podman", "volume", "create", v)
        args = ["podman", "create", "--name", self.cname(), f"--cpus={CPUS}", f"--memory={MEM}",
                "--memory-swap", MEM, "--network", NET, "-p", f"127.0.0.1:{self.host_port}:{self.inner_port}",
                "-v", f"{MUSIC}:{self.music_mount}:ro"]
        for v, p in self.vols():
            args += ["-v", f"{v}:{p}"]
        for k, v in self.env.items():
            args += ["-e", f"{k}={v}"]
        sh(*args, *self.extra_args, self.image)

    def pin(self):
        """Rootless podman has no cpuset controller, so pin by affinity: every process and thread of the
        container, right after it starts (threads created later inherit it)."""
        if not CPUSET:
            return
        for p in tree_pids(self.pid()):
            subprocess.run(["taskset", "-a", "-cp", CPUSET, str(p)], capture_output=True)

    def start(self):
        t0 = time.time()
        sh("podman", "start", self.cname())
        self.pin()
        return t0

    def pid(self):
        return int(sh("podman", "inspect", "-f", "{{.State.Pid}}", self.cname()))

    def stop(self):
        sh("podman", "stop", "-t", "10", self.cname(), check=False)

    def destroy(self):
        sh("podman", "rm", "-f", self.cname(), check=False)
        for v, _ in self.vols():
            sh("podman", "volume", "rm", "-f", v, check=False)

    def logs(self):
        return sh("podman", "logs", self.cname(), check=False)

    def wait_ready(self, t0):
        return wait_http(self.host_port, self.ping_path, t0=t0)

    def setup(self):
        pass

    def start_scan(self):
        pass

    def scan_seconds_from_logs(self):
        """A server's own account of how long its scan took, if its log has one (None otherwise)."""
        return None

    def scan_done(self):
        r = get_json(self.host_port, "/rest/getScanStatus?" + subsonic_qs(self.user_name, self.user_pass))[1]
        ss = r["subsonic-response"]["scanStatus"]
        return (not ss.get("scanning"), ss.get("count", 0))

    def api(self):
        return SubsonicApi(self.host_port, self.user_name, self.user_pass)


class SubsonicApi:
    """The Subsonic endpoints measured for Leyr, Navidrome and gonic."""
    QUERIES = ["love", "night", "the", "black", "dark", "sun", "fire", "blue", "star", "dream"]

    def __init__(self, port, user, pw):
        self.port, self.user, self.pw = port, user, pw
        self.discover()

    def url(self, ep, **kw):
        return f"/rest/{ep}?" + subsonic_qs(self.user, self.pw, kw)

    def call(self, ep, **kw):
        st, d = get_json(self.port, self.url(ep, **kw))
        return d["subsonic-response"] if isinstance(d, dict) else {"status": "bad", "raw": d}

    def hdrs(self):
        return {}

    def discover(self):
        every = self.call("getAlbumList2", type="alphabeticalByName", size=500, offset=0)["albumList2"].get("album", [])
        albums = every[100:120] if len(every) >= 120 else every[-20:]  # a small library has no 100th album
        self.album_ids = [a["id"] for a in albums]
        self.cover_ids = [a.get("coverArt") or a["id"] for a in albums]
        self.flac_tracks, self.long_track = [], None
        offs = 0
        while len(self.flac_tracks) < 20 and offs < 600:
            lst = self.call("getAlbumList2", type="alphabeticalByName", size=50, offset=offs)["albumList2"].get("album", [])
            if not lst:
                break
            for a in lst:
                for s in self.call("getAlbum", id=a["id"])["album"].get("song", []):
                    if s.get("suffix") == "flac" and s.get("duration", 0) < 900 and len(self.flac_tracks) < 20:
                        self.flac_tracks.append(s["id"])
            offs += 50
        r = self.call("search3", query=LONG_FLAC_HINT, songCount=10, artistCount=0, albumCount=0)
        for song in r.get("searchResult3", {}).get("song", []):
            if song.get("duration", 0) > 25 * 60 and song.get("suffix") == "flac":
                self.long_track = song["id"]

    def endpoints(self):
        n, q = len(self.album_ids), self.QUERIES
        return {
            "ping": lambda i: self.url("ping"),
            "getAlbumList2_newest50": lambda i: self.url("getAlbumList2", type="newest", size=50),
            "getArtists": lambda i: self.url("getArtists"),
            "getAlbum": lambda i: self.url("getAlbum", id=self.album_ids[i % n]),
            "search3": lambda i: self.url("search3", query=q[i % len(q)], songCount=20, albumCount=20, artistCount=20),
            "getCoverArt_300": lambda i: self.url("getCoverArt", id=self.cover_ids[i % n], size=300),
        }

    def stream_orig(self, tid):
        return self.url("stream", id=tid, format="raw")

    def stream_mp3(self, tid, offset=0):
        kw = dict(id=tid, format="mp3", maxBitRate=320)
        if offset:
            kw["timeOffset"] = offset
        return self.url("stream", **kw)

    def stream_opus(self, tid, offset=0):
        kw = dict(id=tid, format="opus", maxBitRate=128)
        if offset:
            kw["timeOffset"] = offset
        return self.url("stream", **kw)


class Leyr(Server):
    name = "leyr"
    image = os.environ.get("BENCH_LEYR_IMAGE", "localhost/bench-leyr")
    inner_port = 5176
    host_port = 19176
    volumes = [("data", "/app/data")]
    env = {"LIBRARY_FOLDERS": "/music", "LIBRARY_SCAN_MINUTES": "0",
           "SOULSEERR_BACKGROUND_JOBS": "off", "LEYR_REQUEST_QUEUE": "off",
           "LEYR_ROOMS_RUNTIME": "off", "LEYR_EXPORT_RUNTIME": "off",
           "LEYR_BACKGROUND_RUNTIME": "off",
           "SOULSEERR_JWT_SECRET": "bench-only-secret-0123456789abcdef",
           "MUSICBRAINZ_USER_AGENT": "leyr-bench/0 (local benchmark)"}

    def __init__(self):
        # BENCH_LEYR_ENV="NAME=value,NAME=value": extra variables for an experiment (never in a published run)
        self.env = dict(self.env)
        for kv in filter(None, os.environ.get("BENCH_LEYR_ENV", "").split(",")):
            k, _, v = kv.partition("=")
            self.env[k] = v

    def scan_seconds_from_logs(self):
        return leyr_scan_seconds(self.logs())

    # BENCH_LEYR_BIN runs a locally built leyr-server directly instead of the image: no container,
    # so no cgroup limit (BENCH_CPU_LIST pins it to cores with taskset). The same environment as the
    # container, MALLOC_ARENA_MAX=2 included. Needs ffmpeg on PATH and BENCH_DATA, a scratch directory.
    native = os.environ.get("BENCH_LEYR_BIN")

    def _data(self):
        return os.environ["BENCH_DATA"]

    def create(self):
        if not self.native:
            return super().create()
        os.makedirs(self._data(), exist_ok=True)

    def start(self):
        if not self.native:
            return super().start()
        env = dict(os.environ)
        env.update(self.env)
        env.update(MALLOC_ARENA_MAX="2", PORT=str(self.host_port), LIBRARY_FOLDERS=MUSIC,
                   SOULSEERR_DATA_DIR=self._data(),
                   SOULSEERR_DB_PATH=os.path.join(self._data(), "soulseerr.db"),
                   LEYR_SERVER_ROOT=self._data())
        cmd = [self.native]
        if os.environ.get("BENCH_CPU_LIST"):
            cmd = ["taskset", "-c", os.environ["BENCH_CPU_LIST"]] + cmd
        self.log = open(os.path.join(self._data(), "server.log"), "ab")
        self.proc = subprocess.Popen(cmd, env=env, cwd=self._data(), stdout=self.log, stderr=self.log)
        return time.time()

    def pid(self):
        return self.proc.pid if self.native else super().pid()

    def stop(self):
        if not self.native:
            return super().stop()
        if getattr(self, "proc", None) and self.proc.poll() is None:
            self.proc.terminate()
            try:
                self.proc.wait(15)
            except subprocess.TimeoutExpired:
                self.proc.kill()

    def destroy(self):
        if not self.native:
            return super().destroy()
        self.stop()
        import shutil
        heap = os.path.join(self._data(), "dhat-heap.json")  # a --features dhat-heap build writes this at exit
        if os.path.exists(heap) and os.environ.get("BENCH_KEEP_DHAT"):
            shutil.copy(heap, os.environ["BENCH_KEEP_DHAT"])
        shutil.rmtree(self._data(), ignore_errors=True)

    def logs(self):
        if not self.native:
            return super().logs()
        try:
            return open(os.path.join(self._data(), "server.log"), errors="replace").read()
        except OSError:
            return ""

    def wait_ready(self, t0):
        # /api/health is what the image's own HEALTHCHECK uses
        return wait_http(self.host_port, "/api/health", t0=t0)

    def setup(self):
        p = self.host_port
        st, d = get_json(p, "/api/auth/register", {"Content-Type": "application/json"}, "POST",
                         json.dumps({"username": self.user_name, "password": self.user_pass}))
        assert st in (200, 201), (st, d)
        tok = d.get("token")
        st, d = get_json(p, "/api/users/me/apps/password",
                         {"Content-Type": "application/json", "Authorization": f"Bearer {tok}"}, "POST",
                         json.dumps({"label": "bench"}))
        assert st in (200, 201), (st, d)
        self.user_pass = d.get("password") or d.get("appPassword")
        assert self.user_pass, d


def leyr_scan_seconds(log):
    """Leyr starts scanning on its own at boot and finishes in a second or two, before the harness has started
    polling, so a poll-based time would only measure the polling. Its log has the real figure: the
    'native music library open' line to the 'library scan finished' line."""
    import datetime
    def ts(line):
        return datetime.datetime.fromisoformat(line.split()[0].replace("Z", "+00:00"))
    start = end = None
    for line in log.splitlines():
        if start is None and "native music library open" in line:
            start = ts(line)
        if start is not None and end is None and "library scan finished" in line:
            end = ts(line)
    return (end - start).total_seconds() if start and end else None


class Navidrome(Server):
    name = "navidrome"
    image = os.environ.get("BENCH_ND_IMAGE", "docker.io/deluan/navidrome@sha256:38dc2727bfcfd5ede290f8ada114fc90368146f265ae4701ddddbcbe2a44ee52")
    inner_port = 4533
    host_port = 19533
    ping_path = "/ping"
    volumes = [("data", "/data")]
    env = {"ND_MUSICFOLDER": "/music", "ND_DATAFOLDER": "/data", "ND_SCANNER_SCHEDULE": "0",
           "ND_ENABLEINSIGHTSCOLLECTOR": "false", "ND_LASTFM_ENABLED": "false",
           "ND_ENABLEEXTERNALSERVICES": "false", "ND_LOGLEVEL": "info"}

    def setup(self):
        st, d = get_json(self.host_port, "/auth/createAdmin", {"Content-Type": "application/json"}, "POST",
                         json.dumps({"username": self.user_name, "password": self.user_pass}))
        assert st == 200, (st, d)

    def start_scan(self):
        get_json(self.host_port, "/rest/startScan?" + subsonic_qs(self.user_name, self.user_pass, {"fullScan": "true"}))


class Gonic(Server):
    name = "gonic"
    image = os.environ.get("BENCH_GONIC_IMAGE", "docker.io/sentriz/gonic@sha256:7b2298d905149ecb7e57cdd3bf7a40082758b9bda2e24454b41c17c99bf25e8f")
    inner_port = 80
    host_port = 19747
    volumes = [("data", "/data"), ("cache", "/cache"), ("pod", "/podcasts"), ("pl", "/playlists")]
    env = {"GONIC_MUSIC_PATH": "/music", "GONIC_PODCAST_PATH": "/podcasts", "GONIC_CACHE_PATH": "/cache",
           "GONIC_PLAYLISTS_PATH": "/playlists", "GONIC_DB_PATH": "/data/gonic.db", "GONIC_SCAN_INTERVAL": "0",
           "GONIC_SCAN_WATCHER_ENABLED": "false", "GONIC_SCAN_AT_START_ENABLED": "false"}

    def wait_ready(self, t0):
        # any HTTP answer (200 or an auth error) from the Subsonic endpoint means the server is up
        return wait_http(self.host_port, "/rest/ping?v=1.16.1&c=x&f=json", ok=lambda st: st in (200, 401), t0=t0)

    def setup(self):
        # gonic creates "admin" on first run; the password is in its log (or admin/admin)
        pw = "admin"
        for _ in range(25):
            m = re.search(r"admin.*?password[^A-Za-z0-9]+([A-Za-z0-9]{6,})", self.logs(), re.I)
            if m:
                pw = m.group(1)
                break
            time.sleep(0.2)
        self.user_name, self.user_pass = "admin", pw

    def start_scan(self):
        get_json(self.host_port, "/rest/startScan?" + subsonic_qs(self.user_name, self.user_pass))


class JellyfinApi:
    QUERIES = SubsonicApi.QUERIES

    def __init__(self, port, token, uid):
        self.port, self.uid = port, uid
        self.h = {"Authorization": f'MediaBrowser Client="bench", Device="bench", DeviceId="bench", Version="1", Token="{token}"'}
        self.discover()

    def hdrs(self):
        return self.h

    def j(self, path):
        return get_json(self.port, path, self.h)[1]

    def discover(self):
        u = self.uid
        every = self.j(f"/Items?userId={u}&IncludeItemTypes=MusicAlbum&Recursive=true&SortBy=SortName&Limit=500&StartIndex=0")["Items"]
        items = every[100:120] if len(every) >= 120 else every[-20:]
        self.album_ids = self.cover_ids = [i["Id"] for i in items]
        self.flac_tracks, self.long_track = [], None
        st = 0
        while len(self.flac_tracks) < 20 and st < 1000:
            d = self.j(f"/Items?userId={u}&IncludeItemTypes=Audio&Recursive=true&SortBy=SortName&Limit=200&StartIndex={st}&Fields=MediaSources")
            for i in d["Items"]:
                ms = (i.get("MediaSources") or [{}])[0]
                if ms.get("Container") == "flac" and i.get("RunTimeTicks", 0) / 1e7 < 900 and len(self.flac_tracks) < 20:
                    self.flac_tracks.append(i["Id"])
            st += 200
        d = self.j(f"/Items?userId={u}&IncludeItemTypes=Audio&Recursive=true&SearchTerm={urllib.parse.quote(LONG_FLAC_HINT)}")
        for i in d["Items"]:
            if i.get("RunTimeTicks", 0) / 1e7 > 25 * 60:
                self.long_track = i["Id"]

    def endpoints(self):
        n, u, q = len(self.album_ids), self.uid, self.QUERIES
        return {
            "ping": lambda i: "/System/Info/Public",
            "getAlbumList2_newest50": lambda i: f"/Items?userId={u}&IncludeItemTypes=MusicAlbum&Recursive=true&SortBy=DateCreated&SortOrder=Descending&Limit=50",
            "getArtists": lambda i: f"/Artists?userId={u}&Recursive=true&SortBy=SortName",
            "getAlbum": lambda i: f"/Items?userId={u}&ParentId={self.album_ids[i % n]}&IncludeItemTypes=Audio&Fields=MediaSources",
            "search3": lambda i: f"/Search/Hints?userId={u}&searchTerm={q[i % len(q)]}&limit=60",
            "getCoverArt_300": lambda i: f"/Items/{self.cover_ids[i % n]}/Images/Primary?maxWidth=300",
        }

    def stream_orig(self, tid):
        return f"/Audio/{tid}/stream?static=true"

    def stream_mp3(self, tid, offset=0):
        p = f"/Audio/{tid}/stream.mp3?audioCodec=mp3&audioBitRate=320000&static=false&deviceId=bench"
        return p + (f"&startTimeTicks={int(offset * 1e7)}" if offset else "")

    def stream_opus(self, tid, offset=0):
        p = f"/Audio/{tid}/stream.opus?audioCodec=opus&audioBitRate=128000&static=false&deviceId=bench"
        return p + (f"&startTimeTicks={int(offset * 1e7)}" if offset else "")


class Jellyfin(Server):
    name = "jellyfin"
    image = os.environ.get("BENCH_JF_IMAGE", "docker.io/jellyfin/jellyfin@sha256:fea863d0cd9ec445985d9c75fc6ed64e38a1d2766cf0c250a33d553f4fc1b771")
    inner_port = 8096
    host_port = 19096
    ping_path = "/System/Info/Public"
    volumes = [("config", "/config"), ("cache", "/cache")]
    music_mount = "/media/music"
    # Every metadata and image provider is switched off per library (no internet lookups). Side effect, found
    # by trying: with the image providers off Jellyfin extracts no album art, so its cover endpoint answers 404.
    # Re-enabling them (with and without DNS) made it query MusicBrainz in a retry loop and was dropped.
    env = {"JELLYFIN_CACHE_DIR": "/cache", "JELLYFIN_CONFIG_DIR": "/config/config",
           "JELLYFIN_DATA_DIR": "/config/data", "JELLYFIN_LOG_DIR": "/config/log"}
    AUTH = 'MediaBrowser Client="bench", Device="bench", DeviceId="bench", Version="1"'
    J = {"Content-Type": "application/json"}

    def hd(self):
        return {"Authorization": self.AUTH + f', Token="{self.token}"'}

    def login(self):
        for _ in range(120):  # the server restarts its web host after the wizard
            try:
                st, d = get_json(self.host_port, "/Users/AuthenticateByName", {**self.J, "Authorization": self.AUTH}, "POST",
                                 json.dumps({"Username": self.user_name, "Pw": self.user_pass}))
            except OSError as e:
                st, d = 0, str(e)
            if st == 200:
                break
            time.sleep(0.5)
        assert st == 200, (st, d)
        self.token, self.uid = d["AccessToken"], d["User"]["Id"]

    def setup(self):
        p, J = self.host_port, self.J
        wait_http(p, "/Startup/Configuration", timeout=120)  # first boot restarts the web host once
        get_json(p, "/Startup/Configuration", J, "POST", json.dumps({"UICulture": "en-US", "MetadataCountryCode": "US", "PreferredMetadataLanguage": "en"}))
        get_json(p, "/Startup/User")
        get_json(p, "/Startup/User", J, "POST", json.dumps({"Name": self.user_name, "Password": self.user_pass}))
        get_json(p, "/Startup/RemoteAccess", J, "POST", json.dumps({"EnableRemoteAccess": True, "EnableAutomaticPortMapping": False}))
        get_json(p, "/Startup/Complete", J, "POST", "{}")
        self.login()
        # no internet metadata fetchers or image downloads, no realtime monitor
        opts = {"LibraryOptions": {"EnableRealtimeMonitor": False, "EnableInternetProviders": False,
                "DownloadImagesInAdvance": False, "SaveLocalMetadata": False,
                "AutomaticRefreshIntervalDays": 0, "PathInfos": [{"Path": "/media/music"}],
                "TypeOptions": [{"Type": t, "MetadataFetchers": [], "ImageFetchers": []}
                                for t in ("MusicArtist", "MusicAlbum", "Audio")]}}
        q = urllib.parse.urlencode({"name": "Music", "collectionType": "music", "refreshLibrary": "false"})
        st, d = get_json(p, "/Library/VirtualFolders?" + q, {**J, **self.hd()}, "POST", json.dumps(opts))
        assert st in (200, 204), (st, d)

    def start_scan(self):
        Conn(self.host_port).req("/Library/Refresh", self.hd(), "POST", "")

    def scan_done(self):
        st, d = get_json(self.host_port, "/ScheduledTasks?isHidden=false", self.hd())
        running = any(t.get("Key") == "RefreshLibrary" and t.get("State") != "Idle" for t in d)
        st, c = get_json(self.host_port, "/Items/Counts", self.hd())
        return (not running, c.get("SongCount", 0) if isinstance(c, dict) else 0)

    def api(self):
        return JellyfinApi(self.host_port, self.token, self.uid)


class PlexApi:
    """Plex Media Server's own API, unclaimed and token-free (the harness client is on an allowed network)."""
    QUERIES = SubsonicApi.QUERIES
    CID = "X-Plex-Client-Identifier=leyr-bench&X-Plex-Product=leyr-bench&X-Plex-Platform=Chrome&X-Plex-Device=bench"

    def __init__(self, port, section):
        self.port, self.sec = port, section
        self.h = {"Accept": "application/json", "X-Plex-Client-Identifier": "leyr-bench", "X-Plex-Product": "leyr-bench"}
        self.discover()

    def hdrs(self):
        return self.h

    def j(self, path):
        return get_json(self.port, path, self.h)[1]["MediaContainer"]

    def discover(self):
        s = self.sec
        total = self.j(f"/library/sections/{s}/all?type=9&X-Plex-Container-Start=0&X-Plex-Container-Size=0").get("totalSize", 0)
        albums = self.j(f"/library/sections/{s}/all?type=9&sort=titleSort&X-Plex-Container-Start={max(0, min(100, total - 20))}&X-Plex-Container-Size=20").get("Metadata", [])
        self.album_ids = [a["ratingKey"] for a in albums]
        self.cover_urls = [a.get("thumb") or f"/library/metadata/{a['ratingKey']}/thumb" for a in albums]
        self.flac_tracks, self.long_track = [], None
        start = 0
        while len(self.flac_tracks) < 20 and start < 4000:
            d = self.j(f"/library/sections/{s}/all?type=10&X-Plex-Container-Start={start}&X-Plex-Container-Size=200")
            for t in d.get("Metadata", []):
                m = (t.get("Media") or [{}])[0]
                if m.get("audioCodec") == "flac" and t.get("duration", 0) < 900000 and len(self.flac_tracks) < 20:
                    self.flac_tracks.append(self.ref(t))
            if len(d.get("Metadata", [])) < 200:
                break
            start += 200
        q = urllib.parse.quote(LONG_FLAC_HINT)
        for t in self.j(f"/library/sections/{s}/all?type=10&title={q}").get("Metadata", []):
            if t.get("duration", 0) > 25 * 60 * 1000:
                self.long_track = self.ref(t)

    @staticmethod
    def ref(t):
        return f"{t['ratingKey']}|{t['Media'][0]['Part'][0]['key']}"

    def endpoints(self):
        n, s, q = len(self.album_ids), self.sec, self.QUERIES
        return {
            "ping": lambda i: "/identity",
            "getAlbumList2_newest50": lambda i: f"/library/sections/{s}/all?type=9&sort=addedAt:desc&X-Plex-Container-Start=0&X-Plex-Container-Size=50",
            "getArtists": lambda i: f"/library/sections/{s}/all?type=8",
            "getAlbum": lambda i: f"/library/metadata/{self.album_ids[i % n]}/children",
            "search3": lambda i: f"/hubs/search?query={q[i % len(q)]}&limit=20&sectionId={s}",
            "getCoverArt_300": lambda i: "/photo/:/transcode?" + urllib.parse.urlencode(
                {"width": 300, "height": 300, "minSize": 1, "upscale": 1, "url": self.cover_urls[i % n]}),
        }

    def stream_orig(self, tid):
        return tid.split("|", 1)[1] + "?download=0"

    def _uni(self, tid, ext, bitrate, offset):
        rk = tid.split("|", 1)[0]
        p = (f"/music/:/transcode/universal/start.{ext}?path=%2Flibrary%2Fmetadata%2F{rk}&mediaIndex=0&partIndex=0"
             f"&protocol=http&directPlay=0&directStream=0&musicBitrate={bitrate}&{self.CID}")
        return p + (f"&offset={int(offset)}" if offset else "")

    def stream_mp3(self, tid, offset=0):
        return self._uni(tid, "mp3", 320, offset)

    def stream_opus(self, tid, offset=0):
        return self._uni(tid, "opus", 128, offset)


class Plex(Server):
    name = "plex"
    image = os.environ.get("BENCH_PLEX_IMAGE", "docker.io/plexinc/pms-docker@sha256:6d778d41daae68f89115aa87a44ae14a96b23e71a2093da173b5a9aac4091336")
    inner_port = 32400
    host_port = 19032
    volumes = [("config", "/config"), ("transcode", "/transcode")]
    # unclaimed (no account): ALLOWED_NETWORKS writes allowedNetworks into Preferences.xml on first start, so
    # requests from the host need no X-Plex-Token. PLEX_UID/GID 0 = the invoking user in rootless podman.
    env = {"TZ": "UTC", "ALLOWED_NETWORKS": "0.0.0.0/0", "PLEX_UID": "0", "PLEX_GID": "0"}
    # background work that would use the network or the CPU after a scan is switched off, like the others
    PREFS = {"GenerateBIFBehavior": "never", "GenerateChapterThumbBehavior": "never", "LoudnessAnalysisBehavior": "never",
             "MusicAnalysisBehavior": "never", "ScheduledLibraryUpdatesEnabled": "0", "FSEventLibraryUpdatesEnabled": "0",
             "DlnaEnabled": "0", "PublishServerOnPlexOnlineKey": "0", "logDebug": "0", "OnDeckWindow": "1",
             "autoEmptyTrash": "0", "GdmEnabled": "0", "TranscoderQuality": "0"}
    J = {"Accept": "application/json"}
    section = None

    def wait_ready(self, t0):
        # /identity answers (with startState) while plugins are still loading; it drops startState once up
        deadline = time.time() + 180
        while time.time() < deadline:
            try:
                st, d, _ = Conn(self.host_port).req("/identity", self.J)
                if st == 200 and b"startState" not in d:
                    return time.time() - t0
            except Exception:
                pass
            time.sleep(0.01)
        raise TimeoutError("plex ready")

    def setup(self):
        for k, v in self.PREFS.items():
            Conn(self.host_port).req(f"/:/prefs?{k}={v}", self.J, "PUT")

    def start_scan(self):
        # creating the library starts its first scan. Agent "Personal Media" (tv.plex.agents.none) takes
        # metadata from the files only: the Plex Music agent looks everything up on Plex's servers.
        q = urllib.parse.urlencode({"name": "Music", "type": "artist", "agent": "tv.plex.agents.none", "scanner": "Plex Music",
                                    "language": "xn", "location": "/music"})
        st, d, _ = Conn(self.host_port).req("/library/sections?" + q, self.J, "POST")
        assert st in (200, 201), (st, d[:200])
        self.section = json.loads(d)["MediaContainer"]["Directory"][0]["key"]

    def scan_done(self):
        st, d, _ = Conn(self.host_port).req("/activities", self.J)
        busy = b"library.update" in d
        st, d, _ = Conn(self.host_port).req(f"/library/sections/{self.section}/all?type=10&X-Plex-Container-Start=0&X-Plex-Container-Size=0", self.J)
        return (not busy, json.loads(d)["MediaContainer"].get("totalSize", 0))

    def api(self):
        return PlexApi(self.host_port, self.section)


SERVERS = {"leyr": Leyr, "navidrome": Navidrome, "gonic": Gonic, "jellyfin": Jellyfin, "plex": Plex}


# ---------------------------------------------------------------- load + stream
def run_load(port, urls, hdrs, n, conc):
    lat, codes, idx, lock = [0.0] * n, {}, [0], threading.Lock()

    def worker():
        c = Conn(port)
        while True:
            with lock:
                i = idx[0]
                idx[0] += 1
            if i >= n:
                return
            t = time.perf_counter()
            try:
                st, _, _ = c.req(urls(i), hdrs)
            except Exception:
                st = 0
            lat[i] = (time.perf_counter() - t) * 1000
            with lock:
                codes[st] = codes.get(st, 0) + 1

    t0 = time.perf_counter()
    th = [threading.Thread(target=worker) for _ in range(conc)]
    [t.start() for t in th]
    [t.join() for t in th]
    r = summ(lat)
    r.update({"conc": conc, "codes": codes, "rps": n / (time.perf_counter() - t0)})
    return r


def ttfb(port, path, hdrs):
    """ms from sending the request to the first body byte; status, content type, first 4 bytes."""
    c = http.client.HTTPConnection("127.0.0.1", port, timeout=180)
    t = time.perf_counter()
    c.request("GET", path, headers=hdrs)
    r = c.getresponse()
    first = r.read(1)
    ms = (time.perf_counter() - t) * 1000
    head = first + (r.read(3) if first else b"")
    ct, st = r.getheader("Content-Type"), r.status
    c.close()
    return ms, st, ct, head


# ---------------------------------------------------------------- phases
def wait_scan(s, timeout=3600):
    """Wait until the server says the scan is finished, the track count is stable, and the container's
    processes have gone quiet (under 0.15 CPU-seconds in 6 s). The last condition matters for Jellyfin,
    whose library-scan task ends before its metadata/lyric/image work queue has drained. The returned
    time is the start of the quiet window."""
    t0, last, stable, busy = time.time(), -1, 0, False
    pid = s.pid()
    while time.time() - t0 < timeout:
        try:
            done, cnt = s.scan_done()
        except Exception:
            done, cnt = False, last
        busy = busy or not done
        stable = stable + 1 if (done and cnt > 0 and cnt == last) else 0
        last = cnt
        if stable >= 3 and (busy or stable >= 40):
            ta, c0 = time.time(), cpu_seconds(pid)
            time.sleep(6)
            if cpu_seconds(pid) - c0 < 0.15:
                return ta, cnt
            stable = 0
            continue
        time.sleep(0.25)
    raise TimeoutError("scan")


def ensure_network():
    if sh("podman", "network", "ls", "--filter", f"name=^{NET}$", "--format", "{{.Name}}") != NET:
        sh("podman", "network", "create", "--internal", NET)


def egress_check(s):
    """From inside the container's network namespace, try to reach outside hosts. All must fail; a control
    request from the host namespace must succeed, or the test proves nothing. The harness aborts otherwise."""
    pid = s.pid()
    probes = ["http://1.1.1.1/", "https://9.9.9.9/", "http://8.8.8.8/"]
    def reach(ns):
        out = []
        for u in probes:
            cmd = (["podman", "unshare", "nsenter", "-t", str(pid), "-n"] if ns else []) + \
                  ["curl", "-sS", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}", u]
            r = subprocess.run(cmd, capture_output=True, text=True)
            out.append({"url": u, "http_code": r.stdout.strip(), "exit": r.returncode})
        return out
    inside, control = reach(True), reach(False)
    ok = all(x["exit"] != 0 for x in inside) and any(x["exit"] == 0 for x in control)
    res = {"inside_container": inside, "host_control": control, "blocked": ok}
    if not ok:
        raise RuntimeError(f"egress check failed: {res}")
    return res


def phase_scan(s):
    """Fresh volume -> boot -> setup -> scan. Records timings and RSS."""
    s.destroy()
    s.create()
    t0 = s.start()
    boot = s.wait_ready(t0)
    egress = egress_check(s)
    smp = Sampler(s.pid())
    smp.start()
    s.setup()
    ts = time.time()
    s.start_scan()
    tend, cnt = wait_scan(s)
    res = {"egress_check": egress, "first_boot_ready_s": boot, "scan_s_from_trigger": tend - ts,
           "scan_s_from_container_start": tend - t0, "tracks": cnt}
    log_s = s.scan_seconds_from_logs()
    if log_s is not None:
        res["scan_s_log"] = log_s  # preferred over the poll-based figure where present (see leyr_scan_seconds)
    res.update(smp.stop())
    return res


def phase_main(s, idle_s, quick=False):
    res = {"scan_run": phase_scan(s)}
    t_end = time.time()
    print("scan done", res["scan_run"], flush=True)
    time.sleep(idle_s)
    res["idle"] = proc_stats(s.pid())
    if BREAKDOWN:
        res["idle"]["breakdown"] = rss_breakdown.snapshot(s.pid())
    res["idle"]["cgroup_mem_mb"] = cgroup_mem_mb(s.cname())
    res["idle"]["seconds_after_scan"] = time.time() - t_end
    print("idle", res["idle"], flush=True)
    if s.name == "jellyfin":
        s.login()
    api = s.api()
    hd, eps, port = api.hdrs(), api.endpoints(), s.host_port
    res["streamable"] = {"flac_tracks": len(api.flac_tracks), "long_track": bool(api.long_track)}
    for f in eps.values():  # warm every endpoint and every rotated id
        for i in range(40):
            Conn(port).req(f(i), hd)
    res["latency"] = {n: {"sequential": run_load(port, f, hd, 200, 1)} for n, f in eps.items()}
    smp = Sampler(s.pid(), 0.25)
    smp.start()
    for n, f in eps.items():
        res["latency"][n]["concurrent16"] = run_load(port, f, hd, 800, 16)
    res["rss_under_load"] = smp.stop()
    res["after_load"] = proc_stats(s.pid())
    print("load done", flush=True)
    if quick:  # --phase load: stop before the stream and restart series
        return res

    st = {}

    def series(label, mk, n, tracks):
        xs, bad, meta = [], [], None
        for i in range(n):
            try:
                ms, code, ct, head = ttfb(port, mk(tracks[i % len(tracks)], i), hd)
            except Exception as e:
                bad.append(str(e)[:80])
                continue
            if code in (200, 206):
                xs.append(ms)
                meta = {"content_type": ct, "first4": head.hex()}
            else:
                bad.append(str(code))
        r = summ(xs)
        r.update({"nfail": len(bad), "failures": bad[:3], "sample": meta, "first_ms": xs[0] if xs else None})
        st[label] = r

    # 20 different tracks per series, so a server's transcode cache (if it has one) never answers a repeat.
    series("flac_original", lambda t, i: api.stream_orig(t), 20, api.flac_tracks)
    series("flac_to_mp3_320", lambda t, i: api.stream_mp3(t), 20, api.flac_tracks)
    series("flac_to_opus_128", lambda t, i: api.stream_opus(t), 20, api.flac_tracks)
    if api.long_track:
        lt = [api.long_track]
        series("long_flac_original", lambda t, i: api.stream_orig(t), 10, lt)
        # one track, so repeats of the same request can hit a transcode cache: report the first request on its own
        series("long_flac_mp3_320_from_start_repeated", lambda t, i: api.stream_mp3(t), 10, lt)
        # a different offset each time (1500 s + 7 s * i): every request is a cold transcode
        series("long_flac_mp3_320_seek_1500s_plus", lambda t, i: api.stream_mp3(t, 1500 + 7 * i), 10, lt)
        series("long_flac_opus_128_seek_1500s_plus", lambda t, i: api.stream_opus(t, 1500 + 7 * i), 10, lt)
        # the same offset repeated, the case a cache would help
        series("long_flac_mp3_320_seek_1500s_repeated", lambda t, i: api.stream_mp3(t, 1500), 10, lt)
    res["stream_ttfb_ms"] = st
    cs = []
    for _ in range(5):
        s.stop()
        time.sleep(1)
        t0 = s.start()
        cs.append(s.wait_ready(t0) * 1000)
        time.sleep(2)
    res["cold_start_populated_ms"] = {**summ(cs), "all": cs}
    return res


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("server", choices=SERVERS)
    ap.add_argument("--phase", default="main", choices=["main", "scan", "load"])
    ap.add_argument("--run", type=int, default=1)
    ap.add_argument("--idle", type=int, default=300)
    ap.add_argument("--out", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "results"))
    a = ap.parse_args()
    os.makedirs(a.out, exist_ok=True)
    ensure_network()
    s = SERVERS[a.server]()
    try:
        res = {"scan_run": phase_scan(s)} if a.phase == "scan" else phase_main(s, a.idle, quick=a.phase == "load")
        res.update({"server": a.server, "image": s.image, "limits": {"cpus": CPUS, "memory": MEM}, "network": NET, "library": LIBRARY,
                    "log_tail": s.logs()[-1500:]})
        fn = os.path.join(a.out, f"{a.server}-{a.phase}-{a.run}.json")
        json.dump(res, open(fn, "w"), indent=1)
        print("wrote", fn)
    finally:
        s.destroy()


if __name__ == "__main__":
    main()
