Sylpheed/tools/re-capture/kf_record_census.py

#!/usr/bin/env python3
"""Disc-wide test of the UI placement-group record layout.

A placement group in a screen bundle is an 8-byte header `{u32 element_index,
u32 frame_count}` followed by `frame_count` records of 40 bytes, each
`{u32 time; 36-byte pose}`.  The time word therefore **precedes** the pose it
belongs to.

Our parser's block window starts at the pose, so under the old reading a
block's `+36` word was taken as *its own* time — off by one, which left the
group's final pose (the end of every fade-out) untimed and made the group look
4 bytes short.

This script tests the corrected reading against the whole disc, with controls:

  A. monotonicity — prepending the lead-in word to the shifted time series must
     give a non-decreasing sequence, for every group.
  B. the non-zero lead-ins — a lead-in that is a *time* must be strictly less
     than the next time.  Control: swap in another group's lead-in from the
     same bundle.
  C. ramp linearity — within a monotone alpha ramp of >=3 segments, is
     d(alpha)/d(time) constant?  Keyframe interpolation is linear
     (`docs/re/ui-keyframe-time-unit.md`), so a correct time assignment should
     make multi-keyframe ramps come out at a constant rate far more often than
     an incorrect one.

Usage:  python3 tools/re-capture/kf_record_census.py "$SYLPHEED_DISC"/dat/*.pak
"""
import collections
import os
import random
import struct
import sys
import zlib


def be32(b, o):
    return struct.unpack_from(">I", b, o)[0]


def load_pak(pak_path):
    """Entries of an IPFB archive, transparently un-Z1'd. See sylpheed-formats::pak."""
    idx = open(pak_path, "rb").read()
    if idx[:4] != b"IPFB":
        return []
    n = be32(idx, 4)
    data = b""
    base = os.path.splitext(pak_path)[0]
    for i in range(100):
        p = f"{base}.p{i:02d}"
        if not os.path.exists(p):
            break
        data += open(p, "rb").read()
    out = []
    for k in range(n):
        _h, off, sz = struct.unpack_from(">III", idx, 0x10 + 12 * k)
        blob = data[off : off + sz]
        if blob[:2] == b"Z1":
            try:
                blob = zlib.decompress(blob[0x0A:])
            except Exception:
                blob = b""
        out.append(blob)
    return out


def elem_name(b, o):
    s = b[o : o + 28]
    z = s.find(b"\0")
    return (s[:z] if z >= 0 else s).decode("latin1")


def groups(bundle):
    """(element_index, name, frames, lead_in, W, alphas) per placement group.

    `W[k]` is the word at the k-th 40-byte stride's `+36`; `W[-1]` is None
    because it lies outside the group.  Under the corrected reading the pose
    times are `[lead_in] + W[:-1]`.
    """
    n = be32(bundle, 0x14)
    names = [elem_name(bundle, 0x20 + i * 60) for i in range(n)]
    pos = 0x20 + n * 60
    out = []
    for _ in range(n):
        if pos + 8 > len(bundle):
            break
        idx, frames = be32(bundle, pos), be32(bundle, pos + 4)
        if idx >= n or frames == 0 or frames > 4096:
            break
        lead_in = be32(bundle, pos + 8)
        first = pos + 12
        end = first + frames * 40 - 4
        if end > len(bundle):
            break
        W, alphas = [], []
        for k in range(frames):
            blk = first + k * 40
            W.append(be32(bundle, blk + 36) if blk + 40 <= end else None)
            alphas.append(be32(bundle, blk) >> 24)
        out.append((idx, names[idx], frames, lead_in, W, alphas))
        pos = end
    return out


def is_build(raw):
    if raw[:4] != b"RATC" or len(raw) < 0x20:
        return False
    n = be32(raw, 0x14)
    return 0 < n <= 4096 and 0x20 + n * 60 <= len(raw)


def monotone_ramps(alphas, min_segments=3):
    out, i, n = [], 0, len(alphas)
    while i < n - 1:
        if alphas[i] == alphas[i + 1]:
            i += 1
            continue
        d = 1 if alphas[i + 1] > alphas[i] else -1
        j = i + 1
        while j < n - 1 and (alphas[j + 1] - alphas[j]) * d > 0:
            j += 1
        if j - i >= min_segments:
            out.append((i, j))
        i = j
    return out


def constant_rate(times, alphas, a, b, tol=0.06):
    rates = []
    for k in range(a, b):
        dt = times[k + 1] - times[k]
        if dt <= 0:
            return None
        rates.append(abs(alphas[k + 1] - alphas[k]) / dt)
    mean = sum(rates) / len(rates)
    if mean == 0:
        return None
    return max(abs(r - mean) for r in rates) / mean <= tol


def main(paks):
    rnd = random.Random(20260829)
    per_bundle = collections.defaultdict(list)
    total = mono = 0
    ramp_new = [0, 0]
    ramp_old = [0, 0]
    for p in paks:
        for ei, raw in enumerate(load_pak(p)):
            if not is_build(raw):
                continue
            try:
                gs = groups(raw)
            except Exception:
                continue
            for _idx, nm, _frames, lead_in, W, alphas in gs:
                rest = W[:-1]
                if not rest or any(w is None for w in rest):
                    continue
                total += 1
                t_new = [lead_in] + rest
                if all(t_new[i] <= t_new[i + 1] for i in range(len(t_new) - 1)):
                    mono += 1
                per_bundle[(os.path.basename(p), ei)].append((lead_in, rest, nm))
                for a, b in monotone_ramps(alphas):
                    r = constant_rate(t_new, alphas, a, b)
                    if r is not None:
                        ramp_new[1] += 1
                        ramp_new[0] += r
                # old reading: `+36` is the block's own time, last pose untimed
                a_old = alphas[:-1]
                for a, b in monotone_ramps(a_old):
                    r = constant_rate(rest, a_old, a, b)
                    if r is not None:
                        ramp_old[1] += 1
                        ramp_old[0] += r

    nz = nz_ok = ctrl = ctrl_ok = 0
    gaps = collections.Counter()
    for rows in per_bundle.values():
        pool = [l for l, _, _ in rows]
        for lead_in, rest, _nm in rows:
            if lead_in == 0:
                continue
            nz += 1
            nz_ok += lead_in < rest[0]
            gaps[rest[0] - lead_in] += 1
            for _ in range(10):
                ctrl += 1
                ctrl_ok += rnd.choice(pool) < rest[0]

    pct = lambda a, b: f"{100 * a / b:.3f}%" if b else "n/a"
    print(f"paks scanned            : {len(paks)}")
    print(f"placement groups        : {total}")
    print()
    print("A. lead-in prepended to the shifted times is non-decreasing")
    print(f"   {mono}/{total} = {pct(mono, total)}")
    print()
    print("B. non-zero lead-in is strictly less than the next time")
    print(f"   {nz_ok}/{nz} = {pct(nz_ok, nz)}")
    print(f"   control (another group's lead-in, same bundle): "
          f"{ctrl_ok}/{ctrl} = {pct(ctrl_ok, ctrl)}")
    print(f"   gap to the next time, most common: {gaps.most_common(8)}")
    print()
    print("C. constant d(alpha)/d(time) across a multi-segment ramp")
    print(f"   corrected (time precedes pose): {ramp_new[0]}/{ramp_new[1]} = "
          f"{pct(ramp_new[0], ramp_new[1])}")
    print(f"   old       (+36 is own time)   : {ramp_old[0]}/{ramp_old[1]} = "
          f"{pct(ramp_old[0], ramp_old[1])}")


if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    main(sys.argv[1:])