#!/usr/bin/env python3 """Turn the gdb wait-object dumps into a table, discarding what does not check out. The two waiting functions do not put the same thing in %rbx (see freeze_waitobj.sh), so they are parsed separately: Wait rbx = this -> [rbx] is the vtable WaitMultiple rbx = XObject** -> [[rbx+8i]] is object i's vtable %ebp holds the count only at the prologue: WaitMultiple reuses it at 8fc158 (`mov 0x10(%rax),%ebp`), and it reads 0 at the point these captures interrupt. So the array length is NOT taken from a register -- entries are read until one stops resolving, which is the same self-validating rule used for everything else here. Every reading is validated the same way: gdb's `info symbol` must resolve it to a `vtable for ...` symbol. A polymorphic object's first word always is one, so anything else is a misread and is COUNTED but not interpreted. That check is what the reading is worth -- without it a stale register looks like a result. """ import re, sys, collections VT = re.compile(r'vtable for ([\w:]+) \+ (\d+)') HDR = re.compile(r'=== (\w+) T(\d+) (Wait|WaitMultiple) f(\d+) ===') def parse(tag): """-> list of (thread, kind, count, [vtable-or-None per slot])""" try: txt = open('/tmp/fz-obj-%s.txt' % tag, errors='replace').read() except FileNotFoundError: return [] out, cur = [], None for line in txt.splitlines(): line = line.replace('(gdb) ', '') h = HDR.search(line) if h: if cur: out.append(cur) cur = dict(th=h.group(2), kind=h.group(3), count=None, slots=[], bad=0) continue if not cur: continue m = re.search(r'rbp\s+0x[0-9a-f]+\s+(\d+)', line) if m: cur['count'] = int(m.group(1)) & 0xffffffff if 'info symbol' in line or line.startswith('$'): continue v = VT.search(line) if v: cur['slots'].append(v.group(1)) elif 'No symbol matches' in line: cur['slots'].append(None); cur['bad'] += 1 if cur: out.append(cur) return out def report(tag): recs = parse(tag) print('=== %s: %d wait frames ===' % (tag, len(recs))) if not recs: return collections.Counter() tally = collections.Counter() for r in recs: # take entries up to the first one that did not resolve live = [] for s_ in r['slots']: if s_ is None: break live.append(s_) for s in live: tally[s if s else ''] += 1 print(' T%-4s %-13s n=%d %s' % ( r['th'], r['kind'], len(live), ', '.join(live) or '(nothing readable)')) print(' --- objects waited on:') for k, c in tally.most_common(): print(' %-45s %d' % (k, c)) return tally def per_thread(tag): d = {} for r in parse(tag): live = [] for s_ in r['slots']: if s_ is None: break live.append(s_.replace('xe::kernel::', '')) d[int(r['th'])] = '%s(%s)' % (r['kind'], ','.join(live)) return d def diff_threads(a, b): """The tally alone hides the signature -- WHICH thread moved is the result.""" x, y = per_thread(a), per_thread(b) print('=== per-thread %s -> %s ===' % (a, b)) print(' %-6s %-32s %-32s' % ('thread', a, b)) for t in sorted(set(x) | set(y), reverse=True): fa, fb = x.get(t, '--'), y.get(t, '--') print(' T%-5d %-32s %-32s %s' % (t, fa, fb, '' if fa == fb else ' <-- CHANGED')) def stability(tags): """Which thread states hold STILL across repeated samples of one healthy run? Written after a frozen-vs-healthy diff was read as a signature and did not reproduce: the healthy state varies between instants too, so a difference of two samples is not yet a difference of two states. Anything that moves here is disqualified as freeze evidence before it is ever used as such. """ snaps = [(t, per_thread(t)) for t in tags] snaps = [(t, d) for t, d in snaps if d] if len(snaps) < 2: print('need at least 2 usable captures, got %d' % len(snaps)); return threads = sorted({t for _, d in snaps for t in d}, reverse=True) print('=== stability across %d healthy captures: %s ===' % ( len(snaps), ', '.join(t for t, _ in snaps))) stable = moved = 0 for th in threads: vals = [d.get(th, '--') for _, d in snaps] uniq = sorted(set(vals)) if len(uniq) == 1: stable += 1 print(' T%-5d STABLE %s' % (th, uniq[0])) else: moved += 1 print(' T%-5d VARIES %s' % (th, ' | '.join(vals))) print(' --- %d stable, %d vary across healthy play' % (stable, moved)) print(' Only a thread in the STABLE set can carry a frozen-state signature;') print(' a VARIES thread differing when frozen proves nothing.') def dist(n): """Compare N healthy captures against N frozen ones, per thread. The point of doing it this way: a thread only counts as a freeze signature if the set of states it takes while FROZEN is disjoint from the set it takes while HEALTHY. Two earlier "signatures" died because a single healthy sample happened to differ -- a distribution cannot be fooled that way. """ H = [per_thread('h%d' % i) for i in range(1, n + 1)] F = [per_thread('f%d' % i) for i in range(1, n + 1)] H = [d for d in H if d]; F = [d for d in F if d] if not H or not F: print('need both halves: %d healthy, %d frozen' % (len(H), len(F))); return threads = sorted({t for d in H + F for t in d}, reverse=True) print('=== healthy(%d) vs frozen(%d) distributions ===' % (len(H), len(F))) sig = [] for th in threads: hs = {d.get(th, '--') for d in H} fs = {d.get(th, '--') for d in F} mark = '' if not (hs & fs): mark = ' <== SIGNATURE (disjoint)'; sig.append(th) print(' T%-5d healthy{%s} frozen{%s}%s' % ( th, ' , '.join(sorted(hs)), ' , '.join(sorted(fs)), mark)) print(' --- %d thread(s) whose frozen states never occur while healthy' % len(sig)) if not sig: print(' No signature: every frozen state is one healthy play also produces.') if __name__ == '__main__': if sys.argv[1:2] == ['--dist']: dist(int(sys.argv[2])); sys.exit(0) if sys.argv[1:2] == ['--stability']: stability(sys.argv[2:]); sys.exit(0) tallies = {t: report(t) for t in (sys.argv[1:] or ['healthy'])} if len(tallies) > 1: a, b = list(tallies) print('=== %s -> %s ===' % (a, b)) keys = set(tallies[a]) | set(tallies[b]) for k in sorted(keys): x, y = tallies[a][k], tallies[b][k] print(' %-45s %3d -> %-3d %s' % (k, x, y, '' if x == y else ' CHANGED')) diff_threads(a, b)