re: Stage\script.tbl's 11 non-MISSION fields, and the mission dialogue table

structures/mission-script-ssb.md owns this manifest and names its 40
fields but never read the 11 that are not MISSION<n> = StageNN.ssb.
They are DIALOG_MESSAGE, DIALOG_LOCAL_STRING, FONT (+size), TEXT_POS,
TEXT_LINES and the five pgmsg_*.prt.  Two sibling records were also
unread: GP_TEST (PATH = dat\GP_TEST\, a debug archive not on the disc)
and TEXTS (a second text style).

Probed 7 values x 33 prefixes x 41 archives.  One resolves:
message\MissionDialogMessage.tbl, in all six GP_MAIN_GAME_* paks --
200 records, 25280 bytes, every name S<NN>_P<n>_<KIND> with five kinds
40 each (HINT_PAUSE, HINT, OBJECTIVE, GRAPH, LOSE), fields positional
and tagged 0..3, each value a message key.  An index from (stage,
phase, kind) to the localised strings, on the same S<NN>_P<n> keying
the ISL corpus already uses.

Control: the 40 stage-phases span stages 1-16 and 24-29 -- a subset of
the 28 shipped, and the six with no hints are exactly 18-23, the
tutorials.  A fifth independent route to the story/tutorial split, and
it gives the phase count per stage.

The other six do not resolve, with the control in the same sweep:
MissionDialog_local_string.tbl and all five pgmsg_*.prt are not a pak
entry under any of the 33 prefixes, while
message\MissionDialogMessage.tbl and Stage\script.tbl both resolve in
6 archives.

name_hash is CASE-INSENSITIVE (message\ == Message\); tag_hash is not.

New structure doc, artefact and regenerator; the other five artefacts
regenerate byte-identical.

tools/re-capture/script_manifest.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""`Stage\\script.tbl` -- the mission-script manifest, and what its values point at.
+
+Regenerates docs/re/data/script-manifest.txt.
+"""
+import sys, os, glob, re, collections
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import unitgroup as U
+from unit_substructures import pak_entries
+
+def main():
+    mg = glob.glob('/work/sylph_extract/**/GP_MAIN_GAME_E.pak', recursive=True)[0]
+    E = dict(pak_entries(mg))
+    recs = U.parse(E[U.name_hash('Stage\\script.tbl')])
+
+    print("# Stage\\script.tbl -- the mission-script manifest")
+    print("# Regenerate: python3 tools/re-capture/script_manifest.py")
+    print("# See docs/re/structures/mission-script-manifest.md")
+
+    print("\n## the three records")
+    for r in recs:
+        n = U.named(r)
+        print("   %-10s %2d fields" % (r['squadron'], len(r['fields'])))
+        if r['squadron'] == 'SCRIPTS': continue
+        for k, v in sorted(n.items()):
+            print("        %-22s %r" % (k, v))
+
+    sc = [r for r in recs if r['squadron'] == 'SCRIPTS'][0]
+    print("\n## SCRIPTS -- the 11 non-MISSION fields, read for the first time")
+    for t, n, v in sorted(sc['fields'], key=lambda x: (x[1] or '')):
+        if n and n.startswith('MISSION') and v.endswith('.ssb'): continue
+        print("   %-22s %r" % (n, v))
+
+    idx = {}
+    for pk in sorted(glob.glob('/work/sylph_extract/**/*.pak', recursive=True)):
+        idx[os.path.basename(pk)] = {h for h, b in pak_entries(pk)}
+    targets = ['MissionDialogMessage.tbl', 'MissionDialog_local_string.tbl',
+               'pgmsg_start.prt', 'pgmsg_end.prt', 'pgmsg_update.prt',
+               'pgmsg_failed.prt', 'pgmsg_restart.prt']
+    pre = ['', 'Stage\\', 'stage\\', 'message\\', 'msg\\', 'dialog\\', 'script\\',
+           'SCRIPTS\\', 'GP_SCRIPT\\', '2d\\', 'ui\\', 'prt\\', 'view\\', 'dat\\', 'hidden\\']
+    for l in ['eng', 'jpn', 'fra', 'deu', 'ita', 'esp']:
+        pre += [l + '\\', 'Stage\\' + l + '\\', 'message\\' + l + '\\']
+    print("\n## do those values resolve as pak entries?  (%d prefixes x %d archives)"
+          % (len(pre), len(idx)))
+    for t in targets:
+        hits = sorted({p + t for p in pre if any(U.name_hash(p + t) in s for s in idx.values())})
+        if hits:
+            for hname in hits[:1]:
+                where = sorted(pk for pk, s in idx.items() if U.name_hash(hname) in s)
+                print("   %-32s FOUND as %-40s in %d archives" % (t, hname, len(where)))
+        else:
+            print("   %-32s not found" % t)
+    c = U.name_hash('Stage\\script.tbl')
+    print("   CONTROL Stage\\script.tbl resolves in %d archives"
+          % sum(1 for s in idx.values() if c in s))
+    print("   NOTE name_hash is CASE-INSENSITIVE (message\\ == Message\\); tag_hash is not.")
+
+    b = E[U.name_hash('message\\MissionDialogMessage.tbl')]
+    dr = U.parse(b)
+    pat = re.compile(r'^S(\d+)_P(\d+)_(.+)$')
+    fam = collections.Counter(); st = collections.defaultdict(set); nf = collections.Counter()
+    for r in dr:
+        m = pat.match(r['squadron'])
+        if m:
+            fam[m.group(3)] += 1
+            st[int(m.group(1))].add(int(m.group(2)))
+        nf[len(r['fields'])] += 1
+    print("\n## message\\MissionDialogMessage.tbl -- %d records, %d bytes" % (len(dr), len(b)))
+    print("   name families S<NN>_P<n>_<KIND>: %s" % dict(fam))
+    print("   field counts: %s" % dict(nf))
+    print("   stages present (%d): %s" % (len(st), sorted(st)))
+    print("   phases per stage: %s" % {k: len(v) for k, v in sorted(st.items())})
+    H = {h for h, _ in pak_entries(mg)}
+    ship = {i for i in range(0, 40) if U.name_hash('stage\\UnitGroup_S%02d.tbl' % i) in H}
+    print("   CONTROL subset of the %d shipped stages: %s ; shipped with NO hints: %s"
+          % (len(ship), set(st) <= ship, sorted(ship - set(st))))
+    r0 = [r for r in dr if r['squadron'].endswith('_HINT_PAUSE')][0]
+    print("\n   sample %s:" % r0['squadron'])
+    for t, n, v in r0['fields']:
+        print("        tag %-3s %-8s %r" % (t, n or '<none>', v))
+
+if __name__ == "__main__":
+    main()