feat: bench.py --outer kick mode for kick-in-kick benchmarks
This commit is contained in:
parent
85a2c716ee
commit
506afee745
1 changed files with 99 additions and 46 deletions
145
bench.py
145
bench.py
|
|
@ -7,10 +7,16 @@ use) and reports timing + resource numbers as a human-readable table,
|
|||
optionally also as JSON.
|
||||
|
||||
Usage:
|
||||
./bench.py [--mode auto|nested|standalone] [--repeat N] [--cycles N]
|
||||
./bench.py [--mode auto|nested|standalone] [--outer host|kick]
|
||||
[--repeat N] [--cycles N]
|
||||
[--tabs N] [--filter SUBSTR] [--list] [--json OUT.json]
|
||||
[--quick]
|
||||
|
||||
--outer selects what the measured nested compositor nests into: the
|
||||
ambient host (labwc etc.) or a standalone-headless kick booted by the
|
||||
script (kick-in-kick via _spawn_nested_in). Only valid with
|
||||
--mode nested.
|
||||
|
||||
Benchmarks (all wall-clock unless noted):
|
||||
startup_1tab_ms spawn -> first tab mapped+active (1 wltest tab)
|
||||
startup_2tab_ms spawn -> both tabs mapped (2 wltest tabs)
|
||||
|
|
@ -56,6 +62,41 @@ class BenchError(RuntimeError):
|
|||
pass
|
||||
|
||||
|
||||
# Shared standalone-headless outer for --outer kick (kick-in-kick):
|
||||
# booted once per run, kept alive across benchmarks; each measured
|
||||
# session is a fresh inner nested into it. Closed at the end of main().
|
||||
_OUTER = None
|
||||
|
||||
|
||||
def _ensure_outer():
|
||||
"""Boot the shared standalone outer (with one keeper tab) if needed."""
|
||||
global _OUTER
|
||||
if _OUTER is not None and _OUTER.proc.poll() is None:
|
||||
return _OUTER
|
||||
if _OUTER is not None:
|
||||
_OUTER.close()
|
||||
_OUTER = None
|
||||
try:
|
||||
outer = T._spawn_standalone({}, _tabs(1)[0])
|
||||
except BaseException as e:
|
||||
raise BenchError(f"outer spawn failed: {e}") from e
|
||||
try:
|
||||
_wait_for(outer, r"active app=\d+", 30.0)
|
||||
_wait_for(outer, re.escape(MAP_RE + "0"), 30.0)
|
||||
except (BenchError, TimeoutError) as e:
|
||||
outer.close()
|
||||
raise BenchError(f"outer never settled: {e}") from e
|
||||
_OUTER = outer
|
||||
return _OUTER
|
||||
|
||||
|
||||
def _close_outer():
|
||||
global _OUTER
|
||||
if _OUTER is not None:
|
||||
_OUTER.close()
|
||||
_OUTER = None
|
||||
|
||||
|
||||
def _tabs(n):
|
||||
return ["%s '%s'" % (T.WL, COLORS[i % len(COLORS)]) for i in range(n)]
|
||||
|
||||
|
|
@ -74,9 +115,11 @@ def _resolve_mode(mode):
|
|||
return mode
|
||||
|
||||
|
||||
def _spawn(mode, tabs):
|
||||
def _spawn(mode, args, tabs):
|
||||
try:
|
||||
if mode == "nested":
|
||||
if args.outer == "kick":
|
||||
return T._spawn_nested_in(_ensure_outer(), *tabs)
|
||||
return T._nested_session(*tabs)
|
||||
return T._spawn_standalone({}, *tabs)
|
||||
except BaseException as e: # pytest.skip/fail raise BaseException subclasses
|
||||
|
|
@ -140,7 +183,7 @@ def _assert_alive(s):
|
|||
def bench_startup_1tab(mode, args):
|
||||
tabs = _tabs(1)
|
||||
t0 = time.perf_counter()
|
||||
s = _spawn(mode, tabs)
|
||||
s = _spawn(mode, args, tabs)
|
||||
try:
|
||||
_wait_for(s, r"active app=0\b", 30.0)
|
||||
_wait_for(s, re.escape(MAP_RE + "0"), 30.0)
|
||||
|
|
@ -154,7 +197,7 @@ def bench_startup_1tab(mode, args):
|
|||
def bench_startup_2tab(mode, args):
|
||||
tabs = _tabs(2)
|
||||
t0 = time.perf_counter()
|
||||
s = _spawn(mode, tabs)
|
||||
s = _spawn(mode, args, tabs)
|
||||
try:
|
||||
_wait_for(s, r"active app=\d+", 30.0)
|
||||
_wait_for(s, re.escape(MAP_RE + "0"), 30.0)
|
||||
|
|
@ -170,7 +213,7 @@ def bench_startup_8tab(mode, args):
|
|||
n = args.tabs
|
||||
tabs = _tabs(n)
|
||||
t0 = time.perf_counter()
|
||||
s = _spawn(mode, tabs)
|
||||
s = _spawn(mode, args, tabs)
|
||||
try:
|
||||
_wait_for(s, r"active app=\d+", 30.0)
|
||||
for i in range(n):
|
||||
|
|
@ -182,8 +225,8 @@ def bench_startup_8tab(mode, args):
|
|||
s.close()
|
||||
|
||||
|
||||
def _boot_2tab_settled(mode):
|
||||
s = _spawn(mode, _tabs(2))
|
||||
def _boot_2tab_settled(mode, args):
|
||||
s = _spawn(mode, args, _tabs(2))
|
||||
_wait_for(s, r"active app=\d+", 30.0)
|
||||
_wait_for(s, re.escape(MAP_RE + "0"), 30.0)
|
||||
_wait_for(s, re.escape(MAP_RE + "1"), 30.0)
|
||||
|
|
@ -193,7 +236,7 @@ def _boot_2tab_settled(mode):
|
|||
|
||||
def bench_tab_cycle(mode, args):
|
||||
samples = []
|
||||
s = _boot_2tab_settled(mode)
|
||||
s = _boot_2tab_settled(mode, args)
|
||||
try:
|
||||
for i in range(args.cycles):
|
||||
cur_m = ACTIVE_RE.findall(s.read_log())
|
||||
|
|
@ -211,7 +254,7 @@ def bench_tab_cycle(mode, args):
|
|||
|
||||
def bench_input_ack(mode, args):
|
||||
samples = []
|
||||
s = _boot_2tab_settled(mode)
|
||||
s = _boot_2tab_settled(mode, args)
|
||||
try:
|
||||
for _ in range(args.cycles):
|
||||
pos = len(s.read_log())
|
||||
|
|
@ -228,7 +271,7 @@ def bench_input_ack(mode, args):
|
|||
|
||||
def bench_rapid_cycle(mode, args):
|
||||
n = 20
|
||||
s = _boot_2tab_settled(mode)
|
||||
s = _boot_2tab_settled(mode, args)
|
||||
try:
|
||||
cur_m = ACTIVE_RE.findall(s.read_log())
|
||||
cur = int(cur_m[-1]) if cur_m else 1
|
||||
|
|
@ -247,7 +290,7 @@ def bench_rapid_cycle(mode, args):
|
|||
|
||||
def bench_frame_dump(mode, args):
|
||||
samples = []
|
||||
s = _boot_2tab_settled(mode)
|
||||
s = _boot_2tab_settled(mode, args)
|
||||
try:
|
||||
for _ in range(3):
|
||||
t0 = time.perf_counter()
|
||||
|
|
@ -263,7 +306,7 @@ def bench_frame_dump(mode, args):
|
|||
|
||||
|
||||
def bench_rss_2tab(mode, args):
|
||||
s = _spawn(mode, _tabs(2))
|
||||
s = _spawn(mode, args, _tabs(2))
|
||||
try:
|
||||
_wait_for(s, r"active app=\d+", 30.0)
|
||||
_wait_for(s, re.escape(MAP_RE + "0"), 30.0)
|
||||
|
|
@ -279,7 +322,7 @@ def bench_rss_2tab(mode, args):
|
|||
|
||||
def bench_rss_8tab(mode, args):
|
||||
n = args.tabs
|
||||
s = _spawn(mode, _tabs(n))
|
||||
s = _spawn(mode, args, _tabs(n))
|
||||
try:
|
||||
_wait_for(s, r"active app=\d+", 30.0)
|
||||
for i in range(n):
|
||||
|
|
@ -329,6 +372,10 @@ def main(argv=None):
|
|||
ap.add_argument("--mode", default="auto",
|
||||
choices=["auto", "nested", "standalone"],
|
||||
help="compositor mode (auto: nested with a host, else headless standalone)")
|
||||
ap.add_argument("--outer", default="host",
|
||||
choices=["host", "kick"],
|
||||
help="what a nested session nests into: the ambient host "
|
||||
"or a standalone-headless kick (kick-in-kick)")
|
||||
ap.add_argument("--repeat", type=int, default=3,
|
||||
help="outer repeats for boot/static benchmarks (default 3)")
|
||||
ap.add_argument("--cycles", type=int, default=10,
|
||||
|
|
@ -356,42 +403,47 @@ def main(argv=None):
|
|||
|
||||
_check_binaries()
|
||||
mode = _resolve_mode(args.mode)
|
||||
print(f"bench: mode={mode} repeat={args.repeat} "
|
||||
if args.outer == "kick" and mode != "nested":
|
||||
raise BenchError("--outer kick needs --mode nested")
|
||||
print(f"bench: mode={mode} outer={args.outer} repeat={args.repeat} "
|
||||
f"cycles={args.cycles} tabs={args.tabs}", flush=True)
|
||||
|
||||
results = {}
|
||||
failed = 0
|
||||
for name, unit, desc, fn, kind in BENCHMARKS:
|
||||
if args.filter and args.filter not in name:
|
||||
continue
|
||||
# Static benchmarks measure once; boot benchmarks repeat per spawn;
|
||||
# session benchmarks gather inner samples per spawn, repeated.
|
||||
outer = 1 if kind == "static" else args.repeat
|
||||
samples, extras, err = [], {}, None
|
||||
t0 = time.perf_counter()
|
||||
for _ in range(outer):
|
||||
try:
|
||||
s, e = fn(mode, args)
|
||||
except (BenchError, TimeoutError) as ex:
|
||||
err = str(ex)
|
||||
break
|
||||
samples.extend(s)
|
||||
extras.update(e)
|
||||
wall_ms = (time.perf_counter() - t0) * 1000.0
|
||||
if err is not None or not samples:
|
||||
failed += 1
|
||||
results[name] = {"unit": unit, "description": desc,
|
||||
"error": err or "no samples", "extras": extras}
|
||||
print(f" {name:20s} FAILED: {err or 'no samples'}", flush=True)
|
||||
continue
|
||||
summary = _summarize(samples)
|
||||
summary.update({"unit": unit, "description": desc,
|
||||
"samples": samples, "extras": extras,
|
||||
"wall_ms": wall_ms})
|
||||
results[name] = summary
|
||||
print(f" {name:20s} mean={summary['mean']:10.2f} {unit:9s} "
|
||||
f"(median={summary['median']:.2f} min={summary['min']:.2f} "
|
||||
f"max={summary['max']:.2f} n={summary['runs']})", flush=True)
|
||||
try:
|
||||
for name, unit, desc, fn, kind in BENCHMARKS:
|
||||
if args.filter and args.filter not in name:
|
||||
continue
|
||||
# Static benchmarks measure once; boot benchmarks repeat per spawn;
|
||||
# session benchmarks gather inner samples per spawn, repeated.
|
||||
reps = 1 if kind == "static" else args.repeat
|
||||
samples, extras, err = [], {}, None
|
||||
t0 = time.perf_counter()
|
||||
for _ in range(reps):
|
||||
try:
|
||||
s, e = fn(mode, args)
|
||||
except (BenchError, TimeoutError) as ex:
|
||||
err = str(ex)
|
||||
break
|
||||
samples.extend(s)
|
||||
extras.update(e)
|
||||
wall_ms = (time.perf_counter() - t0) * 1000.0
|
||||
if err is not None or not samples:
|
||||
failed += 1
|
||||
results[name] = {"unit": unit, "description": desc,
|
||||
"error": err or "no samples", "extras": extras}
|
||||
print(f" {name:20s} FAILED: {err or 'no samples'}", flush=True)
|
||||
continue
|
||||
summary = _summarize(samples)
|
||||
summary.update({"unit": unit, "description": desc,
|
||||
"samples": samples, "extras": extras,
|
||||
"wall_ms": wall_ms})
|
||||
results[name] = summary
|
||||
print(f" {name:20s} mean={summary['mean']:10.2f} {unit:9s} "
|
||||
f"(median={summary['median']:.2f} min={summary['min']:.2f} "
|
||||
f"max={summary['max']:.2f} n={summary['runs']})", flush=True)
|
||||
finally:
|
||||
_close_outer()
|
||||
|
||||
# Human table.
|
||||
print("\n%-20s %5s %12s %12s %12s %12s %-9s" %
|
||||
|
|
@ -409,7 +461,8 @@ def main(argv=None):
|
|||
r["min"], r["max"], unit))
|
||||
|
||||
if args.json:
|
||||
payload = {"mode": mode, "repeat": args.repeat,
|
||||
payload = {"mode": mode, "outer": args.outer,
|
||||
"repeat": args.repeat,
|
||||
"cycles": args.cycles, "tabs": args.tabs,
|
||||
"results": results}
|
||||
with open(args.json, "w") as f:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue