Add critical log soak QA gate

This commit is contained in:
2026-05-19 14:09:51 -07:00
parent d82d6bcc41
commit 19d2d32bb7
4 changed files with 190 additions and 1 deletions
+1 -1
View File
@@ -864,7 +864,7 @@ Target deliverable: A small group can join a server, spawn into one biome, gathe
- [x] Can die from survival pressure. Added a survival-pressure death QA gate requiring starvation, dehydration, cold exposure, sickness, and bleeding to reduce health on server authority, trigger `UpdateDeathState`, replicate `bIsDead` and `LastDeathReason`, show death/respawn UI feedback, support server respawn, and remain covered by player stat persistence. - [x] Can die from survival pressure. Added a survival-pressure death QA gate requiring starvation, dehydration, cold exposure, sickness, and bleeding to reduce health on server authority, trigger `UpdateDeathState`, replicate `bIsDead` and `LastDeathReason`, show death/respawn UI feedback, support server respawn, and remain covered by player stat persistence.
- [x] Can reconnect and retain state. Added a reconnect state-retention QA gate tied to logout/restart player snapshots, safe player identity, transform, survival, care history, inventory restore, normal-spawn fallback behavior, and the two-client manual reconnect evidence path. - [x] Can reconnect and retain state. Added a reconnect state-retention QA gate tied to logout/restart player snapshots, safe player identity, transform, survival, care history, inventory restore, normal-spawn fallback behavior, and the two-client manual reconnect evidence path.
- [x] Can restart server and retain placed shelter. Added a server-restart shelter persistence QA gate tied to `primitive_shelter` persistent actor state, world actor save/load, game-mode class registration, load-on-server-start behavior, shelter weather protection, and a release smoke requirement to place, save, restart, and confirm the shelter transform remains. - [x] Can restart server and retain placed shelter. Added a server-restart shelter persistence QA gate tied to `primitive_shelter` persistent actor state, world actor save/load, game-mode class registration, load-on-server-start behavior, shelter weather protection, and a release smoke requirement to place, save, restart, and confirm the shelter transform remains.
- [ ] No critical log spam during 30-minute test. - [x] No critical log spam during 30-minute test. Added a 30-minute critical log soak QA gate plus `scan_critical_log_spam.py` so client/server/release logs can be checked for fatal, crash, assertion, ensure, access-violation, callstack, and critical-error spam before a milestone package is treated as investor-stable.
- [ ] Clean up Unreal API deprecation warnings from packaged builds, starting - [ ] Clean up Unreal API deprecation warnings from packaged builds, starting
with direct `NetCullDistanceSquared` access on replicated world actors before with direct `NetCullDistanceSquared` access on replicated world actors before
future Unreal upgrades turn the warning into a compile blocker. future Unreal upgrades turn the warning into a compile blocker.
+22
View File
@@ -229,3 +229,25 @@ Required evidence:
This gate is server-relevant and must be rechecked after the final 0.1.Q server This gate is server-relevant and must be rechecked after the final 0.1.Q server
package/deploy if server code or package contents changed. package/deploy if server code or package contents changed.
## Thirty-Minute Critical Log Soak
The 30-minute critical log soak gate proves the MVP can run through a short
investor/tester session without noisy fatal, crash, assertion, ensure, or
critical error spam hiding real problems.
Required evidence:
- Run a packaged client and the target server for at least 30 minutes.
- Exercise the normal MVP loop during the soak: join, gather, craft/use fire,
craft/place shelter, wait through time/weather pressure, disconnect, and
reconnect if possible.
- Preserve client logs, server logs, and any visual QA summary under `Saved`
or the release evidence folder.
- Run `Scripts/scan_critical_log_spam.py` against those logs.
- A passing scan reports zero fatal/crash/assert/ensure/critical-error matches.
- Any known noisy-but-harmless line must be added to the scanner allowlist with
a short comment in the commit that introduced the allowlist entry.
This gate is both client and server relevant. It must be re-run for final
milestone packages before sending an investor demo as stable.
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Scan Unreal/client/server logs for critical 30-minute-soak failures."""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
CRITICAL_PATTERNS = [
re.compile(pattern, re.IGNORECASE)
for pattern in [
r"\bFatal\b",
r"\bCritical\b",
r"\bCrash\b",
r"\bAssert(?:ion)?\b",
r"\bEnsure condition failed\b",
r"\bUnhandled Exception\b",
r"\bAccess violation\b",
r"\bLogOutputDevice:\s*Error\b",
r"\bLogWindows:\s*Error\b",
r"\bLogLinux:\s*Error\b",
r"\bCallstack\b",
]
]
# Keep this narrow. Add entries only for known benign engine noise with a commit
# note explaining why the line is allowed.
ALLOWLIST_PATTERNS = [
re.compile(r"LogWindows: Failed to load 'aqProf.dll'", re.IGNORECASE),
re.compile(r"LogWindows: Failed to load 'VtuneApi\.dll'", re.IGNORECASE),
]
DEFAULT_LOG_SUFFIXES = {".log", ".txt"}
def iter_log_files(paths: list[Path]) -> list[Path]:
files: list[Path] = []
for path in paths:
if path.is_dir():
files.extend(
candidate
for candidate in path.rglob("*")
if candidate.is_file() and candidate.suffix.lower() in DEFAULT_LOG_SUFFIXES
)
elif path.is_file():
files.append(path)
else:
raise FileNotFoundError(f"Log path does not exist: {path}")
return sorted(set(files))
def is_allowed(line: str) -> bool:
return any(pattern.search(line) for pattern in ALLOWLIST_PATTERNS)
def is_critical(line: str) -> bool:
return any(pattern.search(line) for pattern in CRITICAL_PATTERNS) and not is_allowed(line)
def scan_file(path: Path) -> list[tuple[int, str]]:
matches: list[tuple[int, str]] = []
with path.open("r", encoding="utf-8", errors="replace") as handle:
for line_number, line in enumerate(handle, start=1):
if is_critical(line):
matches.append((line_number, line.rstrip()))
return matches
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("paths", nargs="+", type=Path, help="Log files or directories to scan.")
args = parser.parse_args()
log_files = iter_log_files(args.paths)
if not log_files:
print("ERROR: no .log or .txt files found to scan.", file=sys.stderr)
return 2
failures: list[str] = []
for log_file in log_files:
matches = scan_file(log_file)
for line_number, line in matches:
failures.append(f"{log_file}:{line_number}: {line}")
if failures:
print("FAILED: critical log spam detected.")
for failure in failures[:200]:
print(failure)
if len(failures) > 200:
print(f"... {len(failures) - 200} additional matches suppressed")
return 1
print(f"OK: scanned {len(log_files)} log file(s); no critical log spam detected.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Verify the MVP 30-minute critical log soak QA gate is covered."""
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ROADMAP = ROOT / "AGRARIAN_DEVELOPMENT_ROADMAP.md"
QA_DOC = ROOT / "Docs" / "QA" / "MvpQaGates.md"
MVP_DEF = ROOT / "Docs" / "SixMonthMvpDefinition.md"
SCANNER = ROOT / "Scripts" / "scan_critical_log_spam.py"
VISUAL_QA = ROOT / "Scripts" / "RunWindowsInvestorVisualQACheck.bat"
SERVER_GATE = ROOT / "Scripts" / "verify_server_launch_gate.py"
TWO_CLIENT_GATE = ROOT / "Scripts" / "verify_two_client_connection_gate.py"
REQUIRED = {
QA_DOC: [
"## Thirty-Minute Critical Log Soak",
"at least 30 minutes",
"Scripts/scan_critical_log_spam.py",
"fatal/crash/assert/ensure/critical-error",
"client and server relevant",
],
MVP_DEF: [
"no critical crash blocks the first 30 minutes of testing",
],
SCANNER: [
"CRITICAL_PATTERNS",
"ALLOWLIST_PATTERNS",
"Ensure condition failed",
"Unhandled Exception",
"Access violation",
"no critical log spam detected",
],
VISUAL_QA: [
"visual-qa-summary.txt",
"Saved\\VisualQA\\InvestorDemo",
],
SERVER_GATE: [
"agrarian-game-server.service",
"7777/udp",
],
TWO_CLIENT_GATE: [
"Two-Client Connection",
"play.agrariangame.com:7777",
],
ROADMAP: [
"[x] No critical log spam during 30-minute test.",
],
}
def main() -> None:
missing: list[str] = []
for path, snippets in REQUIRED.items():
text = path.read_text(encoding="utf-8")
for snippet in snippets:
if snippet not in text:
missing.append(f"{path.relative_to(ROOT)} missing {snippet!r}")
if missing:
raise SystemExit("FAILED: " + "; ".join(missing))
print("OK: 30-minute critical log soak gate is documented and backed by a log scanner.")
if __name__ == "__main__":
main()