fix(release): require visual selection evidence

This commit is contained in:
Pulse Test 2026-08-28 20:11:04 +01:00
parent 1d15ab60a3
commit a1ae0fa07f
8 changed files with 77 additions and 27 deletions

View file

@ -274,6 +274,7 @@ current HEAD with identical generated demo data.
Return only JSON in this shape, with at most three captures:
{
"schema_version": 1,
"decision": "Why the selected views improve the notes, or why no screenshot adds meaningful value",
"captures": [
{
"id": "lower-case-hyphenated-id",
@ -311,7 +312,9 @@ testid. Actions may be click or wait. Role locators use role and name. Other
locators use value. Every state needs a ready locator for content that must be
visible in the finished image. Locator names and values are literal accessible
text, not regular expressions. Use labels verified against the deterministic
generated demo data. Use no semicolon or em dash characters in public text.
generated demo data. The decision field records the model's evidence-based
reason for selecting these captures or selecting none. Use no semicolon or em
dash characters in public text.
Customer release notes:

View file

@ -3206,6 +3206,30 @@ func TestReleaseNotesGeneratorResolvesChannelSpecificComparisonRanges(t *testing
}
}
func TestReleaseTriggersReevaluateVisualsInsteadOfTrustingSidecars(t *testing.T) {
for _, path := range []string{
repoFile("scripts", "trigger-release.sh"),
repoFile("scripts", "trigger-stable-patch.sh"),
} {
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
text := string(content)
if strings.Contains(text, "VISUAL_PLAN_SIDECAR") {
t.Fatalf("%s must not treat a committed visual sidecar as dispatch evidence", path)
}
for _, required := range []string{
"A committed sidecar is review material, not proof",
"generate-release-notes.sh --visual-plan",
} {
if !strings.Contains(text, required) {
t.Fatalf("%s missing visual reevaluation contract %q", path, required)
}
}
}
}
func assertFileContainsAllNormalized(t *testing.T, path string, required ...string) {
t.Helper()
content, err := os.ReadFile(path)

View file

@ -131,11 +131,12 @@ def _state(value: Any, field: str) -> dict[str, Any]:
def validate_plan(raw: Any) -> dict[str, Any]:
if not isinstance(raw, dict):
raise PlanError("visual plan must be a JSON object")
unknown = set(raw) - {"schema_version", "captures"}
unknown = set(raw) - {"schema_version", "decision", "captures"}
if unknown:
raise PlanError(f"visual plan has unsupported fields: {', '.join(sorted(unknown))}")
if raw.get("schema_version") != 1:
raise PlanError("visual plan schema_version must be 1")
decision = _text(raw.get("decision"), "visual plan decision", maximum=500)
captures = raw.get("captures")
if not isinstance(captures, list) or len(captures) > MAX_CAPTURES:
raise PlanError(f"visual plan captures must be a list with at most {MAX_CAPTURES} entries")
@ -191,7 +192,11 @@ def validate_plan(raw: Any) -> dict[str, Any]:
"after": _state(capture.get("after"), f"{field}.after"),
}
)
return {"schema_version": 1, "captures": normalized_captures}
return {
"schema_version": 1,
"decision": decision,
"captures": normalized_captures,
}
def load_plan(path: str) -> dict[str, Any]:
@ -273,9 +278,10 @@ def json_schema() -> dict[str, Any]:
return {
"type": "object",
"additionalProperties": False,
"required": ["schema_version", "captures"],
"required": ["schema_version", "decision", "captures"],
"properties": {
"schema_version": {"const": 1, "type": "integer"},
"decision": {"type": "string", "minLength": 1, "maxLength": 500},
"captures": {
"type": "array",
"maxItems": MAX_CAPTURES,

View file

@ -22,6 +22,7 @@ RENDERER_SPEC.loader.exec_module(renderer)
def valid_plan():
return {
"schema_version": 1,
"decision": "This settings comparison makes the responsive redesign immediately visible.",
"captures": [
{
"id": "responsive-settings",
@ -79,6 +80,7 @@ class ReleaseNoteVisualPlanTest(unittest.TestCase):
def test_structured_output_schema_carries_public_and_capture_bounds(self):
schema = visuals.json_schema()
self.assertEqual(schema["properties"]["schema_version"]["type"], "integer")
self.assertIn("decision", schema["required"])
captures = schema["properties"]["captures"]
capture = captures["items"]["properties"]
self.assertEqual(captures["maxItems"], visuals.MAX_CAPTURES)
@ -88,6 +90,18 @@ class ReleaseNoteVisualPlanTest(unittest.TestCase):
visuals.MAX_STEPS,
)
def test_empty_capture_plan_requires_an_explicit_model_decision(self):
with self.assertRaisesRegex(visuals.PlanError, "decision"):
visuals.validate_plan({"schema_version": 1, "captures": []})
plan = visuals.validate_plan(
{
"schema_version": 1,
"decision": "The investigated changes have no meaningful static visual state.",
"captures": [],
}
)
self.assertEqual(plan["captures"], [])
def test_current_only_capture_has_one_asset(self):
raw = valid_plan()
raw["captures"][0]["before"] = None

View file

@ -139,13 +139,9 @@ echo ""
# Check 5: Release notes file
NOTES_FILE="${NOTES_FILE_ARG:-/tmp/release_notes_${VERSION}.md}"
VISUAL_PLAN_SIDECAR="${NOTES_FILE}.visuals.json"
if [ -s "$VISUAL_PLAN_SIDECAR" ]; then
VISUAL_PLAN_FILE="$VISUAL_PLAN_SIDECAR"
else
VISUAL_PLAN_FILE=$(mktemp)
rm -f "$VISUAL_PLAN_FILE"
fi
VISUAL_PLAN_FILE=$(mktemp)
rm -f "$VISUAL_PLAN_FILE"
VISUAL_PLAN_GENERATED_THIS_RUN="false"
if [ -f "$NOTES_FILE" ]; then
echo "Found release notes file: ${NOTES_FILE}"
echo ""
@ -166,6 +162,7 @@ else
echo "Generating release notes..."
if RELEASE_NOTE_VISUAL_PLAN_FILE="$VISUAL_PLAN_FILE" \
./scripts/generate-release-notes.sh "$VERSION" > "$NOTES_FILE"; then
VISUAL_PLAN_GENERATED_THIS_RUN="true"
echo "Release notes generated at ${NOTES_FILE}"
echo ""
# Show first few lines
@ -202,8 +199,12 @@ python3 scripts/release_control/render_release_body.py \
--validate-notes-file "$NOTES_FILE"
echo "✓ Release-note Markdown structure validated"
if [ ! -s "$VISUAL_PLAN_FILE" ]; then
if [ "$VISUAL_PLAN_GENERATED_THIS_RUN" != "true" ]; then
# A committed sidecar is review material, not proof that visual investigation
# ran for this dispatch. Always make the release model judge the exact notes
# and comparison range used by the publication request.
./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE"
VISUAL_PLAN_GENERATED_THIS_RUN="true"
fi
python3 scripts/release_control/release_note_visuals.py \
validate --plan "$VISUAL_PLAN_FILE" --output "$VISUAL_PLAN_FILE"
@ -217,7 +218,7 @@ if [ "$VISUAL_CAPTURE_COUNT" -gt 0 ]; then
read -p "Use this visual plan? [Y/n] " -n 1 -r
echo ""
if [[ $REPLY =~ ^[Nn]$ ]]; then
printf '%s\n' '{"schema_version":1,"captures":[]}' > "$VISUAL_PLAN_FILE"
printf '%s\n' '{"schema_version":1,"decision":"The release owner declined the model-selected visual plan.","captures":[]}' > "$VISUAL_PLAN_FILE"
VISUAL_CAPTURE_COUNT=0
fi
fi

View file

@ -107,13 +107,8 @@ if [ ! -s "$NOTES_FILE" ]; then
echo "Canonical release notes are required at ${NOTES_FILE}." >&2
exit 1
fi
VISUAL_PLAN_SIDECAR="${NOTES_FILE}.visuals.json"
if [ -s "$VISUAL_PLAN_SIDECAR" ]; then
VISUAL_PLAN_FILE="$VISUAL_PLAN_SIDECAR"
else
VISUAL_PLAN_FILE=$(mktemp)
rm -f "$VISUAL_PLAN_FILE"
fi
VISUAL_PLAN_FILE=$(mktemp)
rm -f "$VISUAL_PLAN_FILE"
RESOLVER_ARGS=(
--version "$VERSION"
@ -197,9 +192,10 @@ else
--version "$VERSION" \
--validate-notes-file "$NOTES_FILE"
if [ ! -s "$VISUAL_PLAN_FILE" ]; then
./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE"
fi
# A committed sidecar is review material, not proof that visual investigation
# ran for this dispatch. Always make the release model judge the exact notes
# and comparison range used by the publication request.
./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE"
python3 scripts/release_control/release_note_visuals.py \
validate --plan "$VISUAL_PLAN_FILE" --output "$VISUAL_PLAN_FILE"