From 40492a4c46ada8a5828b5f255426d64a3e22c64a Mon Sep 17 00:00:00 2001 From: PSBigBig + MiniPS Date: Wed, 6 May 2026 17:25:31 +0800 Subject: [PATCH] Delete Polaris/experiments/downloads/PP02D_~3.IPY --- Polaris/experiments/downloads/PP02D_~3.IPY | 2424 -------------------- 1 file changed, 2424 deletions(-) delete mode 100644 Polaris/experiments/downloads/PP02D_~3.IPY diff --git a/Polaris/experiments/downloads/PP02D_~3.IPY b/Polaris/experiments/downloads/PP02D_~3.IPY deleted file mode 100644 index c5c9c73d..00000000 --- a/Polaris/experiments/downloads/PP02D_~3.IPY +++ /dev/null @@ -1,2424 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8095d5c4", - "metadata": {}, - "source": [ - "# PP02D_C — Public QA Compact Context Full100 Mini\n", - "\n", - "**Belongs to:** WFGY 5.0 Polaris Protocol \n", - "**Repository:** https://github.com/onestardao/WFGY \n", - "**Polaris Protocol path:** https://github.com/onestardao/WFGY/tree/main/Polaris \n", - "**Experiment page:** https://github.com/onestardao/WFGY/blob/main/Polaris/experiments/README.md\n", - "\n", - "## Experiment spirit\n", - "\n", - "Public QA compact-context supplement. The goal is reproduction / inspection of a scoped public QA signal, not global QA superiority.\n", - "\n", - "## What this Colab tests\n", - "\n", - "Tests public QA compact context under a Full100 Mini MVP runnable Colab: baseline raw context vs Polaris compact context on 100 public QA cases with the primary mini model.\n", - "\n", - "## Published / aligned result summary\n", - "\n", - "- MVP runnable Colab scope: 100 public QA cases × 2 arms × 1 primary model ≈ 200 model-arm output records.\n", - "- `gpt-4.1-mini` baseline answer F1 mean: 0.7978095238.\n", - "- `gpt-4.1-mini` compact-context answer F1 mean: 0.8063095238.\n", - "- `gpt-4.1-mini` baseline support-title F1 mean: 0.8906666667.\n", - "- `gpt-4.1-mini` compact-context support-title F1 mean: 0.8503333333.\n", - "- wrong-source total: baseline 1, compact 2.\n", - "- hallucinated-detail total: baseline 0, compact 0.\n", - "- estimated cost: baseline USD 0.099630, compact USD 0.068566.\n", - "- estimated cost reduction compact vs baseline: 0.3117936365.\n", - "\n", - "## Expected usage / token-cost note\n", - "\n", - "Full100 Mini run is expected to use about 100 cases × 2 arms × 1 model = 200 API calls. Actual cost depends on model and output length.\n", - "\n", - "## Scientific boundary\n", - "\n", - "100-case public QA compact-context evidence only. Not global QA superiority and not proof that compact context is always better.\n", - "\n", - "**Blackfan alignment note:** Do not mix the broader 2-model / 400-record model-ladder aggregate into this Mini notebook's fallback report. If you want the broader 400-record report, publish the ModelLadder Both Colab as a separate advanced notebook.\n", - "\n", - "This Colab is provided for **reproduction, inspection, and adaptation**. You do not have to rerun it to read the published result summary. If you rerun it, model behavior, token usage, cost, and output details may vary.\n", - "\n", - "---\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "574aba2d", - "metadata": {}, - "outputs": [], - "source": [ - "# Polaris Protocol experiment identity\n", - "POLARIS_PROTOCOL_REPO = \"https://github.com/onestardao/WFGY\"\n", - "POLARIS_EXPERIMENT_ID = \"PP02D_C\"\n", - "POLARIS_EXPERIMENT_TITLE = \"PP02D_C — Public QA Compact Context Full100 Mini\"\n", - "POLARIS_EXPERIMENT_ROLE = \"Public QA Compact Context Full100 Mini\"\n", - "POLARIS_CLAIM_BOUNDARY = \"100-case public QA compact-context evidence only. Not global QA superiority and not proof that compact context is always better.\"\n", - "print(f\"{POLARIS_EXPERIMENT_ID} — {POLARIS_EXPERIMENT_TITLE}\")\n", - "print(f\"Repo: {POLARIS_PROTOCOL_REPO}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "3150ca1e", - "metadata": {}, - "source": [ - "# WFGY 5.0 Polaris Protocol\n", - "\n", - "Repository: https://github.com/onestardao/WFGY \n", - "Branch: DD02C \n", - "Experiment: DD02C_ZAI_EXACT_PUBLIC_QA_NONINFERIORITY_V2 \n", - "Notebook version: v2.0.1 MathCore Evidence Controller, FULL100 MINI STRATIFIED preset, smoke-skip fix\n", - "\n", - "This notebook runs a ZAI-aligned public real QA validation experiment on HotpotQA.\n", - "\n", - "It compares two prompt conditions on the same public HotpotQA validation cases.\n", - "\n", - "A_BASELINE_RAW_CONTEXT \n", - "The model receives the raw public context from the dataset.\n", - "\n", - "B_POLARIS_COMPACT_CONTEXT \n", - "The model receives an answer-blind compact context built only from the question and public context text.\n", - "\n", - "The goal is not to claim universal QA superiority. \n", - "The goal is to test whether the compact context preserves real QA quality within a strict non-inferiority margin while reducing input tokens and avoiding wrong-source or hallucination risk. Version v2.0 uses an answer-blind MathCore Evidence Controller. It optimizes evidence selection with explicit cue, target-graph, answer-field, candidate-span, distractor-penalty, redundancy, and token-budget terms. Optional Model Ladder smoke mode remains available.\n", - "\n", - "Claim ceiling: sandbox and smoke checks validate the experiment harness only. Only API outputs on public HotpotQA cases can produce real QA evidence.\n", - "\n", - "This notebook does not store your OpenAI API key in any output file.\n", - "\n", - "## Full30 Mini Preset\n", - "\n", - "This file is preset for the next validation step: 30 public HotpotQA cases × 2 arms × primary small model only. Optional full model ladder remains off by default.\n", - "\n", - "\n", - "## Full100 Mini Stratified Preset\n", - "\n", - "This file is preset for the next validation step: 100 public HotpotQA distractor validation cases × 2 arms × primary small model only.\n", - "\n", - "The sampling strategy is stratified by HotpotQA `type` and `level` fields when enough cases are available. This keeps the run closer to a standard benchmark-style exam rather than a cherry-picked or purely lucky random slice.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d2cbd584", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 1: Install dependencies\n", - "!pip -q install openai datasets tiktoken pandas numpy tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "162365d8", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 2: Imports and fixed configuration\n", - "\n", - "import os\n", - "import re\n", - "import json\n", - "import math\n", - "import time\n", - "import hashlib\n", - "import random\n", - "import string\n", - "import zipfile\n", - "from pathlib import Path\n", - "from datetime import datetime, timezone\n", - "from getpass import getpass\n", - "from typing import Any, Dict, List, Tuple\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from tqdm.auto import tqdm\n", - "\n", - "from datasets import load_dataset\n", - "\n", - "try:\n", - " import tiktoken\n", - "except Exception:\n", - " tiktoken = None\n", - "\n", - "EXPERIMENT_NAME = \"DD02C_ZAI_EXACT_PUBLIC_QA_NONINFERIORITY_V2\"\n", - "BRANCH_NAME = \"DD02C\"\n", - "DATASET_NAME_PRIMARY = \"hotpotqa/hotpot_qa\"\n", - "DATASET_NAME_FALLBACK = \"hotpot_qa\"\n", - "DATASET_CONFIG = \"distractor\"\n", - "DATASET_SPLIT = \"validation\"\n", - "\n", - "SAMPLE_COUNT = 100\n", - "SMOKE_COUNT = 0\n", - "RANDOM_SEED = 520\n", - "COMPACT_MAX_TITLES = 10\n", - "COMPACT_MAX_SENTENCES = 22\n", - "COMPACT_MIN_SENTENCES_PER_SELECTED_TITLE = 1\n", - "COMPACT_TARGET_RATIO = 0.43\n", - "QA_F1_NONINFERIORITY_MARGIN = 0.12\n", - "SUPPORT_TITLE_NONINFERIORITY_MARGIN = 0.16\n", - "HALLUCINATION_MARGIN = 1\n", - "SMOKE_TOKEN_REDUCTION_GATE = 0.25\n", - "STRICT_TOKEN_REDUCTION_GATE = 0.35\n", - "\n", - "OUTPUT_ROOT = Path(\"/content/dd02c_outputs\")\n", - "RUN_ID = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n", - "RUN_DIR = OUTPUT_ROOT / f\"{EXPERIMENT_NAME}_{RUN_ID}\"\n", - "RUN_DIR.mkdir(parents=True, exist_ok=True)\n", - "\n", - "print(\"Experiment:\", EXPERIMENT_NAME)\n", - "print(\"Run directory:\", RUN_DIR)\n", - "# v1.3 model ladder defaults.\n", - "RUN_MODEL_LADDER_SMOKE_DEFAULT = False\n", - "PRIMARY_MODEL_DEFAULT = \"gpt-4.1-mini\"\n", - "COMPARISON_MODEL_DEFAULT = \"gpt-4.1\"\n", - "\n", - "# Editable price table for estimated cost only. Update manually if provider pricing changes.\n", - "MODEL_PRICE_PER_1M_TOKENS = {\n", - " \"gpt-4.1-mini\": {\"input\": 0.40, \"output\": 1.60},\n", - " \"gpt-4.1\": {\"input\": 2.00, \"output\": 8.00},\n", - "}\n", - "\n", - "# v1.8 compact control.\n", - "SLIMBANK_MAX_CANDIDATES = 8\n", - "SLIMBANK_MAX_PRIMARY = 4\n", - "SLIMBANK_MAX_SECONDARY = 3\n", - "SLIMBANK_MAX_DISTRACTOR = 1\n", - "TARGETGRAPH_MAX_LINES = 8\n", - "EVIDENCECARD_MAX_LINES = 22\n", - "\n", - "# v2.0 MathCore weights. These are answer-blind and use only question/context features.\n", - "MATHCORE_WEIGHTS = {\n", - " \"cue_match\": 2.4,\n", - " \"relation_match\": 2.3,\n", - " \"target_bridge\": 2.2,\n", - " \"answer_field_match\": 1.8,\n", - " \"candidate_density\": 1.4,\n", - " \"title_coverage\": 0.9,\n", - " \"span_extractability\": 1.3,\n", - " \"wrong_focus_attraction\": -3.2,\n", - " \"distractor_cluster\": -1.7,\n", - " \"redundancy\": -1.1,\n", - " \"token_cost\": -0.045,\n", - "}\n", - "MATHCORE_MAX_GRAPH_LINES = 7\n", - "MATHCORE_TITLE_SKELETON_TOKEN_SOFT_CAP = 28\n", - "MATHCORE_FIELDLOCK_TOP_ROWS = 8\n", - "\n", - "# Full100 standard-exam sampling controls.\n", - "STRATIFIED_SAMPLE_ENABLED = True\n", - "STRATIFY_BY_FIELDS = [\"type\", \"level\"]\n", - "TARGET_TYPE_RATIO = {\n", - " \"bridge\": 0.70,\n", - " \"comparison\": 0.30,\n", - "}\n", - "MIN_HARD_OR_MEDIUM_RATIO = 0.80\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a9e692fb", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 3: API key input and model ladder controls, runtime memory only\n", - "\n", - "OPENAI_API_KEY = getpass(\"Enter your OpenAI API key. It will stay in runtime memory only: \").strip()\n", - "if not OPENAI_API_KEY:\n", - " raise ValueError(\"Missing OpenAI API key.\")\n", - "\n", - "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n", - "\n", - "PRIMARY_MODEL_INPUT = input(f\"Primary small model, press Enter for default {PRIMARY_MODEL_DEFAULT}: \").strip()\n", - "PRIMARY_MODEL = PRIMARY_MODEL_INPUT or PRIMARY_MODEL_DEFAULT\n", - "\n", - "COMPARISON_MODEL_INPUT = input(f\"Comparison larger model, press Enter for default {COMPARISON_MODEL_DEFAULT}: \").strip()\n", - "COMPARISON_MODEL = COMPARISON_MODEL_INPUT or COMPARISON_MODEL_DEFAULT\n", - "\n", - "MODEL_LADDER_INPUT = input(\"Run optional model ladder smoke? Type YES to run both models. Press Enter for mini-only smoke: \").strip().upper()\n", - "RUN_MODEL_LADDER_SMOKE = MODEL_LADDER_INPUT == \"YES\"\n", - "\n", - "MODEL_NAME = PRIMARY_MODEL # Backward-compatible alias.\n", - "SMOKE_MODELS = [PRIMARY_MODEL]\n", - "if RUN_MODEL_LADDER_SMOKE and COMPARISON_MODEL not in SMOKE_MODELS:\n", - " SMOKE_MODELS.append(COMPARISON_MODEL)\n", - "\n", - "print(\"Primary model:\", PRIMARY_MODEL)\n", - "print(\"Comparison model:\", COMPARISON_MODEL)\n", - "print(\"Run model ladder smoke:\", RUN_MODEL_LADDER_SMOKE)\n", - "print(\"Smoke models:\", SMOKE_MODELS)\n", - "print(\"API key stored in memory only. It will not be written into artifacts.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16c8dbe4", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 4: Utility functions\n", - "\n", - "def write_jsonl(path: Path, rows: List[Dict[str, Any]]) -> None:\n", - " path.parent.mkdir(parents=True, exist_ok=True)\n", - " with path.open(\"w\", encoding=\"utf-8\") as f:\n", - " for row in rows:\n", - " f.write(json.dumps(row, ensure_ascii=False) + \"\\n\")\n", - "\n", - "def read_jsonl(path: Path) -> List[Dict[str, Any]]:\n", - " rows = []\n", - " with path.open(\"r\", encoding=\"utf-8\") as f:\n", - " for line in f:\n", - " if line.strip():\n", - " rows.append(json.loads(line))\n", - " return rows\n", - "\n", - "def sha256_file(path: Path) -> str:\n", - " h = hashlib.sha256()\n", - " with path.open(\"rb\") as f:\n", - " for chunk in iter(lambda: f.read(1024 * 1024), b\"\"):\n", - " h.update(chunk)\n", - " return h.hexdigest()\n", - "\n", - "def normalize_text(s: str) -> str:\n", - " if s is None:\n", - " return \"\"\n", - " s = str(s).lower()\n", - " s = re.sub(r\"\\b(a|an|the)\\b\", \" \", s)\n", - " s = s.translate(str.maketrans(\"\", \"\", string.punctuation))\n", - " s = \" \".join(s.split())\n", - " return s\n", - "\n", - "def f1_score(prediction: str, ground_truth: str) -> float:\n", - " pred_tokens = normalize_text(prediction).split()\n", - " gold_tokens = normalize_text(ground_truth).split()\n", - " if len(pred_tokens) == 0 and len(gold_tokens) == 0:\n", - " return 1.0\n", - " if len(pred_tokens) == 0 or len(gold_tokens) == 0:\n", - " return 0.0\n", - " pred_counts = {}\n", - " gold_counts = {}\n", - " for tok in pred_tokens:\n", - " pred_counts[tok] = pred_counts.get(tok, 0) + 1\n", - " for tok in gold_tokens:\n", - " gold_counts[tok] = gold_counts.get(tok, 0) + 1\n", - " num_same = sum(min(pred_counts.get(tok, 0), gold_counts.get(tok, 0)) for tok in pred_counts)\n", - " if num_same == 0:\n", - " return 0.0\n", - " precision = num_same / len(pred_tokens)\n", - " recall = num_same / len(gold_tokens)\n", - " return 2 * precision * recall / (precision + recall)\n", - "\n", - "def exact_match(prediction: str, ground_truth: str) -> int:\n", - " return int(normalize_text(prediction) == normalize_text(ground_truth))\n", - "\n", - "def set_f1(pred_items: List[str], gold_items: List[str]) -> float:\n", - " pred = {normalize_text(x) for x in pred_items if str(x).strip()}\n", - " gold = {normalize_text(x) for x in gold_items if str(x).strip()}\n", - " if not pred and not gold:\n", - " return 1.0\n", - " if not pred or not gold:\n", - " return 0.0\n", - " tp = len(pred & gold)\n", - " precision = tp / len(pred) if pred else 0.0\n", - " recall = tp / len(gold) if gold else 0.0\n", - " if precision + recall == 0:\n", - " return 0.0\n", - " return 2 * precision * recall / (precision + recall)\n", - "\n", - "def estimate_tokens(text: str) -> int:\n", - " if tiktoken is not None:\n", - " try:\n", - " enc = tiktoken.get_encoding(\"cl100k_base\")\n", - " return len(enc.encode(text))\n", - " except Exception:\n", - " pass\n", - " return int(max(1, len(str(text).split()) * 1.33))\n", - "\n", - "def extract_json_object(text: str) -> Dict[str, Any]:\n", - " text = str(text).strip()\n", - " try:\n", - " return json.loads(text)\n", - " except Exception:\n", - " pass\n", - " start = text.find(\"{\")\n", - " end = text.rfind(\"}\")\n", - " if start >= 0 and end > start:\n", - " return json.loads(text[start:end+1])\n", - " raise ValueError(\"No valid JSON object found.\")\n", - "\n", - "STOPWORDS = set(\"\"\"\n", - "a an the and or but if then else when while where who whom whose which what why how\n", - "is are was were be been being do does did done to of in on for from with by as at\n", - "it its this that these those into about over under after before between among\n", - "\"\"\".split())\n", - "\n", - "def tokenize_keywords(text: str) -> List[str]:\n", - " raw = re.findall(r\"[A-Za-z0-9]+\", str(text).lower())\n", - " return [t for t in raw if len(t) > 2 and t not in STOPWORDS]\n", - "\n", - "def entity_like_tokens(text: str) -> List[str]:\n", - " return re.findall(r\"\\b[A-Z][A-Za-z0-9_\\-]{2,}\\b\", str(text))\n", - "\n", - "def compact_sentence_score(question: str, title: str, sentence: str, sent_idx: int) -> Dict[str, Any]:\n", - " q_tokens = set(tokenize_keywords(question))\n", - " title_tokens = set(tokenize_keywords(title))\n", - " sent_tokens = set(tokenize_keywords(sentence))\n", - " q_overlap = len(q_tokens & sent_tokens)\n", - " title_overlap = len(q_tokens & title_tokens)\n", - " entity_overlap = len(set(entity_like_tokens(question)) & set(entity_like_tokens(sentence)))\n", - " position_bonus = 1.0 if sent_idx <= 1 else 0.25 if sent_idx <= 3 else 0.0\n", - " length_penalty = 0.5 if len(str(sentence).split()) > 45 else 0.0\n", - " score = (3.0 * q_overlap) + (1.5 * title_overlap) + (2.0 * entity_overlap) + position_bonus - length_penalty\n", - " return {\n", - " \"score\": float(score),\n", - " \"q_overlap\": int(q_overlap),\n", - " \"title_overlap\": int(title_overlap),\n", - " \"entity_overlap\": int(entity_overlap),\n", - " \"position_bonus\": float(position_bonus),\n", - " \"length_penalty\": float(length_penalty)\n", - " }\n", - "\n", - "def api_key_pattern_found(text: str) -> bool:\n", - " return bool(re.search(r\"sk-[A-Za-z0-9_\\-]{16,}\", str(text)))\n", - "\n", - "def bridge_terms_from_text(text: str) -> List[str]:\n", - " caps = entity_like_tokens(text)\n", - " keywords = tokenize_keywords(text)\n", - " # Keep medium-length keywords and capitalized entity-like tokens.\n", - " merged = []\n", - " for tok in caps + keywords:\n", - " tok_norm = tok.lower()\n", - " if len(tok_norm) >= 3 and tok_norm not in STOPWORDS:\n", - " merged.append(tok_norm)\n", - " # Stable order de-duplication.\n", - " seen = set()\n", - " out = []\n", - " for tok in merged:\n", - " if tok not in seen:\n", - " seen.add(tok)\n", - " out.append(tok)\n", - " return out\n", - "\n", - "def unsupported_answer_token_ratio(answer: str, context: str) -> float:\n", - " ans_tokens = [t for t in tokenize_keywords(answer) if len(t) >= 3]\n", - " if not ans_tokens:\n", - " return 0.0\n", - " context_tokens = set(tokenize_keywords(context))\n", - " unsupported = [t for t in ans_tokens if t not in context_tokens]\n", - " return len(unsupported) / max(1, len(ans_tokens))\n", - "\n", - "def estimate_cost_usd(model: str, input_tokens: int, output_tokens: int) -> float:\n", - " price = MODEL_PRICE_PER_1M_TOKENS.get(model)\n", - " if not price:\n", - " return float(\"nan\")\n", - " return (input_tokens / 1_000_000.0) * price[\"input\"] + (output_tokens / 1_000_000.0) * price[\"output\"]\n", - "\n", - "def value_per_dollar(answer_f1: float, support_title_f1: float, cost_usd: float) -> float:\n", - " if not cost_usd or math.isnan(cost_usd) or cost_usd <= 0:\n", - " return float(\"nan\")\n", - " quality = 0.70 * answer_f1 + 0.30 * support_title_f1\n", - " return quality / cost_usd\n", - "\n", - "\n", - "def question_type_from_text(question: str) -> str:\n", - " q = str(question).strip().lower()\n", - " if q.startswith(\"who\"):\n", - " return \"who\"\n", - " if q.startswith(\"where\"):\n", - " return \"where\"\n", - " if q.startswith(\"when\"):\n", - " return \"when\"\n", - " if q.startswith(\"which\"):\n", - " return \"which\"\n", - " if q.startswith(\"are \") or q.startswith(\"is \") or q.startswith(\"was \") or q.startswith(\"were \") or q.startswith(\"do \") or q.startswith(\"does \") or q.startswith(\"did \"):\n", - " return \"yes_no\"\n", - " return \"other\"\n", - "\n", - "def question_cue_terms(question: str) -> Dict[str, Any]:\n", - " q = str(question)\n", - " q_lower = q.lower()\n", - " cues = set(tokenize_keywords(q))\n", - " relation_cues = set()\n", - "\n", - " cue_groups = {\n", - " \"where\": [\"where\", \"performed\", \"premiered\", \"located\", \"place\", \"city\", \"venue\", \"country\"],\n", - " \"who\": [\"who\", \"wrote\", \"author\", \"lyrics\", \"written\", \"composer\", \"director\", \"created\", \"founded\"],\n", - " \"when\": [\"when\", \"year\", \"date\", \"released\", \"published\", \"founded\", \"born\", \"died\"],\n", - " \"which\": [\"which\", \"released\", \"first\", \"team\", \"album\", \"film\", \"tour\", \"song\"],\n", - " \"subject_bridge\": [\"subject\", \"subject of\", \"about\", \"based on\", \"concerns\", \"depicts\", \"adapted\"],\n", - " \"theme_bridge\": [\"theme\", \"theme song\", \"lyrics\", \"song\", \"used by\"],\n", - " \"comparison\": [\"first\", \"earlier\", \"later\", \"before\", \"after\", \"released\"],\n", - " }\n", - "\n", - " for group, terms in cue_groups.items():\n", - " if any(term in q_lower for term in terms):\n", - " relation_cues.update(terms)\n", - "\n", - " # Extract title-like capitalized spans from the question, e.g. After Aida, Range Busters.\n", - " title_like = re.findall(r\"\\\\b[A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,4}\\\\b\", q)\n", - " title_like = [x.strip() for x in title_like if len(x.strip()) >= 3]\n", - "\n", - " return {\n", - " \"question_type\": question_type_from_text(question),\n", - " \"keyword_cues\": sorted(cues),\n", - " \"relation_cues\": sorted(set(tokenize_keywords(\" \".join(relation_cues)))),\n", - " \"title_like_mentions\": title_like\n", - " }\n", - "\n", - "def sentence_matches_question_cues(sentence: str, cue_info: Dict[str, Any]) -> Dict[str, Any]:\n", - " sent_lower = str(sentence).lower()\n", - " relation_hits = [c for c in cue_info.get(\"relation_cues\", []) if c.lower() in sent_lower]\n", - " title_hits = [t for t in cue_info.get(\"title_like_mentions\", []) if t.lower() in sent_lower]\n", - " keyword_hits = [k for k in cue_info.get(\"keyword_cues\", []) if k.lower() in sent_lower]\n", - " return {\n", - " \"relation_hits\": relation_hits,\n", - " \"title_hits\": title_hits,\n", - " \"keyword_hits\": keyword_hits,\n", - " \"cue_hit_count\": len(relation_hits) + len(title_hits) + min(3, len(keyword_hits))\n", - " }\n", - "\n", - "def answer_instruction_from_question(question: str) -> str:\n", - " qtype = question_type_from_text(question)\n", - " if qtype == \"who\":\n", - " return \"Answer with the shortest person, group, or organization name span.\"\n", - " if qtype == \"where\":\n", - " return \"Answer with the shortest place, venue, city, or country span required by the question.\"\n", - " if qtype == \"when\":\n", - " return \"Answer with the shortest date or year span.\"\n", - " if qtype == \"which\":\n", - " return \"Answer with the shortest entity title or option span.\"\n", - " if qtype == \"yes_no\":\n", - " return \"Answer exactly yes or no.\"\n", - " return \"Answer with the shortest answer span possible.\"\n", - "\n", - "\n", - "def phrase_norm(x: str) -> str:\n", - " return normalize_text(x)\n", - "\n", - "def clean_candidate_span(span: str) -> str:\n", - " s = str(span).strip()\n", - " s = re.sub(r\"^[\\\\s,;:\\\\-\\\\(\\\\)\\\\[\\\\]\\\\\\\"']+\", \"\", s)\n", - " s = re.sub(r\"[\\\\s,;:\\\\-\\\\(\\\\)\\\\[\\\\]\\\\\\\"']+$\", \"\", s)\n", - " s = re.sub(r\"\\\\s+\", \" \", s)\n", - " return s.strip()\n", - "\n", - "def extract_capitalized_spans(text: str) -> List[str]:\n", - " spans = re.findall(r\"\\\\b[A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+(?:of|the|and|for|in|on|at|de|la|le|du|[A-Z][A-Za-z0-9'\\\\-]*)){0,6}\", str(text))\n", - " cleaned = []\n", - " for s in spans:\n", - " s = clean_candidate_span(s)\n", - " if len(s) >= 3 and len(s.split()) <= 7:\n", - " cleaned.append(s)\n", - " return cleaned\n", - "\n", - "def extract_year_date_spans(text: str) -> List[str]:\n", - " spans = []\n", - " spans += re.findall(r\"\\\\b(?:1[5-9]\\\\d{2}|20\\\\d{2})\\\\b\", str(text))\n", - " spans += re.findall(r\"\\\\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\\\\s+\\\\d{1,2},\\\\s+\\\\d{4}\\\\b\", str(text))\n", - " spans += re.findall(r\"\\\\b\\\\d{1,2}\\\\s+(?:January|February|March|April|May|June|July|August|September|October|November|December)\\\\s+\\\\d{4}\\\\b\", str(text))\n", - " return [clean_candidate_span(x) for x in spans]\n", - "\n", - "def extract_where_spans(text: str) -> List[str]:\n", - " spans = []\n", - " patterns = [\n", - " r\"\\\\b(?:in|at|from|near|located in|based in|performed at|premiered at)\\\\s+([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+(?:of|the|and|for|in|on|at|de|la|le|du|[A-Z][A-Za-z0-9'\\\\-]*)){0,6})\",\n", - " r\"\\\\b([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\\\\s+(?:Opera House|Theatre|Theater|Stadium|University|City|Province|State|County|Hall|Center|Centre)\"\n", - " ]\n", - " for pat in patterns:\n", - " spans += re.findall(pat, str(text))\n", - " return [clean_candidate_span(x) for x in spans if clean_candidate_span(x)]\n", - "\n", - "def extract_who_spans(text: str) -> List[str]:\n", - " spans = []\n", - " patterns = [\n", - " r\"\\\\b(?:by|written by|lyrics by|composed by|directed by|founded by|created by|author is|writer is)\\\\s+([A-Z][A-Za-z'\\\\-]*(?:\\\\s+[A-Z][A-Za-z'\\\\-]*){0,4})\",\n", - " r\"\\\\b([A-Z][A-Za-z'\\\\-]*(?:\\\\s+[A-Z][A-Za-z'\\\\-]*){1,4})\\\\s+(?:wrote|composed|directed|founded|created|authored)\"\n", - " ]\n", - " for pat in patterns:\n", - " spans += re.findall(pat, str(text), flags=re.I)\n", - " spans += extract_capitalized_spans(text)\n", - " return [clean_candidate_span(x) for x in spans if clean_candidate_span(x)]\n", - "\n", - "def extract_candidate_spans_for_question(question: str, sentence: str, title: str) -> List[Dict[str, Any]]:\n", - " qtype = question_type_from_text(question)\n", - " sent = str(sentence)\n", - " title = str(title)\n", - " candidates = []\n", - "\n", - " def add(span, source, bonus=0.0):\n", - " span = clean_candidate_span(span)\n", - " if not span or len(span) < 2:\n", - " return\n", - " if len(span.split()) > 8:\n", - " return\n", - " candidates.append({\n", - " \"span\": span,\n", - " \"source\": source,\n", - " \"bonus\": float(bonus)\n", - " })\n", - "\n", - " if qtype == \"when\":\n", - " for y in extract_year_date_spans(sent):\n", - " add(y, \"date_year_pattern\", 2.0)\n", - " elif qtype == \"where\":\n", - " for x in extract_where_spans(sent):\n", - " add(x, \"where_pattern\", 2.0)\n", - " for x in extract_capitalized_spans(sent):\n", - " if any(place_word in x.lower() for place_word in [\"opera house\", \"theatre\", \"theater\", \"city\", \"university\", \"stadium\", \"hall\", \"center\", \"centre\"]):\n", - " add(x, \"where_entity_pattern\", 1.5)\n", - " elif qtype == \"who\":\n", - " for x in extract_who_spans(sent):\n", - " add(x, \"who_pattern\", 1.5)\n", - " elif qtype == \"yes_no\":\n", - " add(\"yes\", \"yes_no_candidate\", 0.0)\n", - " add(\"no\", \"yes_no_candidate\", 0.0)\n", - " else:\n", - " for x in extract_capitalized_spans(sent):\n", - " add(x, \"capitalized_entity_pattern\", 1.0)\n", - "\n", - " # For which questions, titles and named entities are often valid answer spans.\n", - " if qtype == \"which\":\n", - " add(title, \"context_title_candidate\", 1.0)\n", - " for x in extract_capitalized_spans(sent):\n", - " add(x, \"which_entity_pattern\", 1.3)\n", - "\n", - " return candidates\n", - "\n", - "def build_candidate_answer_bank(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], max_candidates: int = 18) -> List[Dict[str, Any]]:\n", - " question = case[\"question\"]\n", - " cue_info = question_cue_terms(question)\n", - " question_mentions_norm = {phrase_norm(x) for x in cue_info.get(\"title_like_mentions\", [])}\n", - " bank = {}\n", - "\n", - " for row in selected_rows:\n", - " candidates = extract_candidate_spans_for_question(question, row[\"sentence\"], row[\"title\"])\n", - " cue_match = sentence_matches_question_cues(row[\"sentence\"] + \" \" + row[\"title\"], cue_info)\n", - " cue_bonus = 0.35 * cue_match[\"cue_hit_count\"]\n", - " relation_bonus = 0.75 if cue_match[\"relation_hits\"] else 0.0\n", - " title_bonus = 0.50 if cue_match[\"title_hits\"] else 0.0\n", - "\n", - " for cand in candidates:\n", - " span = cand[\"span\"]\n", - " norm = phrase_norm(span)\n", - " if not norm:\n", - " continue\n", - " # Avoid over-selecting the title already mentioned in the question when relation asks for its target.\n", - " mention_penalty = -1.25 if norm in question_mentions_norm else 0.0\n", - " score = (\n", - " float(row.get(\"questioncue_score\", row.get(\"evidencelock_score\", row.get(\"score\", 0.0))))\n", - " + cand.get(\"bonus\", 0.0)\n", - " + cue_bonus\n", - " + relation_bonus\n", - " + title_bonus\n", - " + mention_penalty\n", - " )\n", - " if norm not in bank or score > bank[norm][\"score\"]:\n", - " bank[norm] = {\n", - " \"span\": span,\n", - " \"score\": float(score),\n", - " \"source_title\": row[\"title\"],\n", - " \"source_sentence_id\": f\"{row['title']}::sent_{row['sent_id']}\",\n", - " \"candidate_source\": cand[\"source\"],\n", - " \"question_mentioned_penalty_applied\": mention_penalty < 0\n", - " }\n", - "\n", - " out = sorted(bank.values(), key=lambda x: (-x[\"score\"], x[\"span\"]))[:max_candidates]\n", - " return out\n", - "\n", - "def format_questioncue_compact_context(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], compact_context: str) -> str:\n", - " cue_info = question_cue_terms(case[\"question\"])\n", - " bank = build_candidate_answer_bank(case, selected_rows)\n", - " title_index = [doc[\"title\"] for doc in case[\"context_docs\"]]\n", - "\n", - " cue_lines = [\n", - " \"QuestionCueCard:\",\n", - " f\"question_type: {cue_info['question_type']}\",\n", - " \"relation_cues: \" + \", \".join(cue_info.get(\"relation_cues\", [])[:20]),\n", - " \"title_like_mentions: \" + \", \".join(cue_info.get(\"title_like_mentions\", [])[:12]),\n", - " \"answer_instruction: \" + answer_instruction_from_question(case[\"question\"]),\n", - " \"\",\n", - " \"AllSourceTitleIndex:\",\n", - " ]\n", - " for t in title_index:\n", - " cue_lines.append(f\"- {t}\")\n", - "\n", - " cue_lines.append(\"\")\n", - " cue_lines.append(\"CandidateAnswerBank:\")\n", - " if bank:\n", - " for i, cand in enumerate(bank, start=1):\n", - " cue_lines.append(f\"{i}. span: {cand['span']} | source: {cand['source_sentence_id']} | reason: {cand['candidate_source']}\")\n", - " else:\n", - " cue_lines.append(\"- No high-confidence candidate spans extracted. Answer from EvidenceCards.\")\n", - "\n", - " cue_lines.append(\"\")\n", - " cue_lines.append(\"EvidenceCards:\")\n", - " cue_lines.append(compact_context)\n", - " return \"\\\\n\".join(cue_lines).strip()\n", - "\n", - "\n", - "def relation_pattern_from_question(question: str) -> Dict[str, Any]:\n", - " q = str(question)\n", - " q_lower = q.lower()\n", - " patterns = []\n", - "\n", - " relation_specs = [\n", - " (\"subject_of\", [r\"subject of ([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\", r\"the subject of ([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\"]),\n", - " (\"based_on\", [r\"based on ([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\"]),\n", - " (\"theme_song_of\", [r\"theme song (?:of|for|used by) ([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\"]),\n", - " (\"lyrics_of\", [r\"lyrics (?:to|of|for) ([A-Z][A-Za-z0-9'\\\\-]*(?:\\\\s+[A-Z][A-Za-z0-9'\\\\-]*){0,5})\"]),\n", - " (\"written_by\", [r\"(?:written|wrote|author|lyrics)\"]),\n", - " (\"first_performed\", [r\"first performed\", r\"premiered\", r\"first premiered\"]),\n", - " (\"released_first\", [r\"released first\", r\"first released\", r\"released earlier\"]),\n", - " (\"located_where\", [r\"where\", r\"located\", r\"performed at\", r\"premiered at\"]),\n", - " ]\n", - "\n", - " extracted_focus = []\n", - " for rel, pats in relation_specs:\n", - " for pat in pats:\n", - " if re.search(pat, q, flags=re.I):\n", - " patterns.append(rel)\n", - " for m in re.findall(pat, q):\n", - " if isinstance(m, tuple):\n", - " m = \" \".join([x for x in m if x])\n", - " m = clean_candidate_span(str(m))\n", - " if m and len(m) >= 3 and not m.lower() in [\"where\", \"which\", \"what\", \"who\"]:\n", - " extracted_focus.append(m)\n", - "\n", - " title_like = question_cue_terms(question).get(\"title_like_mentions\", [])\n", - " for t in title_like:\n", - " if t not in extracted_focus:\n", - " extracted_focus.append(t)\n", - "\n", - " return {\n", - " \"relation_patterns\": sorted(set(patterns)),\n", - " \"focus_mentions\": extracted_focus[:8],\n", - " \"question_type\": question_type_from_text(question),\n", - " }\n", - "\n", - "def targetgraph_score_row(question: str, row: Dict[str, Any], relation_info: Dict[str, Any]) -> Dict[str, Any]:\n", - " text = f\"{row.get('title','')} {row.get('sentence','')}\"\n", - " text_lower = text.lower()\n", - "\n", - " focus_hits = []\n", - " for focus in relation_info.get(\"focus_mentions\", []):\n", - " if focus.lower() in text_lower:\n", - " focus_hits.append(focus)\n", - "\n", - " rel_hits = []\n", - " relation_keywords = {\n", - " \"subject_of\": [\"subject\", \"based\", \"opera\", \"play\", \"work\", \"story\", \"about\"],\n", - " \"based_on\": [\"based\", \"adapted\", \"novel\", \"story\", \"work\"],\n", - " \"theme_song_of\": [\"theme\", \"song\", \"used\", \"lyrics\", \"music\"],\n", - " \"lyrics_of\": [\"lyrics\", \"words\", \"written\", \"poem\", \"song\"],\n", - " \"written_by\": [\"written\", \"wrote\", \"author\", \"lyrics\", \"composer\", \"by\"],\n", - " \"first_performed\": [\"first performed\", \"premiered\", \"opera house\", \"theatre\", \"theater\"],\n", - " \"released_first\": [\"released\", \"first\", \"album\", \"film\", \"single\"],\n", - " \"located_where\": [\"located\", \"performed at\", \"premiered at\", \"city\", \"country\", \"opera house\", \"theatre\", \"theater\"],\n", - " }\n", - "\n", - " for rel in relation_info.get(\"relation_patterns\", []):\n", - " for kw in relation_keywords.get(rel, []):\n", - " if kw in text_lower:\n", - " rel_hits.append(kw)\n", - "\n", - " candidate_count = len(extract_candidate_spans_for_question(question, row.get(\"sentence\", \"\"), row.get(\"title\", \"\")))\n", - " focus_bonus = 1.6 * len(set(focus_hits))\n", - " rel_bonus = min(2.4, 0.55 * len(set(rel_hits)))\n", - " candidate_bonus = min(1.4, 0.35 * candidate_count)\n", - "\n", - " return {\n", - " \"focus_hits\": focus_hits,\n", - " \"relation_keyword_hits\": sorted(set(rel_hits))[:12],\n", - " \"candidate_count\": candidate_count,\n", - " \"focus_bonus\": float(focus_bonus),\n", - " \"relation_bonus\": float(rel_bonus),\n", - " \"candidate_bonus\": float(candidate_bonus),\n", - " }\n", - "\n", - "def classify_candidate_role(question: str, cand: Dict[str, Any], relation_info: Dict[str, Any]) -> str:\n", - " span_norm = phrase_norm(cand.get(\"span\", \"\"))\n", - " focus_norms = {phrase_norm(x) for x in relation_info.get(\"focus_mentions\", [])}\n", - " source_text = f\"{cand.get('source_title','')} {cand.get('source_sentence_id','')}\".lower()\n", - "\n", - " if span_norm in focus_norms and any(rel in relation_info.get(\"relation_patterns\", []) for rel in [\"subject_of\", \"based_on\", \"theme_song_of\", \"lyrics_of\"]):\n", - " return \"distractor_focus_mention\"\n", - " if cand.get(\"question_mentioned_penalty_applied\"):\n", - " return \"distractor_question_mention\"\n", - " if cand.get(\"score\", 0) >= 3.0:\n", - " return \"primary\"\n", - " if cand.get(\"score\", 0) >= 1.5:\n", - " return \"secondary\"\n", - " return \"distractor_low_score\"\n", - "\n", - "def slim_candidate_bank(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], max_candidates: int = None) -> List[Dict[str, Any]]:\n", - " max_candidates = max_candidates or SLIMBANK_MAX_CANDIDATES\n", - " relation_info = relation_pattern_from_question(case[\"question\"])\n", - " raw_bank = build_candidate_answer_bank(case, selected_rows, max_candidates=40)\n", - "\n", - " buckets = {\"primary\": [], \"secondary\": [], \"distractor\": []}\n", - " for cand in raw_bank:\n", - " role = classify_candidate_role(case[\"question\"], cand, relation_info)\n", - " cand = dict(cand)\n", - " cand[\"role\"] = role\n", - " if role.startswith(\"primary\"):\n", - " buckets[\"primary\"].append(cand)\n", - " elif role.startswith(\"secondary\"):\n", - " buckets[\"secondary\"].append(cand)\n", - " else:\n", - " buckets[\"distractor\"].append(cand)\n", - "\n", - " for k in buckets:\n", - " buckets[k] = sorted(buckets[k], key=lambda x: (-x.get(\"score\", 0), x.get(\"span\", \"\")))\n", - "\n", - " out = []\n", - " out.extend(buckets[\"primary\"][:SLIMBANK_MAX_PRIMARY])\n", - " out.extend(buckets[\"secondary\"][:SLIMBANK_MAX_SECONDARY])\n", - " out.extend(buckets[\"distractor\"][:SLIMBANK_MAX_DISTRACTOR])\n", - " out = sorted(out, key=lambda x: (0 if x[\"role\"] == \"primary\" else 1 if x[\"role\"] == \"secondary\" else 2, -x.get(\"score\", 0), x.get(\"span\", \"\")))\n", - " return out[:max_candidates]\n", - "\n", - "def build_targetgraph_lines(case: Dict[str, Any], selected_rows: List[Dict[str, Any]]) -> List[str]:\n", - " relation_info = relation_pattern_from_question(case[\"question\"])\n", - " lines = []\n", - " lines.append(\"TargetGraph:\")\n", - " lines.append(\"question_type: \" + relation_info.get(\"question_type\", \"other\"))\n", - " lines.append(\"relation_patterns: \" + \", \".join(relation_info.get(\"relation_patterns\", [])[:8]))\n", - " lines.append(\"focus_mentions: \" + \", \".join(relation_info.get(\"focus_mentions\", [])[:8]))\n", - " lines.append(\"rule: resolve the target entity before answering its requested field\")\n", - " lines.append(\"rule: do not answer with a focus title when the question asks for that title's subject, target, source, theme, author, location, date, or related entity\")\n", - " return lines[:TARGETGRAPH_MAX_LINES]\n", - "\n", - "def format_targetgraph_slimbank_context(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], compact_context: str) -> str:\n", - " cue_info = question_cue_terms(case[\"question\"])\n", - " relation_info = relation_pattern_from_question(case[\"question\"])\n", - " bank = slim_candidate_bank(case, selected_rows)\n", - " title_index = [doc[\"title\"] for doc in case[\"context_docs\"]]\n", - "\n", - " lines = []\n", - " lines.extend(build_targetgraph_lines(case, selected_rows))\n", - " lines.append(\"\")\n", - " lines.append(\"QuestionCueCard:\")\n", - " lines.append(f\"question_type: {cue_info['question_type']}\")\n", - " lines.append(\"relation_cues: \" + \", \".join(cue_info.get(\"relation_cues\", [])[:14]))\n", - " lines.append(\"answer_instruction: \" + answer_instruction_from_question(case[\"question\"]))\n", - " lines.append(\"\")\n", - " lines.append(\"AllSourceTitleIndex:\")\n", - " for t in title_index:\n", - " lines.append(f\"- {t}\")\n", - "\n", - " lines.append(\"\")\n", - " lines.append(\"SlimCandidateAnswerBank:\")\n", - " if bank:\n", - " for i, cand in enumerate(bank, start=1):\n", - " lines.append(f\"{i}. role: {cand.get('role','')} | span: {cand['span']} | source: {cand['source_sentence_id']} | reason: {cand['candidate_source']}\")\n", - " else:\n", - " lines.append(\"- No high-confidence candidate spans extracted. Answer from EvidenceCards.\")\n", - "\n", - " lines.append(\"\")\n", - " lines.append(\"EvidenceCards:\")\n", - " lines.append(compact_context)\n", - " return \"\\\\n\".join(lines).strip()\n", - "\n", - "\n", - "def mathcore_answer_field(question: str) -> str:\n", - " q = str(question).lower()\n", - " qtype = question_type_from_text(question)\n", - " if qtype == \"where\":\n", - " if \"first performed\" in q or \"premiered\" in q:\n", - " return \"performance_location\"\n", - " if \"located\" in q:\n", - " return \"location\"\n", - " return \"place\"\n", - " if qtype == \"who\":\n", - " if \"lyrics\" in q or \"written\" in q or \"wrote\" in q:\n", - " return \"author_or_lyricist\"\n", - " if \"directed\" in q:\n", - " return \"director\"\n", - " return \"person_or_group\"\n", - " if qtype == \"when\":\n", - " return \"date_or_year\"\n", - " if qtype == \"which\":\n", - " if \"released first\" in q or \"first released\" in q:\n", - " return \"earliest_release_entity\"\n", - " return \"entity_title\"\n", - " if qtype == \"yes_no\":\n", - " return \"yes_no\"\n", - " return \"short_entity_span\"\n", - "\n", - "def mathcore_relation_hints(question: str) -> Dict[str, Any]:\n", - " info = relation_pattern_from_question(question)\n", - " q = str(question)\n", - " qlower = q.lower()\n", - "\n", - " # Build field phrases from question words only.\n", - " field = mathcore_answer_field(question)\n", - " field_terms = set(tokenize_keywords(field.replace(\"_\", \" \")))\n", - "\n", - " if \"first performed\" in qlower or \"premiered\" in qlower:\n", - " field_terms.update([\"first\", \"performed\", \"premiered\", \"opera\", \"house\", \"theatre\", \"theater\", \"venue\"])\n", - " if \"lyrics\" in qlower:\n", - " field_terms.update([\"lyrics\", \"words\", \"written\", \"poem\", \"song\"])\n", - " if \"released first\" in qlower or \"first released\" in qlower:\n", - " field_terms.update([\"released\", \"first\", \"earlier\", \"album\", \"film\", \"single\"])\n", - " if \"where\" in qlower:\n", - " field_terms.update([\"where\", \"located\", \"city\", \"country\", \"place\", \"venue\"])\n", - " if \"who\" in qlower:\n", - " field_terms.update([\"who\", \"by\", \"author\", \"writer\", \"composer\", \"director\", \"founder\"])\n", - "\n", - " return {\n", - " **info,\n", - " \"answer_field\": field,\n", - " \"field_terms\": sorted(field_terms),\n", - " }\n", - "\n", - "def mathcore_wrong_focus_penalty(question: str, row: Dict[str, Any], relation_info: Dict[str, Any]) -> float:\n", - " # Penalize rows that strongly describe the focus title itself when the question asks for its target/subject.\n", - " text = f\"{row.get('title','')} {row.get('sentence','')}\".lower()\n", - " focus_mentions = [x.lower() for x in relation_info.get(\"focus_mentions\", [])]\n", - " relation_patterns = set(relation_info.get(\"relation_patterns\", []))\n", - " needs_target = bool(relation_patterns & {\"subject_of\", \"based_on\", \"theme_song_of\", \"lyrics_of\"})\n", - "\n", - " if not needs_target:\n", - " return 0.0\n", - "\n", - " focus_hit = any(f and f in text for f in focus_mentions)\n", - " target_word_hit = any(w in text for w in [\"subject\", \"based\", \"about\", \"adapted\", \"story\", \"opera\", \"work\", \"target\"])\n", - " # If a row only discusses a sibling title or the focus itself but lacks target relation words, penalize.\n", - " if focus_hit and not target_word_hit:\n", - " return 1.0\n", - " return 0.0\n", - "\n", - "def mathcore_answer_field_match(question: str, sentence: str, relation_info: Dict[str, Any]) -> Dict[str, Any]:\n", - " sent = str(sentence).lower()\n", - " field_terms = relation_info.get(\"field_terms\", [])\n", - " hits = [t for t in field_terms if t and t.lower() in sent]\n", - "\n", - " pattern_hits = []\n", - " field = relation_info.get(\"answer_field\", \"\")\n", - " patterns = {\n", - " \"performance_location\": [r\"first performed\", r\"premiered\", r\"opera house\", r\"theatre\", r\"theater\", r\"scala\", r\"venue\"],\n", - " \"author_or_lyricist\": [r\"lyrics\", r\"written by\", r\"words by\", r\"poem\", r\"author\"],\n", - " \"date_or_year\": [r\"\\b(?:1[5-9]\\d{2}|20\\d{2})\\b\"],\n", - " \"place\": [r\"located\", r\"in [A-Z]\", r\"city\", r\"country\"],\n", - " \"earliest_release_entity\": [r\"released\", r\"first\", r\"earlier\"],\n", - " }\n", - " for pat in patterns.get(field, []):\n", - " if re.search(pat, str(sentence), re.I):\n", - " pattern_hits.append(pat)\n", - "\n", - " return {\n", - " \"field_hits\": sorted(set(hits))[:10],\n", - " \"field_pattern_hits\": sorted(set(pattern_hits))[:10],\n", - " \"field_match_count\": len(set(hits)) + len(set(pattern_hits))\n", - " }\n", - "\n", - "def mathcore_span_extractability(question: str, sentence: str, title: str) -> float:\n", - " cands = extract_candidate_spans_for_question(question, sentence, title)\n", - " if not cands:\n", - " return 0.0\n", - " # Prefer a small number of compact spans over a noisy long candidate list.\n", - " short = [c for c in cands if len(c.get(\"span\", \"\").split()) <= 5]\n", - " return min(1.5, 0.35 * len(short) + 0.15 * len(cands))\n", - "\n", - "def mathcore_redundancy_penalty(row: Dict[str, Any], selected_rows: List[Dict[str, Any]]) -> float:\n", - " if not selected_rows:\n", - " return 0.0\n", - " row_terms = set(row.get(\"sent_terms\", []))\n", - " if not row_terms:\n", - " return 0.0\n", - " max_overlap = 0.0\n", - " for s in selected_rows:\n", - " s_terms = set(s.get(\"sent_terms\", []))\n", - " if not s_terms:\n", - " continue\n", - " overlap = len(row_terms & s_terms) / max(1, len(row_terms | s_terms))\n", - " max_overlap = max(max_overlap, overlap)\n", - " return max_overlap\n", - "\n", - "def mathcore_score_row(question: str, row: Dict[str, Any], relation_info: Dict[str, Any], selected_rows: List[Dict[str, Any]] = None) -> Dict[str, Any]:\n", - " selected_rows = selected_rows or []\n", - " text = f\"{row.get('title','')} {row.get('sentence','')}\"\n", - " cue_match = sentence_matches_question_cues(text, question_cue_terms(question))\n", - " tg = targetgraph_score_row(question, row, relation_info)\n", - " field = mathcore_answer_field_match(question, row.get(\"sentence\", \"\"), relation_info)\n", - "\n", - " candidate_density = len(extract_candidate_spans_for_question(question, row.get(\"sentence\", \"\"), row.get(\"title\", \"\")))\n", - " span_extract = mathcore_span_extractability(question, row.get(\"sentence\", \"\"), row.get(\"title\", \"\"))\n", - " wrong_focus = mathcore_wrong_focus_penalty(question, row, relation_info)\n", - " redundancy = mathcore_redundancy_penalty(row, selected_rows)\n", - " token_count = max(1, row.get(\"sentence_token_estimate\", estimate_tokens(row.get(\"sentence\", \"\"))))\n", - "\n", - " score = (\n", - " MATHCORE_WEIGHTS[\"cue_match\"] * min(1.0, cue_match[\"cue_hit_count\"] / 4.0)\n", - " + MATHCORE_WEIGHTS[\"relation_match\"] * min(1.0, len(tg[\"relation_keyword_hits\"]) / 4.0)\n", - " + MATHCORE_WEIGHTS[\"target_bridge\"] * min(1.0, len(tg[\"focus_hits\"]) / 2.0)\n", - " + MATHCORE_WEIGHTS[\"answer_field_match\"] * min(1.0, field[\"field_match_count\"] / 3.0)\n", - " + MATHCORE_WEIGHTS[\"candidate_density\"] * min(1.0, candidate_density / 4.0)\n", - " + MATHCORE_WEIGHTS[\"title_coverage\"] * (1.0 if row.get(\"sent_id\") == 0 else 0.25)\n", - " + MATHCORE_WEIGHTS[\"span_extractability\"] * span_extract\n", - " + MATHCORE_WEIGHTS[\"wrong_focus_attraction\"] * wrong_focus\n", - " + MATHCORE_WEIGHTS[\"redundancy\"] * redundancy\n", - " + MATHCORE_WEIGHTS[\"token_cost\"] * token_count\n", - " )\n", - "\n", - " return {\n", - " \"mathcore_score\": float(score),\n", - " \"cue_hit_count\": cue_match[\"cue_hit_count\"],\n", - " \"field_match_count\": field[\"field_match_count\"],\n", - " \"field_hits\": field[\"field_hits\"],\n", - " \"field_pattern_hits\": field[\"field_pattern_hits\"],\n", - " \"candidate_density\": int(candidate_density),\n", - " \"span_extractability\": float(span_extract),\n", - " \"wrong_focus_penalty\": float(wrong_focus),\n", - " \"redundancy_penalty\": float(redundancy),\n", - " \"relation_keyword_hits\": tg[\"relation_keyword_hits\"],\n", - " \"focus_hits\": tg[\"focus_hits\"],\n", - " }\n", - "\n", - "def mathcore_candidate_bank(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], max_candidates: int = None) -> List[Dict[str, Any]]:\n", - " max_candidates = max_candidates or SLIMBANK_MAX_CANDIDATES\n", - " relation_info = mathcore_relation_hints(case[\"question\"])\n", - " raw_bank = build_candidate_answer_bank(case, selected_rows, max_candidates=36)\n", - "\n", - " scored = []\n", - " focus_norms = {phrase_norm(x) for x in relation_info.get(\"focus_mentions\", [])}\n", - " needs_target = bool(set(relation_info.get(\"relation_patterns\", [])) & {\"subject_of\", \"based_on\", \"theme_song_of\", \"lyrics_of\"})\n", - "\n", - " for cand in raw_bank:\n", - " span_norm = phrase_norm(cand.get(\"span\", \"\"))\n", - " role = classify_candidate_role(case[\"question\"], cand, relation_info)\n", - " score = cand.get(\"score\", 0.0)\n", - " if needs_target and span_norm in focus_norms:\n", - " score -= 3.5\n", - " role = \"distractor_focus_mention\"\n", - " if len(str(cand.get(\"span\", \"\")).split()) <= 5:\n", - " score += 0.6\n", - " else:\n", - " score -= 0.4\n", - " cand = dict(cand)\n", - " cand[\"role\"] = role\n", - " cand[\"mathcore_candidate_score\"] = float(score)\n", - " scored.append(cand)\n", - "\n", - " # Keep an explicit balance of roles, not a noisy long bank.\n", - " prim = [c for c in scored if c[\"role\"] == \"primary\"]\n", - " sec = [c for c in scored if c[\"role\"] == \"secondary\"]\n", - " dist = [c for c in scored if c[\"role\"].startswith(\"distractor\")]\n", - " other = [c for c in scored if c not in prim and c not in sec and c not in dist]\n", - "\n", - " def sortc(xs):\n", - " return sorted(xs, key=lambda x: (-x.get(\"mathcore_candidate_score\", 0), x.get(\"span\", \"\")))\n", - "\n", - " out = []\n", - " out.extend(sortc(prim)[:SLIMBANK_MAX_PRIMARY])\n", - " out.extend(sortc(sec)[:SLIMBANK_MAX_SECONDARY])\n", - " if len(out) < max_candidates:\n", - " out.extend(sortc(other)[:max(0, max_candidates - len(out) - SLIMBANK_MAX_DISTRACTOR)])\n", - " out.extend(sortc(dist)[:SLIMBANK_MAX_DISTRACTOR])\n", - " out = sorted(out, key=lambda x: (-x.get(\"mathcore_candidate_score\", 0), x.get(\"role\", \"\"), x.get(\"span\", \"\")))\n", - " return out[:max_candidates]\n", - "\n", - "def format_mathcore_context(case: Dict[str, Any], selected_rows: List[Dict[str, Any]], compact_context: str) -> str:\n", - " relation_info = mathcore_relation_hints(case[\"question\"])\n", - " bank = mathcore_candidate_bank(case, selected_rows)\n", - " title_index = [doc[\"title\"] for doc in case[\"context_docs\"]]\n", - "\n", - " lines = []\n", - " lines.append(\"MathCoreTargetGraph:\")\n", - " lines.append(\"question_type: \" + relation_info.get(\"question_type\", \"other\"))\n", - " lines.append(\"answer_field: \" + relation_info.get(\"answer_field\", \"short_entity_span\"))\n", - " lines.append(\"relation_patterns: \" + \", \".join(relation_info.get(\"relation_patterns\", [])[:8]))\n", - " lines.append(\"focus_mentions: \" + \", \".join(relation_info.get(\"focus_mentions\", [])[:8]))\n", - " lines.append(\"rule: first resolve focus mention to target entity, then answer the requested answer_field\")\n", - " lines.append(\"rule: if a relation target is requested, the focus mention itself is usually not the final answer\")\n", - " lines.append(\"\")\n", - " lines.append(\"AllSourceTitleIndex:\")\n", - " for t in title_index:\n", - " lines.append(f\"- {t}\")\n", - "\n", - " lines.append(\"\")\n", - " lines.append(\"MathCoreSlimCandidateBank:\")\n", - " if bank:\n", - " for i, cand in enumerate(bank, start=1):\n", - " lines.append(f\"{i}. role: {cand.get('role','')} | span: {cand['span']} | source: {cand['source_sentence_id']}\")\n", - " else:\n", - " lines.append(\"- No compact candidate. Answer from EvidenceCards.\")\n", - "\n", - " lines.append(\"\")\n", - " lines.append(\"EvidenceCards:\")\n", - " lines.append(compact_context)\n", - " return \"\\\\n\".join(lines).strip()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "140de92f", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 5: Load public HotpotQA validation dataset\n", - "\n", - "def load_hotpotqa_validation():\n", - " errors = []\n", - " for dataset_name in [DATASET_NAME_PRIMARY, DATASET_NAME_FALLBACK]:\n", - " try:\n", - " ds = load_dataset(dataset_name, DATASET_CONFIG, split=DATASET_SPLIT)\n", - " print(\"Loaded dataset:\", dataset_name, DATASET_CONFIG, DATASET_SPLIT)\n", - " print(\"Rows:\", len(ds))\n", - " return ds, dataset_name\n", - " except Exception as e:\n", - " errors.append((dataset_name, repr(e)))\n", - " raise RuntimeError(f\"Could not load HotpotQA dataset. Errors: {errors}\")\n", - "\n", - "dataset, loaded_dataset_name = load_hotpotqa_validation()\n", - "print(\"First row keys:\", list(dataset[0].keys()))\n", - "print(\"First row id field sample:\", dataset[0].get(\"id\", dataset[0].get(\"_id\", \"NO_ID_FIELD\")))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fd4e1653", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 6: Normalize HotpotQA row format\n", - "\n", - "def normalize_context(raw_context: Any) -> List[Dict[str, Any]]:\n", - " docs = []\n", - " if isinstance(raw_context, dict):\n", - " titles = raw_context.get(\"title\") or raw_context.get(\"titles\") or []\n", - " sentences = raw_context.get(\"sentences\") or raw_context.get(\"sentence\") or []\n", - " for title, sents in zip(titles, sentences):\n", - " docs.append({\"title\": str(title), \"sentences\": [str(x) for x in list(sents)]})\n", - " elif isinstance(raw_context, list):\n", - " for item in raw_context:\n", - " if isinstance(item, (list, tuple)) and len(item) >= 2:\n", - " docs.append({\"title\": str(item[0]), \"sentences\": [str(x) for x in list(item[1])]})\n", - " elif isinstance(item, dict):\n", - " title = item.get(\"title\", \"\")\n", - " sents = item.get(\"sentences\", item.get(\"text\", []))\n", - " if isinstance(sents, str):\n", - " sents = [sents]\n", - " docs.append({\"title\": str(title), \"sentences\": [str(x) for x in list(sents)]})\n", - " return docs\n", - "\n", - "def normalize_supporting_facts(raw_support: Any) -> List[Dict[str, Any]]:\n", - " facts = []\n", - " if isinstance(raw_support, dict):\n", - " titles = raw_support.get(\"title\") or raw_support.get(\"titles\") or []\n", - " sent_ids = raw_support.get(\"sent_id\") or raw_support.get(\"sent_ids\") or raw_support.get(\"sentence_id\") or []\n", - " for title, sent_id in zip(titles, sent_ids):\n", - " facts.append({\"title\": str(title), \"sent_id\": int(sent_id)})\n", - " elif isinstance(raw_support, list):\n", - " for item in raw_support:\n", - " if isinstance(item, (list, tuple)) and len(item) >= 2:\n", - " facts.append({\"title\": str(item[0]), \"sent_id\": int(item[1])})\n", - " elif isinstance(item, dict):\n", - " facts.append({\"title\": str(item.get(\"title\", \"\")), \"sent_id\": int(item.get(\"sent_id\", item.get(\"sentence_id\", 0)))})\n", - " return facts\n", - "\n", - "def normalize_case(row: Dict[str, Any], idx: int) -> Dict[str, Any]:\n", - " row_id = row.get(\"id\", row.get(\"_id\", f\"row_{idx}\"))\n", - " context_docs = normalize_context(row.get(\"context\"))\n", - " support = normalize_supporting_facts(row.get(\"supporting_facts\", []))\n", - " return {\n", - " \"case_id\": f\"DD02C_{idx:06d}\",\n", - " \"dataset_row_id\": str(row_id),\n", - " \"question\": str(row.get(\"question\", \"\")),\n", - " \"gold_answer\": str(row.get(\"answer\", \"\")),\n", - " \"level\": str(row.get(\"level\", \"\")),\n", - " \"type\": str(row.get(\"type\", \"\")),\n", - " \"context_docs\": context_docs,\n", - " \"gold_supporting_facts\": support,\n", - " \"gold_support_titles\": sorted({f[\"title\"] for f in support if f.get(\"title\")})\n", - " }\n", - "\n", - "def valid_case(c: Dict[str, Any]) -> bool:\n", - " return (\n", - " bool(c[\"question\"].strip())\n", - " and bool(c[\"gold_answer\"].strip())\n", - " and len(c[\"context_docs\"]) >= 2\n", - " and all(\"title\" in d and \"sentences\" in d for d in c[\"context_docs\"])\n", - " )\n", - "\n", - "all_cases = []\n", - "for i in range(len(dataset)):\n", - " c = normalize_case(dataset[i], i)\n", - " if valid_case(c):\n", - " all_cases.append(c)\n", - "\n", - "print(\"Valid normalized cases:\", len(all_cases))\n", - "print(\"Example normalized case keys:\", list(all_cases[0].keys()))\n", - "print(\"Example doc count:\", len(all_cases[0][\"context_docs\"]))\n", - "print(\"Example support titles:\", all_cases[0][\"gold_support_titles\"][:3])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2971a4bb", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 7: Fixed seed stratified sample and manifest\n", - "\n", - "def stable_case_key(c: Dict[str, Any]) -> str:\n", - " return f\"{c.get('type','')}|{c.get('level','')}|{c.get('dataset_row_id','')}|{c.get('question','')[:80]}\"\n", - "\n", - "def stratified_sample_hotpotqa(cases: List[Dict[str, Any]], n: int, seed: int) -> List[Dict[str, Any]]:\n", - " \"\"\"\n", - " Standard-exam style sampling.\n", - "\n", - " Priority:\n", - " 1. Preserve bridge/comparison mix when possible.\n", - " 2. Prefer medium/hard over easy when possible.\n", - " 3. Use a fixed seed.\n", - " 4. Never inspect gold answers for sampling.\n", - " \"\"\"\n", - " rng = random.Random(seed)\n", - "\n", - " valid = cases[:]\n", - " rng.shuffle(valid)\n", - "\n", - " by_type = {}\n", - " for c in valid:\n", - " by_type.setdefault(str(c.get(\"type\", \"\")).lower(), []).append(c)\n", - "\n", - " selected = []\n", - " used_ids = set()\n", - "\n", - " def add_cases(pool: List[Dict[str, Any]], k: int):\n", - " nonlocal selected, used_ids\n", - " # Prefer hard/medium first, then easy.\n", - " hard_medium = [c for c in pool if str(c.get(\"level\", \"\")).lower() in [\"hard\", \"medium\"]]\n", - " easy_other = [c for c in pool if c not in hard_medium]\n", - " rng.shuffle(hard_medium)\n", - " rng.shuffle(easy_other)\n", - " for c in hard_medium + easy_other:\n", - " if len(selected) >= n or k <= 0:\n", - " break\n", - " key = stable_case_key(c)\n", - " if key in used_ids:\n", - " continue\n", - " selected.append(c)\n", - " used_ids.add(key)\n", - " k -= 1\n", - "\n", - " if STRATIFIED_SAMPLE_ENABLED:\n", - " bridge_target = int(round(n * TARGET_TYPE_RATIO.get(\"bridge\", 0.70)))\n", - " comparison_target = n - bridge_target\n", - " add_cases(by_type.get(\"bridge\", []), bridge_target)\n", - " add_cases(by_type.get(\"comparison\", []), comparison_target)\n", - "\n", - " # Fill remaining from all cases if the dataset split lacks enough type balance.\n", - " if len(selected) < n:\n", - " add_cases(valid, n - len(selected))\n", - "\n", - " # Stable final ordering after selection.\n", - " selected = selected[:n]\n", - " selected.sort(key=lambda c: stable_case_key(c))\n", - " return selected\n", - "\n", - "selected_cases = stratified_sample_hotpotqa(all_cases, SAMPLE_COUNT, RANDOM_SEED)\n", - "\n", - "case_manifest = []\n", - "for local_idx, c in enumerate(selected_cases, start=1):\n", - " public_case = {\n", - " \"case_id\": f\"DD02C_{local_idx:06d}\",\n", - " \"dataset_name\": loaded_dataset_name,\n", - " \"dataset_config\": DATASET_CONFIG,\n", - " \"dataset_split\": DATASET_SPLIT,\n", - " \"dataset_row_id\": c[\"dataset_row_id\"],\n", - " \"question\": c[\"question\"],\n", - " \"level\": c.get(\"level\", \"\"),\n", - " \"type\": c.get(\"type\", \"\"),\n", - " \"context_doc_count\": len(c[\"context_docs\"]),\n", - " \"gold_answer_sha256\": hashlib.sha256(c[\"gold_answer\"].encode(\"utf-8\")).hexdigest(),\n", - " \"gold_support_titles_sha256\": hashlib.sha256(json.dumps(c[\"gold_support_titles\"], sort_keys=True).encode(\"utf-8\")).hexdigest(),\n", - " \"sampling_policy\": \"fixed_seed_stratified_by_type_level_no_gold_answer_access\"\n", - " }\n", - " c[\"case_id\"] = public_case[\"case_id\"]\n", - " case_manifest.append(public_case)\n", - "\n", - "write_jsonl(RUN_DIR / \"case_manifest.jsonl\", case_manifest)\n", - "\n", - "manifest_df = pd.DataFrame(case_manifest)\n", - "print(\"Selected cases:\", len(selected_cases))\n", - "print(\"Manifest written:\", RUN_DIR / \"case_manifest.jsonl\")\n", - "display(manifest_df.groupby([\"type\", \"level\"]).size().reset_index(name=\"count\").sort_values([\"type\", \"level\"]))\n", - "display(manifest_df.head())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df7944d8", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 8: Build A raw prompts and B answer-blind compact prompts\n", - "\n", - "MODEL_OUTPUT_SCHEMA_TEXT = \"\"\"\n", - "Return only one JSON object with this schema:\n", - "{\n", - " \"case_id\": \"string\",\n", - " \"arm\": \"A_BASELINE_RAW_CONTEXT or B_POLARIS_COMPACT_CONTEXT\",\n", - " \"answer\": \"shortest final answer span string\",\n", - " \"support_titles\": [\"string\"],\n", - " \"support_sentence_ids\": [\"Title::sent_0\"],\n", - " \"claim_ceiling\": \"ANSWERED or INSUFFICIENT_EVIDENCE\",\n", - " \"confidence\": \"LOW or MEDIUM or HIGH\"\n", - "}\n", - "\n", - "Do not include answer_correct, passed, support_correct, evidence_grounded, or any self-grading field.\n", - "\"\"\".strip()\n", - "\n", - "def format_raw_context(docs: List[Dict[str, Any]]) -> str:\n", - " parts = []\n", - " for doc in docs:\n", - " title = doc[\"title\"]\n", - " parts.append(f\"Title: {title}\")\n", - " for i, sent in enumerate(doc[\"sentences\"]):\n", - " parts.append(f\"{title}::sent_{i}: {sent}\")\n", - " parts.append(\"\")\n", - " return \"\\n\".join(parts).strip()\n", - "\n", - "def select_compact_context(case: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:\n", - " \"\"\"\n", - " Answer-blind MathCore Evidence Controller for HotpotQA.\n", - "\n", - " It does not read gold_answer or supporting_facts.\n", - "\n", - " It optimizes compact context under a token budget:\n", - " maximize cue match, relation match, target bridge, answer-field match,\n", - " candidate density, and title coverage;\n", - " minimize wrong-focus attraction, redundancy, and token cost.\n", - " \"\"\"\n", - " question = case[\"question\"]\n", - " relation_info = mathcore_relation_hints(question)\n", - "\n", - " raw_context_text = format_raw_context(case[\"context_docs\"])\n", - " raw_context_tokens = max(1, estimate_tokens(raw_context_text))\n", - " compact_token_budget = max(480, int(raw_context_tokens * COMPACT_TARGET_RATIO))\n", - "\n", - " rows = []\n", - " for doc in case[\"context_docs\"]:\n", - " title = doc[\"title\"]\n", - " title_terms = set(tokenize_keywords(title))\n", - " for i, sent in enumerate(doc[\"sentences\"]):\n", - " base = compact_sentence_score(question, title, sent, i)\n", - " sent_terms = set(tokenize_keywords(sent))\n", - " sent_entities = set(t.lower() for t in entity_like_tokens(sent))\n", - " row = {\n", - " \"title\": title,\n", - " \"sent_id\": i,\n", - " \"sentence\": sent,\n", - " \"sent_terms\": sorted(sent_terms),\n", - " \"sent_entities\": sorted(sent_entities),\n", - " \"title_terms\": sorted(title_terms),\n", - " \"sentence_token_estimate\": estimate_tokens(sent),\n", - " **base\n", - " }\n", - " row.update(mathcore_score_row(question, row, relation_info, []))\n", - " row[\"score_per_token\"] = row[\"mathcore_score\"] / max(8, row[\"sentence_token_estimate\"])\n", - " rows.append(row)\n", - "\n", - " by_title = {}\n", - " for row in rows:\n", - " by_title.setdefault(row[\"title\"], []).append(row)\n", - "\n", - " title_scores = []\n", - " for title, trs in by_title.items():\n", - " best = max(r[\"mathcore_score\"] for r in trs)\n", - " bridge = max((len(r[\"focus_hits\"]) + len(r[\"relation_keyword_hits\"]) + r[\"field_match_count\"]) for r in trs)\n", - " title_scores.append((title, best + 0.6 * bridge))\n", - " title_scores.sort(key=lambda x: (-x[1], x[0]))\n", - " title_order = [t for t, _ in title_scores]\n", - "\n", - " selected = []\n", - " selected_keys = set()\n", - " current_tokens = 0\n", - "\n", - " def add_row(row, reason, force=False):\n", - " nonlocal current_tokens\n", - " key = (row[\"title\"], row[\"sent_id\"])\n", - " if key in selected_keys:\n", - " return\n", - " row_cost = int(row.get(\"sentence_token_estimate\", estimate_tokens(row[\"sentence\"])))\n", - " if not force and selected and current_tokens + row_cost > compact_token_budget:\n", - " return\n", - " if len(selected) >= min(COMPACT_MAX_SENTENCES, EVIDENCECARD_MAX_LINES):\n", - " return\n", - " new_row = dict(row)\n", - " new_row[\"selection_reason\"] = reason\n", - " selected.append(new_row)\n", - " selected_keys.add(key)\n", - " current_tokens += row_cost\n", - "\n", - " # Minimal title skeleton. Every title gets one compact evidence row.\n", - " for title in title_order:\n", - " trs = by_title[title]\n", - " field_rows = [r for r in trs if r[\"field_match_count\"] > 0 or r[\"candidate_density\"] > 0 or len(r[\"relation_keyword_hits\"]) > 0]\n", - " if field_rows:\n", - " row = sorted(field_rows, key=lambda r: (-r[\"mathcore_score\"], r[\"sentence_token_estimate\"], r[\"sent_id\"]))[0]\n", - " else:\n", - " lead = [r for r in trs if r[\"sent_id\"] == 0]\n", - " best_short = sorted(trs, key=lambda r: (-r[\"score_per_token\"], r[\"sentence_token_estimate\"], r[\"sent_id\"]))[0]\n", - " if lead and lead[0][\"sentence_token_estimate\"] <= MATHCORE_TITLE_SKELETON_TOKEN_SOFT_CAP:\n", - " row = lead[0]\n", - " else:\n", - " row = best_short\n", - " add_row(row, \"mathcore_title_skeleton\", force=True)\n", - "\n", - " # Recompute redundancy-aware scores after skeleton.\n", - " remaining_rows = [r for r in rows if (r[\"title\"], r[\"sent_id\"]) not in selected_keys]\n", - " rescored = []\n", - " for r in remaining_rows:\n", - " rr = dict(r)\n", - " rr.update(mathcore_score_row(question, rr, relation_info, selected))\n", - " rr[\"score_per_token\"] = rr[\"mathcore_score\"] / max(8, rr[\"sentence_token_estimate\"])\n", - " rescored.append(rr)\n", - "\n", - " # FieldLock rows: highest target/field rows, not generic filler.\n", - " fieldlock = sorted(\n", - " [r for r in rescored if r[\"field_match_count\"] > 0 or r[\"candidate_density\"] > 0 or len(r[\"relation_keyword_hits\"]) > 0],\n", - " key=lambda r: (-r[\"mathcore_score\"], -r[\"score_per_token\"], r[\"sentence_token_estimate\"], r[\"title\"], r[\"sent_id\"])\n", - " )\n", - " for row in fieldlock[:MATHCORE_FIELDLOCK_TOP_ROWS]:\n", - " add_row(row, \"mathcore_fieldlock\", force=False)\n", - "\n", - " # Cost-aware fill if budget remains.\n", - " remaining_rows = [r for r in rescored if (r[\"title\"], r[\"sent_id\"]) not in selected_keys]\n", - " fill = sorted(remaining_rows, key=lambda r: (-r[\"score_per_token\"], -r[\"mathcore_score\"], r[\"sentence_token_estimate\"]))\n", - " for row in fill:\n", - " if len(selected) >= min(COMPACT_MAX_SENTENCES, EVIDENCECARD_MAX_LINES):\n", - " break\n", - " if current_tokens >= compact_token_budget:\n", - " break\n", - " if row[\"mathcore_score\"] <= -0.5:\n", - " continue\n", - " add_row(row, \"mathcore_budget_fill\", force=False)\n", - "\n", - " order_map = {title: idx for idx, title in enumerate(title_order)}\n", - " selected = sorted(selected, key=lambda r: (order_map.get(r[\"title\"], 999), r[\"sent_id\"]))\n", - "\n", - " trace = []\n", - " for row in selected:\n", - " trace.append({\n", - " \"case_id\": case[\"case_id\"],\n", - " \"title\": row[\"title\"],\n", - " \"sent_id\": row[\"sent_id\"],\n", - " \"mathcore_score\": row[\"mathcore_score\"],\n", - " \"score_per_token\": row[\"score_per_token\"],\n", - " \"sentence_token_estimate\": row[\"sentence_token_estimate\"],\n", - " \"q_overlap\": row[\"q_overlap\"],\n", - " \"title_overlap\": row[\"title_overlap\"],\n", - " \"entity_overlap\": row[\"entity_overlap\"],\n", - " \"cue_hit_count\": row[\"cue_hit_count\"],\n", - " \"field_match_count\": row[\"field_match_count\"],\n", - " \"field_hits\": row[\"field_hits\"],\n", - " \"field_pattern_hits\": row[\"field_pattern_hits\"],\n", - " \"candidate_density\": row[\"candidate_density\"],\n", - " \"span_extractability\": row[\"span_extractability\"],\n", - " \"wrong_focus_penalty\": row[\"wrong_focus_penalty\"],\n", - " \"redundancy_penalty\": row[\"redundancy_penalty\"],\n", - " \"focus_hits\": row[\"focus_hits\"],\n", - " \"relation_keyword_hits\": row[\"relation_keyword_hits\"],\n", - " \"raw_context_tokens\": raw_context_tokens,\n", - " \"compact_token_budget\": compact_token_budget,\n", - " \"question_type\": relation_info[\"question_type\"],\n", - " \"answer_field\": relation_info[\"answer_field\"],\n", - " \"relation_patterns\": relation_info[\"relation_patterns\"],\n", - " \"focus_mentions\": relation_info[\"focus_mentions\"],\n", - " \"selection_rule\": \"answer_blind_mathcore_evidence_controller\",\n", - " \"selection_reason\": row.get(\"selection_reason\", \"unknown\")\n", - " })\n", - "\n", - " return selected, trace\n", - "\n", - "\n", - "def format_compact_context(selected_rows: List[Dict[str, Any]]) -> str:\n", - " parts = []\n", - " current_title = None\n", - " for row in selected_rows:\n", - " if row[\"title\"] != current_title:\n", - " current_title = row[\"title\"]\n", - " parts.append(f\"SourceCard: {current_title}\")\n", - " parts.append(f\"{row['title']}::sent_{row['sent_id']}: {row['sentence']}\")\n", - " return \"\\n\".join(parts).strip()\n", - "\n", - "def build_prompt(case: Dict[str, Any], arm: str, context_text: str) -> str:\n", - " return f\"\"\"\n", - "You are answering a public HotpotQA validation question.\n", - "\n", - "Case ID:\n", - "{case[\"case_id\"]}\n", - "\n", - "Arm:\n", - "{arm}\n", - "\n", - "Question:\n", - "{case[\"question\"]}\n", - "\n", - "Context:\n", - "{context_text}\n", - "\n", - "Rules:\n", - "Use only the provided context.\n", - "Connect evidence across SourceCards when the question requires multiple hops.\n", - "{answer_instruction_from_question(case[\"question\"])}\n", - "Do not answer with a full sentence unless the answer is yes or no.\n", - "Do not return INSUFFICIENT_EVIDENCE when the provided context contains a plausible answer span.\n", - "If the context is truly insufficient, set claim_ceiling to INSUFFICIENT_EVIDENCE.\n", - "Cite support_titles and support_sentence_ids from the provided context.\n", - "Return JSON only.\n", - "\n", - "{MODEL_OUTPUT_SCHEMA_TEXT}\n", - "\"\"\".strip()\n", - "\n", - "public_prompts = []\n", - "compact_traces = []\n", - "\n", - "for case in selected_cases:\n", - " raw_context = format_raw_context(case[\"context_docs\"])\n", - " a_prompt = build_prompt(case, \"A_BASELINE_RAW_CONTEXT\", raw_context)\n", - " public_prompts.append({\n", - " \"case_id\": case[\"case_id\"],\n", - " \"arm\": \"A_BASELINE_RAW_CONTEXT\",\n", - " \"prompt\": a_prompt,\n", - " \"estimated_token_in\": estimate_tokens(a_prompt),\n", - " \"prompt_sha256\": hashlib.sha256(a_prompt.encode(\"utf-8\")).hexdigest()\n", - " })\n", - "\n", - " compact_rows, trace = select_compact_context(case)\n", - " compact_context = format_compact_context(compact_rows)\n", - " compact_context = format_mathcore_context(case, compact_rows, compact_context)\n", - " b_prompt = build_prompt(case, \"B_POLARIS_COMPACT_CONTEXT\", compact_context)\n", - " public_prompts.append({\n", - " \"case_id\": case[\"case_id\"],\n", - " \"arm\": \"B_POLARIS_COMPACT_CONTEXT\",\n", - " \"prompt\": b_prompt,\n", - " \"estimated_token_in\": estimate_tokens(b_prompt),\n", - " \"prompt_sha256\": hashlib.sha256(b_prompt.encode(\"utf-8\")).hexdigest()\n", - " })\n", - " compact_traces.extend(trace)\n", - "\n", - "write_jsonl(RUN_DIR / \"public_prompts.jsonl\", public_prompts)\n", - "write_jsonl(RUN_DIR / \"compact_selection_trace.jsonl\", compact_traces)\n", - "\n", - "token_summary = pd.DataFrame(public_prompts).groupby(\"arm\")[\"estimated_token_in\"].agg([\"mean\", \"sum\", \"min\", \"max\"]).reset_index()\n", - "display(token_summary)\n", - "\n", - "a_sum = token_summary[token_summary[\"arm\"] == \"A_BASELINE_RAW_CONTEXT\"][\"sum\"].iloc[0]\n", - "b_sum = token_summary[token_summary[\"arm\"] == \"B_POLARIS_COMPACT_CONTEXT\"][\"sum\"].iloc[0]\n", - "print(\"Estimated compact token reduction:\", round(1 - b_sum / a_sum, 4))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e50dfcf2", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 9: Gold metadata leakage audit\n", - "\n", - "FORBIDDEN_METADATA_TERMS = [\n", - " \"gold_answer\",\n", - " \"gold support\",\n", - " \"gold_support\",\n", - " \"supporting_facts\",\n", - " \"supporting facts label\",\n", - " \"oracle support\",\n", - " \"answer key\",\n", - " \"correct answer is\"\n", - "]\n", - "\n", - "SELF_GRADING_REQUEST_TERMS = [\n", - " \"grade your answer\",\n", - " \"score your answer\",\n", - " \"score yourself\",\n", - " \"self grade\",\n", - " \"self-grade\",\n", - " \"judge whether your answer is correct\",\n", - " \"return whether the answer is correct\",\n", - " \"return answer_correct\",\n", - " \"set answer_correct\",\n", - " \"return passed\",\n", - " \"set passed\"\n", - "]\n", - "\n", - "def leakage_audit_prompt(record: Dict[str, Any]) -> Dict[str, Any]:\n", - " prompt_lower = record[\"prompt\"].lower()\n", - " forbidden_hits = [term for term in FORBIDDEN_METADATA_TERMS if term in prompt_lower]\n", - " self_grade_hits = [term for term in SELF_GRADING_REQUEST_TERMS if term in prompt_lower]\n", - " return {\n", - " \"case_id\": record[\"case_id\"],\n", - " \"arm\": record[\"arm\"],\n", - " \"prompt_sha256\": record[\"prompt_sha256\"],\n", - " \"forbidden_metadata_hits\": forbidden_hits,\n", - " \"self_grading_instruction_hits\": self_grade_hits,\n", - " \"metadata_leakage_fail\": bool(forbidden_hits),\n", - " \"self_grading_fail\": bool(self_grade_hits),\n", - " \"api_key_pattern_found\": api_key_pattern_found(record[\"prompt\"])\n", - " }\n", - "\n", - "leakage_rows = [leakage_audit_prompt(r) for r in public_prompts]\n", - "write_jsonl(RUN_DIR / \"leakage_audit.jsonl\", leakage_rows)\n", - "\n", - "leakage_df = pd.DataFrame(leakage_rows)\n", - "display(leakage_df.groupby(\"arm\")[[\"metadata_leakage_fail\", \"self_grading_fail\", \"api_key_pattern_found\"]].sum())\n", - "\n", - "if leakage_df[\"metadata_leakage_fail\"].any():\n", - " raise RuntimeError(\"Gold metadata leakage detected. Stop before API.\")\n", - "if leakage_df[\"self_grading_fail\"].any():\n", - " raise RuntimeError(\"Self-grading instruction leakage detected. Stop before API.\")\n", - "if leakage_df[\"api_key_pattern_found\"].any():\n", - " raise RuntimeError(\"API key-like pattern detected in prompt. Stop before API.\")\n", - "\n", - "print(\"Leakage audit: PASS\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b21bf117", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 10: Model call and parser\n", - "\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(api_key=OPENAI_API_KEY)\n", - "\n", - "SYSTEM_PROMPT = \"\"\"\n", - "You are a strict public QA answerer.\n", - "Return only valid JSON.\n", - "Do not self-grade.\n", - "Do not include hidden analysis.\n", - "Use only the provided context.\n", - "\"\"\".strip()\n", - "\n", - "REQUIRED_OUTPUT_KEYS = {\n", - " \"case_id\",\n", - " \"arm\",\n", - " \"answer\",\n", - " \"support_titles\",\n", - " \"support_sentence_ids\",\n", - " \"claim_ceiling\",\n", - " \"confidence\"\n", - "}\n", - "\n", - "FORBIDDEN_OUTPUT_KEYS = {\n", - " \"answer_correct\",\n", - " \"passed\",\n", - " \"support_correct\",\n", - " \"evidence_grounded\",\n", - " \"final_verdict\",\n", - " \"score_passed\"\n", - "}\n", - "\n", - "def call_model(prompt: str) -> str:\n", - " response = client.chat.completions.create(\n", - " model=MODEL_NAME,\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n", - " {\"role\": \"user\", \"content\": prompt}\n", - " ],\n", - " temperature=0,\n", - " response_format={\"type\": \"json_object\"}\n", - " )\n", - " return response.choices[0].message.content\n", - "\n", - "def validate_model_output(obj: Dict[str, Any], expected_case_id: str, expected_arm: str) -> Tuple[bool, List[str]]:\n", - " errors = []\n", - " missing = sorted(REQUIRED_OUTPUT_KEYS - set(obj.keys()))\n", - " forbidden = sorted(FORBIDDEN_OUTPUT_KEYS & set(obj.keys()))\n", - " if missing:\n", - " errors.append(f\"missing_required_keys={missing}\")\n", - " if forbidden:\n", - " errors.append(f\"forbidden_self_grading_keys={forbidden}\")\n", - " if str(obj.get(\"case_id\")) != expected_case_id:\n", - " errors.append(\"case_id_mismatch\")\n", - " if str(obj.get(\"arm\")) != expected_arm:\n", - " errors.append(\"arm_mismatch\")\n", - " if not isinstance(obj.get(\"support_titles\", []), list):\n", - " errors.append(\"support_titles_not_list\")\n", - " if not isinstance(obj.get(\"support_sentence_ids\", []), list):\n", - " errors.append(\"support_sentence_ids_not_list\")\n", - " if obj.get(\"claim_ceiling\") not in [\"ANSWERED\", \"INSUFFICIENT_EVIDENCE\"]:\n", - " errors.append(\"invalid_claim_ceiling\")\n", - " if obj.get(\"confidence\") not in [\"LOW\", \"MEDIUM\", \"HIGH\"]:\n", - " errors.append(\"invalid_confidence\")\n", - " return len(errors) == 0, errors\n", - "\n", - "def parse_and_validate_raw(raw_text: str, prompt_record: Dict[str, Any]) -> Dict[str, Any]:\n", - " try:\n", - " obj = extract_json_object(raw_text)\n", - " ok, errors = validate_model_output(obj, prompt_record[\"case_id\"], prompt_record[\"arm\"])\n", - " return {\n", - " \"case_id\": prompt_record[\"case_id\"],\n", - " \"arm\": prompt_record[\"arm\"],\n", - " \"parse_ok\": bool(ok),\n", - " \"parse_errors\": errors,\n", - " \"parsed\": obj\n", - " }\n", - " except Exception as e:\n", - " return {\n", - " \"case_id\": prompt_record[\"case_id\"],\n", - " \"arm\": prompt_record[\"arm\"],\n", - " \"parse_ok\": False,\n", - " \"parse_errors\": [repr(e)],\n", - " \"parsed\": {}\n", - " }\n", - "\n", - "print(\"Parser ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d218180", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 11: API smoke run\n", - "# Full100 preset disables smoke calls to avoid duplicate API spending.\n", - "# This cell still creates empty smoke artifacts so later cells can continue safely.\n", - "\n", - "RUN_API_SMOKE = False\n", - "\n", - "prompt_by_case_arm = {(r[\"case_id\"], r[\"arm\"]): r for r in public_prompts}\n", - "smoke_case_ids = [c[\"case_id\"] for c in selected_cases[:SMOKE_COUNT]]\n", - "smoke_prompts = [r for r in public_prompts if r[\"case_id\"] in smoke_case_ids]\n", - "\n", - "raw_outputs = []\n", - "parser_records = []\n", - "\n", - "if RUN_API_SMOKE and SMOKE_COUNT > 0:\n", - " for model_name in SMOKE_MODELS:\n", - " model_role = \"primary_small_model\" if model_name == PRIMARY_MODEL else \"comparison_larger_model\"\n", - " for rec in tqdm(smoke_prompts, desc=f\"API smoke calls / {model_name}\"):\n", - " raw_text = call_model(rec[\"prompt\"].replace(f\"Arm:\\n{rec['arm']}\", f\"Arm:\\n{rec['arm']}\\nModel role:\\n{model_role}\"))\n", - " raw_outputs.append({\n", - " \"case_id\": rec[\"case_id\"],\n", - " \"arm\": rec[\"arm\"],\n", - " \"model\": model_name,\n", - " \"model_role\": model_role,\n", - " \"prompt_sha256\": rec[\"prompt_sha256\"],\n", - " \"raw_output\": raw_text,\n", - " \"raw_output_sha256\": hashlib.sha256(raw_text.encode(\"utf-8\")).hexdigest(),\n", - " \"estimated_token_in\": rec[\"estimated_token_in\"],\n", - " \"estimated_token_out\": estimate_tokens(raw_text),\n", - " \"estimated_cost_usd\": estimate_cost_usd(model_name, rec[\"estimated_token_in\"], estimate_tokens(raw_text)),\n", - " \"run_stage\": \"smoke\"\n", - " })\n", - "\n", - " for r in raw_outputs:\n", - " parsed = parse_and_validate_raw(r[\"raw_output\"], prompt_by_case_arm[(r[\"case_id\"], r[\"arm\"])])\n", - " parsed[\"model\"] = r[\"model\"]\n", - " parsed[\"model_role\"] = r[\"model_role\"]\n", - " parser_records.append(parsed)\n", - "else:\n", - " print(\"RUN_API_SMOKE is False or SMOKE_COUNT is 0. No smoke API calls made. This is expected for the Full100 preset.\")\n", - "\n", - "write_jsonl(RUN_DIR / \"raw_outputs_smoke.jsonl\", raw_outputs)\n", - "write_jsonl(RUN_DIR / \"parser_records_smoke.jsonl\", parser_records)\n", - "\n", - "parse_df = pd.DataFrame([{\n", - " \"case_id\": r[\"case_id\"],\n", - " \"arm\": r[\"arm\"],\n", - " \"model\": r.get(\"model\", \"\"),\n", - " \"model_role\": r.get(\"model_role\", \"\"),\n", - " \"parse_ok\": r[\"parse_ok\"],\n", - " \"parse_errors\": \";\".join(r[\"parse_errors\"])\n", - "} for r in parser_records], columns=[\n", - " \"case_id\",\n", - " \"arm\",\n", - " \"model\",\n", - " \"model_role\",\n", - " \"parse_ok\",\n", - " \"parse_errors\"\n", - "])\n", - "\n", - "display(parse_df)\n", - "\n", - "if len(parse_df) == 0:\n", - " print(\"Smoke parse pass: SKIPPED / 0 smoke rows\")\n", - "else:\n", - " print(\"Smoke parse pass:\", int(parse_df[\"parse_ok\"].sum()), \"/\", len(parse_df))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ac51f1d", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 12: External scorer\n", - "\n", - "case_by_id = {c[\"case_id\"]: c for c in selected_cases}\n", - "\n", - "def context_text_for_case(case: Dict[str, Any]) -> str:\n", - " return format_raw_context(case[\"context_docs\"])\n", - "\n", - "def score_record(parsed_record: Dict[str, Any], raw_record: Dict[str, Any]) -> Dict[str, Any]:\n", - " case = case_by_id[parsed_record[\"case_id\"]]\n", - " parsed = parsed_record.get(\"parsed\") or {}\n", - " pred_answer = str(parsed.get(\"answer\", \"\"))\n", - " gold_answer = case[\"gold_answer\"]\n", - " support_titles_pred = [str(x) for x in parsed.get(\"support_titles\", [])]\n", - " support_sentence_ids_pred = [str(x) for x in parsed.get(\"support_sentence_ids\", [])]\n", - " gold_titles = case[\"gold_support_titles\"]\n", - " gold_sentence_ids = [f\"{f['title']}::sent_{f['sent_id']}\" for f in case[\"gold_supporting_facts\"]]\n", - " full_context_norm = normalize_text(context_text_for_case(case))\n", - " pred_norm = normalize_text(pred_answer)\n", - "\n", - " unsupported_ratio = unsupported_answer_token_ratio(pred_answer, context_text_for_case(case))\n", - " # Heuristic audit only: do not punish non-extractive wording when answer F1 is already reasonably high.\n", - " if normalize_text(gold_answer) in [\"yes\", \"no\"]:\n", - " hallucinated_detail = 0\n", - " elif f1_score(pred_answer, gold_answer) >= 0.50:\n", - " hallucinated_detail = 0\n", - " elif pred_norm and pred_norm not in full_context_norm and unsupported_ratio > 0.50:\n", - " hallucinated_detail = 1\n", - " else:\n", - " hallucinated_detail = 0\n", - "\n", - " wrong_source_used = 0\n", - " if support_titles_pred:\n", - " pred_title_norm = {normalize_text(x) for x in support_titles_pred}\n", - " gold_title_norm = {normalize_text(x) for x in gold_titles}\n", - " if len(pred_title_norm - gold_title_norm) > 0 and not (pred_title_norm & gold_title_norm):\n", - " wrong_source_used = 1\n", - "\n", - " return {\n", - " \"case_id\": parsed_record[\"case_id\"],\n", - " \"arm\": parsed_record[\"arm\"],\n", - " \"model\": parsed_record.get(\"model\", \"\"),\n", - " \"model_role\": parsed_record.get(\"model_role\", \"\"),\n", - " \"parse_ok\": int(parsed_record[\"parse_ok\"]),\n", - " \"answer_exact_match\": exact_match(pred_answer, gold_answer),\n", - " \"answer_f1\": f1_score(pred_answer, gold_answer),\n", - " \"support_title_f1\": set_f1(support_titles_pred, gold_titles),\n", - " \"support_sentence_f1\": set_f1(support_sentence_ids_pred, gold_sentence_ids),\n", - " \"wrong_source_used\": wrong_source_used,\n", - " \"hallucinated_detail\": hallucinated_detail,\n", - " \"unsupported_answer_token_ratio\": unsupported_ratio,\n", - " \"claim_ceiling\": parsed.get(\"claim_ceiling\", \"\"),\n", - " \"confidence\": parsed.get(\"confidence\", \"\"),\n", - " \"estimated_token_in\": int(raw_record.get(\"estimated_token_in\", 0)),\n", - " \"estimated_token_out\": int(raw_record.get(\"estimated_token_out\", 0)),\n", - " \"estimated_cost_usd\": float(raw_record.get(\"estimated_cost_usd\", estimate_cost_usd(raw_record.get(\"model\", \"\"), int(raw_record.get(\"estimated_token_in\", 0)), int(raw_record.get(\"estimated_token_out\", 0))))),\n", - " \"prompt_sha256\": raw_record.get(\"prompt_sha256\", \"\"),\n", - " \"raw_output_sha256\": raw_record.get(\"raw_output_sha256\", \"\")\n", - " }\n", - "\n", - "def score_raw_outputs(raw_outputs_rows: List[Dict[str, Any]], parser_rows: List[Dict[str, Any]]) -> pd.DataFrame:\n", - " raw_map = {(r[\"case_id\"], r[\"arm\"], r.get(\"model\", \"\")): r for r in raw_outputs_rows}\n", - " score_rows = []\n", - " for p in parser_rows:\n", - " raw = raw_map[(p[\"case_id\"], p[\"arm\"], p.get(\"model\", \"\"))]\n", - " score_rows.append(score_record(p, raw))\n", - " return pd.DataFrame(score_rows)\n", - "\n", - "smoke_scores = score_raw_outputs(raw_outputs, parser_records) if raw_outputs else pd.DataFrame()\n", - "if not smoke_scores.empty:\n", - " smoke_scores.to_csv(RUN_DIR / \"scorer_trace_smoke.csv\", index=False)\n", - " display(smoke_scores.groupby([\"model\", \"model_role\", \"arm\"]).agg({\n", - " \"parse_ok\": \"sum\",\n", - " \"answer_exact_match\": \"mean\",\n", - " \"answer_f1\": \"mean\",\n", - " \"support_title_f1\": \"mean\",\n", - " \"support_sentence_f1\": \"mean\",\n", - " \"wrong_source_used\": \"sum\",\n", - " \"hallucinated_detail\": \"sum\",\n", - " \"estimated_token_in\": \"sum\",\n", - " \"estimated_token_out\": \"sum\",\n", - " \"estimated_cost_usd\": \"sum\"\n", - " }).reset_index())\n", - "else:\n", - " print(\"No smoke scores because no API outputs exist.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b7c0c783", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 13: Smoke verdict gate\n", - "\n", - "def summarize_by_model_arm(df: pd.DataFrame) -> Dict[str, Dict[str, Dict[str, Any]]]:\n", - " out = {}\n", - " if df.empty:\n", - " return out\n", - " grouped = df.groupby([\"model\", \"model_role\", \"arm\"])\n", - " for (model, model_role, arm), g in grouped:\n", - " out.setdefault(model, {})\n", - " cost_sum = float(g[\"estimated_cost_usd\"].sum()) if \"estimated_cost_usd\" in g else float(\"nan\")\n", - " answer_f1 = float(g[\"answer_f1\"].mean())\n", - " support_f1 = float(g[\"support_title_f1\"].mean())\n", - " out[model][arm] = {\n", - " \"model_role\": str(model_role),\n", - " \"rows\": int(len(g)),\n", - " \"parse_ok\": int(g[\"parse_ok\"].sum()),\n", - " \"answer_exact_match_mean\": float(g[\"answer_exact_match\"].mean()),\n", - " \"answer_f1_mean\": answer_f1,\n", - " \"support_title_f1_mean\": support_f1,\n", - " \"support_sentence_f1_mean\": float(g[\"support_sentence_f1\"].mean()),\n", - " \"wrong_source_used_total\": int(g[\"wrong_source_used\"].sum()),\n", - " \"hallucinated_detail_total\": int(g[\"hallucinated_detail\"].sum()),\n", - " \"estimated_token_in_total\": int(g[\"estimated_token_in\"].sum()),\n", - " \"estimated_token_out_total\": int(g[\"estimated_token_out\"].sum()),\n", - " \"estimated_cost_usd_total\": cost_sum,\n", - " \"quality_per_dollar\": value_per_dollar(answer_f1, support_f1, cost_sum)\n", - " }\n", - " return out\n", - "\n", - "def evaluate_model_pair(summary_for_model: Dict[str, Any]) -> Dict[str, Any]:\n", - " a = summary_for_model.get(\"A_BASELINE_RAW_CONTEXT\", {})\n", - " b = summary_for_model.get(\"B_POLARIS_COMPACT_CONTEXT\", {})\n", - " token_reduction = None\n", - " cost_reduction = None\n", - " if a and b and a.get(\"estimated_token_in_total\", 0) > 0:\n", - " token_reduction = 1 - (b[\"estimated_token_in_total\"] / a[\"estimated_token_in_total\"])\n", - " if a and b and a.get(\"estimated_cost_usd_total\", 0) > 0 and not math.isnan(a.get(\"estimated_cost_usd_total\", float(\"nan\"))):\n", - " cost_reduction = 1 - (b[\"estimated_cost_usd_total\"] / a[\"estimated_cost_usd_total\"])\n", - "\n", - " a_rows = max(1, a.get(\"rows\", 1))\n", - " b_rows = max(1, b.get(\"rows\", 1))\n", - " a_wrong_rate = a.get(\"wrong_source_used_total\", 999) / a_rows\n", - " b_wrong_rate = b.get(\"wrong_source_used_total\", 999) / b_rows\n", - " a_hallu_rate = a.get(\"hallucinated_detail_total\", 999) / a_rows\n", - " b_hallu_rate = b.get(\"hallucinated_detail_total\", 999) / b_rows\n", - "\n", - " # For larger runs, use rate-based gates instead of small-sample +1 gates.\n", - " rate_margin = 0.03 if min(a_rows, b_rows) >= 50 else None\n", - "\n", - " if rate_margin is not None:\n", - " wrong_source_gate = b_wrong_rate <= a_wrong_rate + rate_margin\n", - " hallucination_gate = b_hallu_rate <= a_hallu_rate + rate_margin\n", - " else:\n", - " wrong_source_gate = b.get(\"wrong_source_used_total\", 999) <= a.get(\"wrong_source_used_total\", 0) + 1\n", - " hallucination_gate = b.get(\"hallucinated_detail_total\", 999) <= a.get(\"hallucinated_detail_total\", 0) + HALLUCINATION_MARGIN\n", - "\n", - " quality_gate = {\n", - " \"b_answer_f1_not_more_than_margin_below_a\": (b.get(\"answer_f1_mean\", 0) + QA_F1_NONINFERIORITY_MARGIN >= a.get(\"answer_f1_mean\", 0)),\n", - " \"b_support_title_f1_not_more_than_margin_below_a\": (b.get(\"support_title_f1_mean\", 0) + SUPPORT_TITLE_NONINFERIORITY_MARGIN >= a.get(\"support_title_f1_mean\", 0)),\n", - " \"b_wrong_source_rate_not_more_than_a_plus_margin\": wrong_source_gate,\n", - " \"b_hallucination_rate_not_more_than_a_plus_margin\": hallucination_gate,\n", - " \"b_token_reduction_at_least_smoke_gate\": bool(token_reduction is not None and token_reduction >= SMOKE_TOKEN_REDUCTION_GATE),\n", - " \"b_token_reduction_at_least_strict_gate\": bool(token_reduction is not None and token_reduction >= STRICT_TOKEN_REDUCTION_GATE)\n", - " }\n", - "\n", - " required_quality_keys = [\n", - " \"b_answer_f1_not_more_than_margin_below_a\",\n", - " \"b_support_title_f1_not_more_than_margin_below_a\",\n", - " \"b_wrong_source_rate_not_more_than_a_plus_margin\",\n", - " \"b_hallucination_rate_not_more_than_a_plus_margin\",\n", - " \"b_token_reduction_at_least_smoke_gate\"\n", - " ]\n", - " return {\n", - " \"quality_gate\": quality_gate,\n", - " \"quality_ok\": all(quality_gate[k] for k in required_quality_keys),\n", - " \"token_reduction_B_vs_A\": token_reduction,\n", - " \"cost_reduction_B_vs_A\": cost_reduction,\n", - " \"wrong_source_rate_A\": a_wrong_rate,\n", - " \"wrong_source_rate_B\": b_wrong_rate,\n", - " \"hallucination_rate_A\": a_hallu_rate,\n", - " \"hallucination_rate_B\": b_hallu_rate,\n", - " \"rate_margin_used\": rate_margin\n", - " }\n", - "\n", - "def build_verdict(df: pd.DataFrame, stage: str) -> Dict[str, Any]:\n", - " summary = summarize_by_model_arm(df)\n", - " hard_gate = {\n", - " \"stage\": stage,\n", - " \"metadata_leakage_fail_count\": int(pd.DataFrame(leakage_rows)[\"metadata_leakage_fail\"].sum()),\n", - " \"self_grading_fail_count\": int(pd.DataFrame(leakage_rows)[\"self_grading_fail\"].sum()),\n", - " \"api_key_pattern_in_prompts_count\": int(pd.DataFrame(leakage_rows)[\"api_key_pattern_found\"].sum()),\n", - " \"model_self_grading_output_fields_allowed\": False\n", - " }\n", - "\n", - " if not summary:\n", - " return {\n", - " \"experiment_name\": EXPERIMENT_NAME,\n", - " \"stage\": stage,\n", - " \"run_disposition\": \"NO_API_OUTPUTS\",\n", - " \"claim_ceiling\": \"HARNESS_ONLY_NOT_REAL_QA_PROOF\",\n", - " \"hard_gate\": hard_gate,\n", - " \"summary_by_model_arm\": summary\n", - " }\n", - "\n", - " hard_ok = (\n", - " hard_gate[\"metadata_leakage_fail_count\"] == 0\n", - " and hard_gate[\"self_grading_fail_count\"] == 0\n", - " and hard_gate[\"api_key_pattern_in_prompts_count\"] == 0\n", - " )\n", - " parse_all_ok = int(df[\"parse_ok\"].sum()) == len(df)\n", - "\n", - " pair_evaluations = {model: evaluate_model_pair(model_summary) for model, model_summary in summary.items()}\n", - " primary_eval = pair_evaluations.get(PRIMARY_MODEL, {})\n", - " primary_quality_ok = bool(primary_eval.get(\"quality_ok\", False))\n", - "\n", - " ladder_eval = {}\n", - " if RUN_MODEL_LADDER_SMOKE and COMPARISON_MODEL in summary and PRIMARY_MODEL in summary:\n", - " mini_compact = summary[PRIMARY_MODEL].get(\"B_POLARIS_COMPACT_CONTEXT\", {})\n", - " big_raw = summary[COMPARISON_MODEL].get(\"A_BASELINE_RAW_CONTEXT\", {})\n", - " if mini_compact and big_raw:\n", - " mini_quality = 0.70 * mini_compact.get(\"answer_f1_mean\", 0) + 0.30 * mini_compact.get(\"support_title_f1_mean\", 0)\n", - " big_quality = 0.70 * big_raw.get(\"answer_f1_mean\", 0) + 0.30 * big_raw.get(\"support_title_f1_mean\", 0)\n", - " mini_cost = mini_compact.get(\"estimated_cost_usd_total\", float(\"nan\"))\n", - " big_cost = big_raw.get(\"estimated_cost_usd_total\", float(\"nan\"))\n", - " ladder_eval = {\n", - " \"mini_compact_vs_big_raw_quality_delta\": mini_quality - big_quality,\n", - " \"mini_compact_vs_big_raw_cost_ratio\": mini_cost / big_cost if big_cost and not math.isnan(big_cost) else None,\n", - " \"mini_compact_quality_per_dollar\": value_per_dollar(mini_compact.get(\"answer_f1_mean\", 0), mini_compact.get(\"support_title_f1_mean\", 0), mini_cost),\n", - " \"big_raw_quality_per_dollar\": value_per_dollar(big_raw.get(\"answer_f1_mean\", 0), big_raw.get(\"support_title_f1_mean\", 0), big_cost)\n", - " }\n", - "\n", - " disposition = \"GREEN\" if hard_ok and parse_all_ok and primary_quality_ok else \"RED\"\n", - "\n", - " return {\n", - " \"experiment_name\": EXPERIMENT_NAME,\n", - " \"branch\": BRANCH_NAME,\n", - " \"stage\": stage,\n", - " \"run_disposition\": disposition,\n", - " \"claim_ceiling\": \"SMOKE_STAGE_ONLY_NOT_FULL_30_PROOF\" if stage == \"smoke\" else \"30_CASE_PUBLIC_HOTPOTQA_PILOT_EVIDENCE_ONLY\",\n", - " \"dataset\": {\n", - " \"name\": loaded_dataset_name,\n", - " \"config\": DATASET_CONFIG,\n", - " \"split\": DATASET_SPLIT,\n", - " \"sample_count\": SAMPLE_COUNT,\n", - " \"smoke_count\": SMOKE_COUNT,\n", - " \"random_seed\": RANDOM_SEED\n", - " },\n", - " \"models\": {\n", - " \"primary_model\": PRIMARY_MODEL,\n", - " \"comparison_model\": COMPARISON_MODEL,\n", - " \"run_model_ladder_smoke\": RUN_MODEL_LADDER_SMOKE,\n", - " \"smoke_models\": SMOKE_MODELS\n", - " },\n", - " \"hard_gate\": hard_gate,\n", - " \"parse_all_ok\": parse_all_ok,\n", - " \"pair_evaluations\": pair_evaluations,\n", - " \"model_ladder_eval\": ladder_eval,\n", - " \"summary_by_model_arm\": summary,\n", - " \"price_table_used_for_estimated_cost\": MODEL_PRICE_PER_1M_TOKENS\n", - " }\n", - "\n", - "smoke_verdict = build_verdict(smoke_scores, \"smoke\")\n", - "with (RUN_DIR / \"verdict_smoke.json\").open(\"w\", encoding=\"utf-8\") as f:\n", - " json.dump(smoke_verdict, f, ensure_ascii=False, indent=2)\n", - "\n", - "print(json.dumps(smoke_verdict, ensure_ascii=False, indent=2))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6edbbd6e", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 14: Optional full 30 run, only after smoke review\n", - "\n", - "RUN_FULL_30 = True\n", - "RUN_FULL_30_MODEL_LADDER = False\n", - "\n", - "if RUN_FULL_30:\n", - " full_models = [PRIMARY_MODEL]\n", - " if RUN_FULL_30_MODEL_LADDER:\n", - " full_models = SMOKE_MODELS\n", - "\n", - " print(\"Starting full100 standard exam: 100 cases x 2 arms API run.\")\n", - " print(\"Full models:\", full_models)\n", - " raw_outputs_full = []\n", - " for model_name in full_models:\n", - " model_role = \"primary_small_model\" if model_name == PRIMARY_MODEL else \"comparison_larger_model\"\n", - " for rec in tqdm(public_prompts, desc=f\"Full API calls / {model_name}\"):\n", - " raw_text = call_model(rec[\"prompt\"].replace(f\"Arm:\\n{rec['arm']}\", f\"Arm:\\n{rec['arm']}\\nModel role:\\n{model_role}\"))\n", - " raw_outputs_full.append({\n", - " \"case_id\": rec[\"case_id\"],\n", - " \"arm\": rec[\"arm\"],\n", - " \"model\": model_name,\n", - " \"model_role\": model_role,\n", - " \"prompt_sha256\": rec[\"prompt_sha256\"],\n", - " \"raw_output\": raw_text,\n", - " \"raw_output_sha256\": hashlib.sha256(raw_text.encode(\"utf-8\")).hexdigest(),\n", - " \"estimated_token_in\": rec[\"estimated_token_in\"],\n", - " \"estimated_token_out\": estimate_tokens(raw_text),\n", - " \"estimated_cost_usd\": estimate_cost_usd(model_name, rec[\"estimated_token_in\"], estimate_tokens(raw_text)),\n", - " \"run_stage\": \"full_30\"\n", - " })\n", - "\n", - " write_jsonl(RUN_DIR / \"raw_outputs_full_30.jsonl\", raw_outputs_full)\n", - "\n", - " parser_records_full = []\n", - " for r in raw_outputs_full:\n", - " parsed = parse_and_validate_raw(r[\"raw_output\"], prompt_by_case_arm[(r[\"case_id\"], r[\"arm\"])])\n", - " parsed[\"model\"] = r[\"model\"]\n", - " parsed[\"model_role\"] = r[\"model_role\"]\n", - " parser_records_full.append(parsed)\n", - "\n", - " write_jsonl(RUN_DIR / \"parser_records_full_30.jsonl\", parser_records_full)\n", - "\n", - " full_scores = score_raw_outputs(raw_outputs_full, parser_records_full)\n", - " full_scores.to_csv(RUN_DIR / \"scorer_trace_full_30.csv\", index=False)\n", - "\n", - " full_verdict = build_verdict(full_scores, \"full_30\")\n", - " with (RUN_DIR / \"verdict_full_30.json\").open(\"w\", encoding=\"utf-8\") as f:\n", - " json.dump(full_verdict, f, ensure_ascii=False, indent=2)\n", - "\n", - " display(full_scores.groupby([\"model\", \"model_role\", \"arm\"]).agg({\n", - " \"parse_ok\": \"sum\",\n", - " \"answer_exact_match\": \"mean\",\n", - " \"answer_f1\": \"mean\",\n", - " \"support_title_f1\": \"mean\",\n", - " \"support_sentence_f1\": \"mean\",\n", - " \"wrong_source_used\": \"sum\",\n", - " \"hallucinated_detail\": \"sum\",\n", - " \"estimated_token_in\": \"sum\",\n", - " \"estimated_token_out\": \"sum\",\n", - " \"estimated_cost_usd\": \"sum\"\n", - " }).reset_index())\n", - " print(json.dumps(full_verdict, ensure_ascii=False, indent=2))\n", - "else:\n", - " print(\"RUN_FULL_30 is False. Review smoke verdict first. Set RUN_FULL_30 = True only after smoke is acceptable.\")\n", - " print(\"Set RUN_FULL_30_MODEL_LADDER = False only after primary full 30 is acceptable.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c231b7f", - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 15: Package evidence zip and scan for accidental key leakage\n", - "\n", - "def build_hash_manifest(run_dir: Path) -> pd.DataFrame:\n", - " rows = []\n", - " for path in sorted(run_dir.rglob(\"*\")):\n", - " if path.is_file():\n", - " rel = path.relative_to(run_dir).as_posix()\n", - " rows.append({\n", - " \"relative_path\": rel,\n", - " \"size_bytes\": path.stat().st_size,\n", - " \"sha256\": sha256_file(path)\n", - " })\n", - " return pd.DataFrame(rows)\n", - "\n", - "def scan_artifacts_for_api_key(run_dir: Path) -> List[str]:\n", - " suspects = []\n", - " for path in sorted(run_dir.rglob(\"*\")):\n", - " if not path.is_file():\n", - " continue\n", - " try:\n", - " text = path.read_text(encoding=\"utf-8\", errors=\"ignore\")\n", - " except Exception:\n", - " continue\n", - " if api_key_pattern_found(text):\n", - " suspects.append(path.relative_to(run_dir).as_posix())\n", - " return suspects\n", - "\n", - "readme = f\"\"\"\n", - "# {EXPERIMENT_NAME} Evidence Package\n", - "\n", - "Branch: {BRANCH_NAME}\n", - "\n", - "Dataset: {loaded_dataset_name}, config={DATASET_CONFIG}, split={DATASET_SPLIT}\n", - "\n", - "This package contains prompts, raw model outputs if API was run, parser records, scorer traces, leakage audit records, compact selection traces, verdict files, and hash manifests. This v2.0 notebook uses the answer-blind MathCore Evidence Controller and can optionally run a Model Ladder smoke comparison.\n", - "\n", - "Claim ceiling:\n", - "This package is public HotpotQA validation evidence only when full_30 outputs exist. In this Full100 preset, full_30 filenames contain the 100-case run because the notebook reuses the same artifact field names.\n", - "Smoke outputs are disabled by default in the Full100 preset.\n", - "Sandbox or smoke stages must not be described as universal QA proof.\n", - "\n", - "API key policy:\n", - "The OpenAI API key is accepted through getpass and stored only in runtime memory.\n", - "It is not intentionally written to any artifact.\n", - "A simple API key pattern scan is run before packaging.\n", - "\"\"\".strip()\n", - "\n", - "(RUN_DIR / \"README_EVIDENCE.md\").write_text(readme, encoding=\"utf-8\")\n", - "\n", - "hash_df = build_hash_manifest(RUN_DIR)\n", - "hash_df.to_csv(RUN_DIR / \"hash_manifest.csv\", index=False)\n", - "\n", - "suspects = scan_artifacts_for_api_key(RUN_DIR)\n", - "if suspects:\n", - " raise RuntimeError(f\"API key-like pattern found in artifacts: {suspects}\")\n", - "\n", - "zip_path = OUTPUT_ROOT / f\"{EXPERIMENT_NAME}_{RUN_ID}_EVIDENCE.zip\"\n", - "with zipfile.ZipFile(zip_path, \"w\", zipfile.ZIP_DEFLATED) as z:\n", - " for path in sorted(RUN_DIR.rglob(\"*\")):\n", - " if path.is_file():\n", - " z.write(path, arcname=path.relative_to(RUN_DIR).as_posix())\n", - "\n", - "print(\"Evidence zip created:\", zip_path)\n", - "print(\"API key artifact scan: PASS\")\n", - "print(\"Files packaged:\", len(hash_df))\n", - "\n", - "try:\n", - " from google.colab import files\n", - " files.download(str(zip_path))\n", - "except Exception:\n", - " print(\"Not running in Colab or auto-download unavailable. Download the zip from:\", zip_path)" - ] - }, - { - "cell_type": "markdown", - "id": "e77cd638", - "metadata": {}, - "source": [ - "---\n", - "\n", - "# Final Polaris Protocol report\n", - "\n", - "This final section generates a compact report for **PP02D_C — Public QA Compact Context Full100 Mini**.\n", - "\n", - "It produces:\n", - "- `polaris_report_PP02D_C/executive_report.md`\n", - "- `polaris_report_PP02D_C/metrics_summary.csv`\n", - "- `polaris_report_PP02D_C/chart_*.png`\n", - "\n", - "The charts are intentionally simple. They are meant to make the experiment result easy to inspect, not to expand the claim boundary.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "619f67b1", - "metadata": {}, - "outputs": [], - "source": [ - "# Final Polaris Protocol report generator\n", - "# This cell uses the published result summary as a stable reporting fallback.\n", - "# If you adapt the notebook and compute fresh metrics, update POLARIS_PUBLISHED_METRICS before running this cell.\n", - "\n", - "from pathlib import Path\n", - "import json, csv, math, re\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "POLARIS_REPORT_EXPERIMENT_ID = \"PP02D_C\"\n", - "POLARIS_REPORT_TITLE = \"PP02D_C — Public QA Compact Context Full100 Mini\"\n", - "POLARIS_REPORT_REPO = \"https://github.com/onestardao/WFGY\"\n", - "POLARIS_REPORT_PURPOSE = \"Tests public QA compact context under a Full100 Mini MVP runnable Colab: baseline raw context vs Polaris compact context.\"\n", - "POLARIS_REPORT_SPIRIT = \"Public QA compact-context supplement. The goal is reproduction / inspection of a scoped public QA signal, not global QA superiority.\"\n", - "POLARIS_REPORT_CLAIM_BOUNDARY = \"100-case public QA compact-context evidence only. Not global QA superiority and not proof that compact context is always better.\"\n", - "POLARIS_PUBLISHED_SUMMARY = [\n", - " \"MVP runnable Colab scope: 100 public QA cases × 2 arms × 1 primary model ≈ 200 model-arm output records.\",\n", - " \"`gpt-4.1-mini` baseline answer F1 mean: 0.7978095238; compact-context answer F1 mean: 0.8063095238.\",\n", - " \"`gpt-4.1-mini` baseline support-title F1 mean: 0.8906666667; compact-context support-title F1 mean: 0.8503333333.\",\n", - " \"wrong-source total: baseline 1, compact 2; hallucinated-detail total: baseline 0, compact 0.\",\n", - " \"estimated cost: baseline USD 0.099630, compact USD 0.068566; estimated cost reduction compact vs baseline: 0.3117936365.\"\n", - "]\n", - "POLARIS_PUBLISHED_METRICS = {\n", - " \"Cases\": 100,\n", - " \"MVP model-arm records\": 200,\n", - " \"Mini baseline answer F1 mean\": 0.7978095238,\n", - " \"Mini compact answer F1 mean\": 0.8063095238,\n", - " \"Mini baseline support-title F1 mean\": 0.8906666667,\n", - " \"Mini compact support-title F1 mean\": 0.8503333333,\n", - " \"Mini baseline wrong-source total\": 1,\n", - " \"Mini compact wrong-source total\": 2,\n", - " \"Mini baseline hallucinated-detail total\": 0,\n", - " \"Mini compact hallucinated-detail total\": 0,\n", - " \"Mini baseline estimated cost USD\": 0.099630,\n", - " \"Mini compact estimated cost USD\": 0.068566,\n", - " \"Mini estimated cost reduction compact vs baseline\": 0.3117936365\n", - "}\n", - "POLARIS_CHART_SPECS = [\n", - " {\n", - " \"title\": \"Mini answer F1 mean\",\n", - " \"ylabel\": \"F1\",\n", - " \"labels\": [\n", - " \"Baseline\",\n", - " \"Compact\"\n", - " ],\n", - " \"values\": [\n", - " 0.7978095238,\n", - " 0.8063095238\n", - " ]\n", - " },\n", - " {\n", - " \"title\": \"Mini estimated cost USD\",\n", - " \"ylabel\": \"USD\",\n", - " \"labels\": [\n", - " \"Baseline\",\n", - " \"Compact\"\n", - " ],\n", - " \"values\": [\n", - " 0.099630,\n", - " 0.068566\n", - " ]\n", - " },\n", - " {\n", - " \"title\": \"Mini caveat checks\",\n", - " \"ylabel\": \"Value\",\n", - " \"labels\": [\n", - " \"Base support-F1\",\n", - " \"Compact support-F1\",\n", - " \"Base wrong-source\",\n", - " \"Compact wrong-source\",\n", - " \"Base hallucination\",\n", - " \"Compact hallucination\"\n", - " ],\n", - " \"values\": [\n", - " 0.8906666667,\n", - " 0.8503333333,\n", - " 1,\n", - " 2,\n", - " 0,\n", - " 0\n", - " ]\n", - " }\n", - "]\n", - "\n", - "report_dir = Path(f\"polaris_report_{POLARIS_REPORT_EXPERIMENT_ID}\")\n", - "report_dir.mkdir(exist_ok=True)\n", - "\n", - "# Save metrics table.\n", - "metrics_df = pd.DataFrame([\n", - " {\"metric\": key, \"value\": value}\n", - " for key, value in POLARIS_PUBLISHED_METRICS.items()\n", - "])\n", - "metrics_path = report_dir / \"metrics_summary.csv\"\n", - "metrics_df.to_csv(metrics_path, index=False)\n", - "\n", - "# Generate compact charts.\n", - "chart_paths = []\n", - "def _safe_chart_name(title):\n", - " return re.sub(r\"[^a-z0-9_\\\\-]+\", \"_\", title.lower().replace(\" \", \"_\")).strip(\"_\")\n", - "\n", - "for idx, spec in enumerate(POLARIS_CHART_SPECS, start=1):\n", - " labels = spec[\"labels\"]\n", - " values = spec[\"values\"]\n", - " plt.figure(figsize=(max(6, len(labels) * 1.2), 4))\n", - " bars = plt.bar(labels, values)\n", - " plt.title(spec[\"title\"])\n", - " plt.ylabel(spec.get(\"ylabel\", \"Value\"))\n", - " plt.xticks(rotation=25, ha=\"right\")\n", - "\n", - " numeric_values = [float(v) for v in values if isinstance(v, (int, float))]\n", - " max_value = max(numeric_values) if numeric_values else 0.0\n", - " min_value = min(numeric_values) if numeric_values else 0.0\n", - "\n", - " if max_value == 0 and min_value == 0:\n", - " # Make zero-count success charts visually readable instead of looking blank.\n", - " plt.ylim(0, 1)\n", - " for bar, value in zip(bars, values):\n", - " x = bar.get_x() + bar.get_width() / 2\n", - " plt.text(x, 0.05, str(value), ha=\"center\", va=\"bottom\", fontsize=10)\n", - " plt.text(\n", - " 0.5, 0.88,\n", - " \"All tracked counts are 0\",\n", - " transform=plt.gca().transAxes,\n", - " ha=\"center\",\n", - " va=\"center\",\n", - " fontsize=11,\n", - " )\n", - " else:\n", - " upper = max_value * 1.18 if max_value > 0 else 1\n", - " if min_value >= 0:\n", - " plt.ylim(0, upper)\n", - " for bar, value in zip(bars, values):\n", - " x = bar.get_x() + bar.get_width() / 2\n", - " y = bar.get_height()\n", - " label = f\"{value:.4g}\" if isinstance(value, float) else str(value)\n", - " plt.text(x, y + (upper * 0.02 if max_value > 0 else 0.03), label, ha=\"center\", va=\"bottom\", fontsize=9)\n", - "\n", - " plt.tight_layout()\n", - " chart_path = report_dir / f\"chart_{idx:02d}_{_safe_chart_name(spec['title'])}.png\"\n", - " plt.savefig(chart_path, dpi=180, bbox_inches=\"tight\")\n", - " plt.show()\n", - " chart_paths.append(chart_path)\n", - "\n", - "# Save executive markdown report.\n", - "summary_lines = \"\\n\".join([f\"- {item}\" for item in POLARIS_PUBLISHED_SUMMARY])\n", - "chart_lines = \"\\n\".join([f\"- {p.name}\" for p in chart_paths])\n", - "\n", - "report_md = f\"\"\"# {POLARIS_REPORT_TITLE}\n", - "\n", - "**Repository:** {POLARIS_REPORT_REPO} \n", - "**Experiment ID:** {POLARIS_REPORT_EXPERIMENT_ID}\n", - "\n", - "## Experiment spirit\n", - "\n", - "{POLARIS_REPORT_SPIRIT}\n", - "\n", - "## What this tests\n", - "\n", - "{POLARIS_REPORT_PURPOSE}\n", - "\n", - "## Published result summary\n", - "\n", - "{summary_lines}\n", - "\n", - "## Generated files\n", - "\n", - "- metrics_summary.csv\n", - "{chart_lines}\n", - "\n", - "## Claim boundary\n", - "\n", - "{POLARIS_REPORT_CLAIM_BOUNDARY}\n", - "\n", - "This report is designed for reproduction / inspection. It does not convert scoped evidence into universal proof.\n", - "\"\"\"\n", - "\n", - "report_path = report_dir / \"executive_report.md\"\n", - "report_path.write_text(report_md, encoding=\"utf-8\")\n", - "\n", - "print(\"Polaris Protocol report generated:\")\n", - "print(f\"- {report_path}\")\n", - "print(f\"- {metrics_path}\")\n", - "for path in chart_paths:\n", - " print(f\"- {path}\")\n" - ] - } - ], - "metadata": { - "colab": { - "name": "DD02C_ZAI_Exact_Public_QA_Colab_Full100_Mini_Stratified_v2_0_1_MATHCORE_FIXED.ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.x" - }, - "polaris_protocol": { - "claim_boundary": "100-case public QA compact-context evidence only. Not global QA superiority and not proof that compact context is always better.", - "experiment_id": "PP02D_C", - "experiment_title": "PP02D_C — Public QA Compact Context Full100 Mini", - "mvp_official_colab": true, - "repo": "https://github.com/onestardao/WFGY" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}