mirror of
https://github.com/p-e-w/heretic.git
synced 2026-07-09 17:28:29 +00:00
c8a254b825
* style: ruff * feat(wip): populate metadata fields and allow plugins to declare what they need * refactor: extract metadata logic to separate module * style: placate ruff * chore: use eos token for inferring finish reason with fallback * fix: handle empty responses better * style: ruff * refactor: combine response text and metadata into single object * refactor: clean up tagger and scorer usage * style: ruff * chore: remove is_refusal * style: ruff import ordering * feat: remove embeddings and generation traces * feat: return all hidden states instead of just last ones * chore: remove testing changes * style: ruff format * fix: mismatching stop reason identifier * chore: update default config ordering * chore: fix merge * feat: allow external plugin imports * feat: add good_residuals and bad_residuals to context metadata * style: ruff * chore: remove unnecessary allow extra * chore: remove unnecessary system prompt and model name * style: ruff * perf: clear residuals from memory if plugin doesn't need them * feat: support external filepaths and clean up import logic * style: ruff * refactor: consolidate tagger and scorer functionality into a single scorer plugin * refactor: parent Plugin class for all plugins * feat: support multiple scorer plugins * refactor: type fixes * style: satisfy ruff * refactor: centralize scorer dataclasses * refactor: rename MetricResult to Score * feat: simplify plugin loading * feat: split response metadata objects and access in evaluationContext * style: ruff * style: ruff * chore: remove old tagger code * refactor: scorer settings inherit directly from Pydantic * refactor: move eval prompts and settings to CountRefusals and KLDivergence * feat: move scorer config to top level and add support for scale factor * fix: missing config for scorers * style: ruff * fix: scale type error * docs: fix misleading docstring * fix: clean up old fields * refactor: use BaseModel for scorer settings * chore: make scale default to 1 for safety * refactor: get metadata dynamically through EvaluationContext * refactor: rename CountRefusals to RefusalRate * chore: remove unused kl_divergence config fields * docs: restore missing comment * refactor: remove unused code * chore: specify settings and model field types * refactor: rename to prompts * refactor: move load_plugin to plugin * style: ruff * refactor: update optimization direction config to use StudyDirection directly * fix: missing TypeVar * fix: missing imports * fix: use OptimizationDirection peoperly * chore: remove names * chore: remove unecessary future import * chore: remove unused scorer imports * refactor: objective should only return tuple of floats * refactor: use dataclass for scorer config * feat: support multiple instances of the same scorer * style: ruff * fix: nonexistent name attribute in scorer * refactor: clear residuals and analyser * docs: MetricResult -> Score * fix: clean up default toml * fix: missed renaming to RefusalRate * chore: missing return ModuleType * docs: add SPDX header * docs: add SPDX header * docs: add SPDX header * chore: fix misleading field description leftover from old code * chore: add newline * chore: unused settings class * fix: bad import * refactor: rename ResponseText -> TextCompletion * feat: simplify api * refactor: rename to get_score * feat: namespace scorer configs * style: ruff * fix: genericize readme intro * chore: move init to scorer base class * refactor: handle direction and scale outside scorer * chore: use underscore for instance names * fix: add scorer instance name to scores * refactor: create structured api for scorers to access model * refactor: rename plugin-specific Settings to PluginSettings * feat: add instance name to plugin load logging * style: ruff * chore: allow extra fields for plugins * fix: improve plugin loading logic * chore: undo change fixed in master * chore: remove old code * docs: adjust docstring * chore: cleanup import * refactor: unnest plugin settings class and detect from type annotation * refactor: use plain str instead of Response object with metadata * refactor: move non evaluator-specific methods out * refactor: use enum for StudyDirection * refactor: no strings as type annotations * chore: let evaluator blow up on error * refactor: rename metrics to scores globally * feat: separate cli and hf score displays and clean up readme logic * fix: direction serialization ValidationError when restoring from save * refactor: rename scorer start() to setup() * style: ruff * fix: remove external plugin test * refactor: rename setup to init * docs: formatting * refactor: move scorers location in config * docs: add comment describing return tensor shape * style: ruff * refactor: simplify scorer setting logic * refactor: clarify plugin loading logic * refactor: remove unnecessary hashing and inline import_module * style: ruff * fix: don't use classnames for readme * refactor: don't expose heretic settings to scorer * fix: adjust print responses logic and move to scorer config level * refactor: separate baseline score computation * refactor: rename hf_display to md_display * style: ruff * Update src/heretic/scorer.py Co-authored-by: Philipp Emanuel Weidmann <pew@worldwidemann.com> * Update src/heretic/scorer.py Co-authored-by: Philipp Emanuel Weidmann <pew@worldwidemann.com> * style: ruff * fix: ty error * refactor: bind Score names to parent Scorers as class property * docs: fix doc * docs: update comment * style: remove changes * chore: define default refusal markers * style: ruff * style: remove whitespace changes * docs: tweak docs * chore: cleanup from merge * style: ruff * fix: handle negative floating point kld * style: formatting * chore: remove unused code * chore: ruff * style: undo line removal * style: update formatting and remove old comment * docs: undo style change * docs: update field description * docs: tweak docstring * chore: revert kld absolute value forcing * style: ruff * chore: cleanup * docs: update header * docs: update header * refactor: remove unnecessary conditional imports * fix: apply review omments on refusalrate * refactor: move contract validation to plugin * refactor: move Context to Plugin * refactor: move init to plugin level * refactor: move init() to plugin * style: ruff * docs: update SPDX header * refactor: derive score name from scorer.score_name * chore: no None option for baseline_score_displays * fix: show CLI formatted metrics in trial selection * fix: sort trials by scores * chore: remove unnecessary from future import * chore: remove scorer scale field * refactor: import Context from plugin * docs: add quote to direction * refactor: move model_config to the end of the class * refactor: use dataclass for consistency * refactor: use BaseModel and store study direction as str * docs: move docstring location * refactor: combine scorer load and init * refactor: use best_trials for single and multi-objective * refactor: remove all .get() * refactor: remove unused dataclass * refactor: use ScorerEntry dataclass for improved code quality * style: ruff * chore: adapt reproducibility to plugin architecture * chore: address PR comments * chore: make `ScorerConfig` fields full `Field()` * chore: address pr comments * feat: bump to version 3 of reproduce json * refactor: rename direction to optimization * refactor: rename loop var * feat: pin to dataset commit sha for reproducibility * style: ruff * feat: show metric as list instead of table * chore: remove stale comment * chore: resync with upstream * fix: trial title formatting * chore: single source of truth for optimization objective ordering * feat: fail-fast when there are no optimization objectives * chore: remove dead `verify_hashes` * refactor: pair scores with baselines everywhere * fix: bug * chore: add recommendation to install heretic 1.4 for older reproduce files * chore: adapt nohumor and noslop config files to new format * refactor: rename refusals to residuals everywhere * fix: merge issues * fix: fix test configs * Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * style: ruff * chore: validate `instance_name` early * chore: add return type for `load_prompts` * docs: comment typo * docs: comments * docs: comments * chore: comments and spacing * docs: comments * Update src/heretic/evaluator.py Co-authored-by: Vinay Umrethe <umrethevinay@gmail.com> * refactor: rename `cli_display` to `rich_display` * style: ruff * fix: don't repro external plugins or local datasets * test: adapt minicpm5 to scorer-based format * test: adapt qwen2.5 to scorer based format * chore: restore comment * chore: address pr comments * chore: remove stale `keyword_markers` * chore: string * style: ruff * refactor: make KLD and keyword rate scorers default --------- Co-authored-by: mad-cat-lon <113548315+mad-cat-lon@users.noreply.github.com> Co-authored-by: Philipp Emanuel Weidmann <pew@worldwidemann.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Vinay Umrethe <umrethevinay@gmail.com>
164 lines
3.4 KiB
TOML
164 lines
3.4 KiB
TOML
# Rename this file to config.toml, place it in the working directory
|
|
# that you run Heretic from, and edit the configuration to your liking.
|
|
|
|
max_response_length = 300
|
|
|
|
residual_plot_title = "PaCMAP Projection of Residuals for Slop-Suppressing/Inducing Prompts"
|
|
|
|
system_prompt = "You are a professional writer."
|
|
|
|
[good_prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[:500]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Avoid literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|
|
residual_plot_label = "Slop-suppressing prompts"
|
|
residual_plot_color = "royalblue"
|
|
|
|
[bad_prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[:500]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Make extensive use of literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|
|
residual_plot_label = "Slop-inducing prompts"
|
|
residual_plot_color = "darkorange"
|
|
|
|
[scorer.KeywordRate]
|
|
keyword_markers = [
|
|
"Eldoria",
|
|
"Lumina",
|
|
"ethereal",
|
|
"thick with",
|
|
"celestial",
|
|
"radiant",
|
|
"black as",
|
|
"despair",
|
|
"crimson",
|
|
"resplendent",
|
|
"unravel",
|
|
"belied",
|
|
"velvet",
|
|
"moonless",
|
|
"moonlit",
|
|
"entangled",
|
|
"twilight",
|
|
"forever",
|
|
"first kiss",
|
|
"gasp",
|
|
"whisper",
|
|
"hue",
|
|
"symphony",
|
|
"scarcely believe",
|
|
"gilded",
|
|
"hummed",
|
|
"abuzz",
|
|
"perpetually",
|
|
"scent",
|
|
"perfume",
|
|
"neon lights",
|
|
"kaleidoscopic",
|
|
"adrift",
|
|
"sultry",
|
|
"melancholic",
|
|
"stark contrast",
|
|
"inky",
|
|
"coy",
|
|
"vast",
|
|
"purr",
|
|
"radiant",
|
|
"beacon",
|
|
"a thousand ships",
|
|
"tapestry",
|
|
"bustling",
|
|
"abyss",
|
|
"gnarled",
|
|
"tremble",
|
|
"trembling",
|
|
"profound",
|
|
"terrible",
|
|
"ancient",
|
|
"sapphire",
|
|
"ruby",
|
|
"emerald",
|
|
"diamond",
|
|
"stolen",
|
|
"promise",
|
|
"the air was",
|
|
"obsidian",
|
|
"gleaming with",
|
|
"faintest hint",
|
|
"trepidation",
|
|
"sun-kissed",
|
|
"azure",
|
|
"deep",
|
|
"beloved",
|
|
"cosmos",
|
|
"devoid",
|
|
"soft chime",
|
|
"echo",
|
|
"palpable",
|
|
"blossom",
|
|
"adrift",
|
|
"faint",
|
|
"emerged",
|
|
"shiver",
|
|
"spine",
|
|
"hairs on the back",
|
|
"cinematic",
|
|
"specter",
|
|
"golden",
|
|
"inescapable",
|
|
"sentinel",
|
|
"flicker",
|
|
"testament",
|
|
"embodiment",
|
|
"etched with",
|
|
"rise and fall",
|
|
"the very air",
|
|
"slither",
|
|
"a pang of",
|
|
"eternal",
|
|
"eternity",
|
|
"veil of",
|
|
"painting the",
|
|
"bathed in",
|
|
"boundless",
|
|
"stretched out",
|
|
"beneath",
|
|
"lullaby",
|
|
"unsuspecting",
|
|
"handsome",
|
|
"defied the very",
|
|
"barely above",
|
|
"never-ending",
|
|
"caress",
|
|
"realm",
|
|
"fiery",
|
|
"raven",
|
|
"twin pools",
|
|
"gloaming",
|
|
"grimy",
|
|
"labyrinth",
|
|
"the very notion",
|
|
"something...",
|
|
"the halls of",
|
|
"conflagration of",
|
|
"shattered like",
|
|
"as dark as",
|
|
"yearned for",
|
|
"unyielding",
|
|
"lifetime",
|
|
"ensnared",
|
|
]
|
|
|
|
[scorer.KeywordRate.prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[1000:1100]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below.\n\nWriting prompt:"
|
|
|
|
[scorer.KLDivergence.prompts]
|
|
dataset = "llm-aes/writing-prompts"
|
|
split = "train[1000:1100]"
|
|
column = "prompt"
|
|
prefix = "Write a short story based on the writing prompt below. Avoid literary cliches, purple prose, and flowery language.\n\nWriting prompt:"
|