ai-space/heretic
3
0
Fork 0
mirror of https://github.com/p-e-w/heretic.git synced 2026-06-02 05:03:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: recheck prefix after inserting predefined (#194)

Browse Source
This commit is contained in:
Spiky Moth
2026-02-27 03:37:33 +01:00
committed by GitHub
parent cb4ef3fdfc
commit 303ba9d978
1 changed files with 29 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/heretic/main.py
+29 -13
View File
@@ -377,7 +377,8 @@ def run():
print() print()
print("Checking for common response prefix...") print("Checking for common response prefix...")
responses = model.get_responses_batched(good_prompts[:100] + bad_prompts[:100]) prefix_check_prompts = good_prompts[:100] + bad_prompts[:100]
responses = model.get_responses_batched(prefix_check_prompts)
# Despite being located in os.path, commonprefix actually performs # Despite being located in os.path, commonprefix actually performs
# a naive string operation without any path-specific logic, # a naive string operation without any path-specific logic,
@@ -388,24 +389,39 @@ def run():
model.response_prefix = commonprefix(responses).rstrip(" ") model.response_prefix = commonprefix(responses).rstrip(" ")
# Suppress CoT output. # Suppress CoT output.
if model.response_prefix.startswith("<think>"): recheck_prefix = False
# Most thinking models. if model.response_prefix:
model.response_prefix = "<think></think>" # When using any of the predefined prefixes below, we need to check that
elif model.response_prefix.startswith("<|channel|>analysis<|message|>"): # the prefix is actually complete (e.g. not missing a trailing newline).
# gpt-oss. recheck_prefix = True
model.response_prefix = "<|channel|>analysis<|message|><|end|><|start|>assistant<|channel|>final<|message|>" if model.response_prefix.startswith("<think>"):
elif model.response_prefix.startswith("<thought>"): # Most thinking models.
# Unknown, suggested by user. model.response_prefix = "<think></think>"
model.response_prefix = "<thought></thought>" elif model.response_prefix.startswith("<|channel|>analysis<|message|>"):
elif model.response_prefix.startswith("[THINK]"): # gpt-oss.
# Unknown, suggested by user. model.response_prefix = "<|channel|>analysis<|message|><|end|><|start|>assistant<|channel|>final<|message|>"
model.response_prefix = "[THINK][/THINK]" elif model.response_prefix.startswith("<thought>"):
# Unknown, suggested by user.
model.response_prefix = "<thought></thought>"
elif model.response_prefix.startswith("[THINK]"):
# Unknown, suggested by user.
model.response_prefix = "[THINK][/THINK]"
else:
recheck_prefix = False
if model.response_prefix: if model.response_prefix:
print(f"* Prefix found: [bold]{model.response_prefix!r}[/]") print(f"* Prefix found: [bold]{model.response_prefix!r}[/]")
else: else:
print("* None found") print("* None found")
if recheck_prefix:
print("* Rechecking with prefix...")
responses = model.get_responses_batched(prefix_check_prompts)
additional_prefix = commonprefix(responses).rstrip(" ")
if additional_prefix:
model.response_prefix += additional_prefix
print(f"* Extended prefix found: [bold]{model.response_prefix!r}[/]")
evaluator = Evaluator(settings, model) evaluator = Evaluator(settings, model)
if settings.evaluate_model is not None: if settings.evaluate_model is not None: