ai-space/heretic
3
0
Fork 0
mirror of https://github.com/p-e-w/heretic.git synced 2026-06-01 20:58:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(ara): set batch size on HFLM object

Browse Source
This commit is contained in:
Philipp Emanuel Weidmann
2026-04-01 14:34:21 +05:30
parent f7a456bd0c
commit 3b70fe5dfa
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/heretic/evaluator.py
+5 -2
View File
@@ -98,11 +98,14 @@ class Evaluator:
def get_score(self) -> tuple[tuple[float, float], float, int]:
if self.settings.use_piqa:
print(" * Running PIQA benchmark...")
hflm = HFLM(pretrained=self.model.model, tokenizer=self.model.tokenizer) # ty:ignore[invalid-argument-type]
hflm = HFLM(
pretrained=self.model.model, # ty:ignore[invalid-argument-type]
tokenizer=self.model.tokenizer, # ty:ignore[invalid-argument-type]
batch_size="auto",
)
results = lm_eval.simple_evaluate(
model=hflm,
tasks=["piqa"],
batch_size="auto",
)
piqa_acc_norm: float = results["results"]["piqa"]["acc_norm,none"]
print(f" * PIQA acc_norm: [bold]{piqa_acc_norm:.4f}[/]")
src/heretic/main.py
+1 -1
View File
@@ -1052,6 +1052,7 @@ def run():
hflm = HFLM(
pretrained=model.model, # ty:ignore[invalid-argument-type]
tokenizer=model.tokenizer, # ty:ignore[invalid-argument-type]
batch_size="auto",
)
table = Table()
@@ -1075,7 +1076,6 @@ def run():
results = lm_eval.simple_evaluate(
model=hflm,
tasks=[benchmark.task],
batch_size="auto",
)
return results["results"][benchmark.task]