mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2025-09-10 15:29:39 +00:00
⚡ add humaneval support
This commit is contained in:
parent
216a63b858
commit
dc10480ef6
7 changed files with 130 additions and 4 deletions
15
ktransformers/tests/humaneval/evaluation.py
Normal file
15
ktransformers/tests/humaneval/evaluation.py
Normal file
|
@ -0,0 +1,15 @@
|
|||
# reference: https://github.com/declare-lab/instruct-eval/blob/main/human_eval/main.py#L35
|
||||
def filter_code(completion: str) -> str:
|
||||
# The program tends to overwrite, we only take the first function
|
||||
completion = completion.lstrip("\n")
|
||||
# we also remove ```python\n and ```
|
||||
completion = completion.replace("```python\n", "").replace("```", "")
|
||||
if 'if __name__ == "__main__":' in completion:
|
||||
completion = completion.split('if __name__ == "__main__":')[0]
|
||||
if "# Example usage" in completion:
|
||||
completion = completion.split("# Example usage")[0]
|
||||
return completion.split("\n\n")[0]
|
||||
|
||||
|
||||
def fix_indents(text: str) -> str:
|
||||
return text.replace("\t", " ")
|
Loading…
Add table
Add a link
Reference in a new issue