Skip to content

Commit

Permalink
trans Chinese comments to English
Browse files Browse the repository at this point in the history
Signed-off-by: IcyFeather <mengzhuo.happy@gmail.com>
  • Loading branch information
IcyFeather233 committed Oct 27, 2024
1 parent 59fba19 commit 3f69cdf
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,13 @@
__all__ = ["acc"]

def get_last_letter(input_string):
# 检查输入是否为空或只包含非字母字符
if not input_string or not any(char.isalpha() for char in input_string):
return None

# 倒序遍历字符串,找到最后一个字母
for char in reversed(input_string):
if 'A' <= char <= 'D':
return char

# 如果没有找到字母,返回None
return None


Expand All @@ -35,10 +32,8 @@ def acc(y_true, y_pred):
y_pred = [get_last_letter(pred) for pred in y_pred]
y_true = [get_last_letter(pred) for pred in y_true]

# 使用列表推导来比较两个列表中的元素是否相同
same_elements = [y_pred[i] == y_true[i] for i in range(len(y_pred))]

# 计算相同元素的数量
acc = sum(same_elements) / len(same_elements)

return acc
Original file line number Diff line number Diff line change
Expand Up @@ -19,29 +19,24 @@
__all__ = ["llm_judgement"]

def extract_comprehensive_score(input_str):
# 使用正则表达式匹配综合得分及其分数
# extract overall points
match = re.search(r"'综合得分': (\d+)", input_str)
if match:
# 提取分数并返回
return int(match.group(1))
else:
# 如果没有找到匹配项,返回None或其他适当的值
return None


@ClassFactory.register(ClassType.GENERAL, alias="llm_judgement")
def llm_judgement(y_true, y_pred):
y_pred = [extract_comprehensive_score(pred) for pred in y_pred]

# 过滤掉None值(如果有)
valid_scores = [score for score in y_pred if score is not None]

LOGGER.info(f"Extracted {len(valid_scores)} datas from {len(y_pred)} datas")

# 计算平均值
if valid_scores:
average_score = sum(valid_scores) / len(valid_scores)
return average_score
else:
# 如果没有有效的分数,返回None或其他适当的值
return -1
50 changes: 40 additions & 10 deletions examples/llm_simple_qa/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,46 @@ The data of simple-qa example structure is:
`train_data/data.jsonl` is empty, and the `test_data/data.jsonl` is as follows:

```
{"question": "如果小明有5个苹果,他给了小华3个,那么小明还剩下多少个苹果?\nA. 2个\nB. 3个\nC. 4个\nD. 5个", "answer": "A"}
{"question": "下列哪个数是最小的质数?\nA. 0\nB. 1\nC. 2\nD. 4", "answer": "C"}
{"question": "一个长方形的长是10厘米,宽是5厘米,它的周长是多少厘米?\nA. 20厘米\nB. 30厘米\nC. 40厘米\nD. 50厘米", "answer": "B"}
{"question": "下列哪个分数是最接近1的?\nA. 1/2\nB. 3/4\nC. 4/5\nD. 5/6", "answer": "D"}
{"question": "如果一个数加上10等于30,那么这个数是多少?\nA. 20\nB. 21\nC. 22\nD. 23", "answer": "A"}
{"question": "下列哪个算式的结果最大?\nA. 3 + 4\nB. 5 - 2\nC. 6 * 2\nD. 7 ÷ 2", "answer": "C"}
{"question": "一个班级有24个学生,如果每个学生都带了2本书,那么总共有多少本书?\nA. 48本\nB. 36本\nC. 24本\nD. 12本", "answer": "A"}
{"question": "下列哪个是正确的乘法口诀?\nA. 三三得七\nB. 四四十六\nC. 五五二十五\nD. 六六三十六", "answer": "B"}
{"question": "如果一个数是另一个数的3倍,并且这个数是15,那么另一个数是多少?\nA. 5\nB. 10\nC. 15\nD. 45", "answer": "A"}
{"question": "下列哪个图形的周长最长?\nA. 正方形\nB. 长方形\nC. 圆形\nD. 三角形", "answer": "C"}
{
"question": "If Xiao Ming has 5 apples, and he gives 3 to Xiao Hua, how many apples does Xiao Ming have left?\nA. 2\nB. 3\nC. 4\nD. 5",
"answer": "A"
}
{
"question": "Which of the following numbers is the smallest prime number?\nA. 0\nB. 1\nC. 2\nD. 4",
"answer": "C"
}
{
"question": "A rectangle has a length of 10 centimeters and a width of 5 centimeters, what is its perimeter in centimeters?\nA. 20 centimeters\nB. 30 centimeters\nC. 40 centimeters\nD. 50 centimeters",
"answer": "B"
}
{
"question": "Which of the following fractions is closest to 1?\nA. 1/2\nB. 3/4\nC. 4/5\nD. 5/6",
"answer": "D"
}
{
"question": "If a number plus 10 equals 30, what is the number?\nA. 20\nB. 21\nC. 22\nD. 23",
"answer": "A"
}
{
"question": "Which of the following expressions has the largest result?\nA. 3 + 4\nB. 5 - 2\nC. 6 * 2\nD. 7 ÷ 2",
"answer": "C"
}
{
"question": "A class has 24 students, and if each student brings 2 books, how many books are there in total?\nA. 48\nB. 36\nC. 24\nD. 12",
"answer": "A"
}
{
"question": "Which of the following is the correct multiplication rhyme?\nA. Three threes are seven\nB. Four fours are sixteen\nC. Five fives are twenty-five\nD. Six sixes are thirty-six",
"answer": "B"
}
{
"question": "If one number is three times another number, and this number is 15, what is the other number?\nA. 5\nB. 10\nC. 15\nD. 45",
"answer": "A"
}
{
"question": "Which of the following shapes has the longest perimeter?\nA. Square\nB. Rectangle\nC. Circle\nD. Triangle",
"answer": "C"
}
```

### Prepare Environment
Expand Down
7 changes: 1 addition & 6 deletions examples/llm_simple_qa/testenv/acc.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,13 @@
__all__ = ["acc"]

def get_last_letter(input_string):
# 检查输入是否为空或只包含非字母字符
if not input_string or not any(char.isalpha() for char in input_string):
return None

# 倒序遍历字符串,找到最后一个字母
for char in reversed(input_string):
if 'A' <= char <= 'D':
return char

# 如果没有找到字母,返回None

return None


Expand All @@ -36,10 +33,8 @@ def acc(y_true, y_pred):
print(y_true)
print(y_pred)

# 使用列表推导来比较两个列表中的元素是否相同
same_elements = [y_pred[i] == y_true[i] for i in range(len(y_pred))]

# 计算相同元素的数量
acc = sum(same_elements) / len(same_elements)

return acc

0 comments on commit 3f69cdf

Please sign in to comment.