From 33a3809df26cb1191bd2198d1217e0d9a139c836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Svanhv=C3=ADt?= Date: Tue, 7 Nov 2023 13:42:30 +0000 Subject: [PATCH 1/2] icelandic gec eval --- .gitattributes | 1 + .../registry/data/icelandic-sentences-gec/samples.jsonl | 3 +++ evals/registry/evals/icelandic-sentences-gec.yaml | 9 +++++++++ 3 files changed, 13 insertions(+) create mode 100644 evals/registry/data/icelandic-sentences-gec/samples.jsonl create mode 100644 evals/registry/evals/icelandic-sentences-gec.yaml diff --git a/.gitattributes b/.gitattributes index 9f45a92b72..c55e5547d6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ evals/registry/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text +evals/registry/data/icelandic-sentences-gec/samples.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/evals/registry/data/icelandic-sentences-gec/samples.jsonl b/evals/registry/data/icelandic-sentences-gec/samples.jsonl new file mode 100644 index 0000000000..d9d5097a3f --- /dev/null +++ b/evals/registry/data/icelandic-sentences-gec/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e8f63f51c5d5e80f48a8bfe4a4785decbb915174a9ea02763aec66af5f4c189 +size 126174 diff --git a/evals/registry/evals/icelandic-sentences-gec.yaml b/evals/registry/evals/icelandic-sentences-gec.yaml new file mode 100644 index 0000000000..b0300c1ebb --- /dev/null +++ b/evals/registry/evals/icelandic-sentences-gec.yaml @@ -0,0 +1,9 @@ +icelandic-sentences-gec: + id: icelandic-sentences-gec.dev.v0 + description: Test the model's ability to correctly identify whether an Icelandic sentence contains a grammatical error. + metrics: [accuracy] + +icelandic-sentences-gec.dev.v0: + class: evals.elsuite.basic.match:Match + args: + samples_jsonl: icelandic-sentences-gec/samples.jsonl From 948576a5965d8793baa6c6334e66241f2dbb6030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Svanhv=C3=ADt=20Lilja=20Ing=C3=B3lfsd=C3=B3ttir?= <24214261+svanhvitlilja@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:16:09 +0000 Subject: [PATCH 2/2] Update .gitattributes --- .gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index c55e5547d6..9f45a92b72 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ evals/registry/data/**/*.jsonl filter=lfs diff=lfs merge=lfs -text -evals/registry/data/icelandic-sentences-gec/samples.jsonl filter=lfs diff=lfs merge=lfs -text