Spaces:
Running
Running
add tests file
Browse files
tests.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from evaluate import load
|
| 3 |
+
|
| 4 |
+
solution_sample1 = json.load(open("test_examples/solutions_problem_1.json", "r"))
|
| 5 |
+
solution_sample2 = json.load(open("test_examples/solutions_problem_2.json", "r"))
|
| 6 |
+
single_solutions = [solution_sample1[:1], solution_sample2[:1]]
|
| 7 |
+
multiple_solutions = [solution_sample1[:3], solution_sample2[:3]]
|
| 8 |
+
|
| 9 |
+
metric = load("loubnabnl/apps_metric")
|
| 10 |
+
result_1 = metric.compute(predictions=single_solutions, level="all")
|
| 11 |
+
result_2 = metric.compute(predictions=multiple_solutions, level="all", k_list=[1, 2, 3])
|
| 12 |
+
|
| 13 |
+
assert result_1 == {'avg_accuracy': 1.0, 'strict_accuracy': 1.0, 'pass_at_k': None}
|
| 14 |
+
assert result_2 == {'avg_accuracy': None, 'strict_accuracy': None, 'pass_at_k': {'pass@1': 1.0, 'pass@2': 1.0, 'pass@3': 1.0}}
|