Update README.md

README.md

@@ -83,3 +83,115 @@ the output:
 - SyntheticWhichIsGreater5k
   - 機械的に合成した、二つの小数のどちらが大きいかを回答する問題 5,000問
  
+下の二つのデータは、[math_problem.py](https://huggingface.co/p1atdev/llm-jp-3-3.7b-instruct2-R27/blob/main/math_problem.py) の関数を使って以下のように合成しました。
+
+```py
+def generate_int_problem(
+    num_generation: int,
+    max_int: int,
+    min_int: int,
+    max_terms: int,
+):
+    for i in range(num_generation):
+        text, tex, result = create_integer_arithmetic_problem(
+            max_val=max_int,
+            min_val=min_int,
+            max_terms=max_terms,
+        )
+        formula = random.choice([text, tex])
+        templates = [
+            "{formula} = ?",
+            "{formula} を計算してください。",
+            "次の式を計算し、計算結果を解答してください。\n{formula}",
+            "計算して\n{formula}",
+            "次の式を計算してください。\n{formula}",
+            "次の式の答えは何ですか？\n{formula}",
+            "? に当てはまる数字を答えてください。\n{formula}",
+            "{formula}\n計算して",
+            "{formula}\n↑の答えを求めてください。",
+        ]
+        instruction = random.choice(templates).format(formula=formula)
+
+        yield {
+            "ground_truth": str(result),
+            "instruction": instruction,
+            "source": "synthetic_int_problem",
+            "answer_dtype": "int",
+            "skip_check": False,
+        }
+
+def generate_wig_problem(
+    num_generation: int,
+    max_num: float,
+    min_num: float,
+    precision: int,
+):
+    for i in range(num_generation):
+        num_1, num_2, greater = create_two_decimals(
+            min_val=min_num,
+            max_val=max_num,
+            precision=precision,
+        )
+        templates = [
+            "次の数字のうち、どちらが大きいですか？\n{num_1}\n{num_2}",
+            "{num_1} と {num_2} のうちどちらが大きいですか？",
+            "{num_1} と {num_2} はどっちが大きい？",
+            "大きいほうを選んで: {num_1} {num_2}",
+            "次の数値を比較し、大きい方を選んでください。\n{num_1} {num_2}",
+        ]
+        instruction = random.choice(templates).format(num_1=num_1, num_2=num_2)
+
+        yield {
+            "ground_truth": str(greater),
+            "instruction": instruction,
+            "source": "synthetic_which_is_greater",
+            "answer_dtype": "float",
+            "skip_check": False,
+        }
+
+
+# generate dataset
+ds_easy_int = Dataset.from_generator(
+    generate_int_problem,
+    gen_kwargs={
+        "num_generation": 5000,
+        "max_int": 10,
+        "min_int": -10,
+        "max_terms": 5,
+    },
+)
+assert isinstance(ds_easy_int, Dataset)
+print("easy_int:", ds_easy_int)
+
+ds_wig = Dataset.from_generator(
+    generate_wig_problem,
+    gen_kwargs={
+        "num_generation": 5000,
+        "max_num": 100.0,
+        "min_num": -100.0,
+        "precision": 3,
+    },
+)
+assert isinstance(ds_wig, Dataset)
+print("wig:", ds_wig)
+
+# japanese gsm8k
+ds_gsm8k = load_dataset("p1atdev/gsm8k-ja-slim", split="train")
+assert isinstance(ds_gsm8k, Dataset)
+ds_gsm8k = ds_gsm8k.filter(filter_gsm8k_ja, batched=True)
+ds_gsm8k = ds_gsm8k.map(
+    map_gsm8k_ja_instruction, remove_columns=ds_gsm8k.column_names
+)
+print("gsm8k:", ds_gsm8k)
+
+# concat
+ds = concatenate_datasets( # from datasets import concatenate_datasets
+    [
+        ds_gsm8k,
+        ds_easy_int,
+        ds_wig,
+    ]
+)
+print("total:", ds)
+```
+