initial commit

README.md

@@ -0,0 +1,70 @@
+---
+language: tr
+datasets:
+- tquad1
+- tquad2
+- xquad 
+tags: 
+- answer-extraction
+- question-answering
+- question-generation
+- text-generation
+- text2text-generation
+license: cc-by-4.0
+---
+
+# mt5-base for Turkish Question Generation
+Automated question generation and question answering using text-to-text transformers by OBSS AI.
+
+```python
+from core.api import GenerationAPI
+generation_api = GenerationAPI(model_url_or_path='mt5-base-3task-highlight-combined3')
+```
+
+## Overview
+**Language model:** mt5-base 
+**Language:** Turkish
+**Downstream-task:** Extractive QA/QG, Answer Extraction
+**Training data:** TQuADv2-train
+**Code:**  https://github.com/obss/turkish-question-generation
+**Paper:**  https://arxiv.org/abs/2111.06476
+
+## Hyperparameters
+```
+batch_size = 256
+n_epochs = 15
+base_LM_model = "mt5-base"
+max_source_length = 512
+max_target_length = 64
+learning_rate = 1.0e-3
+task_lisst = ["qa", "qg", "ans_ext"]
+qg_format = "highlight"
+``` 
+
+## Performance
+Refer to [paper](https://arxiv.org/abs/2111.06476).
+
+## Usage 🔥
+```python
+from core.api import GenerationAPI
+generation_api = GenerationAPI('mt5-base-3task-highlight-combined3')
+
+context = """
+Bu modelin eğitiminde, Türkçe soru cevap verileri kullanılmıştır.
+Paylaşılan model kullanılarak, Türkçe metinlerden otomatik olarak soru ve cevap
+üretilebilir. Bu proje ile paylaşılan kaynak kodu ile Türkçe Soru Üretme
+/ Soru Cevaplama konularında yeni akademik çalışmalar yapılabilir.
+Projenin detaylarına paylaşılan Github ve Arxiv linklerinden ulaşılabilir.
+"""
+
+# a) Fully Automated Question Generation
+generation_api(task='question-generation', context=context)
+
+# b) Question Answering
+question = "Bu model ne işe yarar?"
+generation_api(task='question-answering', context=context, question=question)
+
+# b) Answer Extraction
+generation_api(task='answer-extraction', context=context)
+```
+

added_tokens.json

@@ -0,0 +1 @@
+{"<sep>": 250100, "<hl>": 250101}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "google/mt5-base",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.10.0.dev0",
+  "use_cache": true,
+  "vocab_size": 250102
+}

experiment_config.yaml

@@ -0,0 +1,107 @@
+_n_gpu: 1
+adafactor: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+cache_dir: null
+dataloader_drop_last: false
+dataloader_num_workers: 0
+dataloader_pin_memory: true
+ddp_find_unused_parameters: null
+debug: []
+deepspeed: null
+disable_tqdm: false
+do_eval: true
+do_predict: false
+do_train: true
+eval_accumulation_steps: 1
+eval_dataset_list:
+- tquad2-valid
+- xquad.tr
+eval_steps: 300
+evaluation_strategy: &id001 !!python/object/apply:transformers.trainer_utils.IntervalStrategy
+- steps
+fp16: false
+fp16_backend: auto
+fp16_full_eval: false
+fp16_opt_level: O1
+freeze_embeddings: false
+gradient_accumulation_steps: 8
+greater_is_better: null
+group_by_length: false
+ignore_data_skip: false
+label_names: null
+label_smoothing_factor: 0
+learning_rate: 0.001
+length_column_name: length
+load_best_model_at_end: false
+local_rank: -1
+log_level: -1
+log_level_replica: -1
+log_on_each_node: true
+logging_dir: runs/mt5-base/mt5base-3task-highlight-combined3/runs/Dec02_01-35-48_palamut3.yonetim
+logging_first_step: false
+logging_steps: 500
+logging_strategy: *id001
+lr_scheduler_type: !!python/object/apply:transformers.trainer_utils.SchedulerType
+- linear
+max_grad_norm: 1.0
+max_source_length: 512
+max_steps: -1
+max_target_length: 64
+metric_for_best_model: null
+model_name_or_path: google/mt5-base
+model_type: mt5
+mp_parameters: ''
+mt5_qg_format: highlight
+mt5_task_list:
+- qa
+- qg
+- ans_ext
+neptune_api_token: null
+neptune_project: null
+neptune_run: null
+no_cuda: false
+num_train_epochs: 15
+output_dir: runs/mt5-base/mt5base-3task-highlight-combined3
+overwrite_output_dir: false
+past_index: -1
+per_device_eval_batch_size: 32
+per_device_train_batch_size: 32
+per_gpu_eval_batch_size: null
+per_gpu_train_batch_size: null
+prediction_loss_only: false
+prepare_data: true
+push_to_hub: false
+push_to_hub_model_id: mt5base-3task-highlight-combined3
+push_to_hub_organization: null
+push_to_hub_token: null
+remove_unused_columns: false
+report_to:
+- wandb
+resume_from_checkpoint: null
+run_name: mt5base-3task-highlight-combined3
+save_on_each_node: false
+save_steps: 500
+save_strategy: *id001
+save_total_limit: 1
+seed: 42
+sharded_ddp: []
+skip_memory_metrics: true
+tokenizer_path: tokenizers/mt5-base
+tpu_metrics_debug: false
+tpu_num_cores: null
+train_dataset_list:
+- tquad2-train
+- tquad2-valid
+- xquad.tr
+train_file_path: data/train_data.pt
+use_legacy_prediction_loop: false
+valid_dataset_list:
+- tquad2-valid
+valid_file_path: data/valid_data.pt
+wandb_id: null
+wandb_project: turkish-qa-qg
+warmup_ratio: 0.0
+warmup_steps: 0
+weight_decay: 0.0

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6294e5764270f9fabd04bc143eba0a9021ddbaea8d53e5965b98d3dd215c86c
+size 2329645913

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": null, "special_tokens_map_file": "/home/patrick/.cache/torch/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276", "name_or_path": "tokenizers/mt5-base", "sp_model_kwargs": {}, "tokenizer_class": "T5Tokenizer"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e70335cd934710e27d3c852faf3390b1a97b781da155fe2485387db8dfad1a6
+size 3119