{ "cells": [ { "metadata": {}, "cell_type": "markdown", "source": [ "## Experiment exp020-4\n", "GBERT-Large, Batch Size 32, Warmup Steps 200, Learning Rate 2e-5" ], "id": "d10bfa50537af75f" }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:08.946184Z", "start_time": "2025-06-25T21:35:47.758549Z" } }, "cell_type": "code", "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.model_selection import train_test_split\n", "from transformers import (\n", " AutoTokenizer,\n", " BertForTokenClassification,\n", "\n", ")\n", "import torch\n", "import os\n", "\n", "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'" ], "id": "9748a35a024779ae", "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2025-06-25 23:36:01.712205: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", "2025-06-25 23:36:01.739052: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", "E0000 00:00:1750887361.774014 1226991 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "E0000 00:00:1750887361.784804 1226991 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "W0000 00:00:1750887361.814945 1226991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", "W0000 00:00:1750887361.815027 1226991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", "W0000 00:00:1750887361.815030 1226991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", "W0000 00:00:1750887361.815032 1226991 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", "2025-06-25 23:36:01.826253: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" ] } ], "execution_count": 1 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:09.575772Z", "start_time": "2025-06-25T21:36:09.530797Z" } }, "cell_type": "code", "source": [ "class SpanClassifierWithStrictF1:\n", " def __init__(self, model_name=\"deepset/gbert-base\"):\n", " self.model_name = model_name\n", " self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "\n", " self.labels =[\n", " \"O\",\n", " \"B-positive feedback\", \"B-compliment\", \"B-affection declaration\", \"B-encouragement\", \"B-gratitude\", \"B-agreement\", \"B-ambiguous\", \"B-implicit\", \"B-group membership\", \"B-sympathy\",\n", " \"I-positive feedback\", \"I-compliment\", \"I-affection declaration\", \"I-encouragement\", \"I-gratitude\", \"I-agreement\", \"I-ambiguous\", \"I-implicit\", \"I-group membership\", \"I-sympathy\"\n", " ]\n", " self.label2id = {label: i for i, label in enumerate(self.labels)}\n", " self.id2label = {i: label for i, label in enumerate(self.labels)}\n", "\n", " def create_dataset(self, comments_df, spans_df):\n", " \"\"\"Erstelle Dataset mit BIO-Labels und speichere Evaluation-Daten\"\"\"\n", " examples = []\n", " eval_data = [] # Für Strict F1 Berechnung\n", "\n", " spans_grouped = spans_df.groupby(['document', 'comment_id'])\n", "\n", " for _, row in comments_df.iterrows():\n", " text = row['comment']\n", " document = row['document']\n", " comment_id = row['comment_id']\n", " key = (document, comment_id)\n", "\n", " # True spans für diesen Kommentar\n", " if key in spans_grouped.groups:\n", " true_spans = [(span_type, int(start), int(end))\n", " for span_type, start, end in\n", " spans_grouped.get_group(key)[['type', 'start', 'end']].values]\n", " else:\n", " true_spans = []\n", "\n", " # Tokenisierung\n", " tokenized = self.tokenizer(text, truncation=True, max_length=512,\n", " return_offsets_mapping=True)\n", "\n", " # BIO-Labels erstellen\n", " labels = self._create_bio_labels(tokenized['offset_mapping'],\n", " spans_grouped.get_group(key)[['start', 'end', 'type']].values\n", " if key in spans_grouped.groups else [])\n", "\n", " examples.append({\n", " 'input_ids': tokenized['input_ids'],\n", " 'attention_mask': tokenized['attention_mask'],\n", " 'labels': labels\n", " })\n", "\n", " # Evaluation-Daten speichern\n", " eval_data.append({\n", " 'text': text,\n", " 'offset_mapping': tokenized['offset_mapping'],\n", " 'true_spans': true_spans,\n", " 'document': document,\n", " 'comment_id': comment_id\n", " })\n", "\n", " return examples, eval_data\n", "\n", " def _create_bio_labels(self, offset_mapping, spans):\n", " \"\"\"Erstelle BIO-Labels für Tokens\"\"\"\n", " labels = [0] * len(offset_mapping) # 0 = \"O\"\n", "\n", " for start, end, type_label in spans:\n", " for i, (token_start, token_end) in enumerate(offset_mapping):\n", " if token_start is None: # Spezielle Tokens\n", " continue\n", "\n", " # Token überlappt mit Span\n", " if token_start < end and token_end > start:\n", " if token_start <= start:\n", " labels[i] = self.label2id[f'B-{type_label}'] # B-compliment\n", " else:\n", " labels[i] = self.label2id[f'I-{type_label}'] # I-compliment\n", "\n", " return labels\n", "\n", " def compute_metrics(self, eval_pred):\n", " \"\"\"Berechne Strict F1 für Trainer\"\"\"\n", " predictions, labels = eval_pred\n", " predictions = np.argmax(predictions, axis=2)\n", "\n", " # Konvertiere Vorhersagen zu Spans\n", " batch_pred_spans = []\n", " batch_true_spans = []\n", "\n", " for i, (pred_seq, label_seq) in enumerate(zip(predictions, labels)):\n", " # Evaluation-Daten für dieses Beispiel\n", " if i < len(self.current_eval_data):\n", " eval_item = self.current_eval_data[i]\n", " text = eval_item['text']\n", " offset_mapping = eval_item['offset_mapping']\n", " true_spans = eval_item['true_spans']\n", "\n", " # Filtere gültige Vorhersagen (keine Padding-Tokens)\n", " valid_predictions = []\n", " valid_offsets = []\n", "\n", " for j, (pred_label, true_label) in enumerate(zip(pred_seq, label_seq)):\n", " if true_label != -100 and j < len(offset_mapping):\n", " valid_predictions.append(pred_label)\n", " valid_offsets.append(offset_mapping[j])\n", "\n", " # Konvertiere zu Spans\n", " pred_spans = self._predictions_to_spans(valid_predictions, valid_offsets, text)\n", " pred_spans_tuples = [(span['type'], span['start'], span['end']) for span in pred_spans]\n", "\n", " batch_pred_spans.append(pred_spans_tuples)\n", " batch_true_spans.append(true_spans)\n", "\n", " # Berechne Strict F1\n", " strict_f1, strict_precision, strict_recall, tp, fp, fn = self._calculate_strict_f1(\n", " batch_true_spans, batch_pred_spans\n", " )\n", "\n", " torch.cuda.memory.empty_cache()\n", "\n", " return {\n", " \"strict_f1\": torch.tensor(strict_f1),\n", " \"strict_precision\": torch.tensor(strict_precision),\n", " \"strict_recall\": torch.tensor(strict_recall),\n", " \"true_positives\": torch.tensor(tp),\n", " \"false_positives\": torch.tensor(fp),\n", " \"false_negatives\": torch.tensor(fn)\n", " }\n", "\n", " def _calculate_strict_f1(self, true_spans_list, pred_spans_list):\n", " \"\"\"Berechne Strict F1 über alle Kommentare\"\"\"\n", " tp, fp, fn = 0, 0, 0\n", "\n", " for true_spans, pred_spans in zip(true_spans_list, pred_spans_list):\n", " # Finde exakte Matches (Typ und Span müssen übereinstimmen)\n", " matches = self._find_exact_matches(true_spans, pred_spans)\n", "\n", " tp += len(matches)\n", " fp += len(pred_spans) - len(matches)\n", " fn += len(true_spans) - len(matches)\n", "\n", " # Berechne Metriken\n", " precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0\n", " recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0\n", " f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0\n", "\n", " return f1, precision, recall, tp, fp, fn\n", "\n", " def _find_exact_matches(self, true_spans, pred_spans):\n", " \"\"\"Finde exakte Matches zwischen True und Predicted Spans\"\"\"\n", " matches = []\n", " used_pred = set()\n", "\n", " for true_span in true_spans:\n", " for i, pred_span in enumerate(pred_spans):\n", " if i not in used_pred and true_span == pred_span:\n", " matches.append((true_span, pred_span))\n", " used_pred.add(i)\n", " break\n", "\n", " return matches\n", "\n", " def _predictions_to_spans(self, predicted_labels, offset_mapping, text):\n", " \"\"\"Konvertiere Token-Vorhersagen zu Spans\"\"\"\n", " spans = []\n", " current_span = None\n", "\n", " for i, label_id in enumerate(predicted_labels):\n", " if i >= len(offset_mapping):\n", " break\n", "\n", " label = self.id2label[label_id]\n", " token_start, token_end = offset_mapping[i]\n", "\n", " if token_start is None:\n", " continue\n", "\n", " if label.startswith('B-'):\n", " if current_span:\n", " spans.append(current_span)\n", " current_span = {\n", " 'type': label[2:],\n", " 'start': token_start,\n", " 'end': token_end,\n", " 'text': text[token_start:token_end]\n", " }\n", " elif label.startswith('I-') and current_span:\n", " current_span['end'] = token_end\n", " current_span['text'] = text[current_span['start']:current_span['end']]\n", " else:\n", " if current_span:\n", " spans.append(current_span)\n", " current_span = None\n", "\n", " if current_span:\n", " spans.append(current_span)\n", "\n", " return spans\n", "\n", " def predict(self, texts):\n", " \"\"\"Vorhersage für neue Texte\"\"\"\n", " if not hasattr(self, 'model'):\n", " raise ValueError(\"Modell muss erst trainiert werden!\")\n", "\n", " predictions = []\n", " device = next(self.model.parameters()).device\n", "\n", " for text in texts:\n", " # Tokenisierung\n", " inputs = self.tokenizer(text, return_tensors=\"pt\", truncation=True,\n", " max_length=512, return_offsets_mapping=True)\n", "\n", " offset_mapping = inputs.pop('offset_mapping')\n", " inputs = {k: v.to(device) for k, v in inputs.items()}\n", "\n", " # Vorhersage\n", " with torch.no_grad():\n", " outputs = self.model(**inputs)\n", "\n", " predicted_labels = torch.argmax(outputs.logits, dim=2)[0].cpu().numpy()\n", "\n", " # Spans extrahieren\n", " spans = self._predictions_to_spans(predicted_labels, offset_mapping[0], text)\n", " predictions.append({'text': text, 'spans': spans})\n", "\n", " return predictions\n", "\n", " def evaluate_strict_f1(self, comments_df, spans_df):\n", " \"\"\"Evaluiere Strict F1 auf Test-Daten\"\"\"\n", " if not hasattr(self, 'model'):\n", " raise ValueError(\"Modell muss erst trainiert werden!\")\n", "\n", " print(\"Evaluiere Strict F1...\")\n", "\n", " # Vorhersagen für alle Kommentare\n", " texts = comments_df['comment'].tolist()\n", " predictions = self.predict(texts)\n", "\n", " # Organisiere True Spans\n", " spans_grouped = spans_df.groupby(['document', 'comment_id'])\n", " true_spans_dict = {}\n", " pred_spans_dict = {}\n", "\n", " for i, (_, row) in enumerate(comments_df.iterrows()):\n", " key = (row['document'], row['comment_id'])\n", "\n", " # True spans\n", " if key in spans_grouped.groups:\n", " true_spans = [(span_type, int(start), int(end))\n", " for span_type, start, end in\n", " spans_grouped.get_group(key)[['type', 'start', 'end']].values]\n", " else:\n", " true_spans = []\n", "\n", " # Predicted spans\n", " pred_spans = [(span['type'], span['start'], span['end'])\n", " for span in predictions[i]['spans']]\n", "\n", " true_spans_dict[key] = true_spans\n", " pred_spans_dict[key] = pred_spans\n", "\n", " # Berechne Strict F1\n", " all_true_spans = list(true_spans_dict.values())\n", " all_pred_spans = list(pred_spans_dict.values())\n", "\n", " f1, precision, recall, tp, fp, fn = self._calculate_strict_f1(all_true_spans, all_pred_spans)\n", "\n", " print(f\"\\nStrict F1 Ergebnisse:\")\n", " print(f\"Precision: {precision:.4f}\")\n", " print(f\"Recall: {recall:.4f}\")\n", " print(f\"F1-Score: {f1:.4f}\")\n", " print(f\"True Positives: {tp}, False Positives: {fp}, False Negatives: {fn}\")\n", "\n", " return {\n", " 'strict_f1': f1,\n", " 'strict_precision': precision,\n", " 'strict_recall': recall,\n", " 'true_positives': tp,\n", " 'false_positives': fp,\n", " 'false_negatives': fn\n", " }\n", "\n", "def convert_spans(row):\n", " spans = row['predicted_spans']\n", " document = row['document']\n", " comment_id = row['comment_id']\n", " return [{'document': document, 'comment_id': comment_id, 'type': span['type'], 'start': span['start'], 'end': span['end']} for span in spans]\n", "\n", "def pred_to_spans(row):\n", " predicted_labels, offset_mapping, text = row['predicted_labels'], row['offset_mapping'], row['comment']\n", " return [classifier._predictions_to_spans(predicted_labels, offset_mapping, text)]" ], "id": "aeef20e483ed583a", "outputs": [], "execution_count": 2 }, { "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2025-06-25T21:36:11.656776Z", "start_time": "2025-06-25T21:36:09.751208Z" } }, "cell_type": "code", "source": [ "comments: pd.DataFrame = pd.read_csv(\"./share-GermEval2025-data/Data/training data/comments.csv\")\n", "task1: pd.DataFrame = pd.read_csv(\"./share-GermEval2025-data/Data/training data/task1.csv\")\n", "task2: pd.DataFrame = pd.read_csv(\"./share-GermEval2025-data/Data/training data/task2.csv\")\n", "comments = comments.merge(task1, on=[\"document\", \"comment_id\"])\n", "spans_grouped = task2.groupby(['document', 'comment_id'])\n", "\n", "test_data: pd.DataFrame = pd.read_csv(\"./share-GermEval2025-data/Data/test data/comments.csv\")\n", "\n", "# check every comment that contain spans if they have overlapping start and end positions\n", "task2['overlap'] = False\n", "overlapping_spans = task2.groupby(['document', 'comment_id'])\n", "for (doc, comment), group in overlapping_spans:\n", " if len(group) > 1:\n", " starts = group['start'].tolist()\n", " ends = group['end'].tolist()\n", " for i in range(len(starts)):\n", " for j in range(i + 1, len(starts)):\n", " if not (ends[i] <= starts[j] or ends[j] <= starts[i]):\n", " task2.loc[(task2['document'] == doc) & (task2['comment_id'] == comment), 'overlap'] = True\n", "\n", "task2 = task2[task2['overlap'] == False].drop(columns=['overlap'])" ], "id": "eb861fa831fc5e66", "outputs": [], "execution_count": 3 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:11.812171Z", "start_time": "2025-06-25T21:36:11.787573Z" } }, "cell_type": "code", "source": [ "from multiset import *\n", "ALL_LABELS = [\"affection declaration\",\"agreement\",\"ambiguous\",\n", " \"compliment\",\"encouragement\",\"gratitude\",\"group membership\",\n", " \"implicit\",\"positive feedback\",\"sympathy\"]\n", "\n", "def fine_grained_flausch_by_label(gold, predicted):\n", " gold['cid']= gold['document']+\"_\"+gold['comment_id'].apply(str)\n", " predicted['cid']= predicted['document']+\"_\"+predicted['comment_id'].apply(str)\n", "\n", " # annotation sets (predicted)\n", " pred_spans = Multiset()\n", " pred_spans_loose = Multiset()\n", " pred_types = Multiset()\n", "\n", " # annotation sets (gold)\n", " gold_spans = Multiset()\n", " gold_spans_loose = Multiset()\n", " gold_types = Multiset()\n", "\n", " for row in predicted.itertuples(index=False):\n", " pred_spans.add((row.cid,row.type,row.start,row.end))\n", " pred_spans_loose.add((row.cid,row.start,row.end))\n", " pred_types.add((row.cid,row.type))\n", " for row in gold.itertuples(index=False):\n", " gold_spans.add((row.cid,row.type,row.start,row.end))\n", " gold_spans_loose.add((row.cid,row.start,row.end))\n", " gold_types.add((row.cid,row.type))\n", "\n", " # precision = true_pos / true_pos + false_pos\n", " # recall = true_pos / true_pos + false_neg\n", " # f_1 = 2 * prec * rec / (prec + rec)\n", "\n", " results = {'TOTAL': {'STRICT': {},'SPANS': {},'TYPES': {}}}\n", " # label-wise evaluation (only for strict and type)\n", " for label in ALL_LABELS:\n", " results[label] = {'STRICT': {},'TYPES': {}}\n", " gold_spans_x = set(filter(lambda x: x[1].__eq__(label), gold_spans))\n", " pred_spans_x = set(filter(lambda x: x[1].__eq__(label), pred_spans))\n", " gold_types_x = set(filter(lambda x: x[1].__eq__(label), gold_types))\n", " pred_types_x = set(filter(lambda x: x[1].__eq__(label), pred_types))\n", "\n", " # strict: spans + type must match\n", " ### NOTE: x and y / x returns 0 if x = 0 and y/x otherwise (test for zero division)\n", " strict_p = float(len(pred_spans_x)) and float( len(gold_spans_x.intersection(pred_spans_x))) / len(pred_spans_x)\n", " strict_r = float(len(gold_spans_x)) and float( len(gold_spans_x.intersection(pred_spans_x))) / len(gold_spans_x)\n", " strict_f = (strict_p + strict_r) and 2 * strict_p * strict_r / (strict_p + strict_r)\n", " results[label]['STRICT']['prec'] = strict_p\n", " results[label]['STRICT']['rec'] = strict_r\n", " results[label]['STRICT']['f1'] = strict_f\n", "\n", " # detection mode: only types must match (per post)\n", " types_p = float(len(pred_types_x)) and float( len(gold_types_x.intersection(pred_types_x))) / len(pred_types_x)\n", " types_r = float(len(gold_types_x)) and float( len(gold_types_x.intersection(pred_types_x))) / len(gold_types_x)\n", " types_f = (types_p + types_r) and 2 * types_p * types_r / (types_p + types_r)\n", " results[label]['TYPES']['prec'] = types_p\n", " results[label]['TYPES']['rec'] = types_r\n", " results[label]['TYPES']['f1'] = types_f\n", "\n", " # Overall evaluation\n", " # strict: spans + type must match\n", " strict_p = float(len(pred_spans)) and float( len(gold_spans.intersection(pred_spans))) / len(pred_spans)\n", " strict_r = float(len(gold_spans)) and float( len(gold_spans.intersection(pred_spans))) / len(gold_spans)\n", " strict_f = (strict_p + strict_r) and 2 * strict_p * strict_r / (strict_p + strict_r)\n", " results['TOTAL']['STRICT']['prec'] = strict_p\n", " results['TOTAL']['STRICT']['rec'] = strict_r\n", " results['TOTAL']['STRICT']['f1'] = strict_f\n", "\n", " # spans: spans must match\n", " spans_p = float(len(pred_spans_loose)) and float( len(gold_spans_loose.intersection(pred_spans_loose))) / len(pred_spans_loose)\n", " spans_r = float(len(gold_spans_loose)) and float( len(gold_spans_loose.intersection(pred_spans_loose))) / len(gold_spans_loose)\n", " spans_f = (spans_p + spans_r) and 2 * spans_p * spans_r / (spans_p + spans_r)\n", " results['TOTAL']['SPANS']['prec'] = spans_p\n", " results['TOTAL']['SPANS']['rec'] = spans_r\n", " results['TOTAL']['SPANS']['f1'] = spans_f\n", "\n", " # detection mode: only types must match (per post)\n", " types_p = float(len(pred_types)) and float( len(gold_types.intersection(pred_types))) / len(pred_types)\n", " types_r = float(len(gold_types)) and float( len(gold_types.intersection(pred_types))) / len(gold_types)\n", " types_f = (types_p + types_r) and 2 * types_p * types_r / (types_p + types_r)\n", " results['TOTAL']['TYPES']['prec'] = types_p\n", " results['TOTAL']['TYPES']['rec'] = types_r\n", " results['TOTAL']['TYPES']['f1'] = types_f\n", "\n", "# print(\"STRICT:\\n \",strict_p,strict_r,strict_f)\n", "# print(\"SPANS:\\n \",spans_p,spans_r,spans_f)\n", "# print(\"TYPES:\\n \",types_p,types_r,types_f)\n", " return(results)" ], "id": "c8d1b091bbe9684c", "outputs": [], "execution_count": 4 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:12.419539Z", "start_time": "2025-06-25T21:36:11.890848Z" } }, "cell_type": "code", "source": "classifier = SpanClassifierWithStrictF1('deepset/gbert-large')", "id": "da0c571795f194fe", "outputs": [], "execution_count": 5 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:28.330177Z", "start_time": "2025-06-25T21:36:12.505718Z" } }, "cell_type": "code", "source": [ "# Dataset neu erstellen für diesen Fold\n", "examples, eval_data = classifier.create_dataset(comments, task2)\n", "train_examples, val_examples = train_test_split(examples, test_size=0.1, random_state=42)\n", "\n", "# Evaluation-Daten entsprechend aufteilen\n", "train_indices, val_indices = train_test_split(range(len(examples)), test_size=0.1, random_state=42)" ], "id": "e66b73a1f8c2f5d5", "outputs": [], "execution_count": 6 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:48.985663Z", "start_time": "2025-06-25T21:36:44.069484Z" } }, "cell_type": "code", "source": [ "classifier.model = BertForTokenClassification.from_pretrained(\n", " 'deepset/gbert-large',\n", " num_labels=len(classifier.labels),\n", " id2label=classifier.id2label,\n", " label2id=classifier.label2id\n", ")\n", "classifier.model.load_state_dict(torch.load('./experiments/exp020/exp020-4-final3_model.pth'))\n", "classifier.model.eval()" ], "id": "c9fa9db4db077569", "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of BertForTokenClassification were not initialized from the model checkpoint at deepset/gbert-large and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] }, { "data": { "text/plain": [ "BertForTokenClassification(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(31102, 1024, padding_idx=0)\n", " (position_embeddings): Embedding(512, 1024)\n", " (token_type_embeddings): Embedding(2, 1024)\n", " (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0-23): 24 x BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSdpaSelfAttention(\n", " (query): Linear(in_features=1024, out_features=1024, bias=True)\n", " (key): Linear(in_features=1024, out_features=1024, bias=True)\n", " (value): Linear(in_features=1024, out_features=1024, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", " (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=1024, out_features=4096, bias=True)\n", " (intermediate_act_fn): GELUActivation()\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=4096, out_features=1024, bias=True)\n", " (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (classifier): Linear(in_features=1024, out_features=21, bias=True)\n", ")" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 8 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:51.516583Z", "start_time": "2025-06-25T21:36:51.501571Z" } }, "cell_type": "code", "source": "len(comments.iloc[val_indices].comment.tolist())", "id": "b329419ee6bef88f", "outputs": [ { "data": { "text/plain": [ "3706" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 9 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:36:52.207348Z", "start_time": "2025-06-25T21:36:52.196304Z" } }, "cell_type": "code", "source": [ "comments_val = comments.iloc[val_indices].copy()\n", "comments_val.reset_index(drop=True, inplace=True)" ], "id": "ee60bf8540dd6bc5", "outputs": [], "execution_count": 10 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:49:53.580160Z", "start_time": "2025-06-25T21:36:52.765429Z" } }, "cell_type": "code", "source": [ "test_comments = comments_val.copy()\n", "\n", "comments_val['gold_spans'] = None\n", "comments_val['predicted_labels'] = None\n", "comments_val['predicted_probs'] = None\n", "comments_val['offset_mapping'] = None\n", "comments_val['text_tokens'] = None\n", "\n", "for idx in range(len(comments_val)): #range(15):\n", " row = comments_val.iloc[idx]\n", " text = row['comment']\n", " key = (row['document'], row['comment_id'])\n", "\n", " text_tokens = classifier.tokenizer.tokenize(text)\n", " comments_val.at[idx, 'text_tokens'] = text_tokens\n", "\n", " device = next(classifier.model.parameters()).device\n", " inputs = classifier.tokenizer(text, return_tensors=\"pt\", truncation=True, max_length=512, return_offsets_mapping=True)\n", "\n", " offset_mapping = inputs.pop('offset_mapping')\n", " comments_val.at[idx, 'offset_mapping'] = offset_mapping.cpu().numpy()[0].tolist()\n", " inputs = {k: v.to(device) for k, v in inputs.items()}\n", "\n", " # Vorhersage\n", " with torch.no_grad():\n", " outputs = classifier.model(**inputs)\n", "\n", " predicted_labels = torch.argmax(outputs.logits, dim=2)[0].cpu().numpy()\n", " predicted_probs = torch.nn.functional.softmax(outputs.logits, dim=2)[0].cpu().numpy()\n", " comments_val.at[idx, 'predicted_labels'] = predicted_labels\n", " comments_val.at[idx, 'predicted_probs'] = predicted_probs\n", "\n", " if key not in spans_grouped.groups:\n", " comments_val.at[idx, 'gold_spans'] = []\n", " pass\n", " else:\n", " spans = spans_grouped.get_group(key).to_dict(orient='records')\n", " comments_val.at[idx, 'gold_spans'] = spans\n", "\n", "# or simply predict like this witout probabilities:\n", "# val_set_predictions = classifier.predict(comments_val.comment.tolist())" ], "id": "a151edc97f745098", "outputs": [], "execution_count": 11 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:49:58.863353Z", "start_time": "2025-06-25T21:49:58.516629Z" } }, "cell_type": "code", "source": [ "comments_val['predicted_spans'] = comments_val.apply(pred_to_spans, axis=1, result_type='expand')\n", "\n", "test_gold_spans = pd.DataFrame((comments_val['gold_spans'].explode().dropna().tolist()))\n", "test_baseline_spans = pd.DataFrame(comments_val.apply(convert_spans, axis=1).explode().dropna().tolist())\n", "print(f\"F1 on ES data before postprocessing {fine_grained_flausch_by_label(test_gold_spans, test_baseline_spans)['TOTAL']['STRICT']}\")" ], "id": "6fdc40f482ad548a", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "F1 on ES data before postprocessing {'prec': 0.7572335920959774, 'rec': 0.7073170731707317, 'f1': 0.7314246762099522}\n" ] } ], "execution_count": 12 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:50:05.717970Z", "start_time": "2025-06-25T21:50:05.546569Z" } }, "cell_type": "code", "source": [ "def build_spans_from_classification(tokens, classification, offset_mapping):\n", " \"\"\"Modified version to work with string tokens and offset mapping\"\"\"\n", " res = []\n", " searching_end = False\n", " temp_res = []\n", " trunc_count = 0\n", " skip_count = 0\n", "\n", " for i, el in enumerate(classification):\n", " # Skip special tokens like CLS, SEP\n", " if i >= len(offset_mapping) or offset_mapping[i][0] is None:\n", " continue\n", "\n", " token_start, token_end = offset_mapping[i]\n", "\n", " if el == 'O' and searching_end is True:\n", " if i != 0 and i != len(tokens) - 1 and tokens[i].startswith('##'):\n", " continue\n", " if i > 0:\n", " prev_end = offset_mapping[i-1][1]\n", " temp_res[1] = prev_end\n", " else:\n", " temp_res[1] = -1\n", " res.append(temp_res)\n", " searching_end = False\n", "\n", " elif el.startswith('B-'):\n", " if i != 0 and i != len(tokens) - 1 and tokens[i].startswith('##'):\n", " continue\n", " if searching_end is True:\n", " if i > 0:\n", " prev_end = offset_mapping[i-1][1]\n", " temp_res[1] = prev_end\n", " else:\n", " temp_res[1] = -1\n", " res.append(temp_res)\n", " trunc_count += 1\n", " split = el.split('-', 1)\n", " label_type = split[1]\n", " temp_res = [token_start, -1, label_type, \"\"] # Changed structure\n", " searching_end = True\n", "\n", " elif el.startswith('I-'):\n", " if searching_end is True and i != 0 and i != len(tokens) - 1 and tokens[i].startswith('##'):\n", " continue\n", " split = el.split('-', 1)\n", " label_type = split[1]\n", " if searching_end is True and label_type != temp_res[2]:\n", " if i > 0:\n", " prev_end = offset_mapping[i-1][1]\n", " temp_res[1] = prev_end\n", " else:\n", " temp_res[1] = -1\n", " res.append(temp_res)\n", " searching_end = False\n", " trunc_count += 1\n", " elif searching_end is False:\n", " skip_count += 1\n", "\n", " if searching_end is True and len(offset_mapping) > 1:\n", " temp_res[1] = offset_mapping[-1][1]\n", " res.append(temp_res)\n", "\n", " return res, skip_count, trunc_count\n", "\n", "def apply_span_classification(row):\n", " \"\"\"Apply classification to the tokens and return spans.\"\"\"\n", " tokens = ['[CLS]', *row['text_tokens'], '[SEP]']\n", " classification = row['predicted_labels']\n", " offset_mapping = row['offset_mapping']\n", "\n", " # Convert classification to BIO format\n", " bio_labels = [classifier.id2label[label] for label in classification]\n", "\n", " print(len(tokens), len(bio_labels), len(offset_mapping))\n", " spans, skip_count, trunc_count = build_spans_from_classification(tokens, bio_labels, offset_mapping)\n", "\n", " document = row['document']\n", " comment_id = row['comment_id']\n", "\n", " return [{'document': document, 'comment_id': comment_id, 'type': span[2], 'start': span[0], 'end': span[1]}\n", " for span in spans if span[0] != -1 and span[1] != -1]\n", "\n", "ge2017_rules_test_pred_spans = pd.DataFrame(comments_val.apply(apply_span_classification, axis=1).explode().dropna().tolist())" ], "id": "594bfe12f7106015", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "8 8 8\n", "17 17 17\n", "12 12 12\n", "51 51 51\n", "32 32 32\n", "26 26 26\n", "11 11 11\n", "57 57 57\n", "18 18 18\n", "61 61 61\n", "93 93 93\n", "3 3 3\n", "4 4 4\n", "12 12 12\n", "17 17 17\n", "14 14 14\n", "5 5 5\n", "15 15 15\n", "54 54 54\n", "3 3 3\n", "8 8 8\n", "26 26 26\n", "8 8 8\n", "8 8 8\n", "25 25 25\n", "8 8 8\n", "14 14 14\n", "25 25 25\n", "4 4 4\n", "47 47 47\n", "16 16 16\n", "7 7 7\n", "10 10 10\n", "16 16 16\n", "23 23 23\n", "4 4 4\n", "46 46 46\n", "4 4 4\n", "16 16 16\n", "8 8 8\n", "11 11 11\n", "6 6 6\n", "7 7 7\n", "21 21 21\n", "13 13 13\n", "51 51 51\n", "20 20 20\n", "35 35 35\n", "20 20 20\n", "15 15 15\n", "16 16 16\n", "13 13 13\n", "28 28 28\n", "11 11 11\n", "12 12 12\n", "87 87 87\n", "7 7 7\n", "8 8 8\n", "26 26 26\n", "7 7 7\n", "40 40 40\n", "6 6 6\n", "58 58 58\n", "11 11 11\n", "4 4 4\n", "6 6 6\n", "30 30 30\n", "19 19 19\n", "54 54 54\n", "36 36 36\n", "6 6 6\n", "12 12 12\n", "24 24 24\n", "4 4 4\n", "11 11 11\n", "7 7 7\n", "9 9 9\n", "17 17 17\n", "33 33 33\n", "22 22 22\n", "9 9 9\n", "23 23 23\n", "17 17 17\n", "9 9 9\n", "6 6 6\n", "14 14 14\n", "137 137 137\n", "10 10 10\n", "6 6 6\n", "10 10 10\n", "6 6 6\n", "8 8 8\n", "9 9 9\n", "19 19 19\n", "9 9 9\n", "4 4 4\n", "31 31 31\n", "10 10 10\n", "8 8 8\n", "6 6 6\n", "24 24 24\n", "4 4 4\n", "11 11 11\n", "4 4 4\n", "34 34 34\n", "9 9 9\n", "21 21 21\n", "5 5 5\n", "10 10 10\n", "16 16 16\n", "8 8 8\n", "8 8 8\n", "50 50 50\n", "18 18 18\n", "5 5 5\n", "7 7 7\n", "4 4 4\n", "5 5 5\n", "8 8 8\n", "9 9 9\n", "19 19 19\n", "23 23 23\n", "6 6 6\n", "3 3 3\n", "22 22 22\n", "15 15 15\n", "10 10 10\n", "5 5 5\n", "49 49 49\n", "31 31 31\n", "8 8 8\n", "27 27 27\n", "29 29 29\n", "13 13 13\n", "3 3 3\n", "33 33 33\n", "21 21 21\n", "3 3 3\n", "16 16 16\n", "18 18 18\n", "93 93 93\n", "14 14 14\n", "6 6 6\n", "96 96 96\n", "6 6 6\n", "18 18 18\n", "11 11 11\n", "21 21 21\n", "9 9 9\n", "19 19 19\n", "10 10 10\n", "4 4 4\n", "19 19 19\n", "4 4 4\n", "6 6 6\n", "18 18 18\n", "11 11 11\n", "6 6 6\n", "9 9 9\n", "50 50 50\n", "45 45 45\n", "4 4 4\n", "20 20 20\n", "8 8 8\n", "10 10 10\n", "59 59 59\n", "24 24 24\n", "4 4 4\n", "6 6 6\n", "8 8 8\n", "55 55 55\n", "32 32 32\n", "15 15 15\n", "4 4 4\n", "10 10 10\n", "9 9 9\n", "37 37 37\n", "29 29 29\n", "18 18 18\n", "4 4 4\n", "67 67 67\n", "14 14 14\n", "10 10 10\n", "10 10 10\n", "8 8 8\n", "7 7 7\n", "21 21 21\n", "9 9 9\n", "7 7 7\n", "9 9 9\n", "14 14 14\n", "11 11 11\n", "4 4 4\n", "52 52 52\n", "20 20 20\n", "21 21 21\n", "9 9 9\n", "18 18 18\n", "8 8 8\n", "10 10 10\n", "8 8 8\n", "7 7 7\n", "11 11 11\n", "4 4 4\n", "3 3 3\n", "8 8 8\n", "6 6 6\n", "30 30 30\n", "14 14 14\n", "20 20 20\n", "5 5 5\n", "5 5 5\n", "6 6 6\n", "21 21 21\n", "137 137 137\n", "6 6 6\n", "10 10 10\n", "25 25 25\n", "20 20 20\n", "45 45 45\n", "11 11 11\n", "17 17 17\n", "13 13 13\n", "13 13 13\n", "48 48 48\n", "24 24 24\n", "12 12 12\n", "4 4 4\n", "6 6 6\n", "6 6 6\n", "63 63 63\n", "20 20 20\n", "18 18 18\n", "26 26 26\n", "33 33 33\n", "6 6 6\n", "9 9 9\n", "19 19 19\n", "10 10 10\n", "5 5 5\n", "14 14 14\n", "8 8 8\n", "30 30 30\n", "7 7 7\n", "36 36 36\n", "6 6 6\n", "10 10 10\n", "9 9 9\n", "9 9 9\n", "20 20 20\n", "77 77 77\n", "3 3 3\n", "10 10 10\n", "26 26 26\n", "5 5 5\n", "15 15 15\n", "3 3 3\n", "11 11 11\n", "7 7 7\n", "5 5 5\n", "37 37 37\n", "12 12 12\n", "4 4 4\n", "10 10 10\n", "18 18 18\n", "15 15 15\n", "19 19 19\n", "22 22 22\n", "13 13 13\n", "15 15 15\n", "11 11 11\n", "8 8 8\n", "43 43 43\n", "20 20 20\n", "23 23 23\n", "16 16 16\n", "71 71 71\n", "18 18 18\n", "6 6 6\n", "110 110 110\n", "12 12 12\n", "21 21 21\n", "14 14 14\n", "30 30 30\n", "25 25 25\n", "11 11 11\n", "19 19 19\n", "15 15 15\n", "13 13 13\n", "3 3 3\n", "14 14 14\n", "60 60 60\n", "12 12 12\n", "11 11 11\n", "39 39 39\n", "10 10 10\n", "4 4 4\n", "3 3 3\n", "5 5 5\n", "10 10 10\n", "23 23 23\n", "4 4 4\n", "6 6 6\n", "6 6 6\n", "5 5 5\n", "7 7 7\n", "8 8 8\n", "24 24 24\n", "12 12 12\n", "4 4 4\n", "7 7 7\n", "15 15 15\n", "5 5 5\n", "10 10 10\n", "4 4 4\n", "8 8 8\n", "8 8 8\n", "68 68 68\n", "8 8 8\n", "54 54 54\n", "53 53 53\n", "22 22 22\n", "11 11 11\n", "8 8 8\n", "11 11 11\n", "14 14 14\n", "5 5 5\n", "27 27 27\n", "40 40 40\n", "13 13 13\n", "47 47 47\n", "11 11 11\n", "13 13 13\n", "5 5 5\n", "11 11 11\n", "20 20 20\n", "23 23 23\n", "8 8 8\n", "4 4 4\n", "4 4 4\n", "5 5 5\n", "25 25 25\n", "23 23 23\n", "23 23 23\n", "23 23 23\n", "19 19 19\n", "8 8 8\n", "4 4 4\n", "3 3 3\n", "13 13 13\n", "12 12 12\n", "10 10 10\n", "11 11 11\n", "38 38 38\n", "32 32 32\n", "64 64 64\n", "19 19 19\n", "9 9 9\n", "3 3 3\n", "14 14 14\n", "17 17 17\n", "8 8 8\n", "24 24 24\n", "10 10 10\n", "7 7 7\n", "10 10 10\n", "14 14 14\n", "17 17 17\n", "4 4 4\n", "8 8 8\n", "22 22 22\n", "55 55 55\n", "12 12 12\n", "20 20 20\n", "12 12 12\n", "16 16 16\n", "5 5 5\n", "8 8 8\n", "4 4 4\n", "11 11 11\n", "25 25 25\n", "15 15 15\n", "28 28 28\n", "11 11 11\n", "8 8 8\n", "10 10 10\n", "5 5 5\n", "55 55 55\n", "46 46 46\n", "10 10 10\n", "6 6 6\n", "13 13 13\n", "11 11 11\n", "14 14 14\n", "65 65 65\n", "18 18 18\n", "5 5 5\n", "5 5 5\n", "117 117 117\n", "5 5 5\n", "13 13 13\n", "10 10 10\n", "7 7 7\n", "143 143 143\n", "3 3 3\n", "4 4 4\n", "7 7 7\n", "27 27 27\n", "12 12 12\n", "17 17 17\n", "15 15 15\n", "5 5 5\n", "19 19 19\n", "17 17 17\n", "38 38 38\n", "11 11 11\n", "18 18 18\n", "25 25 25\n", "23 23 23\n", "7 7 7\n", "21 21 21\n", "10 10 10\n", "17 17 17\n", "4 4 4\n", "4 4 4\n", "11 11 11\n", "4 4 4\n", "12 12 12\n", "26 26 26\n", "9 9 9\n", "11 11 11\n", "3 3 3\n", "15 15 15\n", "7 7 7\n", "14 14 14\n", "23 23 23\n", "36 36 36\n", "5 5 5\n", "59 59 59\n", "11 11 11\n", "9 9 9\n", "9 9 9\n", "22 22 22\n", "11 11 11\n", "18 18 18\n", "12 12 12\n", "81 81 81\n", "32 32 32\n", "10 10 10\n", "37 37 37\n", "3 3 3\n", "22 22 22\n", "65 65 65\n", "16 16 16\n", "22 22 22\n", "25 25 25\n", "24 24 24\n", "128 128 128\n", "9 9 9\n", "14 14 14\n", "5 5 5\n", "18 18 18\n", "8 8 8\n", "45 45 45\n", "20 20 20\n", "95 95 95\n", "9 9 9\n", "33 33 33\n", "8 8 8\n", "17 17 17\n", "6 6 6\n", "9 9 9\n", "12 12 12\n", "3 3 3\n", "18 18 18\n", "8 8 8\n", "7 7 7\n", "25 25 25\n", "6 6 6\n", "6 6 6\n", "8 8 8\n", "36 36 36\n", "6 6 6\n", "22 22 22\n", "15 15 15\n", "6 6 6\n", "16 16 16\n", "57 57 57\n", "13 13 13\n", "7 7 7\n", "30 30 30\n", "6 6 6\n", "15 15 15\n", "29 29 29\n", "9 9 9\n", "18 18 18\n", "5 5 5\n", "10 10 10\n", "4 4 4\n", "59 59 59\n", "29 29 29\n", "102 102 102\n", "20 20 20\n", "4 4 4\n", "19 19 19\n", "28 28 28\n", "4 4 4\n", "8 8 8\n", "10 10 10\n", "5 5 5\n", "18 18 18\n", "33 33 33\n", "16 16 16\n", "12 12 12\n", "37 37 37\n", "5 5 5\n", "9 9 9\n", "13 13 13\n", "27 27 27\n", "4 4 4\n", "6 6 6\n", "95 95 95\n", "4 4 4\n", "14 14 14\n", "9 9 9\n", "3 3 3\n", "11 11 11\n", "6 6 6\n", "5 5 5\n", "8 8 8\n", "12 12 12\n", "5 5 5\n", "14 14 14\n", "24 24 24\n", "4 4 4\n", "7 7 7\n", "9 9 9\n", "5 5 5\n", "11 11 11\n", "6 6 6\n", "89 89 89\n", "11 11 11\n", "81 81 81\n", "8 8 8\n", "8 8 8\n", "75 75 75\n", "33 33 33\n", "118 118 118\n", "32 32 32\n", "69 69 69\n", "9 9 9\n", "20 20 20\n", "6 6 6\n", "4 4 4\n", "14 14 14\n", "10 10 10\n", "23 23 23\n", "6 6 6\n", "22 22 22\n", "6 6 6\n", "24 24 24\n", "8 8 8\n", "34 34 34\n", "22 22 22\n", "28 28 28\n", "8 8 8\n", "4 4 4\n", "3 3 3\n", "19 19 19\n", "6 6 6\n", "13 13 13\n", "31 31 31\n", "12 12 12\n", "5 5 5\n", "23 23 23\n", "21 21 21\n", "8 8 8\n", "21 21 21\n", "20 20 20\n", "14 14 14\n", "15 15 15\n", "10 10 10\n", "6 6 6\n", "15 15 15\n", "3 3 3\n", "8 8 8\n", "84 84 84\n", "54 54 54\n", "4 4 4\n", "7 7 7\n", "39 39 39\n", "25 25 25\n", "29 29 29\n", "13 13 13\n", "5 5 5\n", "26 26 26\n", "8 8 8\n", "32 32 32\n", "30 30 30\n", "11 11 11\n", "31 31 31\n", "15 15 15\n", "39 39 39\n", "16 16 16\n", "4 4 4\n", "26 26 26\n", "11 11 11\n", "7 7 7\n", "12 12 12\n", "9 9 9\n", "15 15 15\n", "12 12 12\n", "21 21 21\n", "10 10 10\n", "27 27 27\n", "4 4 4\n", "4 4 4\n", "14 14 14\n", "22 22 22\n", "5 5 5\n", "13 13 13\n", "6 6 6\n", "13 13 13\n", "30 30 30\n", "38 38 38\n", "15 15 15\n", "52 52 52\n", "8 8 8\n", "39 39 39\n", "10 10 10\n", "29 29 29\n", "9 9 9\n", "8 8 8\n", "162 162 162\n", "19 19 19\n", "8 8 8\n", "7 7 7\n", "6 6 6\n", "36 36 36\n", "8 8 8\n", "18 18 18\n", "8 8 8\n", "23 23 23\n", "8 8 8\n", "5 5 5\n", "19 19 19\n", "18 18 18\n", "12 12 12\n", "32 32 32\n", "9 9 9\n", "6 6 6\n", "9 9 9\n", "43 43 43\n", "7 7 7\n", "3 3 3\n", "23 23 23\n", "14 14 14\n", "4 4 4\n", "14 14 14\n", "11 11 11\n", "11 11 11\n", "9 9 9\n", "41 41 41\n", "12 12 12\n", "21 21 21\n", "4 4 4\n", "47 47 47\n", "16 16 16\n", "4 4 4\n", "51 51 51\n", "63 63 63\n", "55 55 55\n", "6 6 6\n", "11 11 11\n", "10 10 10\n", "24 24 24\n", "7 7 7\n", "14 14 14\n", "16 16 16\n", "17 17 17\n", "29 29 29\n", "11 11 11\n", "5 5 5\n", "8 8 8\n", "30 30 30\n", "6 6 6\n", "15 15 15\n", "8 8 8\n", "28 28 28\n", "24 24 24\n", "76 76 76\n", "6 6 6\n", "19 19 19\n", "51 51 51\n", "262 262 262\n", "5 5 5\n", "4 4 4\n", "49 49 49\n", "8 8 8\n", "7 7 7\n", "9 9 9\n", "9 9 9\n", "11 11 11\n", "6 6 6\n", "31 31 31\n", "23 23 23\n", "4 4 4\n", "10 10 10\n", "28 28 28\n", "21 21 21\n", "6 6 6\n", "8 8 8\n", "20 20 20\n", "19 19 19\n", "4 4 4\n", "6 6 6\n", "9 9 9\n", "38 38 38\n", "9 9 9\n", "140 140 140\n", "25 25 25\n", "5 5 5\n", "16 16 16\n", "4 4 4\n", "10 10 10\n", "22 22 22\n", "23 23 23\n", "25 25 25\n", "13 13 13\n", "15 15 15\n", "14 14 14\n", "11 11 11\n", "4 4 4\n", "10 10 10\n", "28 28 28\n", "6 6 6\n", "8 8 8\n", "14 14 14\n", "8 8 8\n", "49 49 49\n", "17 17 17\n", "11 11 11\n", "10 10 10\n", "6 6 6\n", "15 15 15\n", "30 30 30\n", "15 15 15\n", "52 52 52\n", "7 7 7\n", "7 7 7\n", "38 38 38\n", "16 16 16\n", "15 15 15\n", "12 12 12\n", "8 8 8\n", "16 16 16\n", "4 4 4\n", "8 8 8\n", "12 12 12\n", "11 11 11\n", "11 11 11\n", "3 3 3\n", "37 37 37\n", "11 11 11\n", "6 6 6\n", "23 23 23\n", "14 14 14\n", "4 4 4\n", "53 53 53\n", "10 10 10\n", "35 35 35\n", "6 6 6\n", "16 16 16\n", "10 10 10\n", "17 17 17\n", "13 13 13\n", "4 4 4\n", "6 6 6\n", "18 18 18\n", "25 25 25\n", "5 5 5\n", "10 10 10\n", "12 12 12\n", "13 13 13\n", "12 12 12\n", "4 4 4\n", "36 36 36\n", "28 28 28\n", "10 10 10\n", "10 10 10\n", "23 23 23\n", "15 15 15\n", "7 7 7\n", "10 10 10\n", "32 32 32\n", "11 11 11\n", "15 15 15\n", "16 16 16\n", "7 7 7\n", "15 15 15\n", "21 21 21\n", "5 5 5\n", "14 14 14\n", "15 15 15\n", "3 3 3\n", "13 13 13\n", "6 6 6\n", "32 32 32\n", "15 15 15\n", "9 9 9\n", "5 5 5\n", "20 20 20\n", "7 7 7\n", "13 13 13\n", "7 7 7\n", "33 33 33\n", "15 15 15\n", "11 11 11\n", "9 9 9\n", "28 28 28\n", "6 6 6\n", "8 8 8\n", "21 21 21\n", "19 19 19\n", "10 10 10\n", "28 28 28\n", "9 9 9\n", "4 4 4\n", "9 9 9\n", "9 9 9\n", "4 4 4\n", "10 10 10\n", "6 6 6\n", "7 7 7\n", "12 12 12\n", "16 16 16\n", "26 26 26\n", "18 18 18\n", "6 6 6\n", "4 4 4\n", "5 5 5\n", "4 4 4\n", "5 5 5\n", "4 4 4\n", "19 19 19\n", "20 20 20\n", "30 30 30\n", "29 29 29\n", "10 10 10\n", "33 33 33\n", "21 21 21\n", "14 14 14\n", "11 11 11\n", "8 8 8\n", "23 23 23\n", "12 12 12\n", "11 11 11\n", "13 13 13\n", "16 16 16\n", "11 11 11\n", "114 114 114\n", "4 4 4\n", "7 7 7\n", "13 13 13\n", "19 19 19\n", "9 9 9\n", "21 21 21\n", "7 7 7\n", "27 27 27\n", "40 40 40\n", "5 5 5\n", "8 8 8\n", "12 12 12\n", "41 41 41\n", "16 16 16\n", "50 50 50\n", "9 9 9\n", "12 12 12\n", "19 19 19\n", "7 7 7\n", "61 61 61\n", "134 134 134\n", "15 15 15\n", "3 3 3\n", "18 18 18\n", "12 12 12\n", "11 11 11\n", "16 16 16\n", "23 23 23\n", "7 7 7\n", "15 15 15\n", "5 5 5\n", "6 6 6\n", "15 15 15\n", "4 4 4\n", "39 39 39\n", "16 16 16\n", "4 4 4\n", "16 16 16\n", "9 9 9\n", "11 11 11\n", "18 18 18\n", "5 5 5\n", "29 29 29\n", "52 52 52\n", "10 10 10\n", "21 21 21\n", "7 7 7\n", "11 11 11\n", "11 11 11\n", "4 4 4\n", "62 62 62\n", "67 67 67\n", "10 10 10\n", "10 10 10\n", "11 11 11\n", "8 8 8\n", "14 14 14\n", "5 5 5\n", "16 16 16\n", "38 38 38\n", "7 7 7\n", "12 12 12\n", "8 8 8\n", "4 4 4\n", "7 7 7\n", "15 15 15\n", "19 19 19\n", "12 12 12\n", "9 9 9\n", "18 18 18\n", "11 11 11\n", "14 14 14\n", "21 21 21\n", "21 21 21\n", "27 27 27\n", "13 13 13\n", "10 10 10\n", "10 10 10\n", "12 12 12\n", "5 5 5\n", "9 9 9\n", "27 27 27\n", "16 16 16\n", "18 18 18\n", "24 24 24\n", "13 13 13\n", "11 11 11\n", "6 6 6\n", "6 6 6\n", "39 39 39\n", "19 19 19\n", "7 7 7\n", "5 5 5\n", "8 8 8\n", "48 48 48\n", "16 16 16\n", "12 12 12\n", "11 11 11\n", "16 16 16\n", "7 7 7\n", "21 21 21\n", "11 11 11\n", "4 4 4\n", "59 59 59\n", "7 7 7\n", "12 12 12\n", "13 13 13\n", "14 14 14\n", "17 17 17\n", "13 13 13\n", "20 20 20\n", "23 23 23\n", "19 19 19\n", "11 11 11\n", "17 17 17\n", "9 9 9\n", "8 8 8\n", "10 10 10\n", "20 20 20\n", "9 9 9\n", "18 18 18\n", "5 5 5\n", "17 17 17\n", "24 24 24\n", "12 12 12\n", "6 6 6\n", "13 13 13\n", "19 19 19\n", "4 4 4\n", "7 7 7\n", "24 24 24\n", "16 16 16\n", "13 13 13\n", "45 45 45\n", "13 13 13\n", "25 25 25\n", "9 9 9\n", "5 5 5\n", "11 11 11\n", "10 10 10\n", "9 9 9\n", "16 16 16\n", "48 48 48\n", "23 23 23\n", "4 4 4\n", "43 43 43\n", "7 7 7\n", "5 5 5\n", "13 13 13\n", "3 3 3\n", "5 5 5\n", "6 6 6\n", "13 13 13\n", "44 44 44\n", "5 5 5\n", "15 15 15\n", "4 4 4\n", "37 37 37\n", "43 43 43\n", "8 8 8\n", "12 12 12\n", "28 28 28\n", "19 19 19\n", "14 14 14\n", "15 15 15\n", "50 50 50\n", "9 9 9\n", "17 17 17\n", "35 35 35\n", "7 7 7\n", "4 4 4\n", "4 4 4\n", "6 6 6\n", "15 15 15\n", "8 8 8\n", "6 6 6\n", "13 13 13\n", "30 30 30\n", "5 5 5\n", "5 5 5\n", "19 19 19\n", "22 22 22\n", "82 82 82\n", "8 8 8\n", "14 14 14\n", "11 11 11\n", "63 63 63\n", "19 19 19\n", "10 10 10\n", "21 21 21\n", "6 6 6\n", "13 13 13\n", "15 15 15\n", "9 9 9\n", "34 34 34\n", "90 90 90\n", "10 10 10\n", "6 6 6\n", "8 8 8\n", "28 28 28\n", "7 7 7\n", "7 7 7\n", "15 15 15\n", "11 11 11\n", "3 3 3\n", "14 14 14\n", "12 12 12\n", "15 15 15\n", "12 12 12\n", "4 4 4\n", "22 22 22\n", "32 32 32\n", "14 14 14\n", "6 6 6\n", "10 10 10\n", "44 44 44\n", "3 3 3\n", "17 17 17\n", "10 10 10\n", "15 15 15\n", "30 30 30\n", "51 51 51\n", "7 7 7\n", "34 34 34\n", "9 9 9\n", "22 22 22\n", "9 9 9\n", "6 6 6\n", "29 29 29\n", "5 5 5\n", "13 13 13\n", "7 7 7\n", "10 10 10\n", "8 8 8\n", "8 8 8\n", "5 5 5\n", "13 13 13\n", "50 50 50\n", "7 7 7\n", "25 25 25\n", "21 21 21\n", "9 9 9\n", "16 16 16\n", "71 71 71\n", "6 6 6\n", "23 23 23\n", "18 18 18\n", "5 5 5\n", "4 4 4\n", "12 12 12\n", "8 8 8\n", "29 29 29\n", "20 20 20\n", "12 12 12\n", "4 4 4\n", "5 5 5\n", "11 11 11\n", "29 29 29\n", "11 11 11\n", "7 7 7\n", "16 16 16\n", "42 42 42\n", "4 4 4\n", "6 6 6\n", "4 4 4\n", "33 33 33\n", "18 18 18\n", "33 33 33\n", "7 7 7\n", "19 19 19\n", "11 11 11\n", "14 14 14\n", "52 52 52\n", "19 19 19\n", "30 30 30\n", "8 8 8\n", "6 6 6\n", "4 4 4\n", "14 14 14\n", "11 11 11\n", "8 8 8\n", "8 8 8\n", "8 8 8\n", "16 16 16\n", "19 19 19\n", "8 8 8\n", "4 4 4\n", "3 3 3\n", "6 6 6\n", "18 18 18\n", "7 7 7\n", "10 10 10\n", "13 13 13\n", "4 4 4\n", "13 13 13\n", "12 12 12\n", "35 35 35\n", "57 57 57\n", "41 41 41\n", "15 15 15\n", "21 21 21\n", "4 4 4\n", "4 4 4\n", "4 4 4\n", "13 13 13\n", "4 4 4\n", "12 12 12\n", "9 9 9\n", "4 4 4\n", "9 9 9\n", "4 4 4\n", "8 8 8\n", "4 4 4\n", "17 17 17\n", "56 56 56\n", "30 30 30\n", "22 22 22\n", "18 18 18\n", "10 10 10\n", "4 4 4\n", "33 33 33\n", "40 40 40\n", "13 13 13\n", "22 22 22\n", "13 13 13\n", "5 5 5\n", "3 3 3\n", "12 12 12\n", "16 16 16\n", "11 11 11\n", "38 38 38\n", "6 6 6\n", "13 13 13\n", "91 91 91\n", "17 17 17\n", "23 23 23\n", "41 41 41\n", "3 3 3\n", "17 17 17\n", "7 7 7\n", "12 12 12\n", "13 13 13\n", "15 15 15\n", "4 4 4\n", "27 27 27\n", "9 9 9\n", "30 30 30\n", "26 26 26\n", "4 4 4\n", "7 7 7\n", "14 14 14\n", "10 10 10\n", "17 17 17\n", "29 29 29\n", "26 26 26\n", "32 32 32\n", "80 80 80\n", "5 5 5\n", "10 10 10\n", "58 58 58\n", "12 12 12\n", "12 12 12\n", "8 8 8\n", "8 8 8\n", "24 24 24\n", "15 15 15\n", "16 16 16\n", "35 35 35\n", "21 21 21\n", "120 120 120\n", "10 10 10\n", "9 9 9\n", "20 20 20\n", "11 11 11\n", "14 14 14\n", "4 4 4\n", "9 9 9\n", "33 33 33\n", "5 5 5\n", "25 25 25\n", "12 12 12\n", "14 14 14\n", "20 20 20\n", "4 4 4\n", "130 130 130\n", "10 10 10\n", "9 9 9\n", "13 13 13\n", "15 15 15\n", "13 13 13\n", "7 7 7\n", "16 16 16\n", "13 13 13\n", "6 6 6\n", "15 15 15\n", "12 12 12\n", "9 9 9\n", "26 26 26\n", "32 32 32\n", "14 14 14\n", "14 14 14\n", "8 8 8\n", "14 14 14\n", "20 20 20\n", "6 6 6\n", "9 9 9\n", "16 16 16\n", "12 12 12\n", "18 18 18\n", "23 23 23\n", "17 17 17\n", "25 25 25\n", "12 12 12\n", "14 14 14\n", "29 29 29\n", "6 6 6\n", "18 18 18\n", "33 33 33\n", "9 9 9\n", "29 29 29\n", "7 7 7\n", "22 22 22\n", "18 18 18\n", "30 30 30\n", "15 15 15\n", "9 9 9\n", "16 16 16\n", "62 62 62\n", "8 8 8\n", "29 29 29\n", "3 3 3\n", "34 34 34\n", "4 4 4\n", "11 11 11\n", "7 7 7\n", "8 8 8\n", "9 9 9\n", "23 23 23\n", "7 7 7\n", "9 9 9\n", "159 159 159\n", "16 16 16\n", "28 28 28\n", "12 12 12\n", "67 67 67\n", "15 15 15\n", "4 4 4\n", "11 11 11\n", "3 3 3\n", "15 15 15\n", "13 13 13\n", "45 45 45\n", "11 11 11\n", "20 20 20\n", "19 19 19\n", "42 42 42\n", "12 12 12\n", "5 5 5\n", "9 9 9\n", "11 11 11\n", "9 9 9\n", "9 9 9\n", "4 4 4\n", "12 12 12\n", "24 24 24\n", "8 8 8\n", "18 18 18\n", "15 15 15\n", "6 6 6\n", "4 4 4\n", "16 16 16\n", "17 17 17\n", "5 5 5\n", "12 12 12\n", "40 40 40\n", "14 14 14\n", "22 22 22\n", "4 4 4\n", "19 19 19\n", "30 30 30\n", "7 7 7\n", "27 27 27\n", "13 13 13\n", "5 5 5\n", "18 18 18\n", "11 11 11\n", "15 15 15\n", "22 22 22\n", "10 10 10\n", "14 14 14\n", "12 12 12\n", "52 52 52\n", "33 33 33\n", "21 21 21\n", "26 26 26\n", "5 5 5\n", "8 8 8\n", "10 10 10\n", "6 6 6\n", "14 14 14\n", "7 7 7\n", "19 19 19\n", "8 8 8\n", "6 6 6\n", "42 42 42\n", "19 19 19\n", "3 3 3\n", "11 11 11\n", "10 10 10\n", "16 16 16\n", "20 20 20\n", "14 14 14\n", "27 27 27\n", "6 6 6\n", "46 46 46\n", "131 131 131\n", "11 11 11\n", "11 11 11\n", "4 4 4\n", "9 9 9\n", "8 8 8\n", "5 5 5\n", "13 13 13\n", "10 10 10\n", "4 4 4\n", "15 15 15\n", "70 70 70\n", "10 10 10\n", "20 20 20\n", "10 10 10\n", "21 21 21\n", "8 8 8\n", "15 15 15\n", "8 8 8\n", "13 13 13\n", "20 20 20\n", "14 14 14\n", "11 11 11\n", "18 18 18\n", "11 11 11\n", "16 16 16\n", "11 11 11\n", "31 31 31\n", "10 10 10\n", "35 35 35\n", "6 6 6\n", "14 14 14\n", "5 5 5\n", "51 51 51\n", "20 20 20\n", "30 30 30\n", "26 26 26\n", "8 8 8\n", "12 12 12\n", "5 5 5\n", "7 7 7\n", "14 14 14\n", "26 26 26\n", "6 6 6\n", "28 28 28\n", "13 13 13\n", "14 14 14\n", "10 10 10\n", "13 13 13\n", "4 4 4\n", "28 28 28\n", "45 45 45\n", "4 4 4\n", "5 5 5\n", "15 15 15\n", "5 5 5\n", "9 9 9\n", "12 12 12\n", "10 10 10\n", "6 6 6\n", "12 12 12\n", "6 6 6\n", "3 3 3\n", "3 3 3\n", "20 20 20\n", "13 13 13\n", "15 15 15\n", "20 20 20\n", "28 28 28\n", "62 62 62\n", "6 6 6\n", "11 11 11\n", "7 7 7\n", "8 8 8\n", "26 26 26\n", "19 19 19\n", "9 9 9\n", "25 25 25\n", "7 7 7\n", "23 23 23\n", "16 16 16\n", "8 8 8\n", "4 4 4\n", "4 4 4\n", "12 12 12\n", "12 12 12\n", "8 8 8\n", "13 13 13\n", "27 27 27\n", "25 25 25\n", "6 6 6\n", "171 171 171\n", "9 9 9\n", "16 16 16\n", "12 12 12\n", "6 6 6\n", "8 8 8\n", "14 14 14\n", "13 13 13\n", "13 13 13\n", "25 25 25\n", "36 36 36\n", "17 17 17\n", "4 4 4\n", "15 15 15\n", "36 36 36\n", "16 16 16\n", "5 5 5\n", "14 14 14\n", "85 85 85\n", "4 4 4\n", "10 10 10\n", "5 5 5\n", "15 15 15\n", "27 27 27\n", "12 12 12\n", "22 22 22\n", "4 4 4\n", "10 10 10\n", "6 6 6\n", "14 14 14\n", "6 6 6\n", "20 20 20\n", "26 26 26\n", "3 3 3\n", "29 29 29\n", "5 5 5\n", "10 10 10\n", "4 4 4\n", "15 15 15\n", "20 20 20\n", "6 6 6\n", "9 9 9\n", "77 77 77\n", "6 6 6\n", "23 23 23\n", "8 8 8\n", "21 21 21\n", "117 117 117\n", "8 8 8\n", "22 22 22\n", "4 4 4\n", "8 8 8\n", "21 21 21\n", "7 7 7\n", "42 42 42\n", "8 8 8\n", "15 15 15\n", "7 7 7\n", "5 5 5\n", "8 8 8\n", "4 4 4\n", "5 5 5\n", "10 10 10\n", "11 11 11\n", "4 4 4\n", "19 19 19\n", "16 16 16\n", "8 8 8\n", "76 76 76\n", "15 15 15\n", "17 17 17\n", "7 7 7\n", "13 13 13\n", "5 5 5\n", "17 17 17\n", "3 3 3\n", "6 6 6\n", "57 57 57\n", "4 4 4\n", "51 51 51\n", "31 31 31\n", "8 8 8\n", "12 12 12\n", "12 12 12\n", "23 23 23\n", "4 4 4\n", "5 5 5\n", "8 8 8\n", "42 42 42\n", "19 19 19\n", "10 10 10\n", "4 4 4\n", "10 10 10\n", "15 15 15\n", "9 9 9\n", "30 30 30\n", "31 31 31\n", "19 19 19\n", "8 8 8\n", "15 15 15\n", "9 9 9\n", "7 7 7\n", "11 11 11\n", "7 7 7\n", "13 13 13\n", "9 9 9\n", "4 4 4\n", "35 35 35\n", "69 69 69\n", "12 12 12\n", "15 15 15\n", "11 11 11\n", "59 59 59\n", "16 16 16\n", "5 5 5\n", "19 19 19\n", "20 20 20\n", "6 6 6\n", "18 18 18\n", "4 4 4\n", "3 3 3\n", "4 4 4\n", "39 39 39\n", "6 6 6\n", "5 5 5\n", "13 13 13\n", "8 8 8\n", "15 15 15\n", "7 7 7\n", "35 35 35\n", "8 8 8\n", "8 8 8\n", "9 9 9\n", "5 5 5\n", "4 4 4\n", "16 16 16\n", "3 3 3\n", "6 6 6\n", "4 4 4\n", "18 18 18\n", "44 44 44\n", "14 14 14\n", "37 37 37\n", "6 6 6\n", "4 4 4\n", "11 11 11\n", "13 13 13\n", "9 9 9\n", "14 14 14\n", "9 9 9\n", "8 8 8\n", "6 6 6\n", "47 47 47\n", "19 19 19\n", "54 54 54\n", "7 7 7\n", "40 40 40\n", "4 4 4\n", "8 8 8\n", "4 4 4\n", "7 7 7\n", "10 10 10\n", "14 14 14\n", "7 7 7\n", "75 75 75\n", "18 18 18\n", "19 19 19\n", "49 49 49\n", "3 3 3\n", "27 27 27\n", "6 6 6\n", "20 20 20\n", "13 13 13\n", "10 10 10\n", "6 6 6\n", "9 9 9\n", "17 17 17\n", "19 19 19\n", "9 9 9\n", "4 4 4\n", "16 16 16\n", "15 15 15\n", "23 23 23\n", "13 13 13\n", "15 15 15\n", "64 64 64\n", "8 8 8\n", "16 16 16\n", "18 18 18\n", "8 8 8\n", "15 15 15\n", "7 7 7\n", "3 3 3\n", "27 27 27\n", "6 6 6\n", "14 14 14\n", "28 28 28\n", "8 8 8\n", "22 22 22\n", "11 11 11\n", "39 39 39\n", "20 20 20\n", "12 12 12\n", "79 79 79\n", "20 20 20\n", "16 16 16\n", "15 15 15\n", "22 22 22\n", "22 22 22\n", "18 18 18\n", "6 6 6\n", "15 15 15\n", "6 6 6\n", "19 19 19\n", "49 49 49\n", "4 4 4\n", "46 46 46\n", "11 11 11\n", "23 23 23\n", "15 15 15\n", "26 26 26\n", "33 33 33\n", "20 20 20\n", "14 14 14\n", "30 30 30\n", "26 26 26\n", "36 36 36\n", "15 15 15\n", "71 71 71\n", "41 41 41\n", "7 7 7\n", "8 8 8\n", "6 6 6\n", "6 6 6\n", "35 35 35\n", "64 64 64\n", "4 4 4\n", "7 7 7\n", "14 14 14\n", "4 4 4\n", "15 15 15\n", "14 14 14\n", "67 67 67\n", "9 9 9\n", "4 4 4\n", "6 6 6\n", "4 4 4\n", "15 15 15\n", "12 12 12\n", "22 22 22\n", "8 8 8\n", "14 14 14\n", "13 13 13\n", "12 12 12\n", "11 11 11\n", "12 12 12\n", "5 5 5\n", "5 5 5\n", "9 9 9\n", "16 16 16\n", "27 27 27\n", "7 7 7\n", "43 43 43\n", "14 14 14\n", "9 9 9\n", "11 11 11\n", "8 8 8\n", "12 12 12\n", "4 4 4\n", "14 14 14\n", "11 11 11\n", "7 7 7\n", "9 9 9\n", "28 28 28\n", "17 17 17\n", "25 25 25\n", "15 15 15\n", "23 23 23\n", "21 21 21\n", "10 10 10\n", "13 13 13\n", "8 8 8\n", "12 12 12\n", "34 34 34\n", "79 79 79\n", "23 23 23\n", "78 78 78\n", "5 5 5\n", "20 20 20\n", "20 20 20\n", "47 47 47\n", "20 20 20\n", "17 17 17\n", "5 5 5\n", "12 12 12\n", "13 13 13\n", "12 12 12\n", "30 30 30\n", "5 5 5\n", "12 12 12\n", "32 32 32\n", "34 34 34\n", "8 8 8\n", "7 7 7\n", "6 6 6\n", "10 10 10\n", "13 13 13\n", "19 19 19\n", "7 7 7\n", "16 16 16\n", "4 4 4\n", "9 9 9\n", "18 18 18\n", "7 7 7\n", "12 12 12\n", "9 9 9\n", "11 11 11\n", "19 19 19\n", "4 4 4\n", "9 9 9\n", "87 87 87\n", "18 18 18\n", "10 10 10\n", "13 13 13\n", "8 8 8\n", "12 12 12\n", "8 8 8\n", "3 3 3\n", "22 22 22\n", "8 8 8\n", "15 15 15\n", "10 10 10\n", "29 29 29\n", "12 12 12\n", "5 5 5\n", "4 4 4\n", "15 15 15\n", "20 20 20\n", "5 5 5\n", "15 15 15\n", "10 10 10\n", "21 21 21\n", "24 24 24\n", "36 36 36\n", "44 44 44\n", "19 19 19\n", "13 13 13\n", "12 12 12\n", "37 37 37\n", "41 41 41\n", "6 6 6\n", "20 20 20\n", "12 12 12\n", "7 7 7\n", "29 29 29\n", "17 17 17\n", "3 3 3\n", "21 21 21\n", "16 16 16\n", "18 18 18\n", "35 35 35\n", "14 14 14\n", "4 4 4\n", "24 24 24\n", "88 88 88\n", "35 35 35\n", "49 49 49\n", "10 10 10\n", "7 7 7\n", "49 49 49\n", "36 36 36\n", "38 38 38\n", "7 7 7\n", "13 13 13\n", "12 12 12\n", "22 22 22\n", "31 31 31\n", "20 20 20\n", "11 11 11\n", "14 14 14\n", "42 42 42\n", "9 9 9\n", "53 53 53\n", "12 12 12\n", "19 19 19\n", "25 25 25\n", "5 5 5\n", "6 6 6\n", "13 13 13\n", "20 20 20\n", "12 12 12\n", "6 6 6\n", "7 7 7\n", "14 14 14\n", "19 19 19\n", "20 20 20\n", "10 10 10\n", "17 17 17\n", "4 4 4\n", "16 16 16\n", "11 11 11\n", "7 7 7\n", "7 7 7\n", "51 51 51\n", "10 10 10\n", "14 14 14\n", "8 8 8\n", "3 3 3\n", "5 5 5\n", "13 13 13\n", "78 78 78\n", "17 17 17\n", "10 10 10\n", "9 9 9\n", "41 41 41\n", "64 64 64\n", "10 10 10\n", "24 24 24\n", "23 23 23\n", "13 13 13\n", "30 30 30\n", "4 4 4\n", "14 14 14\n", "17 17 17\n", "23 23 23\n", "41 41 41\n", "5 5 5\n", "4 4 4\n", "25 25 25\n", "11 11 11\n", "34 34 34\n", "47 47 47\n", "30 30 30\n", "6 6 6\n", "7 7 7\n", "13 13 13\n", "10 10 10\n", "24 24 24\n", "10 10 10\n", "29 29 29\n", "20 20 20\n", "23 23 23\n", "4 4 4\n", "4 4 4\n", "19 19 19\n", "6 6 6\n", "6 6 6\n", "36 36 36\n", "58 58 58\n", "21 21 21\n", "13 13 13\n", "6 6 6\n", "9 9 9\n", "11 11 11\n", "99 99 99\n", "19 19 19\n", "14 14 14\n", "20 20 20\n", "10 10 10\n", "13 13 13\n", "48 48 48\n", "6 6 6\n", "16 16 16\n", "14 14 14\n", "44 44 44\n", "4 4 4\n", "8 8 8\n", "11 11 11\n", "6 6 6\n", "77 77 77\n", "10 10 10\n", "16 16 16\n", "13 13 13\n", "30 30 30\n", "15 15 15\n", "15 15 15\n", "18 18 18\n", "10 10 10\n", "9 9 9\n", "27 27 27\n", "10 10 10\n", "5 5 5\n", "31 31 31\n", "3 3 3\n", "46 46 46\n", "12 12 12\n", "6 6 6\n", "7 7 7\n", "22 22 22\n", "7 7 7\n", "7 7 7\n", "16 16 16\n", "3 3 3\n", "18 18 18\n", "9 9 9\n", "33 33 33\n", "16 16 16\n", "30 30 30\n", "4 4 4\n", "4 4 4\n", "6 6 6\n", "6 6 6\n", "5 5 5\n", "8 8 8\n", "8 8 8\n", "20 20 20\n", "6 6 6\n", "27 27 27\n", "4 4 4\n", "18 18 18\n", "3 3 3\n", "14 14 14\n", "16 16 16\n", "40 40 40\n", "50 50 50\n", "15 15 15\n", "9 9 9\n", "22 22 22\n", "4 4 4\n", "6 6 6\n", "18 18 18\n", "6 6 6\n", "23 23 23\n", "13 13 13\n", "8 8 8\n", "14 14 14\n", "5 5 5\n", "11 11 11\n", "7 7 7\n", "6 6 6\n", "15 15 15\n", "3 3 3\n", "23 23 23\n", "10 10 10\n", "12 12 12\n", "18 18 18\n", "21 21 21\n", "20 20 20\n", "11 11 11\n", "10 10 10\n", "18 18 18\n", "47 47 47\n", "10 10 10\n", "11 11 11\n", "9 9 9\n", "17 17 17\n", "10 10 10\n", "10 10 10\n", "21 21 21\n", "7 7 7\n", "4 4 4\n", "5 5 5\n", "17 17 17\n", "5 5 5\n", "22 22 22\n", "32 32 32\n", "6 6 6\n", "13 13 13\n", "22 22 22\n", "14 14 14\n", "5 5 5\n", "3 3 3\n", "4 4 4\n", "9 9 9\n", "8 8 8\n", "14 14 14\n", "68 68 68\n", "25 25 25\n", "17 17 17\n", "21 21 21\n", "12 12 12\n", "92 92 92\n", "17 17 17\n", "8 8 8\n", "102 102 102\n", "15 15 15\n", "3 3 3\n", "3 3 3\n", "23 23 23\n", "22 22 22\n", "20 20 20\n", "18 18 18\n", "5 5 5\n", "13 13 13\n", "25 25 25\n", "8 8 8\n", "32 32 32\n", "3 3 3\n", "9 9 9\n", "10 10 10\n", "21 21 21\n", "6 6 6\n", "6 6 6\n", "37 37 37\n", "6 6 6\n", "21 21 21\n", "9 9 9\n", "10 10 10\n", "29 29 29\n", "8 8 8\n", "23 23 23\n", "12 12 12\n", "73 73 73\n", "13 13 13\n", "8 8 8\n", "21 21 21\n", "4 4 4\n", "6 6 6\n", "22 22 22\n", "6 6 6\n", "70 70 70\n", "46 46 46\n", "40 40 40\n", "38 38 38\n", "11 11 11\n", "13 13 13\n", "13 13 13\n", "7 7 7\n", "33 33 33\n", "9 9 9\n", "30 30 30\n", "5 5 5\n", "9 9 9\n", "19 19 19\n", "3 3 3\n", "5 5 5\n", "20 20 20\n", "14 14 14\n", "4 4 4\n", "6 6 6\n", "14 14 14\n", "10 10 10\n", "5 5 5\n", "101 101 101\n", "11 11 11\n", "6 6 6\n", "13 13 13\n", "9 9 9\n", "11 11 11\n", "10 10 10\n", "33 33 33\n", "5 5 5\n", "12 12 12\n", "9 9 9\n", "8 8 8\n", "11 11 11\n", "3 3 3\n", "4 4 4\n", "23 23 23\n", "20 20 20\n", "10 10 10\n", "6 6 6\n", "10 10 10\n", "34 34 34\n", "22 22 22\n", "3 3 3\n", "12 12 12\n", "3 3 3\n", "11 11 11\n", "55 55 55\n", "21 21 21\n", "8 8 8\n", "60 60 60\n", "6 6 6\n", "13 13 13\n", "10 10 10\n", "9 9 9\n", "5 5 5\n", "3 3 3\n", "6 6 6\n", "27 27 27\n", "9 9 9\n", "13 13 13\n", "14 14 14\n", "21 21 21\n", "13 13 13\n", "44 44 44\n", "6 6 6\n", "15 15 15\n", "98 98 98\n", "7 7 7\n", "13 13 13\n", "29 29 29\n", "47 47 47\n", "54 54 54\n", "7 7 7\n", "8 8 8\n", "4 4 4\n", "13 13 13\n", "22 22 22\n", "30 30 30\n", "6 6 6\n", "11 11 11\n", "22 22 22\n", "20 20 20\n", "9 9 9\n", "8 8 8\n", "10 10 10\n", "8 8 8\n", "12 12 12\n", "10 10 10\n", "14 14 14\n", "15 15 15\n", "12 12 12\n", "9 9 9\n", "11 11 11\n", "14 14 14\n", "40 40 40\n", "12 12 12\n", "10 10 10\n", "9 9 9\n", "7 7 7\n", "3 3 3\n", "122 122 122\n", "9 9 9\n", "7 7 7\n", "9 9 9\n", "11 11 11\n", "6 6 6\n", "5 5 5\n", "29 29 29\n", "9 9 9\n", "12 12 12\n", "25 25 25\n", "46 46 46\n", "6 6 6\n", "14 14 14\n", "16 16 16\n", "14 14 14\n", "7 7 7\n", "17 17 17\n", "6 6 6\n", "33 33 33\n", "11 11 11\n", "40 40 40\n", "5 5 5\n", "17 17 17\n", "12 12 12\n", "17 17 17\n", "8 8 8\n", "19 19 19\n", "11 11 11\n", "5 5 5\n", "4 4 4\n", "17 17 17\n", "18 18 18\n", "13 13 13\n", "19 19 19\n", "19 19 19\n", "7 7 7\n", "18 18 18\n", "17 17 17\n", "8 8 8\n", "12 12 12\n", "8 8 8\n", "7 7 7\n", "6 6 6\n", "10 10 10\n", "16 16 16\n", "6 6 6\n", "4 4 4\n", "5 5 5\n", "15 15 15\n", "14 14 14\n", "44 44 44\n", "5 5 5\n", "6 6 6\n", "30 30 30\n", "26 26 26\n", "10 10 10\n", "8 8 8\n", "34 34 34\n", "11 11 11\n", "9 9 9\n", "18 18 18\n", "5 5 5\n", "16 16 16\n", "10 10 10\n", "23 23 23\n", "8 8 8\n", "8 8 8\n", "14 14 14\n", "19 19 19\n", "7 7 7\n", "20 20 20\n", "11 11 11\n", "9 9 9\n", "7 7 7\n", "24 24 24\n", "8 8 8\n", "15 15 15\n", "4 4 4\n", "20 20 20\n", "3 3 3\n", "36 36 36\n", "6 6 6\n", "10 10 10\n", "7 7 7\n", "11 11 11\n", "6 6 6\n", "11 11 11\n", "5 5 5\n", "30 30 30\n", "18 18 18\n", "8 8 8\n", "7 7 7\n", "3 3 3\n", "13 13 13\n", "3 3 3\n", "10 10 10\n", "9 9 9\n", "15 15 15\n", "10 10 10\n", "15 15 15\n", "6 6 6\n", "8 8 8\n", "62 62 62\n", "15 15 15\n", "13 13 13\n", "27 27 27\n", "20 20 20\n", "9 9 9\n", "11 11 11\n", "29 29 29\n", "9 9 9\n", "16 16 16\n", "41 41 41\n", "8 8 8\n", "9 9 9\n", "9 9 9\n", "10 10 10\n", "77 77 77\n", "92 92 92\n", "8 8 8\n", "6 6 6\n", "10 10 10\n", "92 92 92\n", "10 10 10\n", "12 12 12\n", "4 4 4\n", "19 19 19\n", "23 23 23\n", "20 20 20\n", "17 17 17\n", "10 10 10\n", "14 14 14\n", "8 8 8\n", "11 11 11\n", "7 7 7\n", "19 19 19\n", "25 25 25\n", "3 3 3\n", "9 9 9\n", "29 29 29\n", "7 7 7\n", "34 34 34\n", "5 5 5\n", "8 8 8\n", "14 14 14\n", "25 25 25\n", "9 9 9\n", "9 9 9\n", "4 4 4\n", "30 30 30\n", "20 20 20\n", "3 3 3\n", "18 18 18\n", "6 6 6\n", "7 7 7\n", "6 6 6\n", "9 9 9\n", "19 19 19\n", "46 46 46\n", "10 10 10\n", "35 35 35\n", "25 25 25\n", "9 9 9\n", "14 14 14\n", "12 12 12\n", "34 34 34\n", "14 14 14\n", "13 13 13\n", "9 9 9\n", "18 18 18\n", "18 18 18\n", "14 14 14\n", "8 8 8\n", "4 4 4\n", "5 5 5\n", "3 3 3\n", "5 5 5\n", "9 9 9\n", "3 3 3\n", "9 9 9\n", "20 20 20\n", "6 6 6\n", "10 10 10\n", "21 21 21\n", "12 12 12\n", "10 10 10\n", "7 7 7\n", "10 10 10\n", "9 9 9\n", "34 34 34\n", "32 32 32\n", "17 17 17\n", "4 4 4\n", "10 10 10\n", "22 22 22\n", "15 15 15\n", "14 14 14\n", "17 17 17\n", "12 12 12\n", "8 8 8\n", "11 11 11\n", "15 15 15\n", "13 13 13\n", "17 17 17\n", "58 58 58\n", "11 11 11\n", "29 29 29\n", "20 20 20\n", "4 4 4\n", "12 12 12\n", "6 6 6\n", "26 26 26\n", "20 20 20\n", "21 21 21\n", "6 6 6\n", "11 11 11\n", "20 20 20\n", "13 13 13\n", "16 16 16\n", "4 4 4\n", "4 4 4\n", "10 10 10\n", "19 19 19\n", "24 24 24\n", "21 21 21\n", "7 7 7\n", "6 6 6\n", "12 12 12\n", "16 16 16\n", "36 36 36\n", "17 17 17\n", "15 15 15\n", "24 24 24\n", "4 4 4\n", "18 18 18\n", "3 3 3\n", "15 15 15\n", "16 16 16\n", "9 9 9\n", "23 23 23\n", "13 13 13\n", "5 5 5\n", "9 9 9\n", "28 28 28\n", "28 28 28\n", "8 8 8\n", "19 19 19\n", "47 47 47\n", "86 86 86\n", "17 17 17\n", "13 13 13\n", "5 5 5\n", "9 9 9\n", "11 11 11\n", "29 29 29\n", "5 5 5\n", "11 11 11\n", "4 4 4\n", "4 4 4\n", "55 55 55\n", "15 15 15\n", "5 5 5\n", "48 48 48\n", "8 8 8\n", "14 14 14\n", "11 11 11\n", "11 11 11\n", "7 7 7\n", "49 49 49\n", "10 10 10\n", "13 13 13\n", "33 33 33\n", "19 19 19\n", "6 6 6\n", "21 21 21\n", "12 12 12\n", "3 3 3\n", "10 10 10\n", "37 37 37\n", "4 4 4\n", "8 8 8\n", "21 21 21\n", "8 8 8\n", "33 33 33\n", "8 8 8\n", "5 5 5\n", "12 12 12\n", "38 38 38\n", "21 21 21\n", "25 25 25\n", "5 5 5\n", "18 18 18\n", "7 7 7\n", "4 4 4\n", "7 7 7\n", "37 37 37\n", "12 12 12\n", "22 22 22\n", "37 37 37\n", "9 9 9\n", "7 7 7\n", "13 13 13\n", "9 9 9\n", "10 10 10\n", "38 38 38\n", "6 6 6\n", "4 4 4\n", "11 11 11\n", "45 45 45\n", "23 23 23\n", "8 8 8\n", "4 4 4\n", "26 26 26\n", "31 31 31\n", "18 18 18\n", "7 7 7\n", "94 94 94\n", "16 16 16\n", "12 12 12\n", "3 3 3\n", "4 4 4\n", "4 4 4\n", "8 8 8\n", "14 14 14\n", "4 4 4\n", "4 4 4\n", "7 7 7\n", "9 9 9\n", "17 17 17\n", "4 4 4\n", "18 18 18\n", "10 10 10\n", "18 18 18\n", "6 6 6\n", "29 29 29\n", "7 7 7\n", "79 79 79\n", "12 12 12\n", "13 13 13\n", "5 5 5\n", "29 29 29\n", "7 7 7\n", "9 9 9\n", "17 17 17\n", "35 35 35\n", "8 8 8\n", "6 6 6\n", "9 9 9\n", "10 10 10\n", "15 15 15\n", "34 34 34\n", "6 6 6\n", "28 28 28\n", "3 3 3\n", "6 6 6\n", "11 11 11\n", "12 12 12\n", "7 7 7\n", "4 4 4\n", "13 13 13\n", "4 4 4\n", "90 90 90\n", "13 13 13\n", "10 10 10\n", "6 6 6\n", "3 3 3\n", "19 19 19\n", "14 14 14\n", "7 7 7\n", "40 40 40\n", "5 5 5\n", "7 7 7\n", "8 8 8\n", "34 34 34\n", "63 63 63\n", "6 6 6\n", "7 7 7\n", "16 16 16\n", "12 12 12\n", "21 21 21\n", "8 8 8\n", "52 52 52\n", "14 14 14\n", "23 23 23\n", "6 6 6\n", "37 37 37\n", "21 21 21\n", "4 4 4\n", "6 6 6\n", "51 51 51\n", "6 6 6\n", "8 8 8\n", "7 7 7\n", "8 8 8\n", "6 6 6\n", "5 5 5\n", "7 7 7\n", "5 5 5\n", "42 42 42\n", "9 9 9\n", "23 23 23\n", "7 7 7\n", "24 24 24\n", "12 12 12\n", "12 12 12\n", "8 8 8\n", "11 11 11\n", "15 15 15\n", "7 7 7\n", "14 14 14\n", "17 17 17\n", "8 8 8\n", "4 4 4\n", "36 36 36\n", "4 4 4\n", "14 14 14\n", "8 8 8\n", "6 6 6\n", "54 54 54\n", "9 9 9\n", "13 13 13\n", "22 22 22\n", "10 10 10\n", "70 70 70\n", "25 25 25\n", "6 6 6\n", "3 3 3\n", "12 12 12\n", "23 23 23\n", "7 7 7\n", "3 3 3\n", "9 9 9\n", "15 15 15\n", "16 16 16\n", "11 11 11\n", "21 21 21\n", "21 21 21\n", "19 19 19\n", "77 77 77\n", "8 8 8\n", "32 32 32\n", "6 6 6\n", "10 10 10\n", "38 38 38\n", "85 85 85\n", "8 8 8\n", "28 28 28\n", "12 12 12\n", "12 12 12\n", "14 14 14\n", "12 12 12\n", "12 12 12\n", "33 33 33\n", "4 4 4\n", "45 45 45\n", "15 15 15\n", "8 8 8\n", "11 11 11\n", "18 18 18\n", "18 18 18\n", "4 4 4\n", "8 8 8\n", "4 4 4\n", "4 4 4\n", "22 22 22\n", "4 4 4\n", "6 6 6\n", "67 67 67\n", "8 8 8\n", "9 9 9\n", "43 43 43\n", "10 10 10\n", "15 15 15\n", "11 11 11\n", "25 25 25\n", "4 4 4\n", "9 9 9\n", "13 13 13\n", "9 9 9\n", "21 21 21\n", "11 11 11\n", "4 4 4\n", "10 10 10\n", "23 23 23\n", "14 14 14\n", "32 32 32\n", "16 16 16\n", "10 10 10\n", "7 7 7\n", "16 16 16\n", "37 37 37\n", "16 16 16\n", "179 179 179\n", "8 8 8\n", "10 10 10\n", "5 5 5\n", "39 39 39\n", "3 3 3\n", "9 9 9\n", "9 9 9\n", "18 18 18\n", "9 9 9\n", "13 13 13\n", "3 3 3\n", "108 108 108\n", "14 14 14\n", "13 13 13\n", "15 15 15\n", "17 17 17\n", "6 6 6\n", "9 9 9\n", "13 13 13\n", "36 36 36\n", "4 4 4\n", "27 27 27\n", "15 15 15\n", "10 10 10\n", "58 58 58\n", "6 6 6\n", "23 23 23\n", "13 13 13\n", "24 24 24\n", "32 32 32\n", "15 15 15\n", "6 6 6\n", "12 12 12\n", "14 14 14\n", "5 5 5\n", "28 28 28\n", "4 4 4\n", "45 45 45\n", "18 18 18\n", "10 10 10\n", "20 20 20\n", "16 16 16\n", "8 8 8\n", "10 10 10\n", "24 24 24\n", "12 12 12\n", "11 11 11\n", "13 13 13\n", "10 10 10\n", "12 12 12\n", "8 8 8\n", "84 84 84\n", "23 23 23\n", "12 12 12\n", "23 23 23\n", "5 5 5\n", "5 5 5\n", "17 17 17\n", "14 14 14\n", "15 15 15\n", "4 4 4\n", "25 25 25\n", "6 6 6\n", "6 6 6\n", "19 19 19\n", "18 18 18\n", "21 21 21\n", "8 8 8\n", "4 4 4\n", "22 22 22\n", "4 4 4\n", "13 13 13\n", "10 10 10\n", "13 13 13\n", "28 28 28\n", "23 23 23\n", "9 9 9\n", "22 22 22\n", "31 31 31\n", "30 30 30\n", "37 37 37\n", "10 10 10\n", "22 22 22\n", "13 13 13\n", "9 9 9\n", "8 8 8\n", "23 23 23\n", "24 24 24\n", "14 14 14\n", "5 5 5\n", "17 17 17\n", "3 3 3\n", "8 8 8\n", "7 7 7\n", "16 16 16\n", "17 17 17\n", "18 18 18\n", "16 16 16\n", "4 4 4\n", "4 4 4\n", "17 17 17\n", "5 5 5\n", "8 8 8\n", "16 16 16\n", "4 4 4\n", "4 4 4\n", "25 25 25\n", "13 13 13\n", "12 12 12\n", "8 8 8\n", "15 15 15\n", "29 29 29\n", "6 6 6\n", "38 38 38\n", "21 21 21\n", "3 3 3\n", "10 10 10\n", "8 8 8\n", "17 17 17\n", "6 6 6\n", "6 6 6\n", "7 7 7\n", "74 74 74\n", "8 8 8\n", "31 31 31\n", "4 4 4\n", "6 6 6\n", "10 10 10\n", "59 59 59\n", "56 56 56\n", "13 13 13\n", "11 11 11\n", "6 6 6\n", "6 6 6\n", "11 11 11\n", "11 11 11\n", "28 28 28\n", "23 23 23\n", "9 9 9\n", "5 5 5\n", "9 9 9\n", "11 11 11\n", "6 6 6\n", "11 11 11\n", "3 3 3\n", "19 19 19\n", "12 12 12\n", "10 10 10\n", "5 5 5\n", "4 4 4\n", "10 10 10\n", "28 28 28\n", "15 15 15\n", "18 18 18\n", "19 19 19\n", "20 20 20\n", "8 8 8\n", "9 9 9\n", "12 12 12\n", "64 64 64\n", "9 9 9\n", "152 152 152\n", "7 7 7\n", "18 18 18\n", "48 48 48\n", "6 6 6\n", "24 24 24\n", "15 15 15\n", "4 4 4\n", "48 48 48\n", "39 39 39\n", "6 6 6\n", "5 5 5\n", "8 8 8\n", "21 21 21\n", "6 6 6\n", "77 77 77\n", "41 41 41\n", "6 6 6\n", "6 6 6\n", "32 32 32\n", "9 9 9\n", "9 9 9\n", "17 17 17\n", "10 10 10\n", "20 20 20\n", "11 11 11\n", "26 26 26\n", "14 14 14\n", "17 17 17\n", "32 32 32\n", "20 20 20\n", "20 20 20\n", "32 32 32\n", "13 13 13\n", "7 7 7\n", "21 21 21\n", "16 16 16\n", "14 14 14\n", "21 21 21\n", "11 11 11\n", "4 4 4\n", "25 25 25\n", "8 8 8\n", "16 16 16\n", "31 31 31\n", "8 8 8\n", "8 8 8\n", "5 5 5\n", "14 14 14\n", "37 37 37\n", "3 3 3\n", "6 6 6\n", "5 5 5\n", "8 8 8\n", "22 22 22\n", "5 5 5\n", "17 17 17\n", "8 8 8\n", "18 18 18\n", "13 13 13\n", "14 14 14\n", "9 9 9\n", "4 4 4\n", "5 5 5\n", "22 22 22\n", "10 10 10\n", "24 24 24\n", "15 15 15\n", "34 34 34\n", "11 11 11\n", "26 26 26\n", "47 47 47\n", "5 5 5\n", "4 4 4\n", "11 11 11\n", "12 12 12\n", "48 48 48\n", "15 15 15\n", "10 10 10\n", "19 19 19\n", "9 9 9\n", "18 18 18\n", "11 11 11\n", "16 16 16\n", "6 6 6\n", "18 18 18\n", "22 22 22\n", "10 10 10\n", "5 5 5\n", "82 82 82\n", "28 28 28\n", "31 31 31\n", "19 19 19\n", "3 3 3\n", "8 8 8\n", "45 45 45\n", "10 10 10\n", "13 13 13\n", "22 22 22\n", "20 20 20\n", "18 18 18\n", "4 4 4\n", "9 9 9\n", "17 17 17\n", "20 20 20\n", "35 35 35\n", "4 4 4\n", "11 11 11\n", "22 22 22\n", "14 14 14\n", "8 8 8\n", "22 22 22\n", "24 24 24\n", "14 14 14\n", "8 8 8\n", "10 10 10\n", "11 11 11\n", "44 44 44\n", "47 47 47\n", "8 8 8\n", "10 10 10\n", "15 15 15\n", "20 20 20\n", "14 14 14\n", "13 13 13\n", "8 8 8\n", "4 4 4\n", "36 36 36\n", "10 10 10\n", "25 25 25\n", "21 21 21\n", "9 9 9\n", "21 21 21\n", "17 17 17\n", "11 11 11\n", "7 7 7\n", "8 8 8\n", "7 7 7\n", "8 8 8\n", "17 17 17\n", "11 11 11\n", "25 25 25\n", "112 112 112\n", "28 28 28\n", "37 37 37\n", "12 12 12\n", "21 21 21\n", "101 101 101\n", "10 10 10\n", "16 16 16\n", "14 14 14\n", "19 19 19\n", "11 11 11\n", "14 14 14\n", "36 36 36\n", "7 7 7\n", "32 32 32\n", "13 13 13\n", "8 8 8\n", "8 8 8\n", "17 17 17\n", "53 53 53\n", "25 25 25\n", "16 16 16\n", "12 12 12\n", "7 7 7\n", "10 10 10\n", "13 13 13\n", "11 11 11\n", "9 9 9\n", "66 66 66\n", "8 8 8\n", "31 31 31\n", "7 7 7\n", "13 13 13\n", "10 10 10\n", "12 12 12\n", "44 44 44\n", "11 11 11\n", "17 17 17\n", "11 11 11\n", "27 27 27\n", "111 111 111\n", "11 11 11\n", "11 11 11\n", "12 12 12\n", "10 10 10\n", "6 6 6\n", "9 9 9\n", "12 12 12\n", "63 63 63\n", "7 7 7\n", "22 22 22\n", "18 18 18\n", "6 6 6\n", "32 32 32\n", "71 71 71\n", "40 40 40\n", "24 24 24\n", "20 20 20\n", "11 11 11\n", "3 3 3\n", "9 9 9\n", "5 5 5\n", "6 6 6\n", "10 10 10\n", "6 6 6\n", "12 12 12\n", "19 19 19\n", "11 11 11\n", "7 7 7\n", "101 101 101\n", "72 72 72\n", "12 12 12\n", "14 14 14\n", "18 18 18\n", "22 22 22\n", "31 31 31\n", "35 35 35\n", "10 10 10\n", "15 15 15\n", "4 4 4\n", "4 4 4\n", "11 11 11\n", "25 25 25\n", "12 12 12\n", "9 9 9\n", "7 7 7\n", "16 16 16\n", "46 46 46\n", "29 29 29\n", "6 6 6\n", "5 5 5\n", "7 7 7\n", "7 7 7\n", "19 19 19\n", "8 8 8\n", "8 8 8\n", "11 11 11\n", "10 10 10\n", "16 16 16\n", "10 10 10\n", "23 23 23\n", "25 25 25\n", "18 18 18\n", "13 13 13\n", "4 4 4\n", "11 11 11\n", "12 12 12\n", "4 4 4\n", "25 25 25\n", "8 8 8\n", "6 6 6\n", "23 23 23\n", "11 11 11\n", "82 82 82\n", "5 5 5\n", "4 4 4\n", "9 9 9\n", "49 49 49\n", "14 14 14\n", "8 8 8\n", "11 11 11\n", "14 14 14\n", "15 15 15\n", "4 4 4\n", "10 10 10\n", "8 8 8\n", "11 11 11\n", "6 6 6\n", "37 37 37\n", "35 35 35\n", "22 22 22\n", "4 4 4\n", "9 9 9\n", "12 12 12\n", "16 16 16\n", "8 8 8\n", "138 138 138\n", "10 10 10\n", "11 11 11\n", "12 12 12\n", "16 16 16\n", "4 4 4\n", "4 4 4\n", "41 41 41\n", "4 4 4\n", "14 14 14\n", "29 29 29\n", "15 15 15\n", "9 9 9\n", "36 36 36\n", "20 20 20\n", "15 15 15\n", "32 32 32\n", "24 24 24\n", "48 48 48\n", "9 9 9\n", "8 8 8\n", "38 38 38\n", "9 9 9\n", "6 6 6\n", "21 21 21\n", "9 9 9\n", "9 9 9\n", "8 8 8\n", "7 7 7\n", "16 16 16\n", "6 6 6\n", "20 20 20\n", "9 9 9\n", "7 7 7\n", "12 12 12\n", "10 10 10\n", "14 14 14\n", "16 16 16\n", "8 8 8\n", "8 8 8\n", "5 5 5\n", "27 27 27\n", "4 4 4\n", "20 20 20\n", "10 10 10\n", "5 5 5\n", "11 11 11\n", "12 12 12\n", "20 20 20\n", "12 12 12\n", "74 74 74\n", "4 4 4\n", "11 11 11\n", "4 4 4\n", "25 25 25\n", "22 22 22\n", "14 14 14\n", "26 26 26\n", "9 9 9\n", "4 4 4\n", "11 11 11\n", "20 20 20\n", "4 4 4\n", "60 60 60\n", "11 11 11\n", "20 20 20\n", "15 15 15\n", "3 3 3\n", "46 46 46\n", "24 24 24\n", "67 67 67\n", "20 20 20\n", "28 28 28\n", "15 15 15\n", "11 11 11\n", "5 5 5\n", "8 8 8\n", "63 63 63\n", "50 50 50\n", "15 15 15\n", "19 19 19\n", "3 3 3\n", "8 8 8\n", "17 17 17\n", "11 11 11\n", "25 25 25\n", "7 7 7\n", "9 9 9\n", "57 57 57\n", "10 10 10\n", "8 8 8\n", "14 14 14\n", "3 3 3\n", "37 37 37\n", "12 12 12\n", "19 19 19\n", "10 10 10\n", "110 110 110\n", "10 10 10\n", "17 17 17\n", "36 36 36\n", "6 6 6\n", "21 21 21\n", "17 17 17\n", "15 15 15\n", "11 11 11\n", "7 7 7\n", "23 23 23\n", "5 5 5\n", "6 6 6\n", "8 8 8\n", "9 9 9\n", "24 24 24\n", "21 21 21\n", "10 10 10\n", "4 4 4\n", "3 3 3\n", "27 27 27\n", "13 13 13\n", "7 7 7\n", "10 10 10\n", "56 56 56\n", "18 18 18\n", "6 6 6\n", "6 6 6\n", "14 14 14\n", "39 39 39\n", "7 7 7\n", "10 10 10\n", "9 9 9\n", "21 21 21\n", "9 9 9\n", "6 6 6\n", "23 23 23\n", "29 29 29\n", "17 17 17\n", "11 11 11\n", "4 4 4\n", "9 9 9\n", "14 14 14\n", "26 26 26\n", "4 4 4\n", "4 4 4\n", "3 3 3\n", "40 40 40\n", "10 10 10\n", "42 42 42\n", "10 10 10\n", "3 3 3\n", "6 6 6\n", "13 13 13\n", "10 10 10\n", "8 8 8\n", "18 18 18\n", "16 16 16\n", "14 14 14\n", "22 22 22\n", "8 8 8\n", "36 36 36\n", "7 7 7\n", "7 7 7\n", "10 10 10\n", "27 27 27\n", "6 6 6\n", "13 13 13\n", "8 8 8\n", "5 5 5\n", "16 16 16\n", "4 4 4\n", "21 21 21\n", "112 112 112\n", "25 25 25\n", "28 28 28\n", "4 4 4\n", "3 3 3\n", "28 28 28\n", "45 45 45\n", "11 11 11\n", "4 4 4\n", "5 5 5\n", "17 17 17\n", "16 16 16\n", "13 13 13\n", "45 45 45\n", "36 36 36\n", "15 15 15\n", "26 26 26\n", "10 10 10\n", "55 55 55\n", "11 11 11\n", "8 8 8\n", "8 8 8\n", "19 19 19\n", "27 27 27\n", "17 17 17\n", "33 33 33\n", "93 93 93\n", "4 4 4\n", "6 6 6\n", "67 67 67\n", "14 14 14\n", "12 12 12\n", "4 4 4\n", "6 6 6\n", "11 11 11\n", "15 15 15\n", "14 14 14\n", "24 24 24\n", "24 24 24\n", "13 13 13\n", "11 11 11\n", "8 8 8\n", "21 21 21\n", "3 3 3\n", "5 5 5\n", "25 25 25\n", "6 6 6\n", "4 4 4\n", "5 5 5\n", "23 23 23\n", "9 9 9\n", "6 6 6\n", "12 12 12\n", "5 5 5\n", "8 8 8\n", "14 14 14\n", "11 11 11\n", "14 14 14\n", "5 5 5\n", "9 9 9\n", "4 4 4\n", "10 10 10\n", "7 7 7\n", "22 22 22\n", "18 18 18\n", "21 21 21\n", "14 14 14\n", "5 5 5\n", "4 4 4\n", "8 8 8\n", "13 13 13\n", "8 8 8\n", "20 20 20\n", "22 22 22\n", "24 24 24\n", "13 13 13\n", "5 5 5\n", "49 49 49\n", "6 6 6\n", "5 5 5\n", "20 20 20\n", "6 6 6\n", "35 35 35\n", "12 12 12\n", "7 7 7\n", "10 10 10\n", "11 11 11\n", "11 11 11\n", "24 24 24\n", "8 8 8\n", "27 27 27\n", "32 32 32\n", "31 31 31\n", "15 15 15\n", "18 18 18\n", "13 13 13\n", "18 18 18\n", "9 9 9\n", "5 5 5\n", "13 13 13\n", "25 25 25\n", "3 3 3\n", "14 14 14\n", "5 5 5\n", "8 8 8\n", "6 6 6\n", "5 5 5\n", "5 5 5\n", "21 21 21\n", "11 11 11\n", "7 7 7\n", "20 20 20\n", "13 13 13\n", "3 3 3\n", "29 29 29\n", "16 16 16\n", "23 23 23\n", "12 12 12\n", "6 6 6\n", "6 6 6\n", "14 14 14\n", "8 8 8\n", "32 32 32\n", "10 10 10\n", "26 26 26\n", "3 3 3\n", "8 8 8\n", "6 6 6\n", "27 27 27\n", "9 9 9\n", "47 47 47\n", "8 8 8\n", "41 41 41\n", "10 10 10\n", "6 6 6\n", "16 16 16\n", "16 16 16\n", "10 10 10\n", "9 9 9\n", "4 4 4\n", "14 14 14\n", "24 24 24\n", "25 25 25\n", "9 9 9\n", "14 14 14\n", "38 38 38\n", "8 8 8\n", "13 13 13\n", "14 14 14\n", "13 13 13\n", "9 9 9\n", "19 19 19\n", "3 3 3\n", "12 12 12\n", "9 9 9\n", "22 22 22\n", "12 12 12\n", "11 11 11\n", "19 19 19\n", "18 18 18\n", "14 14 14\n", "12 12 12\n", "8 8 8\n", "7 7 7\n", "9 9 9\n", "11 11 11\n", "12 12 12\n", "45 45 45\n", "15 15 15\n", "13 13 13\n", "9 9 9\n", "12 12 12\n", "29 29 29\n", "11 11 11\n", "28 28 28\n", "6 6 6\n", "4 4 4\n", "5 5 5\n", "19 19 19\n", "4 4 4\n", "11 11 11\n", "13 13 13\n", "17 17 17\n", "25 25 25\n", "4 4 4\n", "5 5 5\n", "12 12 12\n", "7 7 7\n", "30 30 30\n", "14 14 14\n", "9 9 9\n", "9 9 9\n", "9 9 9\n", "4 4 4\n", "11 11 11\n", "22 22 22\n", "3 3 3\n", "13 13 13\n", "8 8 8\n", "11 11 11\n", "12 12 12\n", "257 257 257\n", "24 24 24\n", "23 23 23\n", "26 26 26\n", "4 4 4\n", "15 15 15\n", "28 28 28\n", "36 36 36\n", "11 11 11\n", "26 26 26\n", "16 16 16\n", "4 4 4\n", "11 11 11\n", "7 7 7\n", "21 21 21\n", "9 9 9\n", "9 9 9\n", "4 4 4\n", "14 14 14\n", "11 11 11\n", "13 13 13\n", "42 42 42\n", "14 14 14\n", "4 4 4\n", "66 66 66\n", "10 10 10\n", "26 26 26\n", "185 185 185\n", "4 4 4\n", "7 7 7\n", "7 7 7\n", "10 10 10\n", "4 4 4\n", "19 19 19\n", "27 27 27\n", "7 7 7\n", "35 35 35\n", "10 10 10\n", "9 9 9\n", "17 17 17\n", "11 11 11\n", "26 26 26\n", "120 120 120\n", "17 17 17\n", "6 6 6\n", "9 9 9\n", "18 18 18\n", "10 10 10\n", "12 12 12\n", "15 15 15\n", "12 12 12\n", "12 12 12\n", "6 6 6\n", "8 8 8\n", "6 6 6\n", "5 5 5\n", "19 19 19\n", "20 20 20\n", "5 5 5\n", "5 5 5\n", "18 18 18\n", "54 54 54\n", "15 15 15\n", "21 21 21\n", "43 43 43\n", "9 9 9\n", "15 15 15\n", "11 11 11\n", "8 8 8\n", "4 4 4\n", "32 32 32\n", "26 26 26\n", "9 9 9\n", "23 23 23\n", "15 15 15\n", "14 14 14\n", "33 33 33\n", "6 6 6\n", "15 15 15\n", "13 13 13\n", "9 9 9\n", "4 4 4\n", "9 9 9\n", "19 19 19\n", "4 4 4\n", "22 22 22\n", "17 17 17\n", "25 25 25\n", "18 18 18\n", "9 9 9\n", "56 56 56\n", "11 11 11\n", "12 12 12\n", "9 9 9\n", "9 9 9\n", "17 17 17\n", "15 15 15\n", "25 25 25\n", "41 41 41\n", "28 28 28\n", "33 33 33\n", "11 11 11\n", "10 10 10\n", "6 6 6\n", "5 5 5\n", "4 4 4\n", "4 4 4\n", "33 33 33\n", "9 9 9\n", "3 3 3\n", "23 23 23\n", "14 14 14\n", "5 5 5\n", "12 12 12\n", "4 4 4\n", "10 10 10\n", "8 8 8\n", "14 14 14\n", "26 26 26\n", "21 21 21\n", "7 7 7\n", "4 4 4\n", "3 3 3\n", "3 3 3\n", "15 15 15\n", "10 10 10\n", "6 6 6\n", "22 22 22\n", "9 9 9\n", "87 87 87\n", "7 7 7\n", "4 4 4\n", "21 21 21\n", "3 3 3\n", "60 60 60\n", "4 4 4\n", "14 14 14\n", "5 5 5\n", "13 13 13\n", "20 20 20\n", "9 9 9\n", "62 62 62\n", "11 11 11\n", "28 28 28\n", "35 35 35\n", "14 14 14\n", "26 26 26\n", "4 4 4\n", "7 7 7\n", "20 20 20\n", "15 15 15\n", "11 11 11\n", "10 10 10\n", "24 24 24\n", "5 5 5\n", "5 5 5\n", "3 3 3\n", "12 12 12\n", "4 4 4\n", "12 12 12\n", "28 28 28\n", "25 25 25\n", "12 12 12\n", "6 6 6\n", "46 46 46\n", "48 48 48\n", "18 18 18\n", "18 18 18\n", "14 14 14\n", "18 18 18\n", "9 9 9\n", "13 13 13\n", "19 19 19\n", "9 9 9\n", "7 7 7\n", "3 3 3\n", "18 18 18\n", "12 12 12\n", "12 12 12\n", "17 17 17\n", "27 27 27\n", "6 6 6\n", "16 16 16\n", "16 16 16\n", "5 5 5\n", "22 22 22\n", "12 12 12\n", "23 23 23\n", "13 13 13\n", "12 12 12\n", "5 5 5\n", "6 6 6\n", "58 58 58\n", "23 23 23\n", "11 11 11\n", "3 3 3\n", "12 12 12\n", "18 18 18\n", "40 40 40\n", "8 8 8\n", "42 42 42\n", "44 44 44\n", "23 23 23\n", "3 3 3\n", "28 28 28\n", "15 15 15\n", "14 14 14\n", "34 34 34\n", "10 10 10\n", "12 12 12\n", "25 25 25\n", "23 23 23\n", "12 12 12\n", "6 6 6\n", "18 18 18\n", "14 14 14\n", "16 16 16\n", "32 32 32\n", "18 18 18\n", "11 11 11\n", "8 8 8\n", "23 23 23\n" ] } ], "execution_count": 13 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T21:50:07.905274Z", "start_time": "2025-06-25T21:50:07.852363Z" } }, "cell_type": "code", "source": [ "print(f\"F1 on ES data before postprocessing {fine_grained_flausch_by_label(test_gold_spans, test_baseline_spans)['TOTAL']['STRICT']}\")\n", "print(f\"F1 on ES data with GE2017 postprocessing {fine_grained_flausch_by_label(test_gold_spans, ge2017_rules_test_pred_spans)['TOTAL']['STRICT']}\")" ], "id": "c2e99dae31d7e359", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "F1 on ES data before postprocessing {'prec': 0.7572335920959774, 'rec': 0.7073170731707317, 'f1': 0.7314246762099522}\n", "F1 on ES data with GE2017 postprocessing {'prec': 0.7649964714184898, 'rec': 0.7145682267633487, 'f1': 0.7389229720518065}\n" ] } ], "execution_count": 14 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T22:20:26.717428Z", "start_time": "2025-06-25T21:50:16.952422Z" } }, "cell_type": "code", "source": [ "test_comments = test_data\n", "\n", "test_comments['predicted_labels'] = None\n", "test_comments['predicted_probs'] = None\n", "test_comments['offset_mapping'] = None\n", "test_comments['text_tokens'] = None\n", "\n", "for idx in range(len(test_comments)): #range(15):\n", " row = test_comments.iloc[idx]\n", " text = row['comment']\n", " key = (row['document'], row['comment_id'])\n", "\n", " text_tokens = classifier.tokenizer.tokenize(text)\n", " test_comments.at[idx, 'text_tokens'] = text_tokens\n", "\n", " device = next(classifier.model.parameters()).device\n", " inputs = classifier.tokenizer(text, return_tensors=\"pt\", truncation=True, max_length=512, return_offsets_mapping=True)\n", "\n", " offset_mapping = inputs.pop('offset_mapping')\n", " test_comments.at[idx, 'offset_mapping'] = offset_mapping.cpu().numpy()[0].tolist()\n", " inputs = {k: v.to(device) for k, v in inputs.items()}\n", "\n", " # Vorhersage\n", " with torch.no_grad():\n", " outputs = classifier.model(**inputs)\n", "\n", " predicted_labels = torch.argmax(outputs.logits, dim=2)[0].cpu().numpy()\n", " predicted_probs = torch.nn.functional.softmax(outputs.logits, dim=2)[0].cpu().numpy()\n", " test_comments.at[idx, 'predicted_labels'] = predicted_labels\n", " test_comments.at[idx, 'predicted_probs'] = predicted_probs" ], "id": "40a52a3ed629de0e", "outputs": [], "execution_count": 15 }, { "metadata": { "ExecuteTime": { "end_time": "2025-06-25T22:20:27.018913Z", "start_time": "2025-06-25T22:20:26.848618Z" } }, "cell_type": "code", "source": "test_comments", "id": "94f29be61115fa59", "outputs": [ { "data": { "text/plain": [ " document comment_id comment \\\n", "0 NDY-004 1 Lol i love lochis \n", "1 NDY-004 2 ihr singt voll gut :) \n", "2 NDY-004 3 Junge fick dich \n", "3 NDY-004 4 Ihr seit die besten \n", "4 NDY-004 5 ihr seit die ALLER besten ich finde euch soooo... \n", "... ... ... ... \n", "9224 NDY-203 522 hihi kannst du mich grüßen 💕 👋 😍 Achso wusstes... \n", "9225 NDY-203 523 #Glocke aktiviert 👑 Ich liebe deine Videos 💍 💎... \n", "9226 NDY-203 524 Bist die beste ❤ Bitte Grüße mich 💕 ❤ 😘 😍 \n", "9227 NDY-203 525 Hi Bonny ❤️ War letztens auf'm Flughafen , und... \n", "9228 NDY-203 526 du bist die beste ich bin neu ich hab dich sof... \n", "\n", " predicted_labels \\\n", "0 [0, 3, 13, 13, 13, 13, 13, 13, 0] \n", "1 [0, 2, 12, 12, 12, 12, 12, 0] \n", "2 [0, 0, 0, 0, 0, 0] \n", "3 [0, 3, 13, 13, 13, 0] \n", "4 [0, 3, 13, 13, 13, 13, 13, 3, 13, 13, 13, 13, ... \n", "... ... \n", "9224 [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 11, 0, 0, 0, 0,... \n", "9225 [0, 0, 0, 0, 0, 1, 11, 11, 11, 11, 11, 0, 0, 0... \n", "9226 [0, 3, 13, 13, 13, 0, 0, 0, 0, 1, 11, 11, 11, 0] \n", "9227 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", "9228 [0, 3, 13, 13, 13, 0, 0, 0, 1, 11, 11, 11, 11,... \n", "\n", " predicted_probs \\\n", "0 [[0.9998258, 1.1969776e-05, 1.1567051e-06, 7.7... \n", "1 [[0.99976057, 6.098534e-05, 4.611753e-06, 3.97... \n", "2 [[0.9999795, 1.4359919e-06, 3.3691788e-07, 6.3... \n", "3 [[0.9998192, 4.859976e-05, 2.3917532e-06, 6.91... \n", "4 [[0.99971765, 4.202885e-05, 1.5942048e-06, 9.6... \n", "... ... \n", "9224 [[0.99931705, 3.137185e-05, 1.2843102e-06, 8.7... \n", "9225 [[0.9997228, 3.5343048e-05, 8.4241697e-07, 4.5... \n", "9226 [[0.99983156, 3.9064038e-05, 1.023527e-06, 4.7... \n", "9227 [[0.9999653, 2.7837193e-06, 2.9389835e-07, 4.8... \n", "9228 [[0.70373356, 0.0005549868, 1.8810852e-05, 2.5... \n", "\n", " offset_mapping \\\n", "0 [[0, 0], [0, 2], [2, 3], [4, 5], [6, 10], [11,... \n", "1 [[0, 0], [0, 3], [4, 9], [10, 14], [15, 18], [... \n", "2 [[0, 0], [0, 5], [6, 7], [7, 10], [11, 15], [0... \n", "3 [[0, 0], [0, 3], [4, 8], [9, 12], [13, 19], [0... \n", "4 [[0, 0], [0, 3], [4, 8], [9, 12], [13, 16], [1... \n", "... ... \n", "9224 [[0, 0], [0, 2], [2, 4], [5, 11], [12, 14], [1... \n", "9225 [[0, 0], [0, 1], [1, 7], [8, 17], [18, 19], [2... \n", "9226 [[0, 0], [0, 4], [5, 8], [9, 14], [15, 16], [1... \n", "9227 [[0, 0], [0, 2], [3, 7], [7, 8], [9, 11], [12,... \n", "9228 [[0, 0], [0, 2], [3, 7], [8, 11], [12, 17], [1... \n", "\n", " text_tokens \n", "0 [Lo, ##l, i, love, loc, ##hi, ##s] \n", "1 [ihr, singt, voll, gut, :, )] \n", "2 [Junge, f, ##ick, dich] \n", "3 [Ihr, seit, die, besten] \n", "4 [ihr, seit, die, ALL, ##ER, besten, ich, finde... \n", "... ... \n", "9224 [hi, ##hi, kannst, du, mich, grü, ##ßen, [UNK]... \n", "9225 [#, Glocke, aktiviert, [UNK], Ich, liebe, dein... \n", "9226 [Bist, die, beste, [UNK], Bitte, Grü, ##ße, mi... \n", "9227 [Hi, Bonn, ##y, [UNK], War, letzten, ##s, auf,... \n", "9228 [du, bist, die, beste, ich, bin, neu, ich, hab... \n", "\n", "[9229 rows x 7 columns]" ], "text/html": [ "
| \n", " | document | \n", "comment_id | \n", "comment | \n", "predicted_labels | \n", "predicted_probs | \n", "offset_mapping | \n", "text_tokens | \n", "
|---|---|---|---|---|---|---|---|
| 0 | \n", "NDY-004 | \n", "1 | \n", "Lol i love lochis | \n", "[0, 3, 13, 13, 13, 13, 13, 13, 0] | \n", "[[0.9998258, 1.1969776e-05, 1.1567051e-06, 7.7... | \n", "[[0, 0], [0, 2], [2, 3], [4, 5], [6, 10], [11,... | \n", "[Lo, ##l, i, love, loc, ##hi, ##s] | \n", "
| 1 | \n", "NDY-004 | \n", "2 | \n", "ihr singt voll gut :) | \n", "[0, 2, 12, 12, 12, 12, 12, 0] | \n", "[[0.99976057, 6.098534e-05, 4.611753e-06, 3.97... | \n", "[[0, 0], [0, 3], [4, 9], [10, 14], [15, 18], [... | \n", "[ihr, singt, voll, gut, :, )] | \n", "
| 2 | \n", "NDY-004 | \n", "3 | \n", "Junge fick dich | \n", "[0, 0, 0, 0, 0, 0] | \n", "[[0.9999795, 1.4359919e-06, 3.3691788e-07, 6.3... | \n", "[[0, 0], [0, 5], [6, 7], [7, 10], [11, 15], [0... | \n", "[Junge, f, ##ick, dich] | \n", "
| 3 | \n", "NDY-004 | \n", "4 | \n", "Ihr seit die besten | \n", "[0, 3, 13, 13, 13, 0] | \n", "[[0.9998192, 4.859976e-05, 2.3917532e-06, 6.91... | \n", "[[0, 0], [0, 3], [4, 8], [9, 12], [13, 19], [0... | \n", "[Ihr, seit, die, besten] | \n", "
| 4 | \n", "NDY-004 | \n", "5 | \n", "ihr seit die ALLER besten ich finde euch soooo... | \n", "[0, 3, 13, 13, 13, 13, 13, 3, 13, 13, 13, 13, ... | \n", "[[0.99971765, 4.202885e-05, 1.5942048e-06, 9.6... | \n", "[[0, 0], [0, 3], [4, 8], [9, 12], [13, 16], [1... | \n", "[ihr, seit, die, ALL, ##ER, besten, ich, finde... | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 9224 | \n", "NDY-203 | \n", "522 | \n", "hihi kannst du mich grüßen 💕 👋 😍 Achso wusstes... | \n", "[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 11, 0, 0, 0, 0,... | \n", "[[0.99931705, 3.137185e-05, 1.2843102e-06, 8.7... | \n", "[[0, 0], [0, 2], [2, 4], [5, 11], [12, 14], [1... | \n", "[hi, ##hi, kannst, du, mich, grü, ##ßen, [UNK]... | \n", "
| 9225 | \n", "NDY-203 | \n", "523 | \n", "#Glocke aktiviert 👑 Ich liebe deine Videos 💍 💎... | \n", "[0, 0, 0, 0, 0, 1, 11, 11, 11, 11, 11, 0, 0, 0... | \n", "[[0.9997228, 3.5343048e-05, 8.4241697e-07, 4.5... | \n", "[[0, 0], [0, 1], [1, 7], [8, 17], [18, 19], [2... | \n", "[#, Glocke, aktiviert, [UNK], Ich, liebe, dein... | \n", "
| 9226 | \n", "NDY-203 | \n", "524 | \n", "Bist die beste ❤ Bitte Grüße mich 💕 ❤ 😘 😍 | \n", "[0, 3, 13, 13, 13, 0, 0, 0, 0, 1, 11, 11, 11, 0] | \n", "[[0.99983156, 3.9064038e-05, 1.023527e-06, 4.7... | \n", "[[0, 0], [0, 4], [5, 8], [9, 14], [15, 16], [1... | \n", "[Bist, die, beste, [UNK], Bitte, Grü, ##ße, mi... | \n", "
| 9227 | \n", "NDY-203 | \n", "525 | \n", "Hi Bonny ❤️ War letztens auf'm Flughafen , und... | \n", "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... | \n", "[[0.9999653, 2.7837193e-06, 2.9389835e-07, 4.8... | \n", "[[0, 0], [0, 2], [3, 7], [7, 8], [9, 11], [12,... | \n", "[Hi, Bonn, ##y, [UNK], War, letzten, ##s, auf,... | \n", "
| 9228 | \n", "NDY-203 | \n", "526 | \n", "du bist die beste ich bin neu ich hab dich sof... | \n", "[0, 3, 13, 13, 13, 0, 0, 0, 1, 11, 11, 11, 11,... | \n", "[[0.70373356, 0.0005549868, 1.8810852e-05, 2.5... | \n", "[[0, 0], [0, 2], [3, 7], [8, 11], [12, 17], [1... | \n", "[du, bist, die, beste, ich, bin, neu, ich, hab... | \n", "
9229 rows × 7 columns
\n", "| \n", " | document | \n", "comment_id | \n", "type | \n", "start | \n", "end | \n", "
|---|---|---|---|---|---|
| 0 | \n", "NDY-004 | \n", "1 | \n", "affection declaration | \n", "0 | \n", "17 | \n", "
| 1 | \n", "NDY-004 | \n", "2 | \n", "compliment | \n", "0 | \n", "21 | \n", "
| 2 | \n", "NDY-004 | \n", "4 | \n", "affection declaration | \n", "0 | \n", "19 | \n", "
| 3 | \n", "NDY-004 | \n", "5 | \n", "affection declaration | \n", "0 | \n", "25 | \n", "
| 4 | \n", "NDY-004 | \n", "5 | \n", "affection declaration | \n", "26 | \n", "56 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 4954 | \n", "NDY-203 | \n", "524 | \n", "positive feedback | \n", "34 | \n", "41 | \n", "
| 4955 | \n", "NDY-203 | \n", "525 | \n", "positive feedback | \n", "229 | \n", "237 | \n", "
| 4956 | \n", "NDY-203 | \n", "526 | \n", "affection declaration | \n", "0 | \n", "17 | \n", "
| 4957 | \n", "NDY-203 | \n", "526 | \n", "positive feedback | \n", "30 | \n", "63 | \n", "
| 4958 | \n", "NDY-203 | \n", "526 | \n", "affection declaration | \n", "105 | \n", "114 | \n", "
4959 rows × 5 columns
\n", "