Duplicate from pierreguillou/Inference-APP-Document-Understanding-at-paragraphlevel-v2

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,16 @@
+---
+title: Document Understanding Inference APP (v2 - paragraph level - LayoutXLM base)
+emoji: 🐢
+colorFrom: blue
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.18.0
+app_file: app.py
+pinned: false
+models:
+- >-
+  pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-paragraphlevel-ml512
+duplicated_from: pierreguillou/Inference-APP-Document-Understanding-at-paragraphlevel-v2
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,202 @@
+import os
+
+# workaround: install old version of pytorch since detectron2 hasn't released packages for pytorch 1.9 (issue: https://github.com/facebookresearch/detectron2/issues/3158)
+# os.system('pip install torch==1.8.0+cu101 torchvision==0.9.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+os.system('pip install -q torch==1.10.0+cu111 torchvision==0.11+cu111 -f https://download.pytorch.org/whl/torch_stable.html')
+
+# install detectron2 that matches pytorch 1.8
+# See https://detectron2.readthedocs.io/tutorials/install.html for instructions
+#os.system('pip install -q detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html')
+os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
+
+import detectron2
+from detectron2.utils.logger import setup_logger
+setup_logger()
+
+import gradio as gr
+import re
+import string
+
+from operator import itemgetter
+import collections
+
+import pypdf
+from pypdf import PdfReader
+from pypdf.errors import PdfReadError
+
+import pdf2image
+from pdf2image import convert_from_path
+import langdetect
+from langdetect import detect_langs
+
+import pandas as pd
+import numpy as np
+import random
+import tempfile
+import itertools
+
+from matplotlib import font_manager
+from PIL import Image, ImageDraw, ImageFont
+import cv2
+
+## files
+
+import sys  
+sys.path.insert(0, 'files/')
+
+import functions
+from functions import *
+
+# update pip
+os.system('python -m pip install --upgrade pip')
+
+## model / feature extractor / tokenizer
+
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# model
+from transformers import LayoutLMv2ForTokenClassification
+
+model_id = "pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-paragraphlevel-ml512"
+
+model = LayoutLMv2ForTokenClassification.from_pretrained(model_id);
+model.to(device);
+
+# feature extractor
+from transformers import LayoutLMv2FeatureExtractor
+feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=False)
+
+# tokenizer
+from transformers import AutoTokenizer
+tokenizer_id = "xlm-roberta-base"
+tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+
+# get labels
+id2label = model.config.id2label
+label2id = model.config.label2id
+num_labels = len(id2label)
+
+# APP outputs 
+def app_outputs(uploaded_pdf):
+    filename, msg, images = pdf_to_images(uploaded_pdf)
+    num_images = len(images)
+
+    if not msg.startswith("Error with the PDF"):
+
+        # Extraction of image data (text and bounding boxes)
+        dataset, texts_lines, texts_pars, texts_lines_par, row_indexes, par_boxes, line_boxes, lines_par_boxes = extraction_data_from_image(images)
+        # prepare our data in the format of the model
+        encoded_dataset = dataset.map(prepare_inference_features_paragraph, batched=True, batch_size=64, remove_columns=dataset.column_names)
+        custom_encoded_dataset = CustomDataset(encoded_dataset, tokenizer)
+        # Get predictions (token level)
+        outputs, images_ids_list, chunk_ids, input_ids, bboxes = predictions_token_level(images, custom_encoded_dataset)
+        # Get predictions (paragraph level)
+        probs_bbox, bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df = predictions_paragraph_level(dataset, outputs, images_ids_list, chunk_ids, input_ids, bboxes)
+        # Get labeled images with lines bounding boxes
+        images = get_labeled_images(dataset, images_ids_list, bboxes_list_dict, probs_dict_dict)
+
+        img_files = list()
+        # get image of PDF without bounding boxes
+        for i in range(num_images):
+            if filename != "files/blank.png": img_file = f"img_{i}_" + filename.replace(".pdf", ".png")
+            else: img_file = filename.replace(".pdf", ".png")
+            img_file = img_file.replace("/", "_")
+            images[i].save(img_file)
+            img_files.append(img_file)
+
+        if num_images < max_imgboxes:
+            img_files += [image_blank]*(max_imgboxes - num_images)
+            images += [Image.open(image_blank)]*(max_imgboxes - num_images)
+            for count in range(max_imgboxes - num_images):
+                df[num_images + count] = pd.DataFrame()
+        else:
+            img_files = img_files[:max_imgboxes]
+            images = images[:max_imgboxes]
+            df = dict(itertools.islice(df.items(), max_imgboxes))
+
+        # save 
+        csv_files = list()
+        for i in range(max_imgboxes):
+            csv_file = f"csv_{i}_" + filename.replace(".pdf", ".csv")
+            csv_file = csv_file.replace("/", "_")
+            csv_files.append(gr.File.update(value=csv_file, visible=True))
+            df[i].to_csv(csv_file, encoding="utf-8", index=False)
+
+    else:  
+        img_files, images, csv_files = [""]*max_imgboxes, [""]*max_imgboxes, [""]*max_imgboxes
+        img_files[0], img_files[1] = image_blank, image_blank
+        images[0], images[1] = Image.open(image_blank), Image.open(image_blank)
+        csv_file = "csv_wo_content.csv"
+        csv_files[0], csv_files[1] = gr.File.update(value=csv_file, visible=True), gr.File.update(value=csv_file, visible=True)
+        df, df_empty = dict(), pd.DataFrame()
+        df[0], df[1] = df_empty.to_csv(csv_file, encoding="utf-8", index=False), df_empty.to_csv(csv_file, encoding="utf-8", index=False)
+    
+    return msg, img_files[0], img_files[1], images[0], images[1], csv_files[0], csv_files[1], df[0], df[1]
+
+# Gradio APP
+with gr.Blocks(title="Inference APP for Document Understanding at paragraph level (v2 - LayoutXLM base)", css=".gradio-container") as demo:
+    gr.HTML("""
+    <div style="font-family:'Times New Roman', 'Serif'; font-size:26pt; font-weight:bold; text-align:center;"><h1>Inference APP for Document Understanding at paragraph level (v2 - LayoutXLM base)</h1></div>
+    <div style="margin-top: 40px"><p>(03/31/2023) This Inference APP uses the <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-paragraphlevel-ml512" target="_blank">model Layout XLM base combined with XLM-RoBERTa base and finetuned on the dataset DocLayNet base at paragraph level</a> (chunk size of 512 tokens).</p></div>
+    <div><p><a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://arxiv.org/abs/2104.08836" target="_blank">LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding</a> is a Document Understanding model that uses both layout and text in order to detect labels of bounding boxes. Combined with the model <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/xlm-roberta-base" target="_blank">XML-RoBERTa base</a>, this finetuned model has the capacity to <b>understand any language</b>. Finetuned on the dataset <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/datasets/pierreguillou/DocLayNet-base" target="_blank">DocLayNet base</a>, it can <b>classifly any bounding box (and its OCR text) to 11 labels</b> (Caption, Footnote, Formula, List-item, Page-footer, Page-header, Picture, Section-header, Table, Text, Title).</p></div>
+    <div><p>It relies on an external OCR engine to get words and bounding boxes from the document image. Thus, let's run in this APP an OCR engine (<a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://github.com/madmaze/pytesseract#python-tesseract" target="_blank">PyTesseract</a>) to get the bounding boxes, then run Layout XLM base (already fine-tuned on the dataset DocLayNet base at paragraph level) on the individual tokens and then, visualize the result at paragraph level!</p></div>
+    <div><p><b>It allows to get all pages of any PDF (of any language) with bounding boxes labeled at paragraph level and the associated dataframes with labeled data (bounding boxes, texts, labels) :-)</b></p></div>
+    <div><p>However, the inference time per page can be high when running the model on CPU due to the number of paragraph predictions to be made. Therefore, to avoid running this APP for too long, <b>only the first 2 pages are processed by this APP</b>. If you want to increase this limit, you can either clone this APP in Hugging Face Space (or run its <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://github.com/piegu/language-models/blob/master/Gradio_inference_on_LayoutXLM_base_model_finetuned_on_DocLayNet_base_in_any_language_at_levelparagraphs_ml512.ipynb" target="_blank">notebook</a> on your own plateform) and change the value of the parameter <code>max_imgboxes</code>, or run the inference notebook "<a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://github.com/piegu/language-models/blob/master/inference_on_LayoutXLM_base_model_finetuned_on_DocLayNet_base_in_any_language_at_levelparagraphs_ml512.ipynb" target="_blank">Document AI | Inference at paragraph level with a Document Understanding model (LayoutXLM base fine-tuned on DocLayNet dataset)</a>" on your own platform as it does not have this limit.</p></div>
+    <div style="margin-top: 20px"><p>More information about the DocLayNet datasets, the finetuning of the model and this APP in the following blog posts:</p>
+    <ul><li>(03/31/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-inference-app-and-fine-tuning-notebook-for-document-understanding-at-paragraph-level-3507af80573d" target="_blank">Document AI | Inference APP and fine-tuning notebook for Document Understanding at paragraph level with LayoutXLM base</a></li><li>(03/25/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-app-to-compare-the-document-understanding-lilt-and-layoutxlm-base-models-at-line-1c53eb481a15" target="_blank">Document AI | APP to compare the Document Understanding LiLT and LayoutXLM (base) models at line level</a></li><li>(03/05/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-inference-app-and-fine-tuning-notebook-for-document-understanding-at-line-level-with-b08fdca5f4dc" target="_blank">Document AI | Inference APP and fine-tuning notebook for Document Understanding at line level with LayoutXLM base</a></li><li>(02/14/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-inference-app-for-document-understanding-at-line-level-a35bbfa98893" target="_blank">Document AI | Inference APP for Document Understanding at line level</a></li><li>(02/10/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-document-understanding-model-at-line-level-with-lilt-tesseract-and-doclaynet-dataset-347107a643b8" target="_blank">Document AI | Document Understanding model at line level with LiLT, Tesseract and DocLayNet dataset</a></li><li>(01/31/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-doclaynet-image-viewer-app-3ac54c19956" target="_blank">Document AI | DocLayNet image viewer APP</a></li><li>(01/27/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-processing-of-doclaynet-dataset-to-be-used-by-layout-models-of-the-hugging-face-hub-308d8bd81cdb" target="_blank">Document AI | Processing of DocLayNet dataset to be used by layout models of the Hugging Face hub (finetuning, inference)</a></li></ul></div> 
+    """)
+    with gr.Row():
+        pdf_file = gr.File(label="PDF")
+    with gr.Row():
+        submit_btn = gr.Button(f"Display first {max_imgboxes} labeled PDF pages")
+        reset_btn = gr.Button(value="Clear")
+    with gr.Row():
+        output_msg = gr.Textbox(label="Output message")
+    with gr.Row():
+        fileboxes = []
+        for num_page in range(max_imgboxes):
+            file_path = gr.File(visible=True, label=f"Image file of the PDF page n°{num_page}")
+            fileboxes.append(file_path)
+    with gr.Row():
+        imgboxes = []
+        for num_page in range(max_imgboxes):
+            img = gr.Image(type="pil", label=f"Image of the PDF page n°{num_page}")
+            imgboxes.append(img)
+    with gr.Row():
+        csvboxes = []
+        for num_page in range(max_imgboxes):
+            csv = gr.File(visible=True, label=f"CSV file at paragraph level (page {num_page})")
+            csvboxes.append(csv)
+    with gr.Row():
+        dfboxes = []
+        for num_page in range(max_imgboxes):
+            df = gr.Dataframe(
+                      headers=["bounding boxes", "texts", "labels"],
+                      datatype=["str", "str", "str"],
+                      col_count=(3, "fixed"), 
+                      visible=True,
+                      label=f"Data of page {num_page}",
+                      type="pandas",
+                      wrap=True
+                    )
+            dfboxes.append(df)
+
+    outputboxes = [output_msg] + fileboxes + imgboxes + csvboxes + dfboxes
+    submit_btn.click(app_outputs, inputs=[pdf_file], outputs=outputboxes)
+    # https://github.com/gradio-app/gradio/pull/2044/files#diff-a91dd2749f68bb7d0099a0f4079a4fd2d10281e299e7b451cb1bb876a7c21975R91
+    reset_btn.click(
+        lambda: [pdf_file.update(value=None), output_msg.update(value=None)] + [filebox.update(value=None) for filebox in fileboxes] + [imgbox.update(value=None) for imgbox in imgboxes] + [csvbox.update(value=None) for csvbox in csvboxes] + [dfbox.update(value=None) for dfbox in dfboxes],
+        inputs=[],
+        outputs=[pdf_file, output_msg] + fileboxes + imgboxes + csvboxes + dfboxes
+        )
+    
+    gr.Examples(
+        [["files/example.pdf"]],
+        [pdf_file],
+        outputboxes,
+        fn=app_outputs,
+        cache_examples=True,
+        )
+    
+demo.launch()

files/functions.py

@@ -0,0 +1,882 @@
+import os
+
+# workaround: install old version of pytorch since detectron2 hasn't released packages for pytorch 1.9 (issue: https://github.com/facebookresearch/detectron2/issues/3158)
+# os.system('pip install torch==1.8.0+cu101 torchvision==0.9.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+os.system('pip install -q torch==1.10.0+cu111 torchvision==0.11+cu111 -f https://download.pytorch.org/whl/torch_stable.html')
+
+# install detectron2 that matches pytorch 1.8
+# See https://detectron2.readthedocs.io/tutorials/install.html for instructions
+#os.system('pip install -q detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html')
+os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
+
+import detectron2
+from detectron2.utils.logger import setup_logger
+setup_logger()
+
+import gradio as gr
+import re
+import string
+import torch
+
+from operator import itemgetter
+import collections
+
+import pypdf
+from pypdf import PdfReader
+from pypdf.errors import PdfReadError
+
+import pdf2image
+from pdf2image import convert_from_path
+import langdetect
+from langdetect import detect_langs
+
+import pandas as pd
+import numpy as np
+import random
+import tempfile
+import itertools
+
+from matplotlib import font_manager
+from PIL import Image, ImageDraw, ImageFont
+import cv2
+
+import pathlib
+from pathlib import Path
+import shutil
+
+# Tesseract
+print(os.popen(f'cat /etc/debian_version').read())
+print(os.popen(f'cat /etc/issue').read())
+print(os.popen(f'apt search tesseract').read())
+import pytesseract
+
+## Key parameters
+
+# categories colors
+label2color = {
+    'Caption': 'brown',
+    'Footnote': 'orange',
+    'Formula': 'gray',
+    'List-item': 'yellow',
+    'Page-footer': 'red',
+    'Page-header': 'red',
+    'Picture': 'violet',
+    'Section-header': 'orange',
+    'Table': 'green',
+    'Text': 'blue',
+    'Title': 'pink'
+    }
+
+# bounding boxes start and end of a sequence
+cls_box = [0, 0, 0, 0]
+sep_box = [1000, 1000, 1000, 1000]
+
+# model
+model_id = "pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-paragraphlevel-ml512"
+
+# tokenizer
+tokenizer_id = "xlm-roberta-base"
+
+# (tokenization) The maximum length of a feature (sequence)
+if str(384) in model_id:
+  max_length = 384 
+elif str(512) in model_id:
+  max_length = 512 
+else:
+  print("Error with max_length of chunks!")
+
+# (tokenization) overlap
+doc_stride = 128 # The authorized overlap between two part of the context when splitting it is needed.
+
+# max PDF page images that will be displayed
+max_imgboxes = 2
+
+# get files
+examples_dir = 'files/'
+Path(examples_dir).mkdir(parents=True, exist_ok=True)
+from huggingface_hub import hf_hub_download
+files = ["example.pdf", "blank.pdf", "blank.png", "languages_iso.csv", "languages_tesseract.csv", "wo_content.png"]
+for file_name in files:
+    path_to_file = hf_hub_download(
+        repo_id = "pierreguillou/Inference-APP-Document-Understanding-at-paragraphlevel-v2",
+        filename = "files/" + file_name,
+        repo_type = "space"
+        )
+    shutil.copy(path_to_file,examples_dir)
+
+# path to files
+image_wo_content = examples_dir + "wo_content.png" # image without content
+pdf_blank = examples_dir + "blank.pdf" # blank PDF
+image_blank = examples_dir + "blank.png" # blank image
+
+## get langdetect2Tesseract dictionary
+t = "files/languages_tesseract.csv"
+l = "files/languages_iso.csv"
+
+df_t = pd.read_csv(t)
+df_l = pd.read_csv(l)
+
+langs_t = df_t["Language"].to_list()
+langs_t = [lang_t.lower().strip().translate(str.maketrans('', '', string.punctuation)) for lang_t in langs_t]
+langs_l = df_l["Language"].to_list()
+langs_l = [lang_l.lower().strip().translate(str.maketrans('', '', string.punctuation)) for lang_l in langs_l]
+langscode_t = df_t["LangCode"].to_list()
+langscode_l = df_l["LangCode"].to_list()
+
+Tesseract2langdetect, langdetect2Tesseract = dict(), dict()
+for lang_t, langcode_t in zip(langs_t,langscode_t):
+  try:
+    if lang_t == "Chinese - Simplified".lower().strip().translate(str.maketrans('', '', string.punctuation)): lang_t = "chinese"
+    index = langs_l.index(lang_t)
+    langcode_l = langscode_l[index]
+    Tesseract2langdetect[langcode_t] = langcode_l
+  except: 
+    continue
+
+langdetect2Tesseract = {v:k for k,v in Tesseract2langdetect.items()}
+
+## model / feature extractor / tokenizer
+
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+from transformers import LayoutLMv2ForTokenClassification # LayoutXLMTokenizerFast, 
+
+model = LayoutLMv2ForTokenClassification.from_pretrained(model_id);
+model.to(device);
+
+# feature extractor
+from transformers import LayoutLMv2FeatureExtractor
+feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=False)
+
+# tokenizer
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+
+# get labels
+id2label = model.config.id2label
+label2id = model.config.label2id
+num_labels = len(id2label)
+
+## General
+
+# get text and bounding boxes from an image
+# https://stackoverflow.com/questions/61347755/how-can-i-get-line-coordinates-that-readed-by-tesseract
+# https://medium.com/geekculture/tesseract-ocr-understanding-the-contents-of-documents-beyond-their-text-a98704b7c655
+def get_data_paragraph(results, factor, conf_min=0):
+
+  data = {}
+  for i in range(len(results['line_num'])):
+    level = results['level'][i]
+    block_num = results['block_num'][i]
+    par_num = results['par_num'][i]
+    line_num = results['line_num'][i]
+    top, left = results['top'][i], results['left'][i]
+    width, height = results['width'][i], results['height'][i]
+    conf = results['conf'][i]
+    text = results['text'][i]
+    if not (text == '' or text.isspace()):
+      if conf >= conf_min:
+        tup = (text, left, top, width, height)
+        if block_num in list(data.keys()):
+          if par_num in list(data[block_num].keys()):
+            if line_num in list(data[block_num][par_num].keys()):
+              data[block_num][par_num][line_num].append(tup)
+            else:
+              data[block_num][par_num][line_num] = [tup]
+          else:
+            data[block_num][par_num] = {}
+            data[block_num][par_num][line_num] = [tup]
+        else:
+            data[block_num] = {}
+            data[block_num][par_num] = {}
+            data[block_num][par_num][line_num] = [tup]
+
+  # get paragraphs dicionnary with list of lines
+  par_data = {}
+  par_idx = 1
+  for _, b  in data.items():
+    for _, p in b.items():
+      line_data = {}
+      line_idx = 1
+      for _, l in p.items():
+        line_data[line_idx] = l
+        line_idx += 1
+      par_data[par_idx] = line_data 
+      par_idx += 1
+
+  # get lines of texts, grouped by paragraph
+  texts_pars = list()
+  row_indexes = list()
+  texts_lines = list()
+  texts_lines_par = list()
+  row_index = 0
+  for _,par in par_data.items():
+    count_lines = 0
+    lines_par = list()
+    for _,line in par.items():
+      if count_lines == 0: row_indexes.append(row_index)
+      line_text = ' '.join([item[0] for item in line])
+      texts_lines.append(line_text)
+      lines_par.append(line_text)
+      count_lines += 1
+      row_index += 1
+    # lines.append("\n")
+    row_index += 1
+    texts_lines_par.append(lines_par)
+    texts_pars.append(' '.join(lines_par))
+  # lines = lines[:-1]
+  
+  # get paragraphes boxes (par_boxes)
+  # get lines boxes (line_boxes)
+  par_boxes = list()
+  par_idx = 1
+  line_boxes, lines_par_boxes = list(), list()
+  line_idx = 1
+  for _, par in par_data.items():
+    xmins, ymins, xmaxs, ymaxs = list(), list(), list(), list()
+    line_boxes_par = list()
+    count_line_par = 0
+    for _, line in par.items():
+      xmin, ymin = line[0][1], line[0][2]
+      xmax, ymax = (line[-1][1] + line[-1][3]), (line[-1][2] + line[-1][4])
+      line_boxes.append([int(xmin/factor), int(ymin/factor), int(xmax/factor), int(ymax/factor)])
+      line_boxes_par.append([int(xmin/factor), int(ymin/factor), int(xmax/factor), int(ymax/factor)])
+      xmins.append(xmin)
+      ymins.append(ymin)
+      xmaxs.append(xmax)
+      ymaxs.append(ymax)
+      line_idx += 1
+      count_line_par += 1
+    xmin, ymin, xmax, ymax = min(xmins), min(ymins), max(xmaxs), max(ymaxs)
+    par_bbox = [int(xmin/factor), int(ymin/factor), int(xmax/factor), int(ymax/factor)]
+    par_boxes.append(par_bbox)
+    lines_par_boxes.append(line_boxes_par)
+    par_idx += 1
+
+  return texts_lines, texts_pars, texts_lines_par, row_indexes, par_boxes, line_boxes, lines_par_boxes 
+
+# rescale image to get 300dpi
+def set_image_dpi_resize(image):
+    """
+    Rescaling image to 300dpi while resizing
+    :param image: An image
+    :return: A rescaled image
+    """
+    length_x, width_y = image.size
+    factor = min(1, float(1024.0 / length_x))
+    size = int(factor * length_x), int(factor * width_y)
+    # image_resize = image.resize(size, Image.Resampling.LANCZOS)
+    image_resize = image.resize(size, Image.LANCZOS)
+    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='1.png')
+    temp_filename = temp_file.name
+    image_resize.save(temp_filename, dpi=(300, 300))
+    return factor, temp_filename
+
+# it is important that each bounding box should be in (upper left, lower right) format.
+# source: https://github.com/NielsRogge/Transformers-Tutorials/issues/129
+def upperleft_to_lowerright(bbox):
+  x0, y0, x1, y1 = tuple(bbox)
+  if bbox[2] < bbox[0]:
+    x0 = bbox[2]
+    x1 = bbox[0] 
+  if bbox[3] < bbox[1]:
+    y0 = bbox[3]
+    y1 = bbox[1] 
+  return [x0, y0, x1, y1]
+
+# convert boundings boxes (left, top, width, height) format to (left, top, left+widght, top+height) format. 
+def convert_box(bbox):
+    x, y, w, h = tuple(bbox) # the row comes in (left, top, width, height) format
+    return [x, y, x+w, y+h] # we turn it into (left, top, left+widght, top+height) to get the actual box 
+
+# LiLT model gets 1000x10000 pixels images
+def normalize_box(bbox, width, height):
+    return [
+        int(1000 * (bbox[0] / width)),
+        int(1000 * (bbox[1] / height)),
+        int(1000 * (bbox[2] / width)),
+        int(1000 * (bbox[3] / height)),
+    ]
+
+# LiLT model gets 1000x10000 pixels images
+def denormalize_box(bbox, width, height):
+    return [
+        int(width * (bbox[0] / 1000)),
+        int(height * (bbox[1] / 1000)),
+        int(width* (bbox[2] / 1000)),
+        int(height * (bbox[3] / 1000)),
+    ]
+
+# get back original size
+def original_box(box, original_width, original_height, coco_width, coco_height):
+    return [
+        int(original_width * (box[0] / coco_width)),
+        int(original_height * (box[1] / coco_height)),
+        int(original_width * (box[2] / coco_width)),
+        int(original_height* (box[3] / coco_height)),
+    ]
+
+def get_blocks(bboxes_block, categories, texts):
+
+ # get list of unique block boxes
+    bbox_block_dict, bboxes_block_list, bbox_block_prec = dict(), list(), list()
+    for count_block, bbox_block in enumerate(bboxes_block):
+      if bbox_block != bbox_block_prec:
+        bbox_block_indexes = [i for i, bbox in enumerate(bboxes_block) if bbox == bbox_block]
+        bbox_block_dict[count_block] = bbox_block_indexes
+        bboxes_block_list.append(bbox_block)
+      bbox_block_prec = bbox_block
+
+    # get list of categories and texts by unique block boxes
+    category_block_list, text_block_list = list(), list()
+    for bbox_block in bboxes_block_list:
+      count_block = bboxes_block.index(bbox_block)
+      bbox_block_indexes = bbox_block_dict[count_block]
+      category_block = np.array(categories, dtype=object)[bbox_block_indexes].tolist()[0]
+      category_block_list.append(category_block)
+      text_block = np.array(texts, dtype=object)[bbox_block_indexes].tolist()
+      text_block = [text.replace("\n","").strip() for text in text_block]
+      if id2label[category_block] == "Text" or id2label[category_block] == "Caption" or id2label[category_block] == "Footnote":
+        text_block = ' '.join(text_block)
+      else:
+        text_block = '\n'.join(text_block)
+      text_block_list.append(text_block)
+
+    return bboxes_block_list, category_block_list, text_block_list
+
+# function to sort bounding boxes
+def get_sorted_boxes(bboxes):
+
+  # sort by y from page top to bottom 
+  sorted_bboxes = sorted(bboxes, key=itemgetter(1), reverse=False)
+  y_list = [bbox[1] for bbox in sorted_bboxes]
+
+  # sort by x from page left to right when boxes with same y
+  if len(list(set(y_list))) != len(y_list):
+    y_list_duplicates_indexes = dict()
+    y_list_duplicates = [item for item, count in collections.Counter(y_list).items() if count > 1]
+    for item in y_list_duplicates:
+      y_list_duplicates_indexes[item] = [i for i, e in enumerate(y_list) if e == item]
+      bbox_list_y_duplicates = sorted(np.array(sorted_bboxes, dtype=object)[y_list_duplicates_indexes[item]].tolist(), key=itemgetter(0), reverse=False)
+      np_array_bboxes = np.array(sorted_bboxes)
+      np_array_bboxes[y_list_duplicates_indexes[item]] = np.array(bbox_list_y_duplicates)
+      sorted_bboxes = np_array_bboxes.tolist()
+
+  return sorted_bboxes
+
+# sort data from y = 0 to end of page (and after, x=0 to end of page when necessary)
+def sort_data(bboxes, categories, texts):
+
+    sorted_bboxes = get_sorted_boxes(bboxes)
+    sorted_bboxes_indexes = [bboxes.index(bbox) for bbox in sorted_bboxes]
+    sorted_categories = np.array(categories, dtype=object)[sorted_bboxes_indexes].tolist()
+    sorted_texts = np.array(texts, dtype=object)[sorted_bboxes_indexes].tolist()
+
+    return sorted_bboxes, sorted_categories, sorted_texts
+
+# sort data from y = 0 to end of page (and after, x=0 to end of page when necessary)
+def sort_data_wo_labels(bboxes, texts):
+
+    sorted_bboxes = get_sorted_boxes(bboxes)
+    sorted_bboxes_indexes = [bboxes.index(bbox) for bbox in sorted_bboxes]
+    sorted_texts = np.array(texts, dtype=object)[sorted_bboxes_indexes].tolist()
+
+    return sorted_bboxes, sorted_texts
+    
+## PDF processing
+
+# get filename and images of PDF pages
+def pdf_to_images(uploaded_pdf):
+
+    # Check if None object
+    if uploaded_pdf is None:
+        path_to_file = pdf_blank
+        filename = path_to_file.replace(examples_dir,"")
+        msg = "Invalid PDF file."
+        images = [Image.open(image_blank)]
+    else:
+        # path to the uploaded PDF
+        path_to_file = uploaded_pdf.name
+        filename = path_to_file.replace("/tmp/","")
+
+        try:
+            PdfReader(path_to_file)
+        except PdfReadError:
+            path_to_file = pdf_blank
+            filename = path_to_file.replace(examples_dir,"")
+            msg = "Invalid PDF file."
+            images = [Image.open(image_blank)]
+        else:
+            try:
+                images = convert_from_path(path_to_file, last_page=max_imgboxes)
+                num_imgs = len(images)
+                msg = f'The PDF "{filename}" was converted into {num_imgs} images.'
+            except:
+                msg = f'Error with the PDF "{filename}": it was not converted into images.'
+                images = [Image.open(image_wo_content)]
+
+    return filename, msg, images
+
+# Extraction of image data (text and bounding boxes)
+def extraction_data_from_image(images):
+
+    num_imgs = len(images)
+
+    if num_imgs > 0:
+
+        # https://pyimagesearch.com/2021/11/15/tesseract-page-segmentation-modes-psms-explained-how-to-improve-your-ocr-accuracy/
+        custom_config = r'--oem 3 --psm 3 -l eng' # default config PyTesseract: --oem 3 --psm 3 -l eng+deu+fra+jpn+por+spa+rus+hin+chi_sim
+        results, texts_lines, texts_pars, texts_lines_par, row_indexes, par_boxes, line_boxes, lines_par_boxes, images_pixels = dict(), dict(), dict(), dict(), dict(), dict(), dict(), dict(), dict()
+        images_ids_list, texts_lines_list, texts_pars_list, texts_lines_par_list, par_boxes_list, line_boxes_list, lines_par_boxes_list, images_list, images_pixels_list, page_no_list, num_pages_list = list(), list(), list(), list(), list(), list(), list(), list(), list(), list(), list()
+        
+        try: 
+            for i,image in enumerate(images):
+                # image preprocessing
+                # https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html
+                img = image.copy()
+                factor, path_to_img = set_image_dpi_resize(img) # Rescaling to 300dpi
+                img = Image.open(path_to_img)
+                img = np.array(img, dtype='uint8') # convert PIL to cv2
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # gray scale image
+                ret,img = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
+                
+                # OCR PyTesseract | get langs of page
+                txt = pytesseract.image_to_string(img, config=custom_config)
+                txt = txt.strip().lower()
+                txt = re.sub(r" +", " ", txt) # multiple space
+                txt = re.sub(r"(\n\s*)+\n+", "\n", txt) # multiple line
+                # txt = os.popen(f'tesseract {img_filepath} - {custom_config}').read()
+                try:
+                    langs = detect_langs(txt)
+                    langs = [langdetect2Tesseract[langs[i].lang] for i in range(len(langs))]
+                    langs_string = '+'.join(langs)
+                except:
+                    langs_string = "eng"
+                langs_string += '+osd'
+                custom_config = f'--oem 3 --psm 3 -l {langs_string}' # default config PyTesseract: --oem 3 --psm 3
+
+                # OCR PyTesseract | get data
+                results[i] = pytesseract.image_to_data(img, config=custom_config, output_type=pytesseract.Output.DICT)
+                # results[i] = os.popen(f'tesseract {img_filepath} - {custom_config}').read()
+
+                # get image pixels
+                images_pixels[i] = feature_extractor(images[i], return_tensors="pt").pixel_values
+
+                texts_lines[i], texts_pars[i], texts_lines_par[i], row_indexes[i], par_boxes[i], line_boxes[i], lines_par_boxes[i] = get_data_paragraph(results[i], factor, conf_min=0)
+                texts_lines_list.append(texts_lines[i])
+                texts_pars_list.append(texts_pars[i])
+                texts_lines_par_list.append(texts_lines_par[i])
+                par_boxes_list.append(par_boxes[i])
+                line_boxes_list.append(line_boxes[i])
+                lines_par_boxes_list.append(lines_par_boxes[i])
+                images_ids_list.append(i)
+                images_pixels_list.append(images_pixels[i])
+                images_list.append(images[i])
+                page_no_list.append(i)
+                num_pages_list.append(num_imgs) 
+
+        except:
+            print(f"There was an error within the extraction of PDF text by the OCR!")
+        else: 
+            from datasets import Dataset
+            dataset = Dataset.from_dict({"images_ids": images_ids_list, "images": images_list, "images_pixels": images_pixels_list, "page_no": page_no_list, "num_pages": num_pages_list, "texts_line": texts_lines_list, "texts_par": texts_pars_list, "texts_lines_par": texts_lines_par_list, "bboxes_par": par_boxes_list, "bboxes_lines_par":lines_par_boxes_list})
+                    
+
+            # print(f"The text data was successfully extracted by the OCR!")
+
+            return dataset, texts_lines, texts_pars, texts_lines_par, row_indexes, par_boxes, line_boxes, lines_par_boxes
+
+## Inference
+
+def prepare_inference_features_paragraph(example, cls_box = cls_box, sep_box = sep_box):
+
+  images_ids_list, chunks_ids_list, input_ids_list, attention_mask_list, bb_list, images_pixels_list = list(), list(), list(), list(), list(), list()
+
+  # get batch
+  # batch_page_hash = example["page_hash"] 
+  batch_images_ids = example["images_ids"]
+  batch_images = example["images"]
+  batch_images_pixels = example["images_pixels"]
+  batch_bboxes_par = example["bboxes_par"]
+  batch_texts_par = example["texts_par"]  
+  batch_images_size = [image.size for image in batch_images]
+
+  batch_width, batch_height = [image_size[0] for image_size in batch_images_size], [image_size[1] for image_size in batch_images_size]
+
+  # add a dimension if not a batch but only one image
+  if not isinstance(batch_images_ids, list): 
+    batch_images_ids = [batch_images_ids]
+    batch_images = [batch_images]
+    batch_images_pixels = [batch_images_pixels]
+    batch_bboxes_par = [batch_bboxes_par]
+    batch_texts_par = [batch_texts_par]
+    batch_width, batch_height = [batch_width], [batch_height] 
+
+  # process all images of the batch
+  for num_batch, (image_id, image_pixels, boxes, texts_par, width, height) in enumerate(zip(batch_images_ids, batch_images_pixels, batch_bboxes_par, batch_texts_par, batch_width, batch_height)):
+    tokens_list = []
+    bboxes_list = []
+
+    # add a dimension if only on image
+    if not isinstance(texts_par, list):
+      texts_par, boxes = [texts_par], [boxes]
+
+    # convert boxes to original
+    normalize_bboxes_par = [normalize_box(upperleft_to_lowerright(box), width, height) for box in boxes]
+
+    # sort boxes with texts
+    # we want sorted lists from top to bottom of the image
+    boxes, texts_par = sort_data_wo_labels(normalize_bboxes_par, texts_par)
+
+    count = 0
+    for box, text_par in zip(boxes, texts_par):
+      tokens_par = tokenizer.tokenize(text_par)
+      num_tokens_par = len(tokens_par) # get number of tokens
+      tokens_list.extend(tokens_par)
+      bboxes_list.extend([box] * num_tokens_par) # number of boxes must be the same as the number of tokens
+
+    # use of return_overflowing_tokens=True / stride=doc_stride
+    # to get parts of image with overlap
+    # source: https://huggingface.co/course/chapter6/3b?fw=tf#handling-long-contexts
+    encodings = tokenizer(" ".join(texts_par), 
+                          truncation=True,
+                          padding="max_length", 
+                          max_length=max_length, 
+                          stride=doc_stride, 
+                          return_overflowing_tokens=True, 
+                          return_offsets_mapping=True
+                          )
+
+    otsm = encodings.pop("overflow_to_sample_mapping")
+    offset_mapping = encodings.pop("offset_mapping")
+
+    # Let's label those examples and get their boxes   
+    sequence_length_prev = 0   
+    for i, offsets in enumerate(offset_mapping):
+      # truncate tokens, boxes and labels based on length of chunk - 2 (special tokens <s> and </s>)
+      sequence_length = len(encodings.input_ids[i]) - 2
+      if i == 0: start = 0
+      else: start += sequence_length_prev - doc_stride
+      end = start + sequence_length
+      sequence_length_prev = sequence_length
+
+      # get tokens, boxes and labels of this image chunk
+      bb = [cls_box] + bboxes_list[start:end] + [sep_box]
+
+      # as the last chunk can have a length < max_length
+      # we must to add [tokenizer.pad_token] (tokens), [sep_box] (boxes) and [-100] (labels)
+      if len(bb) < max_length:
+        bb = bb + [sep_box] * (max_length - len(bb))
+
+      # append results
+      input_ids_list.append(encodings["input_ids"][i])
+      attention_mask_list.append(encodings["attention_mask"][i])
+      bb_list.append(bb)
+      images_ids_list.append(image_id)
+      chunks_ids_list.append(i)
+      images_pixels_list.append(image_pixels)
+        
+  return {
+      "images_ids": images_ids_list,
+      "chunk_ids": chunks_ids_list,
+      "input_ids": input_ids_list,
+      "attention_mask": attention_mask_list,
+      "normalized_bboxes": bb_list,
+      "images_pixels": images_pixels_list
+  }
+
+from torch.utils.data import Dataset
+
+class CustomDataset(Dataset):
+  def __init__(self, dataset, tokenizer):
+    self.dataset = dataset
+    self.tokenizer = tokenizer
+
+  def __len__(self):
+    return len(self.dataset)
+
+  def __getitem__(self, idx):
+    # get item
+    example = self.dataset[idx]
+    encoding = dict()
+    encoding["images_ids"] = example["images_ids"]
+    encoding["chunk_ids"] = example["chunk_ids"]
+    encoding["input_ids"] = example["input_ids"]
+    encoding["attention_mask"] = example["attention_mask"]
+    encoding["bbox"] = example["normalized_bboxes"]
+    encoding["images_pixels"] = example["images_pixels"]
+    
+    return encoding
+
+import torch.nn.functional as F
+
+# get predictions at token level
+def predictions_token_level(images, custom_encoded_dataset):
+
+    num_imgs = len(images)
+    if num_imgs > 0:
+
+        chunk_ids, input_ids, bboxes, pixels_values, outputs, token_predictions  = dict(), dict(), dict(), dict(), dict(), dict()
+        images_ids_list = list()
+
+        for i,encoding in enumerate(custom_encoded_dataset):
+
+            # get custom encoded data
+            image_id = encoding['images_ids']
+            chunk_id = encoding['chunk_ids']
+            input_id = torch.tensor(encoding['input_ids'])[None]
+            attention_mask = torch.tensor(encoding['attention_mask'])[None]
+            bbox = torch.tensor(encoding['bbox'])[None]
+            pixel_values = torch.tensor(encoding["images_pixels"])
+
+            # save data in dictionnaries
+            if image_id not in images_ids_list: images_ids_list.append(image_id)
+
+            if image_id in chunk_ids: chunk_ids[image_id].append(chunk_id)
+            else: chunk_ids[image_id] = [chunk_id]
+
+            if image_id in input_ids: input_ids[image_id].append(input_id)
+            else: input_ids[image_id] = [input_id]
+
+            if image_id in bboxes: bboxes[image_id].append(bbox)
+            else: bboxes[image_id] = [bbox]
+
+            if image_id in pixels_values: pixels_values[image_id].append(pixel_values)
+            else: pixels_values[image_id] = [pixel_values]
+
+            # get prediction with forward pass
+            with torch.no_grad():
+                output = model(
+                    input_ids=input_id.to(device),
+                    attention_mask=attention_mask.to(device),
+                    bbox=bbox.to(device),
+                    image=pixel_values.to(device)
+                    )
+
+            # save probabilities of predictions in dictionnary
+            if image_id in outputs: outputs[image_id].append(F.softmax(output.logits.squeeze(), dim=-1))
+            else: outputs[image_id] = [F.softmax(output.logits.squeeze(), dim=-1)]
+
+        return outputs, images_ids_list, chunk_ids, input_ids, bboxes
+
+    else:
+        print("An error occurred while getting predictions!")
+
+from functools import reduce
+
+# Get predictions (line level)
+def predictions_paragraph_level(dataset, outputs, images_ids_list, chunk_ids, input_ids, bboxes):
+
+    ten_probs_dict, ten_input_ids_dict, ten_bboxes_dict = dict(), dict(), dict()
+    bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df = dict(), dict(), dict(), dict()
+
+    if len(images_ids_list) > 0:
+
+        for i, image_id in enumerate(images_ids_list):
+
+            # get image information
+            images_list = dataset.filter(lambda example: example["images_ids"] == image_id)["images"]
+            image = images_list[0]
+            width, height = image.size
+
+            # get data
+            chunk_ids_list = chunk_ids[image_id]
+            outputs_list = outputs[image_id]
+            input_ids_list = input_ids[image_id]
+            bboxes_list = bboxes[image_id]
+
+            # create zeros tensors
+            ten_probs = torch.zeros((outputs_list[0].shape[0] - 2)*len(outputs_list), outputs_list[0].shape[1])
+            ten_input_ids = torch.ones(size=(1, (outputs_list[0].shape[0] - 2)*len(outputs_list)), dtype =int)
+            ten_bboxes = torch.zeros(size=(1, (outputs_list[0].shape[0] - 2)*len(outputs_list), 4), dtype =int)
+
+            if len(outputs_list) > 1:
+              
+                for num_output, (output, input_id, bbox) in enumerate(zip(outputs_list, input_ids_list, bboxes_list)):
+                    start = num_output*(max_length - 2) - max(0,num_output)*doc_stride
+                    end = start + (max_length - 2)
+                    
+                    if num_output == 0:
+                        ten_probs[start:end,:] += output[1:-1]
+                        ten_input_ids[:,start:end] = input_id[:,1:-1]
+                        ten_bboxes[:,start:end,:] = bbox[:,1:-1,:]
+                    else:
+                        ten_probs[start:start + doc_stride,:] += output[1:1 + doc_stride]
+                        ten_probs[start:start + doc_stride,:] = ten_probs[start:start + doc_stride,:] * 0.5
+                        ten_probs[start + doc_stride:end,:] += output[1 + doc_stride:-1]
+
+                        ten_input_ids[:,start:start + doc_stride] = input_id[:,1:1 + doc_stride]
+                        ten_input_ids[:,start + doc_stride:end] = input_id[:,1 + doc_stride:-1]
+
+                        ten_bboxes[:,start:start + doc_stride,:] = bbox[:,1:1 + doc_stride,:]
+                        ten_bboxes[:,start + doc_stride:end,:] = bbox[:,1 + doc_stride:-1,:]
+              
+            else:
+                ten_probs += outputs_list[0][1:-1] 
+                ten_input_ids = input_ids_list[0][:,1:-1] 
+                ten_bboxes = bboxes_list[0][:,1:-1] 
+
+            ten_probs_list, ten_input_ids_list, ten_bboxes_list = ten_probs.tolist(), ten_input_ids.tolist()[0], ten_bboxes.tolist()[0]
+            bboxes_list = list()
+            input_ids_dict, probs_dict = dict(), dict()
+            bbox_prev = [-100, -100, -100, -100]
+            for probs, input_id, bbox in zip(ten_probs_list, ten_input_ids_list, ten_bboxes_list):
+                bbox = denormalize_box(bbox, width, height)
+                if bbox != bbox_prev and bbox != cls_box and bbox != sep_box and bbox[0] != bbox[2] and bbox[1] != bbox[3]:
+                    bboxes_list.append(bbox)
+                    input_ids_dict[str(bbox)] = [input_id]
+                    probs_dict[str(bbox)] = [probs]
+                elif bbox != cls_box and bbox != sep_box and bbox[0] != bbox[2] and bbox[1] != bbox[3]:
+                    input_ids_dict[str(bbox)].append(input_id)
+                    probs_dict[str(bbox)].append(probs)
+                bbox_prev = bbox
+                
+            probs_bbox = dict()
+            for i,bbox in enumerate(bboxes_list):
+                probs = probs_dict[str(bbox)]
+                probs = np.array(probs).T.tolist()
+            
+                probs_label = list()
+                for probs_list in probs:
+                    prob_label = reduce(lambda x, y: x*y, probs_list)
+                    prob_label = prob_label**(1./(len(probs_list))) # normalization
+                    probs_label.append(prob_label)
+                max_value = max(probs_label)
+                max_index = probs_label.index(max_value)
+                probs_bbox[str(bbox)] = max_index
+
+            bboxes_list_dict[image_id] = bboxes_list
+            input_ids_dict_dict[image_id] = input_ids_dict
+            probs_dict_dict[image_id] = probs_bbox
+
+            df[image_id] = pd.DataFrame()
+            df[image_id]["bboxes"] = bboxes_list
+            df[image_id]["texts"] = [tokenizer.decode(input_ids_dict[str(bbox)]) for bbox in bboxes_list]
+            df[image_id]["labels"] = [id2label[probs_bbox[str(bbox)]] for bbox in bboxes_list]
+
+        return probs_bbox, bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df
+
+    else:
+        print("An error occurred while getting predictions!")
+
+# Get labeled images with lines bounding boxes
+def get_labeled_images(dataset, images_ids_list, bboxes_list_dict, probs_dict_dict):
+
+    labeled_images = list()
+
+    for i, image_id in enumerate(images_ids_list):
+
+        # get image
+        images_list = dataset.filter(lambda example: example["images_ids"] == image_id)["images"]
+        image = images_list[0]
+        width, height = image.size
+
+        # get predicted boxes and labels
+        bboxes_list = bboxes_list_dict[image_id]
+        probs_bbox = probs_dict_dict[image_id]
+
+        draw = ImageDraw.Draw(image)
+        # https://stackoverflow.com/questions/66274858/choosing-a-pil-imagefont-by-font-name-rather-than-filename-and-cross-platform-f
+        font = font_manager.FontProperties(family='sans-serif', weight='bold')
+        font_file = font_manager.findfont(font)
+        font_size = 30
+        font = ImageFont.truetype(font_file, font_size)
+
+        for bbox in bboxes_list:
+              predicted_label = id2label[probs_bbox[str(bbox)]]
+              draw.rectangle(bbox, outline=label2color[predicted_label])
+              draw.text((bbox[0] + 10, bbox[1] - font_size), text=predicted_label, fill=label2color[predicted_label], font=font)
+
+        labeled_images.append(image)
+
+    return labeled_images
+
+# get data of encoded chunk
+def get_encoded_chunk_inference(index_chunk=None):
+
+  # get datasets
+  example = dataset
+  encoded_example = encoded_dataset
+
+  # get randomly a document in dataset
+  if index_chunk == None: index_chunk = random.randint(0, len(encoded_example)-1)
+  encoded_example = encoded_example[index_chunk]
+  encoded_image_ids = encoded_example["images_ids"]
+
+  # get the image
+  example = example.filter(lambda example: example["images_ids"] ==  encoded_image_ids)[0]
+  image = example["images"] # original image
+  width, height = image.size
+  page_no = example["page_no"]
+  num_pages = example["num_pages"]
+
+  # get boxes, texts, categories
+  bboxes, input_ids  = encoded_example["normalized_bboxes"][1:-1], encoded_example["input_ids"][1:-1]
+  bboxes = [denormalize_box(bbox, width, height) for bbox in bboxes]
+  num_tokens = len(input_ids) + 2
+
+  # get unique bboxes and corresponding labels
+  bboxes_list, input_ids_list = list(), list()
+  input_ids_dict = dict()
+  bbox_prev = [-100, -100, -100, -100]
+  for i, (bbox, input_id) in enumerate(zip(bboxes, input_ids)):
+    if bbox != bbox_prev:
+      bboxes_list.append(bbox)
+      input_ids_dict[str(bbox)] = [input_id]
+    else:
+      input_ids_dict[str(bbox)].append(input_id)
+    
+    # start_indexes_list.append(i)
+    bbox_prev = bbox
+  
+  # do not keep "</s><pad><pad>..."
+  if input_ids_dict[str(bboxes_list[-1])][0] == (tokenizer.convert_tokens_to_ids('</s>')):
+    del input_ids_dict[str(bboxes_list[-1])]
+    bboxes_list = bboxes_list[:-1]
+
+  # get texts by line
+  input_ids_list = input_ids_dict.values()
+  texts_list = [tokenizer.decode(input_ids) for input_ids in input_ids_list]
+  
+  # display DataFrame
+  df = pd.DataFrame({"texts": texts_list, "input_ids": input_ids_list, "bboxes": bboxes_list})
+
+  return image, df, num_tokens, page_no, num_pages
+
+# display chunk of PDF image and its data
+def display_chunk_paragraphs_inference(index_chunk=None):
+
+  # get image and image data
+  image, df, num_tokens, page_no, num_pages = get_encoded_chunk_inference(index_chunk=index_chunk)
+
+  # get data from dataframe
+  input_ids = df["input_ids"]
+  texts = df["texts"]
+  bboxes = df["bboxes"]
+
+  print(f'Chunk ({num_tokens} tokens) of the PDF (page: {page_no+1} / {num_pages})\n')
+
+  # display image with bounding boxes
+  print(">> PDF image with bounding boxes of paragraphs\n")
+  draw = ImageDraw.Draw(image)
+            
+  labels = list()
+  for box, text in zip(bboxes, texts):
+      color = "red"
+      draw.rectangle(box, outline=color)
+
+  # resize image to original
+  width, height = image.size
+  image = image.resize((int(0.5*width), int(0.5*height)))
+
+  # convert to cv and display
+  img = np.array(image, dtype='uint8') # PIL to cv2
+  cv2_imshow(img)
+  cv2.waitKey(0)
+
+  # display image dataframe
+  print("\n>> Dataframe of annotated paragraphs\n")
+  cols = ["texts",	"bboxes"]
+  df = df[cols]
+  display(df)

files/languages_iso.csv

@@ -0,0 +1,184 @@
+Language,LangCode
+Abkhazian,ab
+Afar,aa
+Afrikaans,af
+Akan,ak
+Albanian,sq
+Amharic,am
+Arabic,ar
+Aragonese,an
+Armenian,hy
+Assamese,as
+Avaric,av
+Avestan,ae
+Aymara,ay
+Azerbaijani,az
+Bambara,bm
+Bashkir,ba
+Basque,eu
+Belarusian,be
+Bengali,bn
+Bislama,bi
+Bosnian,bs
+Breton,br
+Bulgarian,bg
+Burmese,my
+"Catalan, Valencian",ca
+Chamorro,ch
+Chechen,ce
+"Chichewa, Chewa, Nyanja",ny
+Chinese,zh
+"Church Slavonic, Old Slavonic, Old Church Slavonic",cu
+Chuvash,cv
+Cornish,kw
+Corsican,co
+Cree,cr
+Croatian,hr
+Czech,cs
+Danish,da
+"Divehi, Dhivehi, Maldivian",dv
+"Dutch, Flemish",nl
+Dzongkha,dz
+English,en
+Esperanto,eo
+Estonian,et
+Ewe,ee
+Faroese,fo
+Fijian,fj
+Finnish,fi
+French,fr
+Western Frisian,fy
+Fulah,ff
+"Gaelic, Scottish Gaelic",gd
+Galician,gl
+Ganda,lg
+Georgian,ka
+German,de
+"Greek, Modern (1453–)",el
+"Kalaallisut, Greenlandic",kl
+Guarani,gn
+Gujarati,gu
+"Haitian, Haitian Creole",ht
+Hausa,ha
+Hebrew,he
+Herero,hz
+Hindi,hi
+Hiri Motu,ho
+Hungarian,hu
+Icelandic,is
+Ido,io
+Igbo,ig
+Indonesian,id
+Interlingua (International Auxiliary Language Association),ia
+"Interlingue, Occidental",ie
+Inuktitut,iu
+Inupiaq,ik
+Irish,ga
+Italian,it
+Japanese,ja
+Javanese,jv
+Kannada,kn
+Kanuri,kr
+Kashmiri,ks
+Kazakh,kk
+Central Khmer,km
+"Kikuyu, Gikuyu",ki
+Kinyarwanda,rw
+"Kirghiz, Kyrgyz",ky
+Komi,kv
+Kongo,kg
+Korean,ko
+"Kuanyama, Kwanyama",kj
+Kurdish,ku
+Lao,lo
+Latin,la
+Latvian,lv
+"Limburgan, Limburger, Limburgish",li
+Lingala,ln
+Lithuanian,lt
+Luba-Katanga,lu
+"Luxembourgish, Letzeburgesch",lb
+Macedonian,mk
+Malagasy,mg
+Malay,ms
+Malayalam,ml
+Maltese,mt
+Manx,gv
+Maori,mi
+Marathi,mr
+Marshallese,mh
+Mongolian,mn
+Nauru,na
+"Navajo, Navaho",nv
+North Ndebele,nd
+South Ndebele,nr
+Ndonga,ng
+Nepali,ne
+Norwegian,no
+Norwegian Bokmål,nb
+Norwegian Nynorsk,nn
+"Sichuan Yi, Nuosu",ii
+Occitan,oc
+Ojibwa,oj
+Oriya,or
+Oromo,om
+"Ossetian, Ossetic",os
+Pali,pi
+"Pashto, Pushto",ps
+Persian,fa
+Polish,pl
+Portuguese,pt
+"Punjabi, Panjabi",pa
+Quechua,qu
+"Romanian, Moldavian, Moldovan",ro
+Romansh,rm
+Rundi,rn
+Russian,ru
+Northern Sami,se
+Samoan,sm
+Sango,sg
+Sanskrit,sa
+Sardinian,sc
+Serbian,sr
+Shona,sn
+Sindhi,sd
+"Sinhala, Sinhalese",si
+Slovak,sk
+Slovenian,sl
+Somali,so
+Southern Sotho,st
+"Spanish, Castilian",es
+Sundanese,su
+Swahili,sw
+Swati,ss
+Swedish,sv
+Tagalog,tl
+Tahitian,ty
+Tajik,tg
+Tamil,ta
+Tatar,tt
+Telugu,te
+Thai,th
+Tibetan,bo
+Tigrinya,ti
+Tonga (Tonga Islands),to
+Tsonga,ts
+Tswana,tn
+Turkish,tr
+Turkmen,tk
+Twi,tw
+"Uighur, Uyghur",ug
+Ukrainian,uk
+Urdu,ur
+Uzbek,uz
+Venda,ve
+Vietnamese,vi
+Volapük,vo
+Walloon,wa
+Welsh,cy
+Wolof,wo
+Xhosa,xh
+Yiddish,yi
+Yoruba,yo
+"Zhuang, Chuang",za
+Zulu,zu

files/languages_tesseract.csv

@@ -0,0 +1,127 @@
+Language,LangCode
+Afrikaans,afr
+Amharic,amh
+Arabic,ara
+Assamese,asm
+Azerbaijani,aze
+Azerbaijani - Cyrilic,aze_cyrl
+Belarusian,bel
+Bengali,ben
+Tibetan,bod
+Bosnian,bos
+Breton,bre
+Bulgarian,bul
+Catalan; Valencian,cat
+Cebuano,ceb
+Czech,ces
+Chinese - Simplified,chi_sim
+Chinese - Traditional,chi_tra
+Cherokee,chr
+Corsican,cos
+Welsh,cym
+Danish,dan
+Danish - Fraktur (contrib),dan_frak
+German,deu
+German - Fraktur (contrib),deu_frak
+Dzongkha,dzo
+"Greek, Modern (1453-)",ell
+English,eng
+"English, Middle (1100-1500)",enm
+Esperanto,epo
+Math / equation detection module,equ
+Estonian,est
+Basque,eus
+Faroese,fao
+Persian,fas
+Filipino (old - Tagalog),fil
+Finnish,fin
+French,fra
+German - Fraktur,frk
+"French, Middle (ca.1400-1600)",frm
+Western Frisian,fry
+Scottish Gaelic,gla
+Irish,gle
+Galician,glg
+"Greek, Ancient (to 1453) (contrib)",grc
+Gujarati,guj
+Haitian; Haitian Creole,hat
+Hebrew,heb
+Hindi,hin
+Croatian,hrv
+Hungarian,hun
+Armenian,hye
+Inuktitut,iku
+Indonesian,ind
+Icelandic,isl
+Italian,ita
+Italian - Old,ita_old
+Javanese,jav
+Japanese,jpn
+Kannada,kan
+Georgian,kat
+Georgian - Old,kat_old
+Kazakh,kaz
+Central Khmer,khm
+Kirghiz; Kyrgyz,kir
+Kurmanji (Kurdish - Latin Script),kmr
+Korean,kor
+Korean (vertical),kor_vert
+Kurdish (Arabic Script),kur
+Lao,lao
+Latin,lat
+Latvian,lav
+Lithuanian,lit
+Luxembourgish,ltz
+Malayalam,mal
+Marathi,mar
+Macedonian,mkd
+Maltese,mlt
+Mongolian,mon
+Maori,mri
+Malay,msa
+Burmese,mya
+Nepali,nep
+Dutch; Flemish,nld
+Norwegian,nor
+Occitan (post 1500),oci
+Oriya,ori
+Orientation and script detection module,osd
+Panjabi; Punjabi,pan
+Polish,pol
+Portuguese,por
+Pushto; Pashto,pus
+Quechua,que
+Romanian; Moldavian; Moldovan,ron
+Russian,rus
+Sanskrit,san
+Sinhala; Sinhalese,sin
+Slovak,slk
+Slovak - Fraktur (contrib),slk_frak
+Slovenian,slv
+Sindhi,snd
+Spanish; Castilian,spa
+Spanish; Castilian - Old,spa_old
+Albanian,sqi
+Serbian,srp
+Serbian - Latin,srp_latn
+Sundanese,sun
+Swahili,swa
+Swedish,swe
+Syriac,syr
+Tamil,tam
+Tatar,tat
+Telugu,tel
+Tajik,tgk
+Tagalog (new - Filipino),tgl
+Thai,tha
+Tigrinya,tir
+Tonga,ton
+Turkish,tur
+Uighur; Uyghur,uig
+Ukrainian,ukr
+Urdu,urd
+Uzbek,uzb
+Uzbek - Cyrilic,uzb_cyrl
+Vietnamese,vie
+Yiddish,yid
+Yoruba,yor

packages.txt

@@ -0,0 +1,2 @@
+tesseract-ocr-all
+poppler-utils 

requirements.txt

@@ -0,0 +1,9 @@
+torch
+transformers
+datasets
+pytesseract
+opencv-python
+pdf2image
+pypdf
+langdetect
+gradio