change pipeline to manual model

app.py

@@ -1,12 +1,14 @@
 import cv2
 import gradio as gr
 import numpy as np
+import torch
 from paddleocr import PaddleOCR
 from PIL import Image
-from transformers import pipeline
+from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
 from transformers.pipelines.document_question_answering import apply_tesseract
 
-PIPE = pipeline("document-question-answering", "impira/layoutlm-document-qa")
+MODEL = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa").eval()
+TOKENIZER = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa")
 OCR = PaddleOCR(
     use_angle_cls=True,
     lang="en",
@@ -52,9 +54,34 @@ def predict(image: Image.Image, question: str, ocr_engine: str):
     else:
         raise ValueError(f"Unsupported ocr_engine={ocr_engine}")
 
-    word_boxes = list(zip(words, boxes))
-    result = PIPE(image, question, word_boxes)[0]
-    return result["answer"], result["score"], image_np
+    token_ids = TOKENIZER(question)["input_ids"]
+    token_boxes = [[0] * 4] * (len(token_ids) - 1) + [[1000] * 4]
+
+    token_ids.append(TOKENIZER.sep_token_id)
+    token_boxes.append([1000] * 4)
+
+    for word, box in zip(words, boxes):
+        new_ids = TOKENIZER(word, add_special_tokens=False)["input_ids"]
+        token_ids.extend(new_ids)
+        token_boxes.extend([box] * len(new_ids))
+
+    token_ids.append(TOKENIZER.sep_token_id)
+    token_boxes.append([1000] * 4)
+
+    with torch.inference_mode():
+        outputs = MODEL(
+            input_ids=torch.tensor(token_ids).unsqueeze(0),
+            bbox=torch.tensor(token_boxes).unsqueeze(0),
+        )
+
+    start_scores = outputs.start_logits.squeeze(0).softmax(-1)
+    end_scores = outputs.end_logits.squeeze(0).softmax(-1)
+
+    start_score, start_idx = start_scores.max(-1)
+    end_score, end_idx = end_scores.max(-1)
+    answer = TOKENIZER.decode(token_ids[start_idx : end_idx + 1])
+
+    return answer, start_score, end_score, image_np
 
 
 gr.Interface(
@@ -66,7 +93,9 @@ gr.Interface(
     ],
     outputs=[
         gr.Textbox(label="Answer"),
-        gr.Number(label="Score"),
+        gr.Number(label="Start score"),
+        gr.Number(label="End score"),
         gr.Image(label="OCR results"),
     ],
+    examples=[["example_01.jpg", "When did the sample take place?", PADDLE_OCR_LABEL]],
 ).launch(server_name="0.0.0.0", server_port=7860)