update decode logic

app.py

@@ -7,8 +7,9 @@ from PIL import Image
 from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
 from transformers.pipelines.document_question_answering import apply_tesseract
 
-MODEL = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa").eval()
-TOKENIZER = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa")
+model_tag = "impira/layoutlm-document-qa"
+MODEL = LayoutLMForQuestionAnswering.from_pretrained(model_tag).eval()
+TOKENIZER = AutoTokenizer.from_pretrained(model_tag)
 OCR = PaddleOCR(
     use_angle_cls=True,
     lang="en",
@@ -56,6 +57,7 @@ def predict(image: Image.Image, question: str, ocr_engine: str):
 
     token_ids = TOKENIZER(question)["input_ids"]
     token_boxes = [[0] * 4] * (len(token_ids) - 1) + [[1000] * 4]
+    n_question_tokens = len(token_ids)
 
     token_ids.append(TOKENIZER.sep_token_id)
     token_boxes.append([1000] * 4)
@@ -74,14 +76,19 @@ def predict(image: Image.Image, question: str, ocr_engine: str):
             bbox=torch.tensor(token_boxes).unsqueeze(0),
         )
 
-    start_scores = outputs.start_logits.squeeze(0).softmax(-1)
-    end_scores = outputs.end_logits.squeeze(0).softmax(-1)
+    start_scores = outputs.start_logits.squeeze(0).softmax(-1)[n_question_tokens:]
+    end_scores = outputs.end_logits.squeeze(0).softmax(-1)[n_question_tokens:]
+
+    span_scores = start_scores.view(-1, 1) * end_scores.view(1, -1)
+    span_scores = torch.triu(span_scores)  # don't allow start < end
+
+    score, indices = span_scores.flatten().max(-1)
+    start_idx = n_question_tokens + indices // span_scores.shape[1]
+    end_idx = n_question_tokens + indices % span_scores.shape[1]
 
-    start_score, start_idx = start_scores.max(-1)
-    end_score, end_idx = end_scores.max(-1)
     answer = TOKENIZER.decode(token_ids[start_idx : end_idx + 1])
 
-    return answer, start_score, end_score, image_np
+    return answer, score, image_np
 
 
 gr.Interface(
@@ -93,8 +100,7 @@ gr.Interface(
     ],
     outputs=[
         gr.Textbox(label="Answer"),
-        gr.Number(label="Start score"),
-        gr.Number(label="End score"),
+        gr.Number(label="Score"),
         gr.Image(label="OCR results"),
     ],
     examples=[