add abs preprocess func

app.py

@@ -1,6 +1,8 @@
 import torch
+import nltk
 import validators
 import streamlit as st
+from nltk.tokenize import sent_tokenize
 from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration
 
 # local modules
@@ -11,7 +13,7 @@ from src.abstractive_summarizer import abstractive_summarizer
 # abstractive summarizer model
 @st.cache()
 def load_abs_model():
-    tokenizer = T5Tokenizer.from_pretrained("t5-large")
+    tokenizer = T5Tokenizer.from_pretrained("t5-base")
     model = T5ForConditionalGeneration.from_pretrained("t5-base")
     return tokenizer, model
 
@@ -24,27 +26,30 @@ if __name__ == "__main__":
     summarize_type = st.sidebar.selectbox(
         "Summarization type", options=["Extractive", "Abstractive"]
     )
+    nltk.download("punkt")
 
     inp_text = st.text_input("Enter text or a url here")
 
     is_url = validators.url(inp_text)
     if is_url:
         # complete text, chunks to summarize (list of sentences for long docs)
-        text, text_to_summarize = fetch_article_text(url=inp_text)
+        text, clean_txt = fetch_article_text(url=inp_text)
     else:
-        text_to_summarize = clean_text(inp_text)
+        clean_txt = clean_text(inp_text)
 
     # view summarized text (expander)
     with st.expander("View input text"):
-        st.write(text_to_summarize)
-
+        if is_url:
+            st.write(clean_txt[0])
+        else:
+            st.write(clean_txt)
     summarize = st.button("Summarize")
 
     # called on toggle button [summarize]
     if summarize:
         if summarize_type == "Extractive":
             if is_url:
-                text_to_summarize = " ".join([txt for txt in text_to_summarize])
+                text_to_summarize = " ".join([txt for txt in clean_txt])
             # extractive summarizer
 
             with st.spinner(
@@ -57,16 +62,19 @@ if __name__ == "__main__":
             with st.spinner(
                 text="Creating abstractive summary. This might take a few seconds ..."
             ):
-                abs_tokenizer, abs_model = load_abs_model()
-                summarized_text = abstractive_summarizer(
-                    abs_tokenizer, abs_model, text_to_summarize
-                )
-        elif summarize_type == "Abstractive" and is_url:
-            abs_url_summarizer = pipeline("summarization")
-            tmp_sum = abs_url_summarizer(
-                text_to_summarize, max_length=120, min_length=30, do_sample=False
-            )
-            summarized_text = " ".join([summ["summary_text"] for summ in tmp_sum])
+                if not is_url:
+                    text_to_summarize = sent_tokenize(clean_txt)
+
+        #         abs_tokenizer, abs_model = load_abs_model()
+        #         summarized_text = abstractive_summarizer(
+        #             abs_tokenizer, abs_model, text_to_summarize
+        #         )
+        # elif summarize_type == "Abstractive" and is_url:
+        #     abs_url_summarizer = pipeline("summarization")
+        #     tmp_sum = abs_url_summarizer(
+        #         text_to_summarize, max_length=120, min_length=30, do_sample=False
+        #     )
+        #     summarized_text = " ".join([summ["summary_text"] for summ in tmp_sum])
 
         # final summarized output
         st.subheader("Summarized text")

src/abstractive_summarizer.py

@@ -1,4 +1,5 @@
 import torch
+from nltk.tokenize import sent_tokenize
 from transformers import T5Tokenizer
 
 
@@ -20,3 +21,39 @@ def abstractive_summarizer(tokenizer, model, text):
     abs_summarized_text = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
 
     return abs_summarized_text
+
+
+def preprocess_text_for_abstractive_summarization(tokenizer, text):
+    sentences = sent_tokenize(text)
+
+    # initialize
+    length = 0
+    chunk = ""
+    chunks = []
+    count = -1
+    for sentence in sentences:
+        count += 1
+        combined_length = (
+            len(tokenizer.tokenize(sentence)) + length
+        )  # add the no. of sentence tokens to the length counter
+
+        if combined_length <= tokenizer.max_len_single_sentence:  # if it doesn't exceed
+            chunk += sentence + " "  # add the sentence to the chunk
+            length = combined_length  # update the length counter
+
+            # if it is the last sentence
+            if count == len(sentences) - 1:
+                chunks.append(chunk.strip())  # save the chunk
+
+        else:
+            chunks.append(chunk.strip())  # save the chunk
+
+            # reset
+            length = 0
+            chunk = ""
+
+            # take care of the overflow sentence
+            chunk += sentence + " "
+            length = len(tokenizer.tokenize(sentence))
+
+    return chunks