Fix empty keyphrase bug

extraction/keyphrase_extraction_pipeline.py

@@ -1,24 +0,0 @@
-from transformers import (
-    TokenClassificationPipeline,
-    AutoModelForTokenClassification,
-    AutoTokenizer,
-)
-from transformers.pipelines import AggregationStrategy
-import numpy as np
-
-
-class KeyphraseExtractionPipeline(TokenClassificationPipeline):
-    def __init__(self, model, *args, **kwargs):
-        super().__init__(
-            model=AutoModelForTokenClassification.from_pretrained(model),
-            tokenizer=AutoTokenizer.from_pretrained(model),
-            *args,
-            **kwargs
-        )
-
-    def postprocess(self, model_outputs):
-        results = super().postprocess(
-            model_outputs=model_outputs,
-            aggregation_strategy=AggregationStrategy.SIMPLE,
-        )
-        return np.unique([result.get("word").strip() for result in results])

extraction/keyphrase_generation_pipeline.py

@@ -1,28 +0,0 @@
-from transformers import (
-    Text2TextGenerationPipeline,
-    AutoModelForSeq2SeqLM,
-    AutoTokenizer,
-)
-
-
-class KeyphraseGenerationPipeline(Text2TextGenerationPipeline):
-    def __init__(self, model, keyphrase_sep_token=";", *args, **kwargs):
-        super().__init__(
-            model=AutoModelForSeq2SeqLM.from_pretrained(model),
-            tokenizer=AutoTokenizer.from_pretrained(model),
-            *args,
-            **kwargs
-        )
-        self.keyphrase_sep_token = keyphrase_sep_token
-
-    def postprocess(self, model_outputs):
-        results = super().postprocess(model_outputs=model_outputs)
-        return [
-            [
-                keyphrase.strip()
-                for keyphrase in result.get("generated_text").split(
-                    self.keyphrase_sep_token
-                )
-            ]
-            for result in results
-        ][0]

pipelines/keyphrase_generation_pipeline.py

@@ -24,6 +24,7 @@ class KeyphraseGenerationPipeline(Text2TextGenerationPipeline):
                 for keyphrase in result.get("generated_text").split(
                     self.keyphrase_sep_token
                 )
+                if keyphrase != ""
             ]
             for result in results
         ][0]