fix_morpheme_index

morpheme.py

@@ -1,6 +1,7 @@
 import os
 import json
 import stanza
+import re
 
 nlp = stanza.Pipeline(
     lang="en",
@@ -66,8 +67,38 @@ def normalize_suffix(lemma: str, raw_suf: str, expected_set: set) -> str | None:
     return None
 
 
+def preprocess_text(text: str) -> tuple[str, list[int]]:
+
+    original_words = text.split()
+    
+    position_map = []  # position_map[original_index] = cleaned_index
+    cleaned_words = []
+    
+    for i, word in enumerate(original_words):
+        if re.match(r'\[.*\]', word):
+            position_map.append(-1)
+        else:
+            position_map.append(len(cleaned_words))
+            cleaned_words.append(word)
+    
+    cleaned_text = ' '.join(cleaned_words)
+    return cleaned_text, position_map
+
+
+def calculate_adjusted_index(cleaned_index: int, position_map: list[int]) -> int:
+
+    for original_index, cleaned_pos in enumerate(position_map):
+        if cleaned_pos == cleaned_index:
+            return original_index
+    
+
+    return cleaned_index
+
+
 def extract_inflectional_morphemes(text: str):
-    doc = nlp(text)
+    cleaned_text, position_map = preprocess_text(text)
+    
+    doc = nlp(cleaned_text)
     results = []
 
     for sent in doc.sentences:
@@ -86,7 +117,7 @@ def extract_inflectional_morphemes(text: str):
                     results.append({
                         "word": prev.text + surf,
                         "lemma": prev.lemma,
-                        "index": i - 1,
+                        "index": calculate_adjusted_index(i - 1, position_map),
                         "inflectional_morpheme": "Possessive",
                         "morpheme_form": "'/s",
                     })
@@ -94,7 +125,7 @@ def extract_inflectional_morphemes(text: str):
                     results.append({
                         "word": prev.text + surf,
                         "lemma": prev.lemma,
-                        "index": i - 1,
+                        "index": calculate_adjusted_index(i - 1, position_map),
                         "inflectional_morpheme": "Contraction",
                         "morpheme_form": "'/s",
                     })
@@ -106,7 +137,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": prev.text + surf,
                     "lemma": prev.lemma,
-                    "index": i - 1,
+                    "index": calculate_adjusted_index(i - 1, position_map),
                     "inflectional_morpheme": "Contraction",
                     "morpheme_form": low_txt,
                 })
@@ -120,7 +151,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": surf,
                     "lemma": lem,
-                    "index": i,
+                    "index": calculate_adjusted_index(i, position_map),
                     "inflectional_morpheme": "Possessive",
                     "morpheme_form": morpheme_form,
                 })
@@ -155,7 +186,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": surf,
                     "lemma": lem,
-                    "index": i,
+                    "index": calculate_adjusted_index(i, position_map),
                     "inflectional_morpheme": inflect_type,
                     "morpheme_form": morpheme_form,
                 })
@@ -187,4 +218,4 @@ def annotate_morpheme(session_id, base_dir="session_data"):
 
 
 if __name__ == "__main__":
-    print(extract_inflectional_morphemes("His is more better than mine, he get up in the water. He is take the buses. I like his books."))
+    print(extract_inflectional_morphemes("And he [UH] [UM] the rabbit [UH] makes [UH] sand castle."))

morpheme_omission.py

@@ -1,6 +1,7 @@
 import os
 import json
 import stanza
+import re
 
 nlp = stanza.Pipeline(
     lang="en",
@@ -66,9 +67,39 @@ def normalize_suffix(lemma: str, raw_suf: str, expected_set: set) -> str | None:
     return None
 
 
+def preprocess_text(text: str) -> tuple[str, list[int]]:
+
+    original_words = text.split()
+    
+    position_map = []  # position_map[original_index] = cleaned_index
+    cleaned_words = []
+    
+    for i, word in enumerate(original_words):
+        if re.match(r'\[.*\]', word):
+            position_map.append(-1)
+        else:
+            position_map.append(len(cleaned_words))
+            cleaned_words.append(word)
+    
+    cleaned_text = ' '.join(cleaned_words)
+    return cleaned_text, position_map
+
+
+def calculate_adjusted_index(cleaned_index: int, position_map: list[int]) -> int:
+
+    for original_index, cleaned_pos in enumerate(position_map):
+        if cleaned_pos == cleaned_index:
+            return original_index
+    
+
+    return cleaned_index
+
+
 def extract_inflectional_morphemes(text: str):
 
-    doc = nlp(text)
+    cleaned_text, position_map = preprocess_text(text)
+    
+    doc = nlp(cleaned_text)
     results = []
 
     for sent in doc.sentences:
@@ -85,7 +116,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": prev.text + surf,
                     "lemma": prev.lemma,
-                    "index": i - 1,
+                    "index": calculate_adjusted_index(i - 1, position_map),
                     "inflectional_morpheme": "Possessive"
                     if prev.upos in {"NOUN", "PROPN"} else "Contraction",
                     "morpheme_form": "'/s",
@@ -98,7 +129,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": prev.text + surf,
                     "lemma": prev.lemma,
-                    "index": i - 1,
+                    "index": calculate_adjusted_index(i - 1, position_map),
                     "inflectional_morpheme": "Contraction",
                     "morpheme_form": low_txt,
                 })
@@ -112,7 +143,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": surf,
                     "lemma": lem,
-                    "index": i,
+                    "index": calculate_adjusted_index(i, position_map),
                     "inflectional_morpheme": "Possessive",
                     "morpheme_form": morpheme_form,
                 })
@@ -147,7 +178,7 @@ def extract_inflectional_morphemes(text: str):
                 results.append({
                     "word": surf,
                     "lemma": lem,
-                    "index": i,
+                    "index": calculate_adjusted_index(i, position_map),
                     "inflectional_morpheme": inflect_type,
                     "morpheme_form": morpheme_form,
                 })
@@ -159,7 +190,9 @@ def extract_inflectional_morphemes(text: str):
 
 def extract_morpheme_omissions(text: str):
 
-    doc = nlp(text)
+    cleaned_text, position_map = preprocess_text(text)
+    
+    doc = nlp(cleaned_text)
     omissions = []
 
     for sent in doc.sentences:
@@ -191,7 +224,7 @@ def extract_morpheme_omissions(text: str):
                 omissions.append({
                     "word": surf,
                     "lemma": lem,
-                    "index": i,
+                    "index": calculate_adjusted_index(i, position_map),
                     "inflectional_morpheme": inflect_type,
                     "morpheme_form": "<OMI>", 
                 })
@@ -223,7 +256,7 @@ def annotate_morpheme_omission(session_id, base_dir="session_data"):
 
 
 if __name__ == "__main__":
-    sample = "His is more better than mine. He's going to play. He get up in the water. He is take the buses."
+    sample = "And he [UM] [UM] the rabbit [UM] [UH] [UH] make [UH] sand castle."
     print("Inflectional Morphemes:")
     print(json.dumps(extract_inflectional_morphemes(sample), indent=2, ensure_ascii=False))
     print("\nMorpheme Omissions:")