add_simple_morpheme_omission

main_socket.py

@@ -10,6 +10,7 @@ from repetition import annotate_repetitions
 from syllable import annotate_syllables
 from fillerword import annotate_fillerwords
 from morpheme import annotate_morpheme
+from morpheme_omission import annotate_morpheme_omission
 
 from annotation import annotate_transcript
 
@@ -71,6 +72,7 @@ def process_audio():
     annotate_fillerwords(session_id)
     # annotate_transcript(session_id)
     annotate_morpheme(session_id)
+    annotate_morpheme_omission(session_id)
     
 
 

morpheme.py

@@ -101,7 +101,6 @@ def extract_inflectional_morphemes(text: str):
                 i += 1
                 continue
 
-
             if low_txt in _CONTRACTION_PARTICLES and i > 0:
                 prev = words[i - 1]
                 results.append({
@@ -114,7 +113,6 @@ def extract_inflectional_morphemes(text: str):
                 i += 1
                 continue
 
-
             if feats.get("Poss") == "Yes" and pos in {"PRON", "DET"}:
                 low_lem, low_surf = lem.lower(), surf.lower()
                 suf = get_suffix(low_lem, low_surf)
@@ -129,7 +127,6 @@ def extract_inflectional_morphemes(text: str):
                 i += 1
                 continue
 
-
             inflect_type = None
             if pos == "NOUN" and feats.get("Number") == "Plur":
                 inflect_type = "Plural"
@@ -148,6 +145,10 @@ def extract_inflectional_morphemes(text: str):
                     inflect_type = "Progressive"
 
             if inflect_type:
+                if surf.lower() == lem.lower() and inflect_type not in {"Possessive", "Comparative", "Superlative"}:
+                    i += 1
+                    continue
+
                 raw_suffix = get_suffix(lem, low_txt)
                 canon = normalize_suffix(lem, raw_suffix, _EXPECTED_SUFFIXES[inflect_type])
                 morpheme_form = f"/{canon}" if canon else "<IRR>"
@@ -165,6 +166,7 @@ def extract_inflectional_morphemes(text: str):
 
 
 def annotate_morpheme(session_id, base_dir="session_data"):
+
     base_dir = base_dir or os.getcwd()
     json_file = os.path.join(base_dir, f"{session_id}/{session_id}_transcriptionCW.json")
 
@@ -183,3 +185,6 @@ def annotate_morpheme(session_id, base_dir="session_data"):
     with open(json_file, "w", encoding="utf-8") as f:
         json.dump(data, f, ensure_ascii=False, indent=2)
 
+
+if __name__ == "__main__":
+    print(extract_inflectional_morphemes("His is more better than mine, he get up in the water. He is take the buses. I like his books."))

morpheme_omission.py

@@ -0,0 +1,230 @@
+import os
+import json
+import stanza
+
+nlp = stanza.Pipeline(
+    lang="en",
+    processors="tokenize,pos,lemma",
+    tokenize_pretokenized=False,
+)
+
+_EXPECTED_SUFFIXES = {
+    "Plural":              {"s", "es"},
+    "Possessive":          {"'s", "s"},
+    "Comparative":         {"er"},
+    "Superlative":         {"est"},
+    "3rd Person Singular": {"s", "es"},
+    "Past Tense":          {"ed"},
+    "Past Participle":     {"ed", "en", "n"},
+    "Progressive":         {"ing"},
+    "Gerund":              {"ing"},
+}
+
+_CONTRACTION_PARTICLES = {
+    "'ll": "will",          # we'll, he'll
+    "'d":  "would/had",     # I'd, she'd
+    "'ve": "have",          # we've, they've
+    "'re": "are",           # you're, they're
+    "'m":  "am",            # I'm
+    "n't": "not",           # isn't, didn't
+    "'s":  "is/has",        # what's, she's
+}
+
+_S_TOKENS = {"'s", "’s"}
+
+
+def is_possessive_candidate(tok):
+    return tok.text in _S_TOKENS and tok.upos == "PART"
+
+
+def lcp(a: str, b: str) -> str:
+    i = 0
+    while i < min(len(a), len(b)) and a[i].lower() == b[i].lower():
+        i += 1
+    return a[:i]
+
+
+def strip_doubling(lemma: str, suf: str) -> str:
+    if suf and len(suf) >= 2 and suf[0] == lemma[-1]:
+        cand = suf[1:]
+        if any(cand in v for v in _EXPECTED_SUFFIXES.values()):
+            return cand
+    return suf
+
+
+def get_suffix(lemma: str, surface: str) -> str:
+    return strip_doubling(lemma, surface[len(lcp(lemma, surface)):])
+
+
+def normalize_suffix(lemma: str, raw_suf: str, expected_set: set) -> str | None:
+    if raw_suf in expected_set:
+        return raw_suf
+    if lemma.lower().endswith("y") and raw_suf.startswith("i"):
+        alt = raw_suf[1:]
+        if alt in expected_set:
+            return alt
+    return None
+
+
+def extract_inflectional_morphemes(text: str):
+
+    doc = nlp(text)
+    results = []
+
+    for sent in doc.sentences:
+        words = sent.words
+        i = 0
+        while i < len(words):
+            w = words[i]
+            surf, lem, pos = w.text, w.lemma, w.upos
+            feats = {k: v for k, v in (f.split("=", 1) for f in (w.feats or "").split("|") if "=" in f)}
+            low_txt = surf.lower()
+
+            if is_possessive_candidate(w) and i > 0:
+                prev = words[i - 1]
+                results.append({
+                    "word": prev.text + surf,
+                    "lemma": prev.lemma,
+                    "index": i - 1,
+                    "inflectional_morpheme": "Possessive"
+                    if prev.upos in {"NOUN", "PROPN"} else "Contraction",
+                    "morpheme_form": "'/s",
+                })
+                i += 1
+                continue
+
+            if low_txt in _CONTRACTION_PARTICLES and i > 0:
+                prev = words[i - 1]
+                results.append({
+                    "word": prev.text + surf,
+                    "lemma": prev.lemma,
+                    "index": i - 1,
+                    "inflectional_morpheme": "Contraction",
+                    "morpheme_form": low_txt,
+                })
+                i += 1
+                continue
+
+            if feats.get("Poss") == "Yes" and pos in {"PRON", "DET"}:
+                low_lem, low_surf = lem.lower(), surf.lower()
+                suf = get_suffix(low_lem, low_surf)
+                morpheme_form = "/s" if suf in {"s", "es"} and low_lem + suf == low_surf else "<IRR>"
+                results.append({
+                    "word": surf,
+                    "lemma": lem,
+                    "index": i,
+                    "inflectional_morpheme": "Possessive",
+                    "morpheme_form": morpheme_form,
+                })
+                i += 1
+                continue
+
+            inflect_type = None
+            if pos == "NOUN" and feats.get("Number") == "Plur":
+                inflect_type = "Plural"
+            elif pos == "ADJ" and feats.get("Degree") == "Cmp":
+                inflect_type = "Comparative"
+            elif pos == "ADJ" and feats.get("Degree") == "Sup":
+                inflect_type = "Superlative"
+            elif pos == "VERB" and feats.get("VerbForm") == "Fin" and feats.get("Tense") == "Pres" and feats.get("Person") == "3":
+                inflect_type = "3rd Person Singular"
+            elif pos == "VERB" and feats.get("VerbForm") == "Fin" and feats.get("Tense") == "Past":
+                inflect_type = "Past Tense"
+            elif pos == "VERB" and feats.get("VerbForm") == "Part":
+                if feats.get("Tense") == "Past" or w.xpos == "VBN":
+                    inflect_type = "Past Participle"
+                elif feats.get("Tense") == "Pres" or w.xpos == "VBG":
+                    inflect_type = "Progressive"
+
+            if inflect_type:
+                if surf.lower() == lem.lower() and inflect_type not in {"Possessive", "Comparative", "Superlative"}:
+                    i += 1
+                    continue
+
+                raw_suffix = get_suffix(lem, low_txt)
+                canon = normalize_suffix(lem, raw_suffix, _EXPECTED_SUFFIXES[inflect_type])
+                morpheme_form = f"/{canon}" if canon else "<IRR>"
+                results.append({
+                    "word": surf,
+                    "lemma": lem,
+                    "index": i,
+                    "inflectional_morpheme": inflect_type,
+                    "morpheme_form": morpheme_form,
+                })
+
+            i += 1
+
+    return results
+
+
+def extract_morpheme_omissions(text: str):
+
+    doc = nlp(text)
+    omissions = []
+
+    for sent in doc.sentences:
+        words = sent.words
+        i = 0
+        while i < len(words):
+            w = words[i]
+            surf, lem, pos = w.text, w.lemma, w.upos
+            feats = {k: v for k, v in (f.split("=", 1) for f in (w.feats or "").split("|") if "=" in f)}
+
+            inflect_type = None
+            if pos == "NOUN" and feats.get("Number") == "Plur":
+                inflect_type = "Plural"
+            elif pos == "ADJ" and feats.get("Degree") == "Cmp":
+                inflect_type = "Comparative"
+            elif pos == "ADJ" and feats.get("Degree") == "Sup":
+                inflect_type = "Superlative"
+            elif pos == "VERB" and feats.get("VerbForm") == "Fin" and feats.get("Tense") == "Pres" and feats.get("Person") == "3":
+                inflect_type = "3rd Person Singular"
+            elif pos == "VERB" and feats.get("VerbForm") == "Fin" and feats.get("Tense") == "Past":
+                inflect_type = "Past Tense"
+            elif pos == "VERB" and feats.get("VerbForm") == "Part":
+                if feats.get("Tense") == "Past" or w.xpos == "VBN":
+                    inflect_type = "Past Participle"
+                elif feats.get("Tense") == "Pres" or w.xpos == "VBG":
+                    inflect_type = "Progressive"
+
+            if inflect_type and surf.lower() == lem.lower() and inflect_type not in {"Possessive", "Comparative", "Superlative"}:
+                omissions.append({
+                    "word": surf,
+                    "lemma": lem,
+                    "index": i,
+                    "inflectional_morpheme": inflect_type,
+                    "morpheme_form": "<OMI>", 
+                })
+
+            i += 1
+
+    return omissions
+
+
+def annotate_morpheme_omission(session_id, base_dir="session_data"):
+
+    base_dir = base_dir or os.getcwd()
+    json_file = os.path.join(base_dir, f"{session_id}/{session_id}_transcriptionCW.json")
+
+    if not os.path.exists(json_file):
+        raise FileNotFoundError(f"{json_file} not found, make sure transcription step ran first.")
+
+    with open(json_file, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    segments = data.get("segments", data) if isinstance(data, dict) else data
+
+    for seg in segments:
+        text = seg.get("text", "")
+        seg["morpheme_omissions"] = extract_morpheme_omissions(text)
+
+    with open(json_file, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+if __name__ == "__main__":
+    sample = "His is more better than mine, he get up in the water. He is take the buses."
+    print("Inflectional Morphemes:")
+    print(json.dumps(extract_inflectional_morphemes(sample), indent=2, ensure_ascii=False))
+    print("\nMorpheme Omissions:")
+    print(json.dumps(extract_morpheme_omissions(sample), indent=2, ensure_ascii=False))