detect the script automatically, if not specified

app.py

@@ -1,12 +1,17 @@
 import gradio as gr
 
-from myv_translit import lat2cyr, cyr2lat
+from myv_translit import lat2cyr, cyr2lat, detect_script
 
+DIRECTIONS = ['lat -> кир', 'кир -> lat']
 
-def transliterator(input_text, direction_to_latn=1, joint_acute=True, not_first_e_with_hacek=False, not_soft_l_after_vowels=True):
+
+def transliterator(input_text, direction, joint_acute=True, not_first_e_with_hacek=False, not_soft_l_after_vowels=True):
     first_e_with_hacek = not not_first_e_with_hacek
     soft_l_after_vowels = not not_soft_l_after_vowels
-    if direction_to_latn:
+    if direction is None:
+        code = detect_script(input_text)
+        direction = DIRECTIONS[int(code != 'lat')]
+    if direction == DIRECTIONS[1]:
         result = cyr2lat(input_text, joint_acute=joint_acute, first_e_with_hacek=first_e_with_hacek, soft_l_after_vowels=soft_l_after_vowels)
     else:
         result = lat2cyr(input_text, joint_acute=joint_acute, first_e_with_hacek=first_e_with_hacek, soft_l_after_vowels=soft_l_after_vowels)
@@ -21,14 +26,12 @@ article = """
 - http://valks.erzja.info/2020/04/30/эрзянский-алфавит/
 """
 
-directions = ['lat -> кир', 'кир -> lat']
-
 
 interface = gr.Interface(
     transliterator,
     [
         gr.Textbox(label="Текст", lines=2, placeholder='text to transliterate'),
-        gr.Radio(choices=directions, type="index", interactive=True, label='Направление'),
+        gr.Radio(choices=DIRECTIONS, type="value", interactive=True, label='Направление'),
         gr.Checkbox(value=True, label='L + ́  -> Ĺ'),
         gr.Checkbox(value=False, label='ěrzä -> erzä'),
         gr.Checkbox(value=False, label='peĺks -> pelks'),

myv_translit.py

@@ -1,5 +1,5 @@
 import re
-
+from collections import Counter
 
 _cyr2lat = [
     {'find_what': 'А', 'replacer': 'A', 're': False},
@@ -265,3 +265,38 @@ def cyr2lat(text, joint_acute=True, first_e_with_hacek=True, soft_l_after_vowels
 def lat2cyr(text, joint_acute=True, first_e_with_hacek=True, soft_l_after_vowels=True):
     # todo: support all the optional settings
     return transliterate_with_rules(text, _lat2cyr)
+
+
+CYR_CHARS = 'абвгдеёжзиклмнопрстуфхцчшщъыьэюя'
+BASIC_LAT_CHARS = 'abcdefghijklmnopqrtuvwxyz'
+ACCENT_LAT_CHARS = 'ěäüöśźćńŕťďĺ'
+LAT_CHARS = BASIC_LAT_CHARS + ACCENT_LAT_CHARS
+
+
+def detect_script(text: str, min_prevalence: float = 2.0, min_detectable: float = 0.1) -> str:
+    """ Detect the script of the text.
+    Possible values:
+        - cyr - Cyrillic
+        - lat - Latin
+        - mix - Mixed Cyrillic and Latin script
+        - unk - Unknown script (probably neither Latin nor Cyrillic)
+    """
+    cyr, lat, other = 0, 0, 0
+    char_cnt = Counter(text.lower())
+    for char, cnt in char_cnt.items():
+        if char in CYR_CHARS:
+            cyr += cnt
+        elif char in LAT_CHARS:
+            lat += cnt
+        else:
+            other += cnt
+    total = cyr + lat + other
+    if not total:
+        return 'unk'
+    if (cyr + lat) / total < min_detectable:
+        return 'unk'
+    if cyr >= lat * min_prevalence:
+        return 'cyr'
+    if lat >= cyr * min_prevalence:
+        return 'lat'
+    return 'mix'

test_translit.py

@@ -1,4 +1,4 @@
-from myv_translit import cyr2lat
+from myv_translit import cyr2lat, detect_script
 
 
 def test_join_acute():
@@ -18,6 +18,15 @@ def test_soft_l():
     assert cyr2lat('пелькс', soft_l_after_vowels=False) == 'pelks'
 
 
+def test_detection():
+    assert detect_script('123 456?? 8743 098543 ???...,.! @%%&&& хз') == 'unk'
+    assert detect_script('ěrzä') == 'lat'
+    assert detect_script('ěrzä ю') == 'lat'
+    assert detect_script('ЭРЗЯ') == 'cyr'
+    assert detect_script('ЭРЗЯ d') == 'cyr'
+    assert detect_script('ěrzä эрзянь') == 'mix'
+
+
 # todo: test on a larger corpus
 # todo: test cyclical consistency