Embeddings (#252)

Adding support for embeddings mode.

---------

Authored-by: vykliuk <[email protected]>
Co-authored-by: vykliuk <[email protected]>

Author: jonorthwash

apertium_apy/apy.py

@@ -43,6 +43,7 @@
     BillookupHandler,
     BilsearchHandler,
     CoverageHandler,
+    EmbeddingsHandler,
     GenerateHandler,
     GuesserHandler,
     IdentifyLangHandler,
@@ -142,6 +143,8 @@ def setup_handler(
         handler.bilsearch[lang_pair] = (dirpath, modename)
     for dirpath, modename, lang_pair in modes['billookup']:
         handler.billookup[lang_pair] = (dirpath, modename)
+    for dirpath, modename, lang_pair in modes['embeddings']:
+        handler.embeddings[lang_pair] = (dirpath, modename)
 
     handler.init_pairs_graph()
     handler.init_paths()
@@ -293,6 +296,7 @@ def setup_application(args):
         (r'/pipedebug', PipeDebugHandler),
         (r'/bilsearch', BilsearchHandler),
         (r'/billookup', BillookupHandler),
+        (r'/embeddings', EmbeddingsHandler),
     ]  # type: List[Tuple[str, Type[tornado.web.RequestHandler]]]
 
     if importlib_util.find_spec('streamparser'):

apertium_apy/handlers/__init__.py

@@ -19,3 +19,4 @@
 from apertium_apy.handlers.translate_webpage import TranslateWebpageHandler  # noqa: F401
 from apertium_apy.handlers.bilsearch import BilsearchHandler  # noqa: F401
 from apertium_apy.handlers.billookup import BillookupHandler  # noqa: F401
+from apertium_apy.handlers.embeddings import EmbeddingsHandler  # noqa: F401

apertium_apy/handlers/base.py

@@ -38,6 +38,7 @@ class BaseHandler(tornado.web.RequestHandler):
     guessers = {}  # type: Dict[str, Tuple[str, str]]
     bilsearch = {}
     billookup = {}
+    embeddings = {}
     pairprefs = {}        # type: Dict[str, Dict[str, Dict[str, str]]]
     # (l1, l2): [translation.Pipeline], only contains flushing pairs!
     pipelines = {}  # type: Dict[Tuple[str, str], List[Union[FlushingPipeline, SimplePipeline]]]

apertium_apy/handlers/billookup.py

@@ -58,34 +58,50 @@ def normalize(form):
                 word = form.split("<", 1)[0]
                 tags = re.findall(r"<([^>]+)>", form)
                 if not tags:
-                    return word
+                    return word, []
                 pos = tags[0]
                 subcats = allowed.get(pos, [])
                 filtered = []
+                extra = []
                 for t in tags[1:]:
-                    if t in subcats and t not in filtered:
-                        filtered.append(t)
+                    if t in subcats:
+                        if t not in filtered:
+                            filtered.append(t)
+                    else:
+                        tag_with_brackets = f"<{t}>"
+                        if tag_with_brackets not in extra:
+                            extra.append(tag_with_brackets)
                 tag_str = f"<{pos}>" + "".join(f"<{t}>" for t in filtered)
-                return f"{word}{tag_str}"
+                return f"{word}{tag_str}", extra
 
             consolidated = {}
             for item in raw_results:
                 for src in item:
-                    norm_src = normalize(src)
+                    norm_src, extra_src = normalize(src)
                     if norm_src not in consolidated:
-                        consolidated[norm_src] = []
+                        consolidated[norm_src] = {"targets": {}, "extra_tags": []}
+                    for tag in extra_src:
+                        if tag not in consolidated[norm_src]["extra_tags"]:
+                            consolidated[norm_src]["extra_tags"].append(tag)
                     for tgt in item[src]:
-                        norm_tgt = normalize(tgt)
-                        if norm_tgt not in consolidated[norm_src]:
-                            consolidated[norm_src].append(norm_tgt)
+                        norm_tgt, _ = normalize(tgt)
+                        if norm_tgt not in consolidated[norm_src]["targets"]:
+                            consolidated[norm_src]["targets"][norm_tgt] = True
 
             results = []
-            for src, tgts in consolidated.items():
-                tgt_list = []
-                for t in tgts:
-                    tgt_list.append(t)
-                entry = {}
-                entry[src] = tgt_list
+            for src, data in consolidated.items():
+                tgt_list = list(data["targets"].keys())
+                if data["extra_tags"]:
+                    extra_combined = "".join(data["extra_tags"])
+                    entry = {
+                        src: tgt_list,
+                        "extra-tags": [extra_combined]
+                    }
+                else:
+                    entry = {
+                        src: tgt_list,
+                        "extra-tags": []
+                    }
                 results.append(entry)
 
             self.send_response({
@@ -102,6 +118,6 @@ def normalize(form):
     @gen.coroutine
     def get(self):
         pair = self.get_pair_or_error(self.get_argument('langpair'))
-        
+
         if pair is not None:
             yield self.lookup_and_respond(pair, self.get_argument('q'))

apertium_apy/handlers/embeddings.py

@@ -0,0 +1,61 @@
+import logging
+import os
+from tornado import gen
+
+from apertium_apy.handlers.base import BaseHandler
+from apertium_apy.utils.translation import translate_simple
+from apertium_apy.utils import to_alpha3_code
+
+class EmbeddingsHandler(BaseHandler):
+    def get_pair_or_error(self, langpair):
+        try:
+            l1, l2 = map(to_alpha3_code, langpair.split('|'))
+            in_mode = f"{l1}-{l2}"
+        except ValueError:
+            self.send_error(400, explanation='That pair is invalid, use e.g. eng|spa')
+            return None
+
+        in_mode = self.find_fallback_mode(in_mode, self.pairs)
+        if in_mode not in self.pairs:
+            self.send_error(400, explanation='That pair is not installed')
+            return None
+        return tuple(in_mode.split('-'))
+
+    @gen.coroutine
+    def embed_and_respond(self, pair, query):
+        try:
+            raw_path, mode = self.embeddings["-".join(pair)]
+            path = os.path.abspath(raw_path)
+
+            commands = [['apertium', '-d', path, '-f', 'none', mode]]
+            raw = yield translate_simple(query + '\n', commands)
+
+            segments = [seg for seg in raw.strip().split('^') if seg and '/' in seg]
+            results = []
+
+            for seg in segments:
+                entry = seg.rstrip('~').rstrip('$')
+                parts = entry.split('/')
+                src = parts[0]
+                forms = []
+                for t in parts[1:]:
+                    t_clean = t.split('~', 1)[0]
+                    if t_clean:
+                        forms.append(t_clean)
+                if forms:
+                    results.append({src: forms})
+
+            self.send_response({
+                'responseData': {'embeddingResults': results},
+                'responseDetails': None,
+                'responseStatus': 200,
+            })
+        except Exception:
+            logging.exception('Embedding error in %s-%s', *pair)
+            self.send_error(503, explanation='internal error')
+
+    @gen.coroutine
+    def get(self):
+        pair = self.get_pair_or_error(self.get_argument('langpair'))
+        if pair is not None:
+            yield self.embed_and_respond(pair, self.get_argument('q'))

apertium_apy/handlers/list_modes.py

@@ -47,6 +47,14 @@ def get(self):
                 if self.get_arguments('include_deprecated_codes'):
                     response_data.append({'sourceLanguage': to_alpha2_code(l1), 'targetLanguage': to_alpha2_code(l2)})
             self.send_response({'responseData': response_data, 'responseDetails': None, 'responseStatus': 200})
+        elif query == 'embeddings':
+            response_data = []
+            for pair, (path, modename) in self.embeddings.items():
+                l1, l2 = pair.split('-')
+                response_data.append({'sourceLanguage': l1, 'targetLanguage': l2})
+                if self.get_arguments('include_deprecated_codes'):
+                    response_data.append({'sourceLanguage': to_alpha2_code(l1), 'targetLanguage': to_alpha2_code(l2)})
+            self.send_response({'responseData': response_data, 'responseDetails': None, 'responseStatus': 200})
 
         else:
             self.send_error(400, explanation='Expecting q argument to be one of analysers, generators, guessers, spellers, disambiguators, or pairs')

apertium_apy/mode_search.py

@@ -44,6 +44,7 @@ def search_path(rootpath, include_pairs=True, verbosity=1):
         'guesser': re.compile(r'(({0}(-{0})?)-guess(er)?)\.mode'.format(lang_code)),
         'bilsearch': re.compile(r'({0})-({0})-bilsearch\.mode'.format(lang_code)),
         'billookup': re.compile(r'({0})-({0})-billookup\.mode'.format(lang_code)),
+        'embeddings': re.compile(r'({0})-({0})-embeddings\.mode'.format(lang_code)),
     }
     modes = {
         'pair': [],
@@ -55,6 +56,7 @@ def search_path(rootpath, include_pairs=True, verbosity=1):
         'guesser': [],
         'bilsearch': [],
         'billookup': [],
+        'embeddings': [],
     }  # type: Dict[str, List[Tuple[str, str, str]]]
 
     real_root = os.path.abspath(os.path.realpath(rootpath))
@@ -67,7 +69,7 @@ def search_path(rootpath, include_pairs=True, verbosity=1):
             for mtype, regex in type_re.items():
                 m = regex.match(filename)
                 if m:
-                    if mtype == 'bilsearch' or mtype == 'billookup':
+                    if mtype == 'bilsearch' or mtype == 'billookup' or mtype == 'embeddings':
                         lang_src = to_alpha3_code(m.group(1))
                         lang_trg = to_alpha3_code(m.group(2))
                         lang_pair = f"{lang_src}-{lang_trg}"