dify/api/services/dify_builder/agent/localize.py

"""M2: localize the Dify Builder engine's static strings into the user's language.

Runs in the service layer (may call the LLM) and is injected into the pure
engine as Env callbacks. Value-matches item strings against the static-string
catalog (core.dify_builder.strings); LLM-generated prose (already localized by
M1) never matches and is passed through. Translations are cached per
(source, language) so a language's finite vocabulary is translated once.
"""

import json
import logging
import re
from collections.abc import Callable

from core.dify_builder import strings
from core.dify_builder.models import ConversationItem
from services.dify_builder.agent import llm

logger = logging.getLogger(__name__)

_DETECT_SYSTEM = (
    "Identify the language of the user's text. Reply with ONLY its IETF BCP-47 "
    "code, e.g. en, zh-Hans, zh-Hant, ja, fr, es. No other text."
)

# A plausible BCP-47 language tag: a 2-3 letter primary subtag optionally
# followed by one or more hyphenated subtags (script/region/variant, e.g.
# "zh-Hans", "zh-Hant-HK"). Guards against a model that ignores "reply with
# ONLY the code" and returns a sentence instead.
_LANG_CODE_RE = re.compile(r"^[A-Za-z]{2,3}(-[A-Za-z0-9]{1,8})*$")

# (source_string, language) -> translation, for the fixed static-string catalog
# (core.dify_builder.strings). Module-level and shared across Localizer
# instances/tenants/sessions: entries are catalog strings translated into a
# given target language, independent of any tenant or session state, so a
# language's finite vocabulary really is translated once process-wide rather
# than once per advance-task invocation (a fresh Localizer is constructed per
# invocation). Plain dict is fine under gevent/GIL: a set is atomic, and a
# concurrent duplicate translation of the same key is harmless and idempotent
# -- no lock needed.
_TRANSLATION_CACHE: dict[tuple[str, str], str] = {}


class Localizer:
    def __init__(self, model_provider: Callable[[], object | None]) -> None:
        self._model_provider = model_provider

    # -- detection ----------------------------------------------------------
    def detect_language(self, text: str) -> str:
        if not text or not text.strip():
            return "en"
        try:
            model = self._model_provider()
            if model is None:
                return "en"
            raw = llm.invoke_text(model, system=_DETECT_SYSTEM, user=text[:2000]).strip()
        except Exception:
            logger.exception("dify_builder: language detection failed; defaulting to en")
            return "en"
        # Reject anything that isn't a single short, code-shaped token: the model
        # disobeying "reply with ONLY the code" (e.g. "The language is Japanese.")
        # must fall back to en rather than silently becoming the target language.
        if not raw or re.search(r"\s", raw) or len(raw) > 35:
            return "en"
        code = raw.strip('."')
        if not code or not _LANG_CODE_RE.match(code):
            return "en"
        return code

    # -- localization -------------------------------------------------------
    def localize_items(self, items: list[ConversationItem], language: str) -> list[ConversationItem]:
        if not language or language == "en":
            return items
        # 1) collect the source strings that need translation across all items
        needed: set[str] = set()
        for item in items:
            self._collect(item.payload, language, needed)
        # 2) batch-translate the cache misses in one call, populate the cache
        self._fill_cache(sorted(needed), language)
        # 3) apply
        for item in items:
            item.payload = self._apply(item.payload, language)
        return items

    def _collect(self, value, language: str, needed: set[str]) -> None:
        if isinstance(value, str):
            src = self._source_to_translate(value)
            if src is not None and (src, language) not in _TRANSLATION_CACHE:
                needed.add(src)
        elif isinstance(value, dict):
            for v in value.values():
                self._collect(v, language, needed)
        elif isinstance(value, list):
            for v in value:
                self._collect(v, language, needed)

    def _apply(self, value, language: str):
        if isinstance(value, str):
            return self._translate_value(value, language)
        if isinstance(value, dict):
            return {k: self._apply(v, language) for k, v in value.items()}
        if isinstance(value, list):
            return [self._apply(v, language) for v in value]
        return value

    def _source_to_translate(self, value: str) -> str | None:
        """The catalog source string that would be translated for `value`, or None.
        Plain catalog hit -> the value itself; template hit -> the template form."""
        if value in strings.PLAIN:
            return value
        m = strings.match_template(value)
        if m is not None:
            return m[0].template
        return None

    def _translate_value(self, value: str, language: str) -> str:
        if value in strings.PLAIN:
            return _TRANSLATION_CACHE.get((value, language), value)
        m = strings.match_template(value)
        if m is not None:
            tpl, groups = m
            translated_tpl = _TRANSLATION_CACHE.get((tpl.template, language), tpl.template)
            try:
                return translated_tpl.format(**groups)
            except Exception:
                return value  # translated template dropped a placeholder -> keep original
        return value

    def _fill_cache(self, sources: list[str], language: str) -> None:
        misses = [s for s in sources if (s, language) not in _TRANSLATION_CACHE]
        if not misses:
            return
        try:
            model = self._model_provider()
        except Exception:
            logger.exception("dify_builder: model_provider failed during translation; leaving strings in English")
            return
        if model is None:
            return
        # Correlate translations by numeric INDEX, not by echoing the source strings
        # back as keys: short index keys survive across models where long verbatim
        # string-keys get normalized/truncated. The reply is accepted as an
        # index-keyed object or a positional array, optionally wrapped one level.
        system = (
            f"You are a translation engine. Translate each numbered string into the language "
            f"with BCP-47 code {language}. Preserve any {{placeholder}} tokens EXACTLY. Do not "
            "translate proper nouns or code. Reply with ONLY a JSON object whose keys are the "
            'SAME index numbers (as strings) and whose values are the translations, e.g. '
            '{"0": "<translation of item 0>", "1": "<translation of item 1>"}.'
        )
        user = json.dumps({str(i): s for i, s in enumerate(misses)}, ensure_ascii=False)
        try:
            raw = llm.invoke_json(model, system=system, user=user)
        except Exception:
            logger.exception("dify_builder: batch translation failed; leaving strings in English")
            return
        translations = self._translations_by_index(raw, len(misses))
        for i, src in enumerate(misses):
            out = translations.get(i)
            # Cache only real translations; leave a miss UNCACHED so a later turn
            # retries (self-heals) instead of sticking on the English original.
            if isinstance(out, str) and out.strip():
                _TRANSLATION_CACHE[(src, language)] = out

    @staticmethod
    def _translations_by_index(raw: object, count: int) -> dict[int, str]:
        """Map a model translation reply to ``{index: translation}``, tolerant of shape.

        Accepts an index-keyed object (``{"0": "..", "1": ".."}``), a positional
        array (``["..", ".."]``), or either wrapped one level under a single key
        (a model echoing the request envelope). Unrecognized or missing entries are
        simply absent, so the caller keeps them untranslated (and uncached)."""
        container: object = raw
        # Unwrap one level when the top level is neither index-keyed nor a list.
        if isinstance(raw, dict) and not any(str(i) in raw for i in range(count)):
            container = next((v for v in raw.values() if isinstance(v, (dict, list))), raw)
        out: dict[int, str] = {}
        if isinstance(container, dict):
            for i in range(count):
                value = container.get(str(i))
                if isinstance(value, str):
                    out[i] = value
        elif isinstance(container, list):
            for i in range(min(count, len(container))):
                if isinstance(container[i], str):
                    out[i] = container[i]
        return out