diff --git a/api/services/dify_builder/agent/build.py b/api/services/dify_builder/agent/build.py index bf64f6db751..1cc16510e6c 100644 --- a/api/services/dify_builder/agent/build.py +++ b/api/services/dify_builder/agent/build.py @@ -18,7 +18,7 @@ from core.dify_builder.contract import ResourceOption from core.dify_builder.models import BuildNodesResult, MutationIntent from graphon.enums import BUILT_IN_NODE_TYPES from services.dify_builder import graph_ops -from services.dify_builder.agent import form_schema, graph_translate, llm, resources +from services.dify_builder.agent import form_schema, graph_translate, llm, resources, user_supplied from services.dify_builder.agent.model_resolver import resolve_model_instance from services.dify_builder.agent.resources import ResourceRef from services.workflow_generator_service import WorkflowGeneratorService @@ -41,7 +41,9 @@ def analyze_goal( system = ( "You are a Dify workflow requirements analyst. Given a build goal, propose 3-6 " "clarifying requirement fields SHAPED BY THE GOAL, and a sensible default value per " - f"field. {form_schema.FORM_FIELD_TYPE_GUIDANCE}" + "field. Never invent a URL/endpoint, API key, token, password, or account/resource id " + "the goal does not state -- give such a field an empty string as its default so the " + f"user fills it in. {form_schema.FORM_FIELD_TYPE_GUIDANCE}" 'Reply with ONLY JSON: {"fields": [{"key": "...", "label": "...", "type": "...", ' '"options": ["..."]}], "values": {"": }}.' ) + llm.json_language_instruction("field labels and values") @@ -53,7 +55,65 @@ def analyze_goal( values = data.get("values") if not isinstance(fields, list) or not isinstance(values, dict): return _degraded_form(goal_text) - return {"fields": form_schema.reconcile_form_fields(fields, values), "values": values} + scrubbed_values = _scrub_invented_defaults(values, goal_text) + return {"fields": form_schema.reconcile_form_fields(fields, scrubbed_values), "values": scrubbed_values} + + +# A dict key naming a credential -- checked against ``is_user_supplied_secret`` +# rather than the plain placeholder regex, so a realistic-looking but +# never-stated secret (e.g. a made-up ``sk-live-...`` string) is still caught +# even though it doesn't match any of CREDENTIAL_PLACEHOLDER_RE's shapes. +_CREDENTIAL_KEY_RE = re.compile(r"(?i)(authorization|api[_-]?key|token|secret|password)") + +# Mirrors the prefix ``user_supplied.is_user_supplied_secret`` strips, so a +# literal is checked against CREDENTIAL_PLACEHOLDER_RE the same way regardless +# of which of the two credential checks below is doing the looking. +_AUTH_SCHEME_PREFIX_RE = re.compile(r"^(?:bearer|basic|token)\s+", re.IGNORECASE) + + +def _is_invented_literal(value: Any, key: str | None, goal_text: str, trusted_hosts: set[str]) -> bool: + """True when ``value`` (found under ``key``, possibly nested inside a + field's dict/list default) is a URL or credential the LLM invented + rather than one the goal actually states.""" + if isinstance(value, dict): + return any(_is_invented_literal(v, k, goal_text, trusted_hosts) for k, v in value.items()) + if isinstance(value, list): + return any(_is_invented_literal(item, None, goal_text, trusted_hosts) for item in value) + if not isinstance(value, str): + return False + if user_supplied.url_hosts(value) - trusted_hosts: + return True + stripped = _AUTH_SCHEME_PREFIX_RE.sub("", value, count=1) + if user_supplied.CREDENTIAL_PLACEHOLDER_RE.search(stripped): + return True + if ( + key is not None + and _CREDENTIAL_KEY_RE.search(key) + and not user_supplied.is_user_supplied_secret(value, goal_text) + ): + return True + return False + + +def _scrub_invented_defaults(values: dict[str, Any], goal_text: str) -> dict[str, Any]: + """Blank any top-level field whose default is an invented URL or + credential rather than one the goal states (S5b/F2: analysis prefilled + ``render_api_url=https://api.yourcompany.com/...`` and + ``Authorization: Bearer YOUR_API_KEY`` for a goal that named neither). + + The whole field is replaced with ``""`` -- even one whose default was a + dict -- so the user is prompted to fill it in themselves rather than the + workflow running against a host or secret that doesn't exist. + """ + trusted_hosts = user_supplied.url_hosts(goal_text) + blanked = [key for key, value in values.items() if _is_invented_literal(value, key, goal_text, trusted_hosts)] + if not blanked: + return values + scrubbed = dict(values) + for key in blanked: + scrubbed[key] = "" + logger.info("dify_builder: blanked invented default(s) for %s", ", ".join(sorted(blanked))) + return scrubbed def propose_app_name( diff --git a/api/services/dify_builder/agent/user_supplied.py b/api/services/dify_builder/agent/user_supplied.py new file mode 100644 index 00000000000..b9f51c610b7 --- /dev/null +++ b/api/services/dify_builder/agent/user_supplied.py @@ -0,0 +1,103 @@ +"""Telling a user-supplied URL or credential from one an LLM invented. + +Pure, no I/O -- these helpers make no network or model calls and import +nothing from ``services`` (besides each other), ``flask``, or model-runtime, +so they are safe to call from anywhere in Dify Builder's cognition. + +ESQ1-302 traces to the Builder inventing a plausible-looking endpoint when a +goal gave it none (``https://api.example.com/ppt/generate``); S5b/F2 shows +the same failure one step earlier, in requirements analysis, which also +invented an ``Authorization: Bearer YOUR_API_KEY`` header. These helpers let +callers decide "did the user actually write this?" by comparing literal text +against what the user actually typed -- never by asking an LLM to judge its +own output (no control flow here is keyed on LLM prose). +""" + +import json +import re +from collections.abc import Mapping +from typing import Any +from urllib.parse import urlsplit + +# A URL literal as it appears embedded in freeform prose -- stops at +# whitespace or a quoting/bracketing character so "https://a.com/x." at the +# end of a sentence doesn't pull in trailing punctuation. +_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE) + +# A Dify workflow template reference, e.g. ``{{#node1.url#}}``. It names a +# variable, not a literal value, so it can never be "invented" the way a +# hardcoded URL or secret can -- it's already bound to whatever that +# variable resolves to at runtime. +_TEMPLATE_MARKER = "{{#" + +_AUTH_SCHEME_PREFIX_RE = re.compile(r"^(?:bearer|basic|token)\s+", re.IGNORECASE) + +# Case-insensitive tokens an LLM writes in place of a secret it doesn't have. +CREDENTIAL_PLACEHOLDER_RE = re.compile( + r"YOUR[_-]?(API[_-]?)?(KEY|TOKEN|SECRET)" + r"|(API[_-]?KEY|TOKEN|SECRET)[_-]?HERE" + r"|<[^>]+>" + r"|REPLACE[_-]?ME" + r"|CHANGE[_-]?ME" + r"|x{6,}" + r"|\*{6,}", + re.IGNORECASE, +) + + +def url_hosts(text: str) -> set[str]: + """Lowercase hostnames of every ``http(s)://…`` URL literally present in + ``text``. This is the trust set for a URL check: a host the user's own + text actually names.""" + hosts: set[str] = set() + for raw_url in _URL_RE.findall(text or ""): + try: + host = urlsplit(raw_url).hostname + except ValueError: + continue + if host: + hosts.add(host.lower()) + return hosts + + +def is_user_supplied_url(url: str, trusted_text: str) -> bool: + """True iff ``url`` is something the user actually gave: its host + appears among the URLs literally present in ``trusted_text``. + + A Dify template reference (contains ``{{#``) is a variable, not a + literal, so it is never a fabrication and always returns True. + """ + if not url: + return False + if _TEMPLATE_MARKER in url: + return True + try: + host = urlsplit(url).hostname + except ValueError: + return False + if not host: + return False + return host.lower() in url_hosts(trusted_text) + + +def is_user_supplied_secret(value: str, trusted_text: str) -> bool: + """True iff ``value`` -- after stripping a leading ``Bearer ``/``Basic + ``/``Token `` auth-scheme prefix -- is a literal the user actually + wrote, rather than an LLM's placeholder for a secret it does not have. + """ + remainder = _AUTH_SCHEME_PREFIX_RE.sub("", value or "", count=1) + if not remainder: + return False + if _TEMPLATE_MARKER in remainder: + return True + if CREDENTIAL_PLACEHOLDER_RE.search(remainder): + return False + return remainder in (trusted_text or "") + + +def trusted_text_for(goal_text: str, requirements: Mapping[str, Any]) -> str: + """The text a candidate URL/credential is checked against: what the user + actually submitted -- the goal prose plus the requirements form they + filled in, as they submitted it (never anything an LLM wrote about + either).""" + return f"{goal_text}\n{json.dumps(dict(requirements), ensure_ascii=False)}" diff --git a/api/tests/unit_tests/services/dify_builder/test_build_cognition.py b/api/tests/unit_tests/services/dify_builder/test_build_cognition.py index 7ca62778bf2..d545074f8a2 100644 --- a/api/tests/unit_tests/services/dify_builder/test_build_cognition.py +++ b/api/tests/unit_tests/services/dify_builder/test_build_cognition.py @@ -80,6 +80,110 @@ def test_analyze_goal_degrades_to_generic_field_on_boom(): assert out["values"].get(out["fields"][0]["key"]) == "some goal" +# The real S5b goal (ESQ1-302 / F2): no URL, no credential anywhere in it. +_S5B_GOAL = ( + "Generate a PowerPoint presentation from a topic the user provides: an LLM writes the " + "slide content, then the workflow POSTs the slides as JSON to our company's PPT rendering " + "HTTP API, which returns the .pptx download link." +) + + +def test_scrub_invented_defaults_blanks_the_real_s5b_invented_endpoint_and_credential(): + values = { + "tone": "Professional", + "slide_count": 10, + "render_api_url": "https://api.yourcompany.com/v1/ppt/render", + "api_auth_header": { + "Content-Type": "application/json", + "Authorization": "Bearer YOUR_API_KEY", + }, + "include_speaker_notes": True, + } + + out = build._scrub_invented_defaults(values, _S5B_GOAL) + + assert out["render_api_url"] == "" + assert out["api_auth_header"] == "" + assert out["tone"] == "Professional" + assert out["slide_count"] == 10 + assert out["include_speaker_notes"] is True + + +def test_scrub_invented_defaults_blanks_a_plausible_invented_host(): + values = {"render_api_url": "https://api.pptrender.io/v1/render"} + + out = build._scrub_invented_defaults(values, _S5B_GOAL) + + assert out["render_api_url"] == "" + + +def test_scrub_invented_defaults_keeps_a_url_whose_host_the_goal_names(): + goal = "POST the finished slides to https://render.acme.internal/v2 and return the link" + values = {"render_api_url": "https://render.acme.internal/v2/pptx"} + + out = build._scrub_invented_defaults(values, goal) + + assert out["render_api_url"] == "https://render.acme.internal/v2/pptx" + + +def test_scrub_invented_defaults_keeps_a_credential_the_goal_states(): + goal = "Authenticate with the API key sk-live-abc123" + values = {"api_key": "sk-live-abc123"} + + out = build._scrub_invented_defaults(values, goal) + + assert out["api_key"] == "sk-live-abc123" + + +def test_scrub_invented_defaults_blanks_a_credential_the_goal_does_not_state(): + values = {"api_key": "sk-live-abc123"} + + out = build._scrub_invented_defaults(values, _S5B_GOAL) + + assert out["api_key"] == "" + + +def test_analyze_goal_scrubs_invented_defaults_end_to_end(monkeypatch): + captured: dict[str, str] = {} + raw = { + "fields": [ + {"key": "tone", "label": "Tone", "type": "text"}, + {"key": "slide_count", "label": "Slide count", "type": "number"}, + {"key": "render_api_url", "label": "Render API URL", "type": "text"}, + {"key": "api_auth_header", "label": "Auth header", "type": "json_object"}, + {"key": "include_speaker_notes", "label": "Include speaker notes", "type": "bool"}, + ], + "values": { + "tone": "Professional", + "slide_count": 10, + "render_api_url": "https://api.yourcompany.com/v1/ppt/render", + "api_auth_header": { + "Content-Type": "application/json", + "Authorization": "Bearer YOUR_API_KEY", + }, + "include_speaker_notes": True, + }, + } + + def fake_invoke_json(model, *, system, user, on_reasoning=None): # noqa: ARG001 + captured["system"] = system + return raw + + monkeypatch.setattr(build.llm, "invoke_json", fake_invoke_json) + + out = build.analyze_goal(object(), _S5B_GOAL) + + assert out["values"]["render_api_url"] == "" + assert out["values"]["api_auth_header"] == "" + assert out["values"]["tone"] == "Professional" + assert out["values"]["slide_count"] == 10 + assert out["values"]["include_speaker_notes"] is True + assert ( + "Never invent a URL/endpoint, API key, token, password, or account/resource id the goal does not state" + in captured["system"] + ) + + def test_propose_plan_v1_returns_bullets(): m = _FakeInstance([json.dumps({"plan": ["Ingest", "Summarize", "Emit"]})]) assert build.propose_plan_v1(m, {"x": 1}) == ["Ingest", "Summarize", "Emit"] diff --git a/api/tests/unit_tests/services/dify_builder/test_user_supplied.py b/api/tests/unit_tests/services/dify_builder/test_user_supplied.py new file mode 100644 index 00000000000..af7de9dea99 --- /dev/null +++ b/api/tests/unit_tests/services/dify_builder/test_user_supplied.py @@ -0,0 +1,97 @@ +"""Unit tests for services.dify_builder.agent.user_supplied. + +These are the shared "did the user actually write this, or did the model +invent it" primitives (ESQ1-302 / S5b / F1, F2): they underpin the +requirements-analysis scrub in Task 1 and later tasks' checks on http-request +URLs and headers. Pure functions, no I/O. +""" + +import json + +from services.dify_builder.agent import user_supplied as us + + +def test_url_hosts_extracts_lowercase_hostnames_from_prose(): + text = "Call https://API.Example.com/v1 and also http://other.example.org:8080/x please." + assert us.url_hosts(text) == {"api.example.com", "other.example.org"} + + +def test_url_hosts_empty_set_when_no_urls_present(): + assert us.url_hosts("no urls in this goal at all") == set() + + +def test_url_hosts_empty_text_is_empty_set(): + assert us.url_hosts("") == set() + + +def test_is_user_supplied_url_true_when_host_matches_case_insensitively(): + trusted = "the goal mentions https://render.acme.internal/v2 as the target" + assert us.is_user_supplied_url("https://Render.Acme.Internal/v2/pptx", trusted) is True + + +def test_is_user_supplied_url_false_when_host_absent_from_trusted_text(): + assert us.is_user_supplied_url("https://api.pptrender.io/v1/render", "no url here") is False + + +def test_is_user_supplied_url_empty_url_is_false(): + assert us.is_user_supplied_url("", "https://anything.example.com") is False + + +def test_is_user_supplied_url_dify_template_is_always_true(): + # A template reference is a variable, not a literal -- it can't be "invented". + assert us.is_user_supplied_url("{{#node1.node5_url#}}", "goal has no url at all") is True + + +def test_credential_placeholder_re_matches_known_placeholder_shapes(): + for token in ( + "YOUR_API_KEY", + "YOURKEY", + "your-token", + "API_KEY_HERE", + "TOKEN_HERE", + "", + "REPLACE_ME", + "CHANGE-ME", + "xxxxxxxx", + "******", + ): + assert us.CREDENTIAL_PLACEHOLDER_RE.search(token), token + + +def test_credential_placeholder_re_does_not_match_a_real_looking_secret(): + assert us.CREDENTIAL_PLACEHOLDER_RE.search("sk-live-abc123") is None + + +def test_is_user_supplied_secret_true_when_bearer_stripped_value_is_literal(): + assert us.is_user_supplied_secret("Bearer sk-live-abc123", "the key is sk-live-abc123") is True + + +def test_is_user_supplied_secret_strips_basic_and_token_prefixes_case_insensitively(): + assert us.is_user_supplied_secret("basic sk-live-abc123", "sk-live-abc123") is True + assert us.is_user_supplied_secret("TOKEN sk-live-abc123", "sk-live-abc123") is True + + +def test_is_user_supplied_secret_false_for_placeholder_even_with_bearer_prefix(): + assert us.is_user_supplied_secret("Bearer YOUR_API_KEY", "anything at all") is False + + +def test_is_user_supplied_secret_false_for_empty_or_prefix_only_value(): + assert us.is_user_supplied_secret("", "anything") is False + assert us.is_user_supplied_secret("Bearer ", "anything") is False + + +def test_is_user_supplied_secret_true_for_dify_template_reference(): + assert us.is_user_supplied_secret("{{#node1.api_key#}}", "goal text with nothing matching") is True + + +def test_is_user_supplied_secret_false_when_remainder_not_in_trusted_text(): + assert us.is_user_supplied_secret("sk-live-abc123", "the goal names nothing like that") is False + + +def test_trusted_text_for_combines_goal_and_requirements_json(): + out = us.trusted_text_for("build a thing", {"a": 1, "b": "x"}) + assert out == "build a thing\n" + json.dumps({"a": 1, "b": "x"}, ensure_ascii=False) + + +def test_trusted_text_for_empty_requirements(): + assert us.trusted_text_for("goal only", {}) == "goal only\n{}"