mirror of
https://github.com/langgenius/dify.git
synced 2026-10-04 13:47:55 +08:00
fix(dify-builder): requirements analysis stops inventing endpoints and credentials
S5b (local E2E 2026-09-22, F2): analyze_goal's "a sensible default value per field" led the LLM to prefill render_api_url=https://api.yourcompany.com/... and Authorization: Bearer YOUR_API_KEY for a goal that named neither -- caught only because "yourcompany" happened to match an existing placeholder-URL regex; a plausible invented host would slip through and recreate ESQ1-302's launch failure one step upstream, in the requirements form itself. Adds services/dify_builder/agent/user_supplied.py (pure, no I/O): url_hosts, is_user_supplied_url, CREDENTIAL_PLACEHOLDER_RE, is_user_supplied_secret, trusted_text_for -- shared primitives for telling a literal the user actually wrote from one an LLM invented, used by this task and consumed by later tasks in the same plan. build.analyze_goal now tells the model never to invent such a field's default, and _scrub_invented_defaults deterministically blanks any top-level field whose value contains a URL host, or a credential-shaped or credential-keyed literal, not present in the goal text -- decided on hosts and literal text, never on LLM prose. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fc0f5cf8fd
commit
be24b00a2a
4 files changed
+367
-3
No files matched your search
@@ -80,6 +80,110 @@ def test_analyze_goal_degrades_to_generic_field_on_boom():
|
||||
assert out["values"].get(out["fields"][0]["key"]) == "some goal"
|
||||
|
||||
|
||||
# The real S5b goal (ESQ1-302 / F2): no URL, no credential anywhere in it.
|
||||
_S5B_GOAL = (
|
||||
"Generate a PowerPoint presentation from a topic the user provides: an LLM writes the "
|
||||
"slide content, then the workflow POSTs the slides as JSON to our company's PPT rendering "
|
||||
"HTTP API, which returns the .pptx download link."
|
||||
)
|
||||
|
||||
|
||||
def test_scrub_invented_defaults_blanks_the_real_s5b_invented_endpoint_and_credential():
|
||||
values = {
|
||||
"tone": "Professional",
|
||||
"slide_count": 10,
|
||||
"render_api_url": "https://api.yourcompany.com/v1/ppt/render",
|
||||
"api_auth_header": {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer YOUR_API_KEY",
|
||||
},
|
||||
"include_speaker_notes": True,
|
||||
}
|
||||
|
||||
out = build._scrub_invented_defaults(values, _S5B_GOAL)
|
||||
|
||||
assert out["render_api_url"] == ""
|
||||
assert out["api_auth_header"] == ""
|
||||
assert out["tone"] == "Professional"
|
||||
assert out["slide_count"] == 10
|
||||
assert out["include_speaker_notes"] is True
|
||||
|
||||
|
||||
def test_scrub_invented_defaults_blanks_a_plausible_invented_host():
|
||||
values = {"render_api_url": "https://api.pptrender.io/v1/render"}
|
||||
|
||||
out = build._scrub_invented_defaults(values, _S5B_GOAL)
|
||||
|
||||
assert out["render_api_url"] == ""
|
||||
|
||||
|
||||
def test_scrub_invented_defaults_keeps_a_url_whose_host_the_goal_names():
|
||||
goal = "POST the finished slides to https://render.acme.internal/v2 and return the link"
|
||||
values = {"render_api_url": "https://render.acme.internal/v2/pptx"}
|
||||
|
||||
out = build._scrub_invented_defaults(values, goal)
|
||||
|
||||
assert out["render_api_url"] == "https://render.acme.internal/v2/pptx"
|
||||
|
||||
|
||||
def test_scrub_invented_defaults_keeps_a_credential_the_goal_states():
|
||||
goal = "Authenticate with the API key sk-live-abc123"
|
||||
values = {"api_key": "sk-live-abc123"}
|
||||
|
||||
out = build._scrub_invented_defaults(values, goal)
|
||||
|
||||
assert out["api_key"] == "sk-live-abc123"
|
||||
|
||||
|
||||
def test_scrub_invented_defaults_blanks_a_credential_the_goal_does_not_state():
|
||||
values = {"api_key": "sk-live-abc123"}
|
||||
|
||||
out = build._scrub_invented_defaults(values, _S5B_GOAL)
|
||||
|
||||
assert out["api_key"] == ""
|
||||
|
||||
|
||||
def test_analyze_goal_scrubs_invented_defaults_end_to_end(monkeypatch):
|
||||
captured: dict[str, str] = {}
|
||||
raw = {
|
||||
"fields": [
|
||||
{"key": "tone", "label": "Tone", "type": "text"},
|
||||
{"key": "slide_count", "label": "Slide count", "type": "number"},
|
||||
{"key": "render_api_url", "label": "Render API URL", "type": "text"},
|
||||
{"key": "api_auth_header", "label": "Auth header", "type": "json_object"},
|
||||
{"key": "include_speaker_notes", "label": "Include speaker notes", "type": "bool"},
|
||||
],
|
||||
"values": {
|
||||
"tone": "Professional",
|
||||
"slide_count": 10,
|
||||
"render_api_url": "https://api.yourcompany.com/v1/ppt/render",
|
||||
"api_auth_header": {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer YOUR_API_KEY",
|
||||
},
|
||||
"include_speaker_notes": True,
|
||||
},
|
||||
}
|
||||
|
||||
def fake_invoke_json(model, *, system, user, on_reasoning=None): # noqa: ARG001
|
||||
captured["system"] = system
|
||||
return raw
|
||||
|
||||
monkeypatch.setattr(build.llm, "invoke_json", fake_invoke_json)
|
||||
|
||||
out = build.analyze_goal(object(), _S5B_GOAL)
|
||||
|
||||
assert out["values"]["render_api_url"] == ""
|
||||
assert out["values"]["api_auth_header"] == ""
|
||||
assert out["values"]["tone"] == "Professional"
|
||||
assert out["values"]["slide_count"] == 10
|
||||
assert out["values"]["include_speaker_notes"] is True
|
||||
assert (
|
||||
"Never invent a URL/endpoint, API key, token, password, or account/resource id the goal does not state"
|
||||
in captured["system"]
|
||||
)
|
||||
|
||||
|
||||
def test_propose_plan_v1_returns_bullets():
|
||||
m = _FakeInstance([json.dumps({"plan": ["Ingest", "Summarize", "Emit"]})])
|
||||
assert build.propose_plan_v1(m, {"x": 1}) == ["Ingest", "Summarize", "Emit"]
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Unit tests for services.dify_builder.agent.user_supplied.
|
||||
|
||||
These are the shared "did the user actually write this, or did the model
|
||||
invent it" primitives (ESQ1-302 / S5b / F1, F2): they underpin the
|
||||
requirements-analysis scrub in Task 1 and later tasks' checks on http-request
|
||||
URLs and headers. Pure functions, no I/O.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from services.dify_builder.agent import user_supplied as us
|
||||
|
||||
|
||||
def test_url_hosts_extracts_lowercase_hostnames_from_prose():
|
||||
text = "Call https://API.Example.com/v1 and also http://other.example.org:8080/x please."
|
||||
assert us.url_hosts(text) == {"api.example.com", "other.example.org"}
|
||||
|
||||
|
||||
def test_url_hosts_empty_set_when_no_urls_present():
|
||||
assert us.url_hosts("no urls in this goal at all") == set()
|
||||
|
||||
|
||||
def test_url_hosts_empty_text_is_empty_set():
|
||||
assert us.url_hosts("") == set()
|
||||
|
||||
|
||||
def test_is_user_supplied_url_true_when_host_matches_case_insensitively():
|
||||
trusted = "the goal mentions https://render.acme.internal/v2 as the target"
|
||||
assert us.is_user_supplied_url("https://Render.Acme.Internal/v2/pptx", trusted) is True
|
||||
|
||||
|
||||
def test_is_user_supplied_url_false_when_host_absent_from_trusted_text():
|
||||
assert us.is_user_supplied_url("https://api.pptrender.io/v1/render", "no url here") is False
|
||||
|
||||
|
||||
def test_is_user_supplied_url_empty_url_is_false():
|
||||
assert us.is_user_supplied_url("", "https://anything.example.com") is False
|
||||
|
||||
|
||||
def test_is_user_supplied_url_dify_template_is_always_true():
|
||||
# A template reference is a variable, not a literal -- it can't be "invented".
|
||||
assert us.is_user_supplied_url("{{#node1.node5_url#}}", "goal has no url at all") is True
|
||||
|
||||
|
||||
def test_credential_placeholder_re_matches_known_placeholder_shapes():
|
||||
for token in (
|
||||
"YOUR_API_KEY",
|
||||
"YOURKEY",
|
||||
"your-token",
|
||||
"API_KEY_HERE",
|
||||
"TOKEN_HERE",
|
||||
"<api-key>",
|
||||
"REPLACE_ME",
|
||||
"CHANGE-ME",
|
||||
"xxxxxxxx",
|
||||
"******",
|
||||
):
|
||||
assert us.CREDENTIAL_PLACEHOLDER_RE.search(token), token
|
||||
|
||||
|
||||
def test_credential_placeholder_re_does_not_match_a_real_looking_secret():
|
||||
assert us.CREDENTIAL_PLACEHOLDER_RE.search("sk-live-abc123") is None
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_true_when_bearer_stripped_value_is_literal():
|
||||
assert us.is_user_supplied_secret("Bearer sk-live-abc123", "the key is sk-live-abc123") is True
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_strips_basic_and_token_prefixes_case_insensitively():
|
||||
assert us.is_user_supplied_secret("basic sk-live-abc123", "sk-live-abc123") is True
|
||||
assert us.is_user_supplied_secret("TOKEN sk-live-abc123", "sk-live-abc123") is True
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_false_for_placeholder_even_with_bearer_prefix():
|
||||
assert us.is_user_supplied_secret("Bearer YOUR_API_KEY", "anything at all") is False
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_false_for_empty_or_prefix_only_value():
|
||||
assert us.is_user_supplied_secret("", "anything") is False
|
||||
assert us.is_user_supplied_secret("Bearer ", "anything") is False
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_true_for_dify_template_reference():
|
||||
assert us.is_user_supplied_secret("{{#node1.api_key#}}", "goal text with nothing matching") is True
|
||||
|
||||
|
||||
def test_is_user_supplied_secret_false_when_remainder_not_in_trusted_text():
|
||||
assert us.is_user_supplied_secret("sk-live-abc123", "the goal names nothing like that") is False
|
||||
|
||||
|
||||
def test_trusted_text_for_combines_goal_and_requirements_json():
|
||||
out = us.trusted_text_for("build a thing", {"a": 1, "b": "x"})
|
||||
assert out == "build a thing\n" + json.dumps({"a": 1, "b": "x"}, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_trusted_text_for_empty_requirements():
|
||||
assert us.trusted_text_for("goal only", {}) == "goal only\n{}"
|
||||
Reference in new issue
Block a user