fix(dify-builder): requirements analysis stops inventing endpoints and credentials

S5b (local E2E 2026-09-22, F2): analyze_goal's "a sensible default value per
field" led the LLM to prefill render_api_url=https://api.yourcompany.com/...
and Authorization: Bearer YOUR_API_KEY for a goal that named neither -- caught
only because "yourcompany" happened to match an existing placeholder-URL
regex; a plausible invented host would slip through and recreate ESQ1-302's
launch failure one step upstream, in the requirements form itself.

Adds services/dify_builder/agent/user_supplied.py (pure, no I/O): url_hosts,
is_user_supplied_url, CREDENTIAL_PLACEHOLDER_RE, is_user_supplied_secret,
trusted_text_for -- shared primitives for telling a literal the user actually
wrote from one an LLM invented, used by this task and consumed by later
tasks in the same plan.

build.analyze_goal now tells the model never to invent such a field's
default, and _scrub_invented_defaults deterministically blanks any top-level
field whose value contains a URL host, or a credential-shaped or
credential-keyed literal, not present in the goal text -- decided on hosts
and literal text, never on LLM prose.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Charles YaoandClaude Opus 5 committed 2026-09-22 08:40:33 -07:00
1 parent fc0f5cf8fd
commit be24b00a2a
4 files changed
+367 -3

No files matched your search

@@ -80,6 +80,110 @@ def test_analyze_goal_degrades_to_generic_field_on_boom():
assert out["values"].get(out["fields"][0]["key"]) == "some goal"
# The real S5b goal (ESQ1-302 / F2): no URL, no credential anywhere in it.
_S5B_GOAL = (
"Generate a PowerPoint presentation from a topic the user provides: an LLM writes the "
"slide content, then the workflow POSTs the slides as JSON to our company's PPT rendering "
"HTTP API, which returns the .pptx download link."
)
def test_scrub_invented_defaults_blanks_the_real_s5b_invented_endpoint_and_credential():
values = {
"tone": "Professional",
"slide_count": 10,
"render_api_url": "https://api.yourcompany.com/v1/ppt/render",
"api_auth_header": {
"Content-Type": "application/json",
"Authorization": "Bearer YOUR_API_KEY",
},
"include_speaker_notes": True,
}
out = build._scrub_invented_defaults(values, _S5B_GOAL)
assert out["render_api_url"] == ""
assert out["api_auth_header"] == ""
assert out["tone"] == "Professional"
assert out["slide_count"] == 10
assert out["include_speaker_notes"] is True
def test_scrub_invented_defaults_blanks_a_plausible_invented_host():
values = {"render_api_url": "https://api.pptrender.io/v1/render"}
out = build._scrub_invented_defaults(values, _S5B_GOAL)
assert out["render_api_url"] == ""
def test_scrub_invented_defaults_keeps_a_url_whose_host_the_goal_names():
goal = "POST the finished slides to https://render.acme.internal/v2 and return the link"
values = {"render_api_url": "https://render.acme.internal/v2/pptx"}
out = build._scrub_invented_defaults(values, goal)
assert out["render_api_url"] == "https://render.acme.internal/v2/pptx"
def test_scrub_invented_defaults_keeps_a_credential_the_goal_states():
goal = "Authenticate with the API key sk-live-abc123"
values = {"api_key": "sk-live-abc123"}
out = build._scrub_invented_defaults(values, goal)
assert out["api_key"] == "sk-live-abc123"
def test_scrub_invented_defaults_blanks_a_credential_the_goal_does_not_state():
values = {"api_key": "sk-live-abc123"}
out = build._scrub_invented_defaults(values, _S5B_GOAL)
assert out["api_key"] == ""
def test_analyze_goal_scrubs_invented_defaults_end_to_end(monkeypatch):
captured: dict[str, str] = {}
raw = {
"fields": [
{"key": "tone", "label": "Tone", "type": "text"},
{"key": "slide_count", "label": "Slide count", "type": "number"},
{"key": "render_api_url", "label": "Render API URL", "type": "text"},
{"key": "api_auth_header", "label": "Auth header", "type": "json_object"},
{"key": "include_speaker_notes", "label": "Include speaker notes", "type": "bool"},
],
"values": {
"tone": "Professional",
"slide_count": 10,
"render_api_url": "https://api.yourcompany.com/v1/ppt/render",
"api_auth_header": {
"Content-Type": "application/json",
"Authorization": "Bearer YOUR_API_KEY",
},
"include_speaker_notes": True,
},
}
def fake_invoke_json(model, *, system, user, on_reasoning=None): # noqa: ARG001
captured["system"] = system
return raw
monkeypatch.setattr(build.llm, "invoke_json", fake_invoke_json)
out = build.analyze_goal(object(), _S5B_GOAL)
assert out["values"]["render_api_url"] == ""
assert out["values"]["api_auth_header"] == ""
assert out["values"]["tone"] == "Professional"
assert out["values"]["slide_count"] == 10
assert out["values"]["include_speaker_notes"] is True
assert (
"Never invent a URL/endpoint, API key, token, password, or account/resource id the goal does not state"
in captured["system"]
)
def test_propose_plan_v1_returns_bullets():
m = _FakeInstance([json.dumps({"plan": ["Ingest", "Summarize", "Emit"]})])
assert build.propose_plan_v1(m, {"x": 1}) == ["Ingest", "Summarize", "Emit"]
@@ -0,0 +1,97 @@
"""Unit tests for services.dify_builder.agent.user_supplied.
These are the shared "did the user actually write this, or did the model
invent it" primitives (ESQ1-302 / S5b / F1, F2): they underpin the
requirements-analysis scrub in Task 1 and later tasks' checks on http-request
URLs and headers. Pure functions, no I/O.
"""
import json
from services.dify_builder.agent import user_supplied as us
def test_url_hosts_extracts_lowercase_hostnames_from_prose():
text = "Call https://API.Example.com/v1 and also http://other.example.org:8080/x please."
assert us.url_hosts(text) == {"api.example.com", "other.example.org"}
def test_url_hosts_empty_set_when_no_urls_present():
assert us.url_hosts("no urls in this goal at all") == set()
def test_url_hosts_empty_text_is_empty_set():
assert us.url_hosts("") == set()
def test_is_user_supplied_url_true_when_host_matches_case_insensitively():
trusted = "the goal mentions https://render.acme.internal/v2 as the target"
assert us.is_user_supplied_url("https://Render.Acme.Internal/v2/pptx", trusted) is True
def test_is_user_supplied_url_false_when_host_absent_from_trusted_text():
assert us.is_user_supplied_url("https://api.pptrender.io/v1/render", "no url here") is False
def test_is_user_supplied_url_empty_url_is_false():
assert us.is_user_supplied_url("", "https://anything.example.com") is False
def test_is_user_supplied_url_dify_template_is_always_true():
# A template reference is a variable, not a literal -- it can't be "invented".
assert us.is_user_supplied_url("{{#node1.node5_url#}}", "goal has no url at all") is True
def test_credential_placeholder_re_matches_known_placeholder_shapes():
for token in (
"YOUR_API_KEY",
"YOURKEY",
"your-token",
"API_KEY_HERE",
"TOKEN_HERE",
"<api-key>",
"REPLACE_ME",
"CHANGE-ME",
"xxxxxxxx",
"******",
):
assert us.CREDENTIAL_PLACEHOLDER_RE.search(token), token
def test_credential_placeholder_re_does_not_match_a_real_looking_secret():
assert us.CREDENTIAL_PLACEHOLDER_RE.search("sk-live-abc123") is None
def test_is_user_supplied_secret_true_when_bearer_stripped_value_is_literal():
assert us.is_user_supplied_secret("Bearer sk-live-abc123", "the key is sk-live-abc123") is True
def test_is_user_supplied_secret_strips_basic_and_token_prefixes_case_insensitively():
assert us.is_user_supplied_secret("basic sk-live-abc123", "sk-live-abc123") is True
assert us.is_user_supplied_secret("TOKEN sk-live-abc123", "sk-live-abc123") is True
def test_is_user_supplied_secret_false_for_placeholder_even_with_bearer_prefix():
assert us.is_user_supplied_secret("Bearer YOUR_API_KEY", "anything at all") is False
def test_is_user_supplied_secret_false_for_empty_or_prefix_only_value():
assert us.is_user_supplied_secret("", "anything") is False
assert us.is_user_supplied_secret("Bearer ", "anything") is False
def test_is_user_supplied_secret_true_for_dify_template_reference():
assert us.is_user_supplied_secret("{{#node1.api_key#}}", "goal text with nothing matching") is True
def test_is_user_supplied_secret_false_when_remainder_not_in_trusted_text():
assert us.is_user_supplied_secret("sk-live-abc123", "the goal names nothing like that") is False
def test_trusted_text_for_combines_goal_and_requirements_json():
out = us.trusted_text_for("build a thing", {"a": 1, "b": "x"})
assert out == "build a thing\n" + json.dumps({"a": 1, "b": "x"}, ensure_ascii=False)
def test_trusted_text_for_empty_requirements():
assert us.trusted_text_for("goal only", {}) == "goal only\n{}"