From f53425243fca53d4d80d89cf00b7e1bd25db5b14 Mon Sep 17 00:00:00 2001 From: Charles Yao Date: Sun, 20 Sep 2026 15:21:33 -0700 Subject: [PATCH] feat(dify-builder): mock test inputs unless the workflow needs an upload handle_execution's run_test branch used to stop and ask a human for test data any time none was prepared yet, even when every declared start variable was something the agent could invent. Add needs_upload_inputs(schema) -> bool (true only for a declared file / file-list variable) and gate BUILD_AWAIT_TESTDATA on it: everything else is now mocked inline via env.agent.generate_mock_inputs and the flow goes straight to build.test_and_repair. A schema mixing text and file variables still stops at the gate, since the file can't be invented. Updates three full-build-flow tests whose generated graph has no file start variable, so run_test now lands directly at build.review instead of pausing at the testdata gate. Co-Authored-By: Claude Opus 5 (1M context) --- api/core/dify_builder/handlers_build.py | 52 +++++++------ api/core/dify_builder/handlers_fix.py | 15 ++++ .../core/dify_builder/test_handlers_build.py | 73 ++++++++++++++++--- .../core/dify_builder/test_handlers_fix.py | 44 +++++++++++ 4 files changed, 151 insertions(+), 33 deletions(-) diff --git a/api/core/dify_builder/handlers_build.py b/api/core/dify_builder/handlers_build.py index 3db36010719..d5786f26550 100644 --- a/api/core/dify_builder/handlers_build.py +++ b/api/core/dify_builder/handlers_build.py @@ -46,6 +46,7 @@ from core.dify_builder.handlers_fix import ( merge_known_keys, mint_checkpoint, model_config_error_text, + needs_upload_inputs, perform_revert, start_schema, testdata_form_fields, @@ -546,7 +547,10 @@ def handle_plan_approval(env: Env, turn: Turn, s: Session, fc: DifyBuilderContex def handle_execution(env: Env, turn: Turn, s: Session, fc: DifyBuilderContext) -> StepResult: """(waiting) At rest after the build. ``run_test`` -> build.await_testdata - when no test input is prepared yet (gate), else straight to + ONLY when no test input is prepared yet AND the start schema declares a + file/file-list variable (a human has to supply an upload; nothing else + needs one) -- else the inputs are mocked inline via + ``env.agent.generate_mock_inputs`` and the flow goes straight to build.test_and_repair; ``revert`` (resolved to ``undo``) -> build.reverted: restores the pre-build draft from the checkpoint and invalidates the approvals made since it (via perform_revert).""" @@ -558,26 +562,32 @@ def handle_execution(env: Env, turn: Turn, s: Session, fc: DifyBuilderContext) - if kind == "run_test": if fc.test_input_ref == "": graph, _hash = env.dify.read_graph(s.app_id, turn.actor) - form_items = append_card( - fc, - FormCard( - variant="testdata", - fields=testdata_form_fields(start_schema(graph)), - values={}, - frozen=False, - ), - ) - turn_items = append_card( - fc, - AssistantTurnItem( - turn_id=str(uuid.uuid4()), - stage_id=str(s.current_state), - execution=ExecutionProgress(status="completed"), - reply_text="Provide test inputs (or use mock data) to run the test.", - cards=["form"], - ), - ) - return StepResult(next=PcState.BUILD_AWAIT_TESTDATA, context=fc, items=[*form_items, *turn_items]) + schema = start_schema(graph) + if needs_upload_inputs(schema): + form_items = append_card( + fc, + FormCard( + variant="testdata", + fields=testdata_form_fields(schema), + values={}, + frozen=False, + ), + ) + turn_items = append_card( + fc, + AssistantTurnItem( + turn_id=str(uuid.uuid4()), + stage_id=str(s.current_state), + execution=ExecutionProgress(status="completed"), + reply_text="Provide test inputs (or use mock data) to run the test.", + cards=["form"], + ), + ) + return StepResult(next=PcState.BUILD_AWAIT_TESTDATA, context=fc, items=[*form_items, *turn_items]) + inputs = env.agent.generate_mock_inputs(schema, {}) + ti = TestInput(session_id=s.id, source="mock", inputs=inputs) + env.repo.save_test_input(ti) + fc.test_input_ref = ti.id emit_canvas(env, "start_test_run") items = append_card(fc, DecisionItem(text="Run tests")) return StepResult(next=PcState.BUILD_TEST_AND_REPAIR, context=fc, items=items) diff --git a/api/core/dify_builder/handlers_fix.py b/api/core/dify_builder/handlers_fix.py index 6758052de0a..88779571836 100644 --- a/api/core/dify_builder/handlers_fix.py +++ b/api/core/dify_builder/handlers_fix.py @@ -87,6 +87,7 @@ __all__ = [ "merge_known_keys", "mint_checkpoint", "model_config_error_text", + "needs_upload_inputs", "perform_revert", "start_schema", "testdata_form_fields", @@ -239,6 +240,20 @@ def start_schema(graph: Graph) -> StartSchema: return {"variables": []} +_UPLOAD_VARIABLE_TYPES = frozenset({"file", "file-list"}) + + +def needs_upload_inputs(schema: StartSchema) -> bool: + """Whether any declared start variable takes a file. + + Everything else can be mocked; an upload cannot, so it is the only reason + left to stop and ask a human for test data. + """ + return any( + isinstance(v, dict) and str(v.get("type") or "") in _UPLOAD_VARIABLE_TYPES for v in schema.get("variables", []) + ) + + def _string_list(value: object) -> list[str]: """Copy only string members from a JSON-style list.""" return [item for item in value if isinstance(item, str)] if isinstance(value, list) else [] diff --git a/api/tests/unit_tests/core/dify_builder/test_handlers_build.py b/api/tests/unit_tests/core/dify_builder/test_handlers_build.py index 906c7b5230a..cf7830a2885 100644 --- a/api/tests/unit_tests/core/dify_builder/test_handlers_build.py +++ b/api/tests/unit_tests/core/dify_builder/test_handlers_build.py @@ -1157,13 +1157,10 @@ def test_full_build_flow_goal_to_complete(): assert len(graph["nodes"]) == 4 assert len(graph["edges"]) == 3 - # 5) run_test -> build.await_testdata (gate; no test input prepared yet) + # 5) run_test -> the generated start schema declares no file/file-list + # variable, so inputs are mocked inline (no gate) -> build.test_and_repair + # (working, auto) -> rests at build.review. out = runner.advance(s.id, Turn(action=Action(kind="run_test", base_version=out.version), actor=_actor())) - assert out.current_state == PcState.BUILD_AWAIT_TESTDATA - - # 5b) provide_testdata (mock) -> build.test_and_repair (working, auto) -> rest at build.review - testdata_action = Action(kind="provide_testdata", payload={"mode": "mock"}, base_version=out.version) - out = runner.advance(s.id, Turn(action=testdata_action, actor=_actor())) assert out.current_state == PcState.BUILD_REVIEW # 6) publish_workflow -> build.publish (auto) -> governance_feedback (auto) @@ -1221,10 +1218,8 @@ def test_full_build_flow_keep_draft_reaches_complete_without_publish(): confirm_action = Action(kind="confirm_resources", payload=confirm_payload, base_version=out.version) out = runner.advance(s.id, Turn(action=confirm_action, actor=_actor())) out = runner.advance(s.id, Turn(action=Action(kind="approve_repair", base_version=out.version), actor=_actor())) + # No file/file-list start variable -> mocked inline, no gate. out = runner.advance(s.id, Turn(action=Action(kind="run_test", base_version=out.version), actor=_actor())) - assert out.current_state == PcState.BUILD_AWAIT_TESTDATA - testdata_action = Action(kind="provide_testdata", payload={"mode": "mock"}, base_version=out.version) - out = runner.advance(s.id, Turn(action=testdata_action, actor=_actor())) assert out.current_state == PcState.BUILD_REVIEW out = runner.advance(s.id, Turn(action=Action(kind="keep_draft", base_version=out.version), actor=_actor())) @@ -1272,10 +1267,8 @@ def test_review_continue_adjusting_then_reapprove_is_idempotent(): assert len(dify.graph["nodes"]) == 4 assert len(dify.graph["edges"]) == 3 + # No file/file-list start variable -> mocked inline, no gate. out = runner.advance(s.id, Turn(action=Action(kind="run_test", base_version=out.version), actor=_actor())) - assert out.current_state == PcState.BUILD_AWAIT_TESTDATA - testdata_action = Action(kind="provide_testdata", payload={"mode": "mock"}, base_version=out.version) - out = runner.advance(s.id, Turn(action=testdata_action, actor=_actor())) assert out.current_state == PcState.BUILD_REVIEW # loop back: continue_adjusting (-> re_fix) -> build.initial_plan (re-plan) @@ -1349,6 +1342,62 @@ def test_run_test_skips_gate_when_input_prepared(): assert result.next == PcState.BUILD_TEST_AND_REPAIR +def test_run_test_mocks_its_own_inputs(): + """Nothing to upload, so nothing to ask -- mock the inputs inline and run.""" + from core.dify_builder.handlers_build import handle_execution + from tests.unit_tests.core.dify_builder.fakes import FakeBuildDifyPort + + env, _ = _new_env() + env.dify = FakeBuildDifyPort() + env.dify.graph = { + "nodes": [ + { + "id": "start", + "data": {"type": "start", "variables": [{"variable": "topic", "type": "text-input"}]}, + } + ], + "edges": [], + } + s = _session(entry_mode=EntryMode.BUILD, current_state=PcState.BUILD_EXECUTION) + fc = DifyBuilderContext(test_input_ref="") + result = handle_execution(env, Turn(actor=_actor(), action=Action(kind="run_test")), s, fc) + assert result.next == PcState.BUILD_TEST_AND_REPAIR + assert result.context.test_input_ref # mocked + persisted, not left blank + assert not any(i.kind == "form" for i in result.items) # no gate shown + + +def test_run_test_still_asks_when_a_file_is_declared_among_other_variables(): + """Mixed schema: text can be mocked, but the file variable cannot -- the + gate must still fire so a human can supply the upload.""" + from core.dify_builder.handlers_build import handle_execution + from tests.unit_tests.core.dify_builder.fakes import FakeBuildDifyPort + + env, _ = _new_env() + env.dify = FakeBuildDifyPort() + env.dify.graph = { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "variables": [ + {"variable": "topic", "type": "text-input"}, + {"variable": "doc", "type": "file"}, + ], + }, + } + ], + "edges": [], + } + s = _session(entry_mode=EntryMode.BUILD, current_state=PcState.BUILD_EXECUTION) + fc = DifyBuilderContext(test_input_ref="") + result = handle_execution(env, Turn(actor=_actor(), action=Action(kind="run_test")), s, fc) + assert result.next == PcState.BUILD_AWAIT_TESTDATA + assert result.context.test_input_ref == "" + form = next(i for i in result.items if i.kind == "form") + assert [f["type"] for f in form.payload["fields"]] == ["text-input", "file"] + + def test_await_testdata_mock_prepares_input_and_advances(): from core.dify_builder.handlers_build import handle_await_testdata from tests.unit_tests.core.dify_builder.fakes import FakeBuildDifyPort diff --git a/api/tests/unit_tests/core/dify_builder/test_handlers_fix.py b/api/tests/unit_tests/core/dify_builder/test_handlers_fix.py index 8c43e739b2d..4edbfb51071 100644 --- a/api/tests/unit_tests/core/dify_builder/test_handlers_fix.py +++ b/api/tests/unit_tests/core/dify_builder/test_handlers_fix.py @@ -623,6 +623,50 @@ def test_start_schema_empty_when_no_start_or_no_vars(): assert start_schema({"nodes": [{"id": "s", "data": {"type": "start"}}], "edges": []}) == {"variables": []} +# ---- needs_upload_inputs ----------------------------------------------------- + + +def test_needs_upload_inputs_false_for_an_all_text_schema(): + from core.dify_builder.handlers_fix import needs_upload_inputs + + schema = {"variables": [{"variable": "topic", "type": "text-input"}, {"variable": "count", "type": "number"}]} + assert needs_upload_inputs(schema) is False + + +def test_needs_upload_inputs_false_when_there_are_no_variables(): + from core.dify_builder.handlers_fix import needs_upload_inputs + + assert needs_upload_inputs({"variables": []}) is False + + +def test_needs_upload_inputs_true_for_a_file_variable(): + from core.dify_builder.handlers_fix import needs_upload_inputs + + schema = {"variables": [{"variable": "doc", "type": "file"}]} + assert needs_upload_inputs(schema) is True + + +def test_needs_upload_inputs_true_for_a_file_list_variable(): + from core.dify_builder.handlers_fix import needs_upload_inputs + + schema = {"variables": [{"variable": "docs", "type": "file-list"}]} + assert needs_upload_inputs(schema) is True + + +def test_needs_upload_inputs_true_for_a_mixed_schema(): + """A human still has to supply the file even though everything else in + the schema could be mocked -- the mixed case must still stop at the gate.""" + from core.dify_builder.handlers_fix import needs_upload_inputs + + schema = { + "variables": [ + {"variable": "topic", "type": "text-input"}, + {"variable": "doc", "type": "file"}, + ] + } + assert needs_upload_inputs(schema) is True + + # ---- is_input_failure / testdata_form_fields -------------------------------