chore: dep inject for model in website crawl api (#40156)

api/controllers/console/datasets/website.py

@@ -1,13 +1,12 @@
 from typing import Any, Literal
 
-from flask import request
 from flask_restx import Resource
 from pydantic import BaseModel, RootModel
 
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
 from controllers.console.datasets.error import WebsiteCrawlError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required, model_validate, setup_required
 from libs.login import login_required
 from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService
 
@@ -40,12 +39,11 @@ class WebsiteCrawlApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self):
-        payload = WebsiteCrawlPayload.model_validate(console_ns.payload or {})
-
+    @model_validate(WebsiteCrawlPayload)
+    def post(self, req_data: WebsiteCrawlPayload):
         # Create typed request and validate
         try:
-            api_request = WebsiteCrawlApiRequest.from_args(payload.model_dump())
+            api_request = WebsiteCrawlApiRequest.from_args(req_data.model_dump())
         except ValueError as e:
             raise WebsiteCrawlError(str(e))
 
@@ -69,12 +67,11 @@ class WebsiteCrawlStatusApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def get(self, job_id: str):
-        args = WebsiteCrawlStatusQuery.model_validate(request.args.to_dict())
-
+    @model_validate(WebsiteCrawlStatusQuery)
+    def get(self, req_data: WebsiteCrawlStatusQuery, job_id: str):
         # Create typed request and validate
         try:
-            api_request = WebsiteCrawlStatusApiRequest.from_args(args.model_dump(), job_id)
+            api_request = WebsiteCrawlStatusApiRequest.from_args(req_data.model_dump(), job_id)
         except ValueError as e:
             raise WebsiteCrawlError(str(e))
 

api/tests/unit_tests/controllers/console/datasets/test_website.py

@@ -1,14 +1,15 @@
-from unittest.mock import Mock, PropertyMock, patch
+from unittest.mock import Mock
 
 import pytest
 from flask import Flask
 from pytest_mock import MockerFixture
 
-from controllers.console import console_ns
 from controllers.console.datasets.error import WebsiteCrawlError
 from controllers.console.datasets.website import (
     WebsiteCrawlApi,
+    WebsiteCrawlPayload,
     WebsiteCrawlStatusApi,
+    WebsiteCrawlStatusQuery,
 )
 from services.website_service import (
     WebsiteCrawlApiRequest,
@@ -58,16 +59,9 @@ class TestWebsiteCrawlApi:
             "url": "https://example.com",
             "options": {"depth": 1},
         }
+        req_data = WebsiteCrawlPayload.model_validate(payload)
 
-        with (
-            app.test_request_context("/", json=payload),
-            patch.object(
-                type(console_ns),
-                "payload",
-                new_callable=PropertyMock,
-                return_value=payload,
-            ),
-        ):
+        with app.test_request_context("/", json=payload):
             mock_request = Mock(spec=WebsiteCrawlApiRequest)
             mocker.patch.object(
                 WebsiteCrawlApiRequest,
@@ -81,7 +75,7 @@ class TestWebsiteCrawlApi:
                 return_value={"job_id": "job-1"},
             )
 
-            result, status = method(api)
+            result, status = method(api, req_data)
 
         assert status == 200
         assert result["job_id"] == "job-1"
@@ -95,16 +89,9 @@ class TestWebsiteCrawlApi:
             "url": "bad-url",
             "options": {},
         }
+        req_data = WebsiteCrawlPayload.model_validate(payload)
 
-        with (
-            app.test_request_context("/", json=payload),
-            patch.object(
-                type(console_ns),
-                "payload",
-                new_callable=PropertyMock,
-                return_value=payload,
-            ),
-        ):
+        with app.test_request_context("/", json=payload):
             mocker.patch.object(
                 WebsiteCrawlApiRequest,
                 "from_args",
@@ -112,7 +99,7 @@ class TestWebsiteCrawlApi:
             )
 
             with pytest.raises(WebsiteCrawlError, match="invalid payload"):
-                method(api)
+                method(api, req_data)
 
     def test_crawl_service_error(self, app: Flask, mocker: MockerFixture):
         api = WebsiteCrawlApi()
@@ -123,16 +110,9 @@ class TestWebsiteCrawlApi:
             "url": "https://example.com",
             "options": {},
         }
+        req_data = WebsiteCrawlPayload.model_validate(payload)
 
-        with (
-            app.test_request_context("/", json=payload),
-            patch.object(
-                type(console_ns),
-                "payload",
-                new_callable=PropertyMock,
-                return_value=payload,
-            ),
-        ):
+        with app.test_request_context("/", json=payload):
             mock_request = Mock(spec=WebsiteCrawlApiRequest)
             mocker.patch.object(
                 WebsiteCrawlApiRequest,
@@ -147,7 +127,7 @@ class TestWebsiteCrawlApi:
             )
 
             with pytest.raises(WebsiteCrawlError, match="crawl failed"):
-                method(api)
+                method(api, req_data)
 
 
 class TestWebsiteCrawlStatusApi:
@@ -156,14 +136,9 @@ class TestWebsiteCrawlStatusApi:
         method = unwrap(api.get)
 
         job_id = "job-123"
-        args = {"provider": "firecrawl"}
+        req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
 
         with app.test_request_context("/?provider=firecrawl"):
-            mocker.patch(
-                "controllers.console.datasets.website.request.args.to_dict",
-                return_value=args,
-            )
-
             mock_request = Mock(spec=WebsiteCrawlStatusApiRequest)
             mocker.patch.object(
                 WebsiteCrawlStatusApiRequest,
@@ -177,7 +152,7 @@ class TestWebsiteCrawlStatusApi:
                 return_value={"status": "completed"},
             )
 
-            result, status = method(api, job_id)
+            result, status = method(api, req_data, job_id)
 
         assert status == 200
         assert result["status"] == "completed"
@@ -187,14 +162,9 @@ class TestWebsiteCrawlStatusApi:
         method = unwrap(api.get)
 
         job_id = "job-123"
-        args = {"provider": "firecrawl"}
+        req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
 
         with app.test_request_context("/?provider=firecrawl"):
-            mocker.patch(
-                "controllers.console.datasets.website.request.args.to_dict",
-                return_value=args,
-            )
-
             mocker.patch.object(
                 WebsiteCrawlStatusApiRequest,
                 "from_args",
@@ -202,21 +172,16 @@ class TestWebsiteCrawlStatusApi:
             )
 
             with pytest.raises(WebsiteCrawlError, match="invalid provider"):
-                method(api, job_id)
+                method(api, req_data, job_id)
 
     def test_get_status_service_error(self, app: Flask, mocker: MockerFixture):
         api = WebsiteCrawlStatusApi()
         method = unwrap(api.get)
 
         job_id = "job-123"
-        args = {"provider": "firecrawl"}
+        req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
 
         with app.test_request_context("/?provider=firecrawl"):
-            mocker.patch(
-                "controllers.console.datasets.website.request.args.to_dict",
-                return_value=args,
-            )
-
             mock_request = Mock(spec=WebsiteCrawlStatusApiRequest)
             mocker.patch.object(
                 WebsiteCrawlStatusApiRequest,
@@ -231,4 +196,4 @@ class TestWebsiteCrawlStatusApi:
             )
 
             with pytest.raises(WebsiteCrawlError, match="status lookup failed"):
-                method(api, job_id)
+                method(api, req_data, job_id)