refactor(api): establish dataset application boundaries (#41729)

Co-authored-by: Jyong <jyong@dify.ai>

api/extensions/application_services/data_sources.py

@@ -0,0 +1,124 @@
+"""Composition root for data-source services and shared credential gateways."""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from sqlalchemy.orm import Session, sessionmaker
+
+from configs import dify_config
+from core.datasource.datasource_manager import DatasourceManager
+from core.helper.ssrf_proxy import ssrf_proxy
+from core.plugin.impl.datasource import PluginDatasourceManager
+from core.plugin.impl.oauth import OAuthHandler
+from repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository
+from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository
+from repositories.data_source.oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
+from services.data_source.auth.api_key_gateways import (
+    ProviderApiKeyAuthCredentialValidator,
+    TenantApiKeyAuthCredentialEncryptor,
+)
+from services.data_source.auth.api_key_service import DataSourceApiKeyAuthService
+from services.data_source.binding_application_service import DataSourceBindingApplicationService
+from services.data_source.credential_adapters import (
+    OAuthDatasourceCredentialRefresher,
+    PluginDatasourceCredentialCodec,
+    PluginDatasourceOAuthClientResolver,
+)
+from services.data_source.credential_gateway import (
+    ActorAwareDatasourceCredentialGateway,
+    TrustedStoredDatasourceCredentialGateway,
+)
+from services.data_source.notion_import_adapters import PluginNotionSourceGateway
+from services.data_source.notion_import_application_service import NotionImportApplicationService
+from services.data_source.notion_oauth_gateway import NotionDataSourceGateway
+from services.data_source.oauth_service import DataSourceOAuthService, InvalidDataSourceOAuthProviderError
+from services.data_source.provider_service import DatasourceProviderService
+from services.data_source.website_service import WebsiteService
+from services.knowledge.dataset_access import DatasetAccessService
+
+
+@dataclass(frozen=True, slots=True)
+class DataSourceServices:
+    providers: DatasourceProviderService
+    api_key_auth: DataSourceApiKeyAuthService
+    oauth: Mapping[str, DataSourceOAuthService]
+    bindings: DataSourceBindingApplicationService
+    notion_imports: NotionImportApplicationService
+
+    def resolve_oauth(self, provider: str) -> DataSourceOAuthService:
+        service = self.oauth.get(provider)
+        if service is None:
+            raise InvalidDataSourceOAuthProviderError("Invalid provider")
+        return service
+
+
+@dataclass(frozen=True, slots=True)
+class DataSourceCredentials:
+    providers: DatasourceProviderService
+    actor: ActorAwareDatasourceCredentialGateway
+    stored: TrustedStoredDatasourceCredentialGateway
+
+
+def build_data_source_credentials(*, database_client: sessionmaker[Session]) -> DataSourceCredentials:
+    credentials = SQLAlchemyDatasourceCredentialRepository(session_factory=database_client)
+    provider_manager = PluginDatasourceManager()
+    codec = PluginDatasourceCredentialCodec(provider_manager=provider_manager)
+    refresher = OAuthDatasourceCredentialRefresher(
+        oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=provider_manager),
+        oauth_handler=OAuthHandler(),
+    )
+    return DataSourceCredentials(
+        providers=DatasourceProviderService(credentials=credentials),
+        actor=ActorAwareDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),
+        stored=TrustedStoredDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),
+    )
+
+
+def build_data_source_services(
+    *,
+    database_client: sessionmaker[Session],
+    dataset_access: DatasetAccessService,
+    datasets: SQLAlchemyDatasetRepository,
+    documents: SQLAlchemyDocumentRepository,
+    actor_credentials: ActorAwareDatasourceCredentialGateway,
+    providers: DatasourceProviderService,
+) -> DataSourceServices:
+    oauth_bindings = SQLAlchemyDataSourceOAuthBindingRepository(session_factory=database_client)
+    notion_gateway = NotionDataSourceGateway(
+        client_id=dify_config.NOTION_CLIENT_ID or "",
+        client_secret=dify_config.NOTION_CLIENT_SECRET or "",
+        redirect_uri=dify_config.CONSOLE_API_URL + "/console/api/oauth/data-source/callback/notion",
+        http_client=ssrf_proxy,
+    )
+    return DataSourceServices(
+        providers=providers,
+        api_key_auth=DataSourceApiKeyAuthService(
+            bindings=SQLAlchemyDataSourceApiKeyAuthBindingRepository(session_factory=database_client),
+            validator=ProviderApiKeyAuthCredentialValidator(),
+            encryptor=TenantApiKeyAuthCredentialEncryptor(),
+        ),
+        oauth={
+            "notion": DataSourceOAuthService(
+                provider_name="notion",
+                provider_gateway=notion_gateway,
+                bindings=oauth_bindings,
+                is_internal_provider=dify_config.NOTION_INTEGRATION_TYPE == "internal",
+                internal_access_token=dify_config.NOTION_INTERNAL_SECRET,
+            )
+        },
+        bindings=DataSourceBindingApplicationService(bindings=oauth_bindings),
+        notion_imports=NotionImportApplicationService(
+            dataset_access=dataset_access,
+            datasets=datasets,
+            documents=documents,
+            source=PluginNotionSourceGateway(
+                credentials=actor_credentials, runtime_loader=DatasourceManager.get_datasource_runtime
+            ),
+        ),
+    )
+
+
+def build_website_service(*, database_client: sessionmaker[Session]) -> WebsiteService:
+    return WebsiteService(providers=build_data_source_credentials(database_client=database_client).providers)

api/extensions/application_services/datasets.py

@@ -0,0 +1,32 @@
+"""Shared dataset dependencies for knowledge and data-source use cases."""
+
+from dataclasses import dataclass
+
+from sqlalchemy.orm import Session, sessionmaker
+
+from configs import dify_config
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
+from services.knowledge.dataset_access import DatasetAccessService, WorkspaceRoleReader
+
+
+@dataclass(frozen=True, slots=True)
+class DatasetDependencies:
+    datasets: SQLAlchemyDatasetRepository
+    documents: SQLAlchemyDocumentRepository
+    access: DatasetAccessService
+
+
+def build_dataset_dependencies(
+    *, database_client: sessionmaker[Session], workspace_roles: WorkspaceRoleReader
+) -> DatasetDependencies:
+    datasets = SQLAlchemyDatasetRepository(session_factory=database_client)
+    return DatasetDependencies(
+        datasets=datasets,
+        documents=SQLAlchemyDocumentRepository(session_factory=database_client),
+        access=DatasetAccessService(
+            datasets=datasets,
+            workspace_roles=workspace_roles,
+            legacy_permissions_enabled=not dify_config.RBAC_ENABLED,
+        ),
+    )

api/extensions/application_services/knowledge.py

@@ -1,16 +1,174 @@
-"""Composition of knowledge base application services."""
+"""Composition root for dataset-controller application services."""
+
+from dataclasses import dataclass
+from uuid import uuid4
 
 from sqlalchemy.orm import Session, sessionmaker
 
 from configs import dify_config
+from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
+from core.rag.extractor.entity.datasource_type import DatasourceType
+from libs.helper import generate_text_hash
 from repositories.knowledge.dataset_api_key_repository import DatasetApiKeyRepository
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
+from repositories.knowledge.metadata_repository import SQLAlchemyMetadataRepository
+from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
+from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
 from services.api_token_service import ApiTokenCache
+from services.app.query_service import AppQueryService
+from services.data_source.credential_gateway import (
+    ActorAwareDatasourceCredentialGateway,
+    TrustedStoredDatasourceCredentialGateway,
+)
+from services.data_source.provider_service import DatasourceProviderService
 from services.knowledge.api_key_service import DatasetApiKeyService
+from services.knowledge.dataset_access import DatasetAccessService
+from services.knowledge.dataset_service import DocumentService
+from services.knowledge.datasets.adapters import SQLAlchemyDatasetOperations
+from services.knowledge.datasets.application import DatasetApplicationService
+from services.knowledge.document_sync import DocumentSyncApplicationService
+from services.knowledge.document_sync_adapters import CeleryDocumentSyncDispatcher
+from services.knowledge.documents.adapters import SQLAlchemyDocumentOperations
+from services.knowledge.documents.application import DatasetDocumentApplicationService
+from services.knowledge.external.adapters import SQLAlchemyExternalKnowledgeOperations
+from services.knowledge.external.application import ExternalKnowledgeApplicationService
+from services.knowledge.indexing.adapters.estimate import IndexingEstimateAdapter, SQLAlchemyProcessRuleReader
+from services.knowledge.indexing.adapters.sources import (
+    CompositeStoredSourceResolver,
+    FileSourceAdapter,
+    NotionSourceResolver,
+    WebsiteSourceAdapter,
+)
+from services.knowledge.indexing.estimate import IndexingEstimateApplicationService
+from services.knowledge.metadata.application import MetadataService
+from services.knowledge.segments.adapters import (
+    CelerySegmentBatchImportDispatcher,
+    ModelManagerSegmentGuard,
+    RedisSegmentClient,
+    RedisSegmentIndexingState,
+)
+from services.knowledge.segments.application import DatasetSegmentApplicationService
+from services.knowledge.segments.indexing import SegmentIndexingGateway
+from services.tag_application_service import TagTargetQuery
+from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
+from tasks.delete_segment_from_index_task import delete_segment_from_index_task
+from tasks.disable_segments_from_index_task import disable_segments_from_index_task
+from tasks.document_indexing_sync_task import document_indexing_sync_task
+from tasks.enable_segments_to_index_task import enable_segments_to_index_task
 
 
-def build_dataset_api_key_service(*, database_client: sessionmaker[Session]) -> DatasetApiKeyService:
+@dataclass(frozen=True, slots=True)
+class KnowledgeServices:
+    metadata: MetadataService
+    datasets: DatasetApplicationService
+    external: ExternalKnowledgeApplicationService
+    documents: DatasetDocumentApplicationService
+    document_sync: DocumentSyncApplicationService
+    indexing_estimates: IndexingEstimateApplicationService
+    segments: DatasetSegmentApplicationService
+    pipeline_generator: PipelineGenerator
+
+
+def build_dataset_api_key_service(
+    *, database_client: sessionmaker[Session], dataset_access: DatasetAccessService
+) -> DatasetApiKeyService:
     return DatasetApiKeyService(
         keys=DatasetApiKeyRepository(session_factory=database_client),
         cache=ApiTokenCache,
+        access=dataset_access,
         rbac_enabled=lambda: dify_config.RBAC_ENABLED,
     )
+
+
+def build_knowledge_services(
+    *,
+    database_client: sessionmaker[Session],
+    dataset_access: DatasetAccessService,
+    datasets: SQLAlchemyDatasetRepository,
+    documents: SQLAlchemyDocumentRepository,
+    actor_credentials: ActorAwareDatasourceCredentialGateway,
+    stored_credentials: TrustedStoredDatasourceCredentialGateway,
+    providers: DatasourceProviderService,
+    redis: RedisSegmentClient,
+    tags: TagTargetQuery,
+    app_queries: AppQueryService,
+) -> KnowledgeServices:
+    """Build the dataset-controller knowledge use cases."""
+
+    uploads = SQLAlchemyKnowledgeUploadRepository(session_factory=database_client)
+    notion_sources = NotionSourceResolver(
+        actor_credentials=actor_credentials,
+        stored_credentials=stored_credentials,
+    )
+    file_sources = FileSourceAdapter(uploads=uploads)
+    website_sources = WebsiteSourceAdapter()
+    stored_sources = CompositeStoredSourceResolver(
+        adapters={
+            DatasourceType.FILE.value: file_sources,
+            DatasourceType.NOTION.value: notion_sources,
+            DatasourceType.WEBSITE.value: website_sources,
+        }
+    )
+    segments = SQLAlchemySegmentRepository(session_factory=database_client)
+    segment_index = SegmentIndexingGateway(
+        segments=segments,
+        uploads=uploads,
+        redis=redis,
+        new_session=database_client,
+        delete_task=delete_segment_from_index_task.delay,
+        enable_task=enable_segments_to_index_task.delay,
+        disable_task=disable_segments_from_index_task.delay,
+    )
+    return KnowledgeServices(
+        metadata=MetadataService(
+            store=SQLAlchemyMetadataRepository(session_factory=database_client), dataset_access=dataset_access
+        ),
+        datasets=DatasetApplicationService(
+            dataset_access=dataset_access,
+            operations=SQLAlchemyDatasetOperations(session_factory=database_client, tags=tags, app_queries=app_queries),
+            rbac_enabled=dify_config.RBAC_ENABLED,
+            service_api_url=dify_config.SERVICE_API_URL,
+            vector_store=dify_config.VECTOR_STORE,
+            tidb_fulltext=dify_config.TIDB_VECTOR_ENABLE_FULLTEXT_SEARCH,
+        ),
+        external=ExternalKnowledgeApplicationService(
+            dataset_access=dataset_access,
+            operations=SQLAlchemyExternalKnowledgeOperations(session_factory=database_client),
+        ),
+        documents=DatasetDocumentApplicationService(
+            dataset_access=dataset_access,
+            operations=SQLAlchemyDocumentOperations(session_factory=database_client),
+            metadata_schema=DocumentService.DOCUMENT_METADATA_SCHEMA,
+        ),
+        pipeline_generator=PipelineGenerator(documents=documents, datasource_providers=providers),
+        document_sync=DocumentSyncApplicationService(
+            dataset_access=dataset_access,
+            documents=documents,
+            dispatcher=CeleryDocumentSyncDispatcher(delay=document_indexing_sync_task.delay),
+        ),
+        indexing_estimates=IndexingEstimateApplicationService(
+            dataset_access=dataset_access,
+            datasets=datasets,
+            documents=documents,
+            files=file_sources,
+            websites=website_sources,
+            stored_sources=stored_sources,
+            notion=notion_sources,
+            process_rules=SQLAlchemyProcessRuleReader(session_factory=database_client),
+            runner=IndexingEstimateAdapter(session_factory=database_client),
+        ),
+        segments=DatasetSegmentApplicationService(
+            dataset_access=dataset_access,
+            scopes=datasets,
+            store=segments,
+            index=segment_index,
+            limits=dify_config,
+            text_hash=generate_text_hash,
+            uploads=uploads,
+            model_guard=ModelManagerSegmentGuard(),
+            indexing_state=RedisSegmentIndexingState(redis),
+            batch_dispatcher=CelerySegmentBatchImportDispatcher(delay=batch_create_segment_to_index_task.delay),
+            job_id_factory=lambda: str(uuid4()),
+        ),
+    )

api/extensions/ext_application_services.py

@@ -19,14 +19,23 @@ from configs import dify_config
 from constants.dsl_version import CURRENT_APP_DSL_VERSION
 from constants.languages import languages
 from core.db.session_factory import get_session_maker
-from core.helper.ssrf_proxy import ssrf_proxy
 from core.schemas.schema_manager import SchemaManager
 from core.tools.tool_file_manager import ToolFileManager
 from enums import DeploymentEdition, WebAppAccessMode
 from extensions.application_services.agent import AgentAppServices, build_agent_app_services
 from extensions.application_services.app import AppServices, build_app_api_key_service, build_app_services
+from extensions.application_services.data_sources import (
+    DataSourceServices,
+    build_data_source_credentials,
+    build_data_source_services,
+)
+from extensions.application_services.datasets import build_dataset_dependencies
 from extensions.application_services.installed_app import InstalledAppServices, build_installed_app_services
-from extensions.application_services.knowledge import build_dataset_api_key_service
+from extensions.application_services.knowledge import (
+    KnowledgeServices,
+    build_dataset_api_key_service,
+    build_knowledge_services,
+)
 from extensions.application_services.trial_app import TrialAppServices, build_trial_app_services
 from extensions.ext_redis import RedisClientWrapper, redis_client
 from extensions.ext_storage import storage
@@ -52,8 +61,7 @@ from repositories.app_definition_query_repository import AppDefinitionQueryRepos
 from repositories.app_preview_query_repository import AppPreviewQueryRepository
 from repositories.app_scoped_end_user_repository import AppScopedEndUserRepo
 from repositories.app_statistic_query_repository import AppStatisticQueryRepository
-from repositories.data_source_api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository
-from repositories.data_source_oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository
+from repositories.credentials.query_repository import CredentialQueryRepository
 from repositories.explore_banner_query_repository import ExploreBannerQueryRepository
 from repositories.factory import DifyAPIRepositoryFactory
 from repositories.file_grant_repository import FileGrantRepository
@@ -164,15 +172,10 @@ from services.app_statistic_query import AppStatisticQuery
 from services.app_task_service import AppTaskControlService
 from services.app_tracing_config_gateway import OpsTraceManagerGateway
 from services.app_tracing_config_service import AppTracingConfigService
-from services.auth.data_source_api_key_auth_gateways import (
-    ProviderApiKeyAuthCredentialValidator,
-    TenantApiKeyAuthCredentialEncryptor,
-)
-from services.auth.data_source_api_key_auth_service import DataSourceApiKeyAuthService
 from services.billing_portal_service import BillingPortalService
 from services.billing_service import BillingService
 from services.compliance_download_service import ComplianceDownloadService
-from services.data_source_oauth_service import DataSourceOAuthService, InvalidDataSourceOAuthProviderError
+from services.credentials.query import CredentialQuery
 from services.enterprise.enterprise_service import EnterpriseService
 from services.entities.file_grant_entities import FileGrantLimits
 from services.errors.enterprise import EnterpriseServiceError
@@ -191,7 +194,6 @@ from services.message_suggested_questions_adapters import MessageSuggestedQuesti
 from services.message_suggested_questions_service import MessageSuggestedQuestions
 from services.notification_gateway import BillingNotificationGateway
 from services.notification_service import NotificationService
-from services.notion_data_source_gateway import NotionDataSourceGateway
 from services.oauth_device_adapters import (
     DifyConfigOAuthDeviceSettings,
     EnterpriseOAuthDeviceSSOGateway,
@@ -337,6 +339,7 @@ class AppScopedEndUserServices:
 class ApplicationServices:
     agent_apps: AgentAppServices
     advanced_prompt_templates: AdvancedPromptTemplateService
+    credential_queries: CredentialQuery
     accounts: AccountServices
     app_api_keys: AppApiKeyService
     dataset_api_keys: DatasetApiKeyService
@@ -351,8 +354,8 @@ class ApplicationServices:
     app_tracing_configs: AppTracingConfigService
     billing_portal: BillingPortalService
     compliance_downloads: ComplianceDownloadService
-    data_source_api_key_auth: DataSourceApiKeyAuthService
-    data_source_oauth: Mapping[str, DataSourceOAuthService]
+    data_sources: DataSourceServices
+    knowledge: KnowledgeServices
     app_scoped_end_users: AppScopedEndUserServices
     webapp_access: WebAppAccessQueryService
     web_app_runtime: WebAppRuntimeQueryService
@@ -392,34 +395,6 @@ class ApplicationServices:
     tags: TagApplicationService
     workflow_statistics: WorkflowStatisticQueryService
 
-    def resolve_data_source_oauth(self, provider: str) -> DataSourceOAuthService:
-        service = self.data_source_oauth.get(provider)
-        if service is None:
-            raise InvalidDataSourceOAuthProviderError("Invalid provider")
-        return service
-
-
-def _build_data_source_oauth_services(
-    *,
-    database_client: sessionmaker[Session],
-) -> Mapping[str, DataSourceOAuthService]:
-    notion_data_source = NotionDataSourceGateway(
-        client_id=dify_config.NOTION_CLIENT_ID or "",
-        client_secret=dify_config.NOTION_CLIENT_SECRET or "",
-        redirect_uri=dify_config.CONSOLE_API_URL + "/console/api/oauth/data-source/callback/notion",
-        http_client=ssrf_proxy,
-    )
-    bindings = SQLAlchemyDataSourceOAuthBindingRepository(session_factory=database_client)
-    return {
-        "notion": DataSourceOAuthService(
-            provider_name="notion",
-            provider_gateway=notion_data_source,
-            bindings=bindings,
-            is_internal_provider=dify_config.NOTION_INTEGRATION_TYPE == "internal",
-            internal_access_token=dify_config.NOTION_INTERNAL_SECRET,
-        )
-    }
-
 
 def _build_oauth_server_service(
     *,
@@ -547,7 +522,6 @@ def build_application_services(
     redis: RedisClientWrapper,
 ) -> ApplicationServices:
     installation_state = InstallationStateRepository(session_factory=database_client)
-    data_source_api_key_auth_bindings = SQLAlchemyDataSourceApiKeyAuthBindingRepository(session_factory=database_client)
     app_definition_repository = AppDefinitionQueryRepository(session_factory=database_client)
     app_definitions = AppDefinitionQueryService(
         definitions=app_definition_repository,
@@ -587,6 +561,39 @@ def build_application_services(
         sources=database_catalog, exporter=RosterAgentPackageExporter()
     )
     workspace_query_repository = WorkspaceQueryRepository(session_factory=database_client)
+    workspace_member_repository = WorkspaceMemberQueryRepository(session_factory=database_client)
+    dataset_dependencies = build_dataset_dependencies(
+        database_client=database_client,
+        workspace_roles=workspace_member_repository,
+    )
+    datasource_credentials = build_data_source_credentials(database_client=database_client)
+    data_sources = build_data_source_services(
+        database_client=database_client,
+        dataset_access=dataset_dependencies.access,
+        datasets=dataset_dependencies.datasets,
+        documents=dataset_dependencies.documents,
+        actor_credentials=datasource_credentials.actor,
+        providers=datasource_credentials.providers,
+    )
+    oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis)
+    apps = build_app_services(
+        database_client=database_client,
+        oauth=oauth_server,
+        recommended_packages=recommended_app_packages,
+    )
+    tags = TagApplicationService(tags=TagRepository(session_factory=database_client))
+    knowledge = build_knowledge_services(
+        database_client=database_client,
+        dataset_access=dataset_dependencies.access,
+        datasets=dataset_dependencies.datasets,
+        documents=dataset_dependencies.documents,
+        actor_credentials=datasource_credentials.actor,
+        stored_credentials=datasource_credentials.stored,
+        providers=datasource_credentials.providers,
+        redis=redis,
+        tags=tags,
+        app_queries=apps.queries,
+    )
     app_scoped_end_user_repository = AppScopedEndUserRepo(session_factory=database_client)
     file_service = FileService(session_factory=database_client)
     remote_file_service = RemoteFileService(files=file_service)
@@ -598,8 +605,8 @@ def build_application_services(
     workflow_node_execution_repository = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(
         session_maker=database_client
     )
-    oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis)
     return ApplicationServices(
+        credential_queries=CredentialQueryRepository(session_factory=database_client),
         accounts=AccountServices(
             access=AccountAccessService(
                 accounts=accounts,
@@ -754,11 +761,7 @@ def build_application_services(
                 enabled=dify_config.RBAC_ENABLED,
             ),
         ),
-        apps=build_app_services(
-            database_client=database_client,
-            oauth=oauth_server,
-            recommended_packages=recommended_app_packages,
-        ),
+        apps=apps,
         agent_apps=build_agent_app_services(database_client=database_client),
         advanced_prompt_templates=AdvancedPromptTemplateService(),
         app_definitions=app_definitions,
@@ -774,7 +777,10 @@ def build_application_services(
             sites=AppSiteCommandRepository(session_factory=database_client),
         ),
         app_api_keys=build_app_api_key_service(database_client=database_client),
-        dataset_api_keys=build_dataset_api_key_service(database_client=database_client),
+        dataset_api_keys=build_dataset_api_key_service(
+            database_client=database_client,
+            dataset_access=dataset_dependencies.access,
+        ),
         app_statistics=AppStatisticQueryRepository(session_factory=database_client),
         app_tracing_configs=AppTracingConfigService(
             configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client),
@@ -794,12 +800,8 @@ def build_application_services(
                 redis_client=redis,
             ),
         ),
-        data_source_api_key_auth=DataSourceApiKeyAuthService(
-            bindings=data_source_api_key_auth_bindings,
-            validator=ProviderApiKeyAuthCredentialValidator(),
-            encryptor=TenantApiKeyAuthCredentialEncryptor(),
-        ),
-        data_source_oauth=_build_data_source_oauth_services(database_client=database_client),
+        data_sources=data_sources,
+        knowledge=knowledge,
         app_scoped_end_users=AppScopedEndUserServices(
             commands=AppScopedEndUserService(end_users=app_scoped_end_user_repository),
             queries=AppScopedEndUserQueryService(end_users=app_scoped_end_user_repository),
@@ -905,9 +907,7 @@ def build_application_services(
             plans=DeploymentWorkspacePlanGateway(),
         ),
         workspace_member_queries=WorkspaceMemberQueryService(
-            members=WorkspaceMemberQueryRepository(
-                session_factory=database_client,
-            ),
+            members=workspace_member_repository,
             roles=DeploymentWorkspaceMemberRoleResolver(),
         ),
         workflow_app_logs=WorkflowAppLogQueryService(
@@ -927,9 +927,7 @@ def build_application_services(
             now=lambda: datetime.now(UTC),
             access_token_expire_minutes=dify_config.ACCESS_TOKEN_EXPIRE_MINUTES,
         ),
-        tags=TagApplicationService(
-            tags=TagRepository(session_factory=database_client),
-        ),
+        tags=tags,
         workflow_statistics=WorkflowStatisticQueryService(
             workflow_runs=DifyAPIRepositoryFactory.create_api_workflow_run_repository(
                 session_maker=database_client,

api/extensions/ext_celery.py

@@ -176,6 +176,8 @@ def init_app(app: DifyApp) -> Celery:
         "tasks.trigger_processing_tasks",  # async trigger processing
         "tasks.generate_summary_index_task",  # summary index generation
         "tasks.regenerate_summary_index_task",  # summary index regeneration
+        "tasks.rag_pipeline.rag_pipeline_run_task",  # regular RAG pipeline execution
+        "tasks.rag_pipeline.priority_rag_pipeline_run_task",  # priority RAG pipeline execution
         "tasks.initialize_created_app_rbac_access_task",  # app access initialization
         "tasks.install_default_plugins_task",  # tenant default plugin installation
         "tasks.new_agent_beta_task",  # New Agent Beta eligibility checks