mirror of
https://github.com/langgenius/dify.git
synced 2026-10-04 13:47:55 +08:00
feat(agent): add roster package export foundation (#42066)
This commit is contained in:
1 parent
5e9017dbb8
commit
93895fa7fb
26 files changed
+3549
-370
No files matched your search
@@ -460,6 +460,12 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
|
||||
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
|
||||
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
|
||||
UPLOAD_SKILL_FILE_SIZE_LIMIT=50
|
||||
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=209715200
|
||||
SKILL_PACKAGE_MAX_ENTRIES=5000
|
||||
SKILL_PACKAGE_MAX_SKILL_MD_BYTES=1048576
|
||||
AGENT_PACKAGE_MAX_BYTES=536870912
|
||||
AGENT_PACKAGE_MAX_MANIFEST_BYTES=5242880
|
||||
AGENT_PACKAGE_MAX_ENTRIES=5000
|
||||
|
||||
# Comma-separated list of file extensions blocked from upload for security reasons.
|
||||
# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll).
|
||||
|
||||
@@ -481,6 +481,36 @@ class FileUploadConfig(BaseSettings):
|
||||
default=50,
|
||||
)
|
||||
|
||||
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES: PositiveInt = Field(
|
||||
description="Maximum total uncompressed size of a Skill package in bytes",
|
||||
default=200 * 1024 * 1024,
|
||||
)
|
||||
|
||||
SKILL_PACKAGE_MAX_ENTRIES: PositiveInt = Field(
|
||||
description="Maximum number of entries in a Skill package",
|
||||
default=5000,
|
||||
)
|
||||
|
||||
SKILL_PACKAGE_MAX_SKILL_MD_BYTES: PositiveInt = Field(
|
||||
description="Maximum allowed SKILL.md size in bytes",
|
||||
default=1 * 1024 * 1024,
|
||||
)
|
||||
|
||||
AGENT_PACKAGE_MAX_BYTES: PositiveInt = Field(
|
||||
description="Maximum allowed Agent package size in bytes",
|
||||
default=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
AGENT_PACKAGE_MAX_MANIFEST_BYTES: PositiveInt = Field(
|
||||
description="Maximum allowed size of each Agent package YAML document in bytes",
|
||||
default=5 * 1024 * 1024,
|
||||
)
|
||||
|
||||
AGENT_PACKAGE_MAX_ENTRIES: PositiveInt = Field(
|
||||
description="Maximum number of entries in an Agent package",
|
||||
default=5000,
|
||||
)
|
||||
|
||||
BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
|
||||
description="Maximum number of files allowed in a batch upload operation",
|
||||
default=20,
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Read malformed UTF-8 ZIP names without weakening payload integrity checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import struct
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
_UTF8_FLAG = 1 << 11
|
||||
_END_RECORD = struct.Struct("<4s4H2IH")
|
||||
_CENTRAL_HEADER = struct.Struct("<4s6H3I5H2I")
|
||||
_LOCAL_HEADER = struct.Struct("<4s5H3I2H")
|
||||
_ZIP64_LOCATOR = struct.Struct("<4sIQI")
|
||||
_ZIP64_END_RECORD = struct.Struct("<4sQ2H2I4Q")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Directory:
|
||||
start: int
|
||||
end: int
|
||||
entries: int
|
||||
prefix_size: int
|
||||
|
||||
|
||||
def _locate_directory(content: bytes) -> _Directory:
|
||||
end = content.rfind(b"PK\x05\x06", max(0, len(content) - 65535 - _END_RECORD.size))
|
||||
if end < 0 or end + _END_RECORD.size > len(content):
|
||||
raise zipfile.BadZipFile("Missing ZIP end record")
|
||||
_, disk, directory_disk, disk_entries, entries, size, offset, comment_size = _END_RECORD.unpack_from(content, end)
|
||||
if end + _END_RECORD.size + comment_size != len(content):
|
||||
raise zipfile.BadZipFile("Invalid ZIP comment length")
|
||||
|
||||
locator_position = end - _ZIP64_LOCATOR.size
|
||||
if locator_position >= 0 and content[locator_position : locator_position + 4] == b"PK\x06\x07":
|
||||
_, record_disk, record_offset, disks = _ZIP64_LOCATOR.unpack_from(content, locator_position)
|
||||
if record_disk != 0 or disks != 1:
|
||||
raise zipfile.BadZipFile("Multi-disk ZIP64 archives are not supported")
|
||||
record_position = record_offset
|
||||
if record_position + _ZIP64_END_RECORD.size > locator_position:
|
||||
raise zipfile.BadZipFile("Invalid ZIP64 end record offset")
|
||||
if content[record_position : record_position + 4] != b"PK\x06\x06":
|
||||
# Match zipfile's handling of prepended data: in this case it
|
||||
# locates the fixed-size ZIP64 record immediately before the locator.
|
||||
record_position = locator_position - _ZIP64_END_RECORD.size
|
||||
if record_position < 0:
|
||||
raise zipfile.BadZipFile("Missing ZIP64 end record")
|
||||
signature, record_size, _, _, disk, directory_disk, disk_entries, entries, size, offset = (
|
||||
_ZIP64_END_RECORD.unpack_from(content, record_position)
|
||||
)
|
||||
if (
|
||||
signature != b"PK\x06\x06"
|
||||
or record_size + 12 != locator_position - record_position
|
||||
or offset + size != record_offset
|
||||
):
|
||||
raise zipfile.BadZipFile("Invalid ZIP64 end record")
|
||||
end = record_position
|
||||
elif entries == 0xFFFF or size == 0xFFFFFFFF or offset == 0xFFFFFFFF:
|
||||
raise zipfile.BadZipFile("Missing ZIP64 directory information")
|
||||
|
||||
start = end - size
|
||||
prefix_size = start - offset
|
||||
if disk != 0 or directory_disk != 0 or disk_entries != entries:
|
||||
raise zipfile.BadZipFile("Multi-disk ZIP archives are not supported")
|
||||
if start < 0 or prefix_size < 0 or entries > size // _CENTRAL_HEADER.size:
|
||||
raise zipfile.BadZipFile("Invalid ZIP directory bounds")
|
||||
return _Directory(start=start, end=end, entries=entries, prefix_size=prefix_size)
|
||||
|
||||
|
||||
def _local_header_offset(*, offset: int, file_size: int, compressed_size: int, extra: bytes) -> int:
|
||||
if offset != 0xFFFFFFFF:
|
||||
return offset
|
||||
cursor = 0
|
||||
while cursor + 4 <= len(extra):
|
||||
tag, size = struct.unpack_from("<HH", extra, cursor)
|
||||
cursor += 4
|
||||
if cursor + size > len(extra):
|
||||
raise zipfile.BadZipFile("Truncated ZIP extra field")
|
||||
if tag == 1:
|
||||
# ZIP64 includes only values whose corresponding 32-bit field
|
||||
# contains the sentinel, in size/compressed-size/offset order.
|
||||
skipped = 8 * (int(file_size == 0xFFFFFFFF) + int(compressed_size == 0xFFFFFFFF))
|
||||
if skipped + 8 > size:
|
||||
raise zipfile.BadZipFile("Missing ZIP64 local header offset")
|
||||
return struct.unpack_from("<Q", extra, cursor + skipped)[0]
|
||||
cursor += size
|
||||
raise zipfile.BadZipFile("Missing ZIP64 extra field")
|
||||
|
||||
|
||||
def open_zip_with_replacement_names(content: bytes) -> zipfile.ZipFile:
|
||||
"""Open a bounded ZIP, replacing invalid UTF-8 name bytes with U+FFFD.
|
||||
|
||||
The normal path delegates entirely to zipfile. For a damaged directory,
|
||||
temporarily read affected names as CP437 (a reversible byte mapping), and
|
||||
expose their lossy UTF-8 form through ZipInfo.filename. orig_filename stays
|
||||
byte-preserving so zipfile still checks local/central name consistency.
|
||||
Callers must reject collisions and unsafe paths in the recovered names.
|
||||
"""
|
||||
try:
|
||||
return zipfile.ZipFile(io.BytesIO(content))
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
directory = _locate_directory(content)
|
||||
|
||||
patched = bytearray(content)
|
||||
recovered: dict[int, str] = {}
|
||||
cursor = directory.start
|
||||
for index in range(directory.entries):
|
||||
if cursor + _CENTRAL_HEADER.size > directory.end:
|
||||
raise zipfile.BadZipFile("Truncated ZIP directory")
|
||||
header = _CENTRAL_HEADER.unpack_from(content, cursor)
|
||||
if header[0] != b"PK\x01\x02":
|
||||
raise zipfile.BadZipFile("Invalid ZIP directory header")
|
||||
flags, name_size, extra_size, entry_comment_size = header[3], header[10], header[11], header[12]
|
||||
name_start = cursor + _CENTRAL_HEADER.size
|
||||
next_cursor = name_start + name_size + extra_size + entry_comment_size
|
||||
if next_cursor > directory.end:
|
||||
raise zipfile.BadZipFile("Truncated ZIP directory name")
|
||||
raw_name = content[name_start : name_start + name_size]
|
||||
if flags & _UTF8_FLAG:
|
||||
try:
|
||||
raw_name.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
extra_start = name_start + name_size
|
||||
local_offset = (
|
||||
_local_header_offset(
|
||||
offset=header[16],
|
||||
file_size=header[9],
|
||||
compressed_size=header[8],
|
||||
extra=content[extra_start : extra_start + extra_size],
|
||||
)
|
||||
+ directory.prefix_size
|
||||
)
|
||||
if local_offset < 0 or local_offset + _LOCAL_HEADER.size > directory.start:
|
||||
raise zipfile.BadZipFile("Invalid ZIP local header offset")
|
||||
local = _LOCAL_HEADER.unpack_from(content, local_offset)
|
||||
local_name_start = local_offset + _LOCAL_HEADER.size
|
||||
if (
|
||||
local[0] != b"PK\x03\x04"
|
||||
or local[9] != name_size
|
||||
or content[local_name_start : local_name_start + local[9]] != raw_name
|
||||
or not local[2] & _UTF8_FLAG
|
||||
):
|
||||
raise zipfile.BadZipFile("ZIP local and directory names differ")
|
||||
struct.pack_into("<H", patched, cursor + 8, flags & ~_UTF8_FLAG)
|
||||
struct.pack_into("<H", patched, local_offset + 6, local[2] & ~_UTF8_FLAG)
|
||||
recovered[index] = raw_name.decode("utf-8", errors="replace")
|
||||
cursor = next_cursor
|
||||
if cursor != directory.end:
|
||||
raise zipfile.BadZipFile("ZIP directory length mismatch")
|
||||
|
||||
archive = zipfile.ZipFile(io.BytesIO(patched))
|
||||
for index, info in enumerate(archive.infolist()):
|
||||
if index in recovered:
|
||||
info.filename = recovered[index]
|
||||
archive.NameToInfo = {info.filename: info for info in archive.infolist()}
|
||||
return archive
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from core.tools.tool_file_manager import ToolFileManager
|
||||
from models.agent_config_entities import AgentConfigSkillRefConfig, validate_config_skill_name
|
||||
from services.agent.skill_package_service import NormalizedSkillPackage, SkillPackageError, SkillPackageService
|
||||
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
|
||||
|
||||
|
||||
class ConfigSkillNormalizeService:
|
||||
@@ -32,7 +32,7 @@ class ConfigSkillNormalizeService:
|
||||
requested_name: str | None,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
) -> tuple[AgentConfigSkillRefConfig, NormalizedSkillPackage]:
|
||||
) -> AgentConfigSkillRefConfig:
|
||||
package = self._package.validate_and_normalize(content=content, filename=filename)
|
||||
normalized_name = validate_config_skill_name(requested_name or package.manifest.name)
|
||||
if package.manifest.name != normalized_name:
|
||||
@@ -50,16 +50,13 @@ class ConfigSkillNormalizeService:
|
||||
mimetype="application/zip",
|
||||
filename=f"{normalized_name}.zip",
|
||||
)
|
||||
return (
|
||||
AgentConfigSkillRefConfig(
|
||||
name=normalized_name,
|
||||
description=package.manifest.description,
|
||||
file_id=tool_file.id,
|
||||
size=tool_file.size,
|
||||
hash=package.manifest.hash,
|
||||
mime_type=tool_file.mimetype,
|
||||
),
|
||||
package,
|
||||
return AgentConfigSkillRefConfig(
|
||||
name=normalized_name,
|
||||
description=package.manifest.description,
|
||||
file_id=tool_file.id,
|
||||
size=tool_file.size,
|
||||
hash=package.manifest.hash,
|
||||
mime_type=tool_file.mimetype,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Plugin dependency discovery shared by portable Agent formats."""
|
||||
|
||||
from models.agent_config_entities import (
|
||||
AgentKnowledgeMetadataFilteringConfig,
|
||||
AgentKnowledgeRetrievalConfig,
|
||||
AgentSoulConfig,
|
||||
)
|
||||
from services.plugin.dependencies_analysis import DependenciesAnalysisService
|
||||
|
||||
|
||||
def _extract_knowledge_retrieval_dependencies(retrieval: AgentKnowledgeRetrievalConfig) -> list[str]:
|
||||
if retrieval.mode == "single":
|
||||
if retrieval.model is None:
|
||||
return []
|
||||
return [DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.model.provider)]
|
||||
|
||||
if not retrieval.reranking_enable:
|
||||
return []
|
||||
|
||||
if retrieval.reranking_mode == "reranking_model":
|
||||
if retrieval.reranking_model is None:
|
||||
return []
|
||||
return [DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.reranking_model.provider)]
|
||||
|
||||
if retrieval.reranking_mode == "weighted_score" and retrieval.weights is not None:
|
||||
vector_setting = retrieval.weights.vector_setting
|
||||
embedding_provider = vector_setting.get("embedding_provider_name") if vector_setting is not None else None
|
||||
if embedding_provider:
|
||||
return [DependenciesAnalysisService.analyze_model_provider_dependency(embedding_provider)]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _extract_metadata_filtering_dependencies(
|
||||
metadata_filtering: AgentKnowledgeMetadataFilteringConfig,
|
||||
) -> list[str]:
|
||||
if metadata_filtering.mode != "automatic" or metadata_filtering.metadata_model_config is None:
|
||||
return []
|
||||
return [
|
||||
DependenciesAnalysisService.analyze_model_provider_dependency(metadata_filtering.metadata_model_config.provider)
|
||||
]
|
||||
|
||||
|
||||
def extract_agent_soul_dependencies(soul: AgentSoulConfig) -> list[str]:
|
||||
dependencies: list[str] = []
|
||||
if soul.model is not None:
|
||||
dependencies.append(DependenciesAnalysisService.analyze_model_provider_dependency(soul.model.model_provider))
|
||||
suggested_questions = soul.app_features.suggested_questions_after_answer
|
||||
if suggested_questions is not None and suggested_questions.model is not None:
|
||||
dependencies.append(
|
||||
DependenciesAnalysisService.analyze_model_provider_dependency(suggested_questions.model.provider)
|
||||
)
|
||||
for tool in soul.tools.dify_tools:
|
||||
provider_id = tool.provider_id or (
|
||||
f"{tool.plugin_id}/{tool.provider}" if tool.plugin_id and tool.provider else None
|
||||
)
|
||||
if provider_id:
|
||||
dependencies.append(DependenciesAnalysisService.analyze_tool_dependency(provider_id))
|
||||
for knowledge_set in soul.knowledge.sets:
|
||||
dependencies.extend(_extract_knowledge_retrieval_dependencies(knowledge_set.retrieval))
|
||||
dependencies.extend(_extract_metadata_filtering_dependencies(knowledge_set.metadata_filtering))
|
||||
return dependencies
|
||||
|
||||
|
||||
__all__ = ["extract_agent_soul_dependencies"]
|
||||
@@ -3,12 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal, Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from core.plugin.entities.plugin import PluginDependency
|
||||
from models.agent import Agent
|
||||
from models.agent_config_entities import AgentSoulConfig
|
||||
from models.model import App
|
||||
from services.entities.dsl_entities import make_app_dsl
|
||||
|
||||
AGENT_PACKAGE_SCHEMA_VERSION = 1
|
||||
AGENT_PACKAGE_REF_KEY = "package_ref"
|
||||
@@ -59,6 +62,54 @@ class AgentPackage(BaseModel):
|
||||
workspace_skills: list[AgentPackageWorkspaceSkill] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AgentAppReference(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
package_ref: str = Field(min_length=1)
|
||||
|
||||
|
||||
class AgentAppDsl(BaseModel):
|
||||
"""The existing standalone Agent App DSL, also stored as archive app.yaml."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
version: str = Field(min_length=1)
|
||||
kind: Literal["app"]
|
||||
app: dict[str, Any]
|
||||
agent: AgentAppReference
|
||||
agent_packages: dict[str, AgentPackage]
|
||||
dependencies: list[PluginDependency] = Field(default_factory=list)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_agent_app(self) -> Self:
|
||||
if self.app.get("mode") != "agent":
|
||||
raise ValueError("Agent App DSL requires app.mode=agent")
|
||||
if not isinstance(self.app.get("name"), str) or not self.app["name"]:
|
||||
raise ValueError("Agent App DSL requires an app name")
|
||||
if set(self.agent_packages) != {self.agent.package_ref}:
|
||||
raise ValueError("Agent App DSL must contain exactly the referenced Agent package")
|
||||
return self
|
||||
|
||||
@property
|
||||
def package(self) -> AgentPackage:
|
||||
return self.agent_packages[self.agent.package_ref]
|
||||
|
||||
|
||||
def make_agent_app_dsl(
|
||||
app: App,
|
||||
*,
|
||||
package_ref: str,
|
||||
packages: dict[str, AgentPackage],
|
||||
dependencies: list[PluginDependency],
|
||||
) -> AgentAppDsl:
|
||||
return AgentAppDsl(
|
||||
**make_app_dsl(app),
|
||||
agent=AgentAppReference(package_ref=package_ref),
|
||||
agent_packages=packages,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
|
||||
|
||||
def portable_ref(prefix: str, value: str) -> str:
|
||||
digest = hashlib.sha256(value.encode()).hexdigest()[:16]
|
||||
return f"{prefix}-{digest}"
|
||||
@@ -87,40 +138,15 @@ def _strip_sensitive_values(value: Any) -> Any:
|
||||
return result
|
||||
|
||||
|
||||
def make_portable_agent_package(
|
||||
agent: Agent,
|
||||
agent_soul: AgentSoulConfig,
|
||||
workspace_skills: list[AgentPackageWorkspaceSkill] | None = None,
|
||||
) -> AgentPackage:
|
||||
"""Return a package safe to place in YAML or the system clipboard."""
|
||||
def make_portable_agent_soul(agent_soul: AgentSoulConfig) -> AgentSoulConfig:
|
||||
"""Return an Agent Soul without workspace-local credentials or identities.
|
||||
|
||||
Resource references are intentionally preserved. Container formats decide
|
||||
whether to omit their payloads, as YAML DSL does, or remap them to
|
||||
package-local identifiers, as Roster packages do.
|
||||
"""
|
||||
|
||||
soul_data = agent_soul.model_dump(mode="json")
|
||||
omitted_assets = [
|
||||
AgentPackageOmittedAsset(
|
||||
kind="skill",
|
||||
name=item.name,
|
||||
size=item.size,
|
||||
hash=item.hash,
|
||||
mime_type=item.mime_type,
|
||||
)
|
||||
for item in agent_soul.config_skills
|
||||
]
|
||||
omitted_assets.extend(
|
||||
AgentPackageOmittedAsset(
|
||||
kind="file",
|
||||
name=item.name,
|
||||
size=item.size,
|
||||
hash=item.hash,
|
||||
mime_type=item.mime_type,
|
||||
)
|
||||
for item in agent_soul.config_files
|
||||
)
|
||||
for item in soul_data.get("config_skills", []):
|
||||
item["file_id"] = ""
|
||||
item["is_missing"] = True
|
||||
for item in soul_data.get("config_files", []):
|
||||
item["file_id"] = ""
|
||||
item["is_missing"] = True
|
||||
|
||||
if soul_data.get("model"):
|
||||
soul_data["model"]["credential_ref"] = None
|
||||
@@ -155,7 +181,53 @@ def make_portable_agent_package(
|
||||
for key in ("id", "contact_id", "human_id", "tenant_id"):
|
||||
contact[key] = None
|
||||
|
||||
portable_soul = AgentSoulConfig.model_validate(soul_data)
|
||||
return AgentSoulConfig.model_validate(soul_data)
|
||||
|
||||
|
||||
def make_portable_agent_package(
|
||||
agent: Agent,
|
||||
agent_soul: AgentSoulConfig,
|
||||
workspace_skills: list[AgentPackageWorkspaceSkill] | None = None,
|
||||
*,
|
||||
include_assets: bool = False,
|
||||
) -> AgentPackage:
|
||||
"""Return a portable package for YAML, clipboard, or an asset-bearing archive.
|
||||
|
||||
Archives preserve package-local references to included assets; standalone
|
||||
YAML and clipboard exports mark resources missing because they omit payloads.
|
||||
"""
|
||||
|
||||
portable_soul = make_portable_agent_soul(agent_soul)
|
||||
omitted_assets = [
|
||||
AgentPackageOmittedAsset(
|
||||
kind="skill",
|
||||
name=item.name,
|
||||
size=item.size,
|
||||
hash=item.hash,
|
||||
mime_type=item.mime_type,
|
||||
)
|
||||
for item in agent_soul.config_skills
|
||||
if not include_assets or item.is_missing
|
||||
]
|
||||
omitted_assets.extend(
|
||||
AgentPackageOmittedAsset(
|
||||
kind="file",
|
||||
name=item.name,
|
||||
size=item.size,
|
||||
hash=item.hash,
|
||||
mime_type=item.mime_type,
|
||||
)
|
||||
for item in agent_soul.config_files
|
||||
if not include_assets or item.is_missing
|
||||
)
|
||||
for skill_ref in portable_soul.config_skills:
|
||||
if not include_assets or skill_ref.is_missing:
|
||||
skill_ref.file_id = ""
|
||||
skill_ref.is_missing = True
|
||||
for file_ref in portable_soul.config_files:
|
||||
if not include_assets or file_ref.is_missing:
|
||||
file_ref.file_id = ""
|
||||
file_ref.is_missing = True
|
||||
icon_type = agent.icon_type.value if agent.icon_type is not None else None
|
||||
return AgentPackage(
|
||||
metadata=AgentPackageMetadata(
|
||||
|
||||
@@ -41,6 +41,7 @@ from models.model import App, AppModelConfig
|
||||
from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot, Skill
|
||||
from models.workflow import Workflow
|
||||
from services.agent.agent_soul_state import agent_soul_has_model
|
||||
from services.agent.dependency_service import extract_agent_soul_dependencies
|
||||
from services.agent.dsl_entities import (
|
||||
AGENT_NODE_JOB_DSL_KEY,
|
||||
AGENT_PACKAGE_REF_KEY,
|
||||
@@ -53,7 +54,6 @@ from services.agent.dsl_entities import (
|
||||
from services.agent.knowledge_datasets import get_tenant_knowledge_dataset_rows
|
||||
from services.agent.roster_service import AgentRosterService
|
||||
from services.entities.dsl_entities import DslImportWarning
|
||||
from services.plugin.dependencies_analysis import DependenciesAnalysisService
|
||||
|
||||
|
||||
class AgentPackageImportResult(BaseModel):
|
||||
@@ -375,29 +375,7 @@ class AgentDslService:
|
||||
def extract_package_dependencies(self, packages: Mapping[str, AgentPackage]) -> list[str]:
|
||||
dependencies: list[str] = []
|
||||
for package in packages.values():
|
||||
soul = package.soul
|
||||
if soul.model is not None:
|
||||
dependencies.append(
|
||||
DependenciesAnalysisService.analyze_model_provider_dependency(soul.model.model_provider)
|
||||
)
|
||||
for tool in soul.tools.dify_tools:
|
||||
provider_id = tool.provider_id or (
|
||||
f"{tool.plugin_id}/{tool.provider}" if tool.plugin_id and tool.provider else None
|
||||
)
|
||||
if provider_id:
|
||||
dependencies.append(DependenciesAnalysisService.analyze_tool_dependency(provider_id))
|
||||
for knowledge_set in soul.knowledge.sets:
|
||||
retrieval = knowledge_set.retrieval
|
||||
if retrieval.model is not None:
|
||||
dependencies.append(
|
||||
DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.model.provider)
|
||||
)
|
||||
if retrieval.reranking_model is not None:
|
||||
dependencies.append(
|
||||
DependenciesAnalysisService.analyze_model_provider_dependency(
|
||||
retrieval.reranking_model.provider
|
||||
)
|
||||
)
|
||||
dependencies.extend(extract_agent_soul_dependencies(package.soul))
|
||||
return dependencies
|
||||
|
||||
def _workspace_skills_for_export(
|
||||
|
||||
@@ -53,6 +53,24 @@ class AgentHomeSnapshotTooLargeError(BaseHTTPException):
|
||||
code = 413
|
||||
|
||||
|
||||
class InvalidRosterAgentPackageError(BaseHTTPException):
|
||||
error_code = "invalid_roster_agent_package"
|
||||
description = "The Roster Agent package is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class RosterAgentPackageTooLargeError(BaseHTTPException):
|
||||
error_code = "roster_agent_package_too_large"
|
||||
description = "The Roster Agent package exceeds the size limit."
|
||||
code = 413
|
||||
|
||||
|
||||
class RosterAgentPackageExportFailedError(BaseHTTPException):
|
||||
error_code = "roster_agent_package_export_failed"
|
||||
description = "The Roster Agent package could not be exported."
|
||||
code = 500
|
||||
|
||||
|
||||
class AgentSoulLockedError(BadRequest):
|
||||
description = "Agent Soul is locked for this workflow node."
|
||||
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Strict data contract for portable Roster Agent archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import BinaryIO, Final, Literal, Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from services.agent.dsl_entities import AgentAppDsl
|
||||
|
||||
ROSTER_AGENT_PACKAGE_FORMAT: Final[Literal["dify.roster-agent"]] = "dify.roster-agent"
|
||||
ROSTER_AGENT_PACKAGE_FORMAT_VERSION: Final[Literal[1]] = 1
|
||||
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES = 64 * 1024
|
||||
|
||||
_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RosterAgentPackageMember:
|
||||
size: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class RosterAgentPackageAudit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
ref: str = Field(min_length=1, max_length=255)
|
||||
|
||||
|
||||
class _RosterAgentPackageResource(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
path: str = Field(min_length=1, max_length=255)
|
||||
size: int = Field(ge=0)
|
||||
sha256: str
|
||||
audit: RosterAgentPackageAudit | None = None
|
||||
|
||||
@field_validator("sha256")
|
||||
@classmethod
|
||||
def validate_sha256(cls, value: str) -> str:
|
||||
if _SHA256_PATTERN.fullmatch(value) is None:
|
||||
raise ValueError("sha256 must be a lowercase hexadecimal SHA-256 digest")
|
||||
return value
|
||||
|
||||
|
||||
class RosterAgentPackageApp(_RosterAgentPackageResource):
|
||||
@field_validator("path")
|
||||
@classmethod
|
||||
def validate_app_path(cls, value: str) -> str:
|
||||
if "/" in value or "\\" in value or not value.endswith((".yaml", ".yml")):
|
||||
raise ValueError("app path must be a root-level YAML member")
|
||||
if value.casefold() == "manifest.yaml":
|
||||
raise ValueError("app path must not use the package manifest name")
|
||||
return value
|
||||
|
||||
|
||||
class RosterAgentPackageSkill(_RosterAgentPackageResource):
|
||||
id: str = Field(pattern=r"^s_[0-9]{6}$")
|
||||
scope: Literal["agent_config", "workspace"]
|
||||
name: str = Field(min_length=1, max_length=64)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_skill_path(self) -> Self:
|
||||
if self.path != f"{self.id}.zip":
|
||||
raise ValueError("skill path must be the resource id with a .zip extension")
|
||||
return self
|
||||
|
||||
|
||||
class RosterAgentPackageFile(_RosterAgentPackageResource):
|
||||
id: str = Field(pattern=r"^f_[0-9]{6}$")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_file_metadata(self) -> Self:
|
||||
if self.path == self.id or not self.path.startswith(f"{self.id}."):
|
||||
raise ValueError("file path must start with the resource id and include an extension")
|
||||
if "/" in self.path or "\\" in self.path:
|
||||
raise ValueError("file path must be a root-level archive member")
|
||||
return self
|
||||
|
||||
|
||||
class RosterAgentPackageManifest(BaseModel):
|
||||
"""Package metadata and resource index stored in ``manifest.yaml``."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
format: Literal["dify.roster-agent"]
|
||||
format_version: Literal[1]
|
||||
audit: RosterAgentPackageAudit | None = None
|
||||
apps: list[RosterAgentPackageApp] = Field(min_length=1)
|
||||
skills: list[RosterAgentPackageSkill] = Field(default_factory=list)
|
||||
files: list[RosterAgentPackageFile] = Field(default_factory=list)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_resource_index(self) -> Self:
|
||||
ids = [item.id for item in self.skills] + [item.id for item in self.files]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("resource ids must be unique")
|
||||
paths = [item.path.casefold() for item in [*self.apps, *self.skills, *self.files]]
|
||||
if len(paths) != len(set(paths)):
|
||||
raise ValueError("resource paths must be unique")
|
||||
skill_names = [item.name for item in self.skills]
|
||||
if len(skill_names) != len(set(skill_names)):
|
||||
raise ValueError("skill names must be unique after workspace Skills are localized")
|
||||
|
||||
return self
|
||||
|
||||
def validate_apps(self, apps: Mapping[str, AgentAppDsl]) -> None:
|
||||
"""Resolve all application references against the shared resource index."""
|
||||
if set(apps) != {item.path for item in self.apps}:
|
||||
raise ValueError("app members must match the package app index")
|
||||
skill_by_id = {item.id: item for item in self.skills}
|
||||
file_by_id = {item.id: item for item in self.files}
|
||||
workspace_names = {item.name for item in self.skills if item.scope == "workspace"}
|
||||
referenced_skill_ids: set[str] = set()
|
||||
referenced_file_ids: set[str] = set()
|
||||
referenced_workspace_names: set[str] = set()
|
||||
for app in apps.values():
|
||||
if app.package.soul.schema_version != 1:
|
||||
raise ValueError("unsupported Agent Soul schema version")
|
||||
for skill_ref in app.package.soul.config_skills:
|
||||
if skill_ref.is_missing:
|
||||
continue
|
||||
skill_resource = skill_by_id.get(skill_ref.file_id)
|
||||
if skill_resource is None or skill_resource.scope != "agent_config":
|
||||
raise ValueError("config skill reference must resolve to an agent_config skill")
|
||||
if skill_resource.name != skill_ref.name:
|
||||
raise ValueError("config skill name must match its resource metadata")
|
||||
referenced_skill_ids.add(skill_resource.id)
|
||||
app_workspace_names = [item.name for item in app.package.workspace_skills]
|
||||
if len(app_workspace_names) != len(set(app_workspace_names)):
|
||||
raise ValueError("workspace skill names must be unique in the Agent DSL")
|
||||
if set(app_workspace_names) - workspace_names:
|
||||
raise ValueError("workspace skill references must match the package resource index")
|
||||
referenced_workspace_names.update(app_workspace_names)
|
||||
for file_ref in app.package.soul.config_files:
|
||||
if file_ref.is_missing:
|
||||
continue
|
||||
if file_ref.file_id not in file_by_id:
|
||||
raise ValueError("config file reference must resolve to a package file")
|
||||
referenced_file_ids.add(file_ref.file_id)
|
||||
if {item.id for item in self.skills if item.scope == "agent_config"} - referenced_skill_ids:
|
||||
raise ValueError("agent_config skill resources must be referenced by the Agent Soul")
|
||||
if workspace_names != referenced_workspace_names:
|
||||
raise ValueError("workspace skill references must match the package resource index")
|
||||
if set(file_by_id) - referenced_file_ids:
|
||||
raise ValueError("file resources must be referenced by the Agent Soul")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedRosterAgentPackage:
|
||||
"""Validated archive retained in a bounded spool for later materialization."""
|
||||
|
||||
archive: BinaryIO
|
||||
manifest: RosterAgentPackageManifest
|
||||
apps: dict[str, AgentAppDsl]
|
||||
members: dict[str, RosterAgentPackageMember]
|
||||
invalid_skills: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def close(self) -> None:
|
||||
self.archive.close()
|
||||
|
||||
def __enter__(self) -> PreparedRosterAgentPackage:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RosterAgentPackageExport:
|
||||
"""Download artifact owning its archive until the response closes."""
|
||||
|
||||
archive: BinaryIO
|
||||
filename: str
|
||||
size: int
|
||||
|
||||
def close(self) -> None:
|
||||
self.archive.close()
|
||||
|
||||
def __enter__(self) -> RosterAgentPackageExport:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ROSTER_AGENT_PACKAGE_FORMAT",
|
||||
"ROSTER_AGENT_PACKAGE_FORMAT_VERSION",
|
||||
"ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES",
|
||||
"PreparedRosterAgentPackage",
|
||||
"RosterAgentPackageApp",
|
||||
"RosterAgentPackageAudit",
|
||||
"RosterAgentPackageExport",
|
||||
"RosterAgentPackageFile",
|
||||
"RosterAgentPackageManifest",
|
||||
"RosterAgentPackageMember",
|
||||
"RosterAgentPackageSkill",
|
||||
]
|
||||
@@ -0,0 +1,468 @@
|
||||
"""Export active Roster Agents as portable ``.ifpkg`` archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Callable, Generator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO, Literal, Protocol, cast
|
||||
|
||||
import yaml
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from core.db.session_factory import session_factory
|
||||
from core.plugin.entities.plugin import PluginDependency
|
||||
from extensions.ext_storage import storage
|
||||
from models.agent import (
|
||||
APP_BACKED_AGENT_SOURCES,
|
||||
Agent,
|
||||
AgentConfigDraft,
|
||||
AgentConfigDraftType,
|
||||
AgentConfigSnapshot,
|
||||
AgentScope,
|
||||
AgentStatus,
|
||||
)
|
||||
from models.agent_config_entities import AgentSoulConfig
|
||||
from models.enums import AppStatus
|
||||
from models.model import App, AppMode, UploadFile
|
||||
from models.tools import ToolFile
|
||||
from services.agent.dependency_service import extract_agent_soul_dependencies
|
||||
from services.agent.dsl_entities import (
|
||||
AgentAppDsl,
|
||||
AgentPackageWorkspaceSkill,
|
||||
make_agent_app_dsl,
|
||||
make_portable_agent_package,
|
||||
)
|
||||
from services.agent.errors import (
|
||||
AgentNotFoundError,
|
||||
AgentVersionNotFoundError,
|
||||
RosterAgentPackageExportFailedError,
|
||||
RosterAgentPackageTooLargeError,
|
||||
)
|
||||
from services.agent.roster_package_entities import (
|
||||
ROSTER_AGENT_PACKAGE_FORMAT,
|
||||
ROSTER_AGENT_PACKAGE_FORMAT_VERSION,
|
||||
RosterAgentPackageApp,
|
||||
RosterAgentPackageAudit,
|
||||
RosterAgentPackageExport,
|
||||
RosterAgentPackageFile,
|
||||
RosterAgentPackageManifest,
|
||||
RosterAgentPackageSkill,
|
||||
)
|
||||
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
|
||||
from services.plugin.dependencies_analysis import DependenciesAnalysisService
|
||||
from services.skill_management_service import RuntimeAgentSkillArchive, SkillManagementService
|
||||
|
||||
|
||||
class _Storage(Protocol):
|
||||
def load_stream(self, filename: str) -> Generator[bytes, None, None]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SkillSource:
|
||||
path: str
|
||||
storage_key: str
|
||||
id: str
|
||||
scope: Literal["agent_config", "workspace"]
|
||||
name: str
|
||||
display_name: str | None
|
||||
description: str
|
||||
priority: int | None
|
||||
audit_ref: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FileSource:
|
||||
path: str
|
||||
storage_key: str
|
||||
id: str
|
||||
audit_ref: str
|
||||
|
||||
|
||||
class RosterAgentPackageExporter:
|
||||
"""Collect a Roster Agent through a dedicated read Session and build its archive."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
storage_backend: _Storage = storage,
|
||||
dependency_provider: Callable[[str, list[str]], list[PluginDependency]] | None = None,
|
||||
) -> None:
|
||||
self._storage = storage_backend
|
||||
self._dependency_provider = dependency_provider or DependenciesAnalysisService.generate_dependencies
|
||||
|
||||
def export(self, *, tenant_id: str, agent_id: str) -> RosterAgentPackageExport:
|
||||
with session_factory.create_session() as session:
|
||||
row = session.execute(
|
||||
select(Agent, App)
|
||||
.join(
|
||||
App,
|
||||
(App.id == Agent.app_id) & (App.tenant_id == Agent.tenant_id),
|
||||
)
|
||||
.where(
|
||||
Agent.id == agent_id,
|
||||
Agent.tenant_id == tenant_id,
|
||||
Agent.scope == AgentScope.ROSTER,
|
||||
Agent.source.in_(APP_BACKED_AGENT_SOURCES),
|
||||
Agent.status == AgentStatus.ACTIVE,
|
||||
Agent.app_id.is_not(None),
|
||||
or_(Agent.backing_app_id == Agent.app_id, Agent.backing_app_id.is_(None)),
|
||||
App.mode == AppMode.AGENT,
|
||||
App.status == AppStatus.NORMAL,
|
||||
)
|
||||
.limit(1)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise AgentNotFoundError()
|
||||
agent, app_model = row
|
||||
|
||||
draft = session.scalar(
|
||||
select(AgentConfigDraft)
|
||||
.where(
|
||||
AgentConfigDraft.tenant_id == tenant_id,
|
||||
AgentConfigDraft.agent_id == agent.id,
|
||||
AgentConfigDraft.draft_type == AgentConfigDraftType.DRAFT,
|
||||
AgentConfigDraft.draft_owner_key == "",
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if draft is not None:
|
||||
soul = AgentSoulConfig.model_validate(draft.config_snapshot_dict)
|
||||
else:
|
||||
snapshot = session.scalar(
|
||||
select(AgentConfigSnapshot)
|
||||
.where(
|
||||
AgentConfigSnapshot.id == agent.active_config_snapshot_id,
|
||||
AgentConfigSnapshot.tenant_id == tenant_id,
|
||||
AgentConfigSnapshot.agent_id == agent.id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if snapshot is None:
|
||||
raise AgentVersionNotFoundError()
|
||||
soul = AgentSoulConfig.model_validate(snapshot.config_snapshot_dict)
|
||||
|
||||
resource_soul, skill_sources, file_sources = self._collect_payloads(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
soul=soul,
|
||||
)
|
||||
package = make_portable_agent_package(agent, resource_soul, include_assets=True)
|
||||
app = make_agent_app_dsl(app_model, package_ref="agent_1", packages={"agent_1": package}, dependencies=[])
|
||||
audit = RosterAgentPackageAudit(ref=agent.id)
|
||||
dependency_ids = extract_agent_soul_dependencies(package.soul)
|
||||
|
||||
workspace_skills = SkillManagementService().list_runtime_agent_skill_archives(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent_id,
|
||||
include_draft=draft is not None,
|
||||
)
|
||||
skill_sources.extend(
|
||||
self._workspace_skill_sources(
|
||||
soul=package.soul,
|
||||
archives=workspace_skills,
|
||||
start_index=len(skill_sources),
|
||||
)
|
||||
)
|
||||
package.workspace_skills = [
|
||||
AgentPackageWorkspaceSkill(
|
||||
name=item.name,
|
||||
display_name=item.display_name or "",
|
||||
description=item.description,
|
||||
priority=item.priority,
|
||||
)
|
||||
for item in skill_sources
|
||||
if item.scope == "workspace" and item.priority is not None
|
||||
]
|
||||
app.dependencies = self._dependency_provider(tenant_id, dependency_ids)
|
||||
return self._build_archive(
|
||||
app=app,
|
||||
audit=audit,
|
||||
skill_sources=skill_sources,
|
||||
file_sources=file_sources,
|
||||
)
|
||||
|
||||
def _build_archive(
|
||||
self,
|
||||
*,
|
||||
app: AgentAppDsl,
|
||||
skill_sources: Sequence[_SkillSource],
|
||||
file_sources: Sequence[_FileSource],
|
||||
audit: RosterAgentPackageAudit | None = None,
|
||||
) -> RosterAgentPackageExport:
|
||||
required_entries = len(skill_sources) + len(file_sources) + 2
|
||||
if required_entries > dify_config.AGENT_PACKAGE_MAX_ENTRIES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package has too many members")
|
||||
|
||||
# Ownership is transferred to RosterAgentPackageExport.
|
||||
output = cast(
|
||||
BinaryIO,
|
||||
tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b"), # noqa: SIM115
|
||||
)
|
||||
try:
|
||||
skills: list[RosterAgentPackageSkill] = []
|
||||
files: list[RosterAgentPackageFile] = []
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
||||
total_size = 0
|
||||
nested_uncompressed_size = 0
|
||||
skill_packages = SkillPackageService()
|
||||
sources: list[_SkillSource | _FileSource] = [*skill_sources, *file_sources]
|
||||
for source in sources:
|
||||
remaining_bytes = dify_config.AGENT_PACKAGE_MAX_BYTES - total_size
|
||||
if isinstance(source, _SkillSource):
|
||||
remaining_bytes = min(remaining_bytes, dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024)
|
||||
member_size, member_digest = self._write_storage_member(
|
||||
archive, path=source.path, storage_key=source.storage_key, max_bytes=remaining_bytes
|
||||
)
|
||||
total_size += member_size
|
||||
if isinstance(source, _SkillSource):
|
||||
# Inspect only the embedded Skill; our writer already owns
|
||||
# the outer container, resource sizes, and digests.
|
||||
try:
|
||||
inspection = skill_packages.inspect(content=archive.read(source.path), filename=source.path)
|
||||
except SkillPackageError as exc:
|
||||
raise RosterAgentPackageExportFailedError(
|
||||
f"Roster Agent package contains unusable Skill {source.name!r}"
|
||||
) from exc
|
||||
if inspection.name != source.name:
|
||||
raise RosterAgentPackageExportFailedError(
|
||||
f"Roster Agent package contains unusable Skill {source.name!r}: name mismatch"
|
||||
)
|
||||
nested_uncompressed_size += inspection.uncompressed_size
|
||||
if nested_uncompressed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError(
|
||||
"Roster Agent package nested Skill contents exceed the size limit"
|
||||
)
|
||||
|
||||
skills.append(
|
||||
RosterAgentPackageSkill(
|
||||
id=source.id,
|
||||
scope=source.scope,
|
||||
name=source.name,
|
||||
path=source.path,
|
||||
size=member_size,
|
||||
sha256=member_digest,
|
||||
audit=RosterAgentPackageAudit(ref=source.audit_ref),
|
||||
)
|
||||
)
|
||||
else:
|
||||
files.append(
|
||||
RosterAgentPackageFile(
|
||||
id=source.id,
|
||||
path=source.path,
|
||||
size=member_size,
|
||||
sha256=member_digest,
|
||||
audit=RosterAgentPackageAudit(ref=source.audit_ref),
|
||||
)
|
||||
)
|
||||
|
||||
app_bytes = yaml.safe_dump(
|
||||
app.model_dump(mode="json", exclude_none=True), allow_unicode=True, sort_keys=False
|
||||
).encode("utf-8")
|
||||
manifest = RosterAgentPackageManifest(
|
||||
format=ROSTER_AGENT_PACKAGE_FORMAT,
|
||||
format_version=ROSTER_AGENT_PACKAGE_FORMAT_VERSION,
|
||||
audit=audit,
|
||||
apps=[
|
||||
RosterAgentPackageApp(
|
||||
path="app.yaml", size=len(app_bytes), sha256=hashlib.sha256(app_bytes).hexdigest()
|
||||
)
|
||||
],
|
||||
skills=skills,
|
||||
files=files,
|
||||
)
|
||||
manifest.validate_apps({"app.yaml": app})
|
||||
manifest_bytes = yaml.safe_dump(
|
||||
manifest.model_dump(mode="json", exclude_none=True), allow_unicode=True, sort_keys=False
|
||||
).encode("utf-8")
|
||||
for path, document_bytes in (("app.yaml", app_bytes), ("manifest.yaml", manifest_bytes)):
|
||||
if len(document_bytes) > dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES:
|
||||
raise RosterAgentPackageTooLargeError(f"Roster Agent package {path} exceeds the size limit")
|
||||
total_size += len(document_bytes)
|
||||
if total_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the size limit")
|
||||
archive.writestr(path, document_bytes)
|
||||
|
||||
size = output.tell()
|
||||
if size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the archive size limit")
|
||||
output.seek(0)
|
||||
return RosterAgentPackageExport(
|
||||
archive=output,
|
||||
filename=f"{self._safe_slug(app.package.metadata.name)}.ifpkg",
|
||||
size=size,
|
||||
)
|
||||
except Exception:
|
||||
output.close()
|
||||
raise
|
||||
|
||||
def _collect_payloads(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
soul: AgentSoulConfig,
|
||||
) -> tuple[AgentSoulConfig, list[_SkillSource], list[_FileSource]]:
|
||||
resource_data = soul.model_dump(mode="json")
|
||||
skill_sources: list[_SkillSource] = []
|
||||
file_sources: list[_FileSource] = []
|
||||
|
||||
skill_file_ids = [item.file_id for item in soul.config_skills if not item.is_missing]
|
||||
tool_files = self._tool_files(session=session, tenant_id=tenant_id, file_ids=skill_file_ids)
|
||||
for skill_ref, portable_skill_ref in zip(soul.config_skills, resource_data["config_skills"]):
|
||||
if skill_ref.is_missing:
|
||||
continue
|
||||
resource_id = f"s_{len(skill_sources) + 1:06d}"
|
||||
tool_file = tool_files.get(skill_ref.file_id)
|
||||
if tool_file is None:
|
||||
raise RosterAgentPackageExportFailedError(f"Config skill {skill_ref.name!r} payload is unavailable")
|
||||
path = f"{resource_id}.zip"
|
||||
portable_skill_ref["file_id"] = resource_id
|
||||
portable_skill_ref["is_missing"] = False
|
||||
skill_sources.append(
|
||||
_SkillSource(
|
||||
path=path,
|
||||
storage_key=tool_file.file_key,
|
||||
id=resource_id,
|
||||
scope="agent_config",
|
||||
name=skill_ref.name,
|
||||
display_name=None,
|
||||
description=skill_ref.description,
|
||||
priority=None,
|
||||
audit_ref=tool_file.id,
|
||||
)
|
||||
)
|
||||
|
||||
tool_file_refs = [
|
||||
item.file_id for item in soul.config_files if not item.is_missing and item.file_kind == "tool_file"
|
||||
]
|
||||
upload_file_refs = [
|
||||
item.file_id for item in soul.config_files if not item.is_missing and item.file_kind == "upload_file"
|
||||
]
|
||||
file_tool_files = self._tool_files(session=session, tenant_id=tenant_id, file_ids=tool_file_refs)
|
||||
upload_files = self._upload_files(session=session, tenant_id=tenant_id, file_ids=upload_file_refs)
|
||||
for file_ref, portable_file_ref in zip(soul.config_files, resource_data["config_files"]):
|
||||
if file_ref.is_missing:
|
||||
continue
|
||||
resource_id = f"f_{len(file_sources) + 1:06d}"
|
||||
extension = self._safe_extension(file_ref.name)
|
||||
path = f"{resource_id}{extension}"
|
||||
if file_ref.file_kind == "tool_file":
|
||||
tool_file = file_tool_files.get(file_ref.file_id)
|
||||
if tool_file is None:
|
||||
raise RosterAgentPackageExportFailedError(f"Config file {file_ref.name!r} payload is unavailable")
|
||||
storage_key = tool_file.file_key
|
||||
mime_type = file_ref.mime_type or tool_file.mimetype
|
||||
audit_ref = tool_file.id
|
||||
else:
|
||||
upload_file = upload_files.get(file_ref.file_id)
|
||||
if upload_file is None:
|
||||
raise RosterAgentPackageExportFailedError(f"Config file {file_ref.name!r} payload is unavailable")
|
||||
storage_key = upload_file.key
|
||||
mime_type = file_ref.mime_type or upload_file.mime_type
|
||||
audit_ref = upload_file.id
|
||||
portable_file_ref["file_id"] = resource_id
|
||||
portable_file_ref["is_missing"] = False
|
||||
portable_file_ref["mime_type"] = mime_type or "application/octet-stream"
|
||||
file_sources.append(
|
||||
_FileSource(
|
||||
path=path,
|
||||
storage_key=storage_key,
|
||||
id=resource_id,
|
||||
audit_ref=audit_ref,
|
||||
)
|
||||
)
|
||||
|
||||
return AgentSoulConfig.model_validate(resource_data), skill_sources, file_sources
|
||||
|
||||
@staticmethod
|
||||
def _workspace_skill_sources(
|
||||
*, soul: AgentSoulConfig, archives: Sequence[RuntimeAgentSkillArchive], start_index: int
|
||||
) -> list[_SkillSource]:
|
||||
sources: list[_SkillSource] = []
|
||||
effective_skill_names = {item.name for item in soul.config_skills if not item.is_missing}
|
||||
for archive in sorted(archives, key=lambda item: item.priority):
|
||||
if archive.name in effective_skill_names:
|
||||
continue
|
||||
effective_skill_names.add(archive.name)
|
||||
resource_id = f"s_{start_index + len(sources) + 1:06d}"
|
||||
path = f"{resource_id}.zip"
|
||||
sources.append(
|
||||
_SkillSource(
|
||||
path=path,
|
||||
storage_key=archive.storage_key,
|
||||
id=resource_id,
|
||||
scope="workspace",
|
||||
name=archive.name,
|
||||
display_name=archive.display_name,
|
||||
description=archive.description,
|
||||
priority=archive.priority,
|
||||
audit_ref=archive.version_id,
|
||||
)
|
||||
)
|
||||
return sources
|
||||
|
||||
@staticmethod
|
||||
def _tool_files(*, session: Session, tenant_id: str, file_ids: Sequence[str]) -> dict[str, ToolFile]:
|
||||
if not file_ids:
|
||||
return {}
|
||||
rows = session.scalars(
|
||||
select(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id.in_(set(file_ids)))
|
||||
).all()
|
||||
return {item.id: item for item in rows}
|
||||
|
||||
@staticmethod
|
||||
def _upload_files(*, session: Session, tenant_id: str, file_ids: Sequence[str]) -> dict[str, UploadFile]:
|
||||
if not file_ids:
|
||||
return {}
|
||||
rows = session.scalars(
|
||||
select(UploadFile).where(UploadFile.tenant_id == tenant_id, UploadFile.id.in_(set(file_ids)))
|
||||
).all()
|
||||
return {item.id: item for item in rows}
|
||||
|
||||
def _write_storage_member(
|
||||
self,
|
||||
archive: zipfile.ZipFile,
|
||||
*,
|
||||
path: str,
|
||||
storage_key: str,
|
||||
max_bytes: int,
|
||||
) -> tuple[int, str]:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
try:
|
||||
with archive.open(path, "w", force_zip64=True) as target:
|
||||
for chunk in self._storage.load_stream(storage_key):
|
||||
if not isinstance(chunk, bytes):
|
||||
raise TypeError("storage stream returned a non-bytes chunk")
|
||||
if size + len(chunk) > max_bytes:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package payload exceeds the size limit")
|
||||
size += len(chunk)
|
||||
digest.update(chunk)
|
||||
target.write(chunk)
|
||||
except RosterAgentPackageTooLargeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RosterAgentPackageExportFailedError(f"Unable to read package resource {path!r}") from exc
|
||||
return size, digest.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _safe_extension(filename: str) -> str:
|
||||
extension = os.path.splitext(filename)[1].lower()
|
||||
if not extension or not re.fullmatch(r"\.[a-z0-9]{1,16}", extension):
|
||||
return ".bin"
|
||||
return extension
|
||||
|
||||
@staticmethod
|
||||
def _safe_slug(name: str) -> str:
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
||||
return slug[:80] or "agent"
|
||||
|
||||
|
||||
__all__ = ["RosterAgentPackageExporter"]
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Read and validate portable Roster Agent ``.ifpkg`` archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import posixpath
|
||||
import stat
|
||||
import tempfile
|
||||
import zipfile
|
||||
import zlib
|
||||
from collections.abc import Hashable
|
||||
from typing import Any, BinaryIO, cast, override
|
||||
|
||||
import yaml
|
||||
from pydantic import ValidationError
|
||||
|
||||
from configs import dify_config
|
||||
from constants.dsl_version import CURRENT_APP_DSL_VERSION
|
||||
from services.agent.dsl_entities import AgentAppDsl
|
||||
from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError
|
||||
from services.agent.roster_package_entities import (
|
||||
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES,
|
||||
PreparedRosterAgentPackage,
|
||||
RosterAgentPackageApp,
|
||||
RosterAgentPackageManifest,
|
||||
RosterAgentPackageMember,
|
||||
RosterAgentPackageSkill,
|
||||
)
|
||||
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
|
||||
from services.dsl_version import check_version_compatibility
|
||||
from services.entities.dsl_entities import ImportStatus
|
||||
|
||||
_COPY_CHUNK_SIZE = 1024 * 1024
|
||||
_ALLOWED_COMPRESSIONS = {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}
|
||||
|
||||
|
||||
class RosterAgentPackageReader:
|
||||
"""Produce a validated, bounded package without external side effects."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._skill_packages = SkillPackageService()
|
||||
|
||||
def read(self, source: BinaryIO) -> PreparedRosterAgentPackage:
|
||||
"""Validate the container and record unusable Skill payloads for import warnings."""
|
||||
# Ownership is transferred to PreparedRosterAgentPackage.
|
||||
spool = cast(
|
||||
BinaryIO,
|
||||
tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b"), # noqa: SIM115
|
||||
)
|
||||
try:
|
||||
self._copy_bounded(source, spool)
|
||||
spool.seek(0)
|
||||
invalid_skills: dict[str, str] = {}
|
||||
manifest, apps, members = self._validate_archive(spool, invalid_skills=invalid_skills)
|
||||
spool.seek(0)
|
||||
return PreparedRosterAgentPackage(
|
||||
archive=spool, manifest=manifest, apps=apps, members=members, invalid_skills=invalid_skills
|
||||
)
|
||||
except Exception:
|
||||
spool.close()
|
||||
raise
|
||||
|
||||
def read_member_bytes(self, package: PreparedRosterAgentPackage, path: str, *, max_bytes: int) -> bytes:
|
||||
"""Read one already-validated member while rechecking its size and digest."""
|
||||
|
||||
member = package.members.get(path)
|
||||
if member is None:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package member is unavailable")
|
||||
try:
|
||||
package.archive.seek(0)
|
||||
with zipfile.ZipFile(package.archive) as archive:
|
||||
info = archive.getinfo(path)
|
||||
payload, digest, _ = self._read_member(
|
||||
archive,
|
||||
info,
|
||||
collect=True,
|
||||
max_bytes=max_bytes,
|
||||
expected_size=member.size,
|
||||
)
|
||||
except (KeyError, OSError, zipfile.BadZipFile, EOFError, RuntimeError, ValueError, zlib.error) as exc:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package member is unavailable") from exc
|
||||
if digest != member.sha256:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
|
||||
return payload
|
||||
|
||||
def _validate_archive(
|
||||
self, archive_file: BinaryIO, *, invalid_skills: dict[str, str]
|
||||
) -> tuple[RosterAgentPackageManifest, dict[str, AgentAppDsl], dict[str, RosterAgentPackageMember]]:
|
||||
try:
|
||||
with zipfile.ZipFile(archive_file) as archive:
|
||||
infos = archive.infolist()
|
||||
if not infos:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package is empty")
|
||||
if len(infos) > dify_config.AGENT_PACKAGE_MAX_ENTRIES:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package has too many members")
|
||||
|
||||
info_by_path: dict[str, zipfile.ZipInfo] = {}
|
||||
casefold_paths: set[str] = set()
|
||||
total_uncompressed = 0
|
||||
for info in infos:
|
||||
path = self._validate_member_info(info)
|
||||
if path in info_by_path or path.casefold() in casefold_paths:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package contains duplicate member paths")
|
||||
info_by_path[path] = info
|
||||
casefold_paths.add(path.casefold())
|
||||
total_uncompressed += info.file_size
|
||||
if total_uncompressed > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package uncompressed size exceeds the limit")
|
||||
|
||||
manifest_data, manifest_size = self._read_yaml_document(archive, info_by_path, "manifest.yaml")
|
||||
try:
|
||||
manifest = RosterAgentPackageManifest.model_validate(manifest_data)
|
||||
except ValidationError as exc:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package manifest is invalid") from exc
|
||||
apps: dict[str, AgentAppDsl] = {}
|
||||
members: dict[str, RosterAgentPackageMember] = {}
|
||||
streamed_size = manifest_size
|
||||
for app_resource in manifest.apps:
|
||||
app_data, app_size = self._read_yaml_document(
|
||||
archive, info_by_path, app_resource.path, resource=app_resource
|
||||
)
|
||||
streamed_size += app_size
|
||||
if streamed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError(
|
||||
"Roster Agent package uncompressed size exceeds the limit"
|
||||
)
|
||||
try:
|
||||
app = AgentAppDsl.model_validate(app_data)
|
||||
if check_version_compatibility(app.version, CURRENT_APP_DSL_VERSION) in {
|
||||
ImportStatus.FAILED,
|
||||
ImportStatus.PENDING,
|
||||
}:
|
||||
raise ValueError("unsupported App DSL version")
|
||||
except ValueError as exc:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package app is invalid") from exc
|
||||
apps[app_resource.path] = app
|
||||
members[app_resource.path] = RosterAgentPackageMember(size=app_size, sha256=app_resource.sha256)
|
||||
try:
|
||||
manifest.validate_apps(apps)
|
||||
except ValueError as exc:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package app is invalid") from exc
|
||||
|
||||
expected_paths = {
|
||||
"manifest.yaml",
|
||||
*(item.path for item in manifest.apps),
|
||||
*(item.path for item in manifest.skills),
|
||||
*(item.path for item in manifest.files),
|
||||
}
|
||||
actual_paths = set(info_by_path)
|
||||
if "signature.sig" in actual_paths:
|
||||
if info_by_path["signature.sig"].file_size > ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package signature is too large")
|
||||
actual_paths.remove("signature.sig")
|
||||
if actual_paths != expected_paths:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package members do not match the manifest")
|
||||
|
||||
nested_uncompressed_size = 0
|
||||
signature_info = info_by_path.get("signature.sig")
|
||||
if signature_info is not None:
|
||||
_, _, signature_size = self._read_member(
|
||||
archive,
|
||||
signature_info,
|
||||
collect=False,
|
||||
max_bytes=min(
|
||||
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES,
|
||||
dify_config.AGENT_PACKAGE_MAX_BYTES - streamed_size,
|
||||
),
|
||||
expected_size=signature_info.file_size,
|
||||
)
|
||||
streamed_size += signature_size
|
||||
for resource in [*manifest.skills, *manifest.files]:
|
||||
info = info_by_path[resource.path]
|
||||
payload = b""
|
||||
remaining_package_bytes = dify_config.AGENT_PACKAGE_MAX_BYTES - streamed_size
|
||||
if isinstance(resource, RosterAgentPackageSkill):
|
||||
max_skill_bytes = dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024
|
||||
if info.file_size > max_skill_bytes:
|
||||
raise RosterAgentPackageTooLargeError(
|
||||
f"Roster Agent package Skill {resource.name!r} exceeds the size limit"
|
||||
)
|
||||
try:
|
||||
payload, digest, actual_size = self._read_member(
|
||||
archive,
|
||||
info,
|
||||
collect=True,
|
||||
max_bytes=min(max_skill_bytes, remaining_package_bytes),
|
||||
expected_size=resource.size,
|
||||
)
|
||||
except (InvalidRosterAgentPackageError, zipfile.BadZipFile, EOFError, zlib.error):
|
||||
invalid_skills[resource.id] = "archive_integrity_failed"
|
||||
streamed_size += info.file_size
|
||||
continue
|
||||
else:
|
||||
_, digest, actual_size = self._read_member(
|
||||
archive,
|
||||
info,
|
||||
collect=False,
|
||||
max_bytes=remaining_package_bytes,
|
||||
expected_size=resource.size,
|
||||
)
|
||||
streamed_size += actual_size
|
||||
if digest != resource.sha256:
|
||||
if isinstance(resource, RosterAgentPackageSkill):
|
||||
invalid_skills[resource.id] = "checksum_mismatch"
|
||||
continue
|
||||
raise InvalidRosterAgentPackageError(
|
||||
f"Roster Agent package resource {resource.path!r} failed integrity checks",
|
||||
)
|
||||
members[resource.path] = RosterAgentPackageMember(
|
||||
size=resource.size,
|
||||
sha256=digest,
|
||||
)
|
||||
if isinstance(resource, RosterAgentPackageSkill):
|
||||
try:
|
||||
inspection = self._skill_packages.inspect(content=payload, filename=resource.path)
|
||||
except SkillPackageError as exc:
|
||||
invalid_skills[resource.id] = exc.code
|
||||
continue
|
||||
nested_uncompressed_size += inspection.uncompressed_size
|
||||
if nested_uncompressed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError(
|
||||
"Roster Agent package nested Skill contents exceed the size limit"
|
||||
)
|
||||
if inspection.name != resource.name:
|
||||
invalid_skills[resource.id] = "skill_name_mismatch"
|
||||
return manifest, apps, members
|
||||
except (InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError):
|
||||
raise
|
||||
except (OSError, zipfile.BadZipFile, EOFError, RuntimeError, ValueError, zlib.error) as exc:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package is not a valid ZIP") from exc
|
||||
|
||||
@staticmethod
|
||||
def _copy_bounded(source: BinaryIO, target: BinaryIO) -> None:
|
||||
size = 0
|
||||
while True:
|
||||
chunk = source.read(_COPY_CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
if not isinstance(chunk, bytes):
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package must be binary")
|
||||
size += len(chunk)
|
||||
if size > dify_config.AGENT_PACKAGE_MAX_BYTES:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the archive size limit")
|
||||
target.write(chunk)
|
||||
|
||||
@staticmethod
|
||||
def _validate_member_info(info: zipfile.ZipInfo) -> str:
|
||||
name = info.filename
|
||||
if info.is_dir():
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package must use root-level files")
|
||||
if "\x00" in name or "\\" in name or any(ord(char) < 0x20 for char in name):
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package contains an unsafe path")
|
||||
normalized = posixpath.normpath(name)
|
||||
if normalized != name or normalized in {"", ".", ".."} or normalized.startswith("/") or "/" in normalized:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package contains an unsafe path")
|
||||
if stat.S_ISLNK(info.external_attr >> 16):
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package must not contain symbolic links")
|
||||
if info.flag_bits & 0x1:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package must not contain encrypted members")
|
||||
if info.compress_type not in _ALLOWED_COMPRESSIONS:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package uses unsupported compression")
|
||||
if info.file_size and info.compress_size == 0:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package contains invalid ZIP metadata")
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _read_member(
|
||||
archive: zipfile.ZipFile,
|
||||
info: zipfile.ZipInfo,
|
||||
*,
|
||||
collect: bool,
|
||||
max_bytes: int,
|
||||
expected_size: int,
|
||||
) -> tuple[bytes, str, int]:
|
||||
digest = hashlib.sha256()
|
||||
output = io.BytesIO() if collect else None
|
||||
size = 0
|
||||
with archive.open(info) as member:
|
||||
while chunk := member.read(_COPY_CHUNK_SIZE):
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
raise RosterAgentPackageTooLargeError("Roster Agent package member exceeds the size limit")
|
||||
if size > expected_size:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
|
||||
digest.update(chunk)
|
||||
if output is not None:
|
||||
output.write(chunk)
|
||||
if size != expected_size:
|
||||
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
|
||||
return output.getvalue() if output is not None else b"", digest.hexdigest(), size
|
||||
|
||||
def _read_yaml_document(
|
||||
self,
|
||||
archive: zipfile.ZipFile,
|
||||
infos: dict[str, zipfile.ZipInfo],
|
||||
path: str,
|
||||
*,
|
||||
resource: RosterAgentPackageApp | None = None,
|
||||
) -> tuple[Any, int]:
|
||||
info = infos.get(path)
|
||||
if info is None:
|
||||
raise InvalidRosterAgentPackageError(f"Roster Agent package is missing {path}")
|
||||
if info.file_size > dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES:
|
||||
raise RosterAgentPackageTooLargeError(f"Roster Agent package {path} exceeds the size limit")
|
||||
payload, digest, size = self._read_member(
|
||||
archive,
|
||||
info,
|
||||
collect=True,
|
||||
max_bytes=dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES,
|
||||
expected_size=resource.size if resource is not None else info.file_size,
|
||||
)
|
||||
if resource is not None and digest != resource.sha256:
|
||||
raise InvalidRosterAgentPackageError(f"Roster Agent package {path} failed integrity checks")
|
||||
try:
|
||||
# The custom SafeLoader also rejects duplicate keys and aliases.
|
||||
return yaml.load(payload, Loader=_PackageYamlLoader), size # noqa: S506
|
||||
except (UnicodeDecodeError, yaml.YAMLError, ValueError, RecursionError) as exc:
|
||||
raise InvalidRosterAgentPackageError(
|
||||
f"Roster Agent package {path.removesuffix('.yaml')} is invalid"
|
||||
) from exc
|
||||
|
||||
|
||||
class _PackageYamlLoader(yaml.SafeLoader):
|
||||
"""Reject ambiguous mappings and aliases in untrusted package documents."""
|
||||
|
||||
@override
|
||||
def compose_node(self, parent: yaml.Node | None, index: Any) -> yaml.Node | None:
|
||||
if self.check_event(yaml.AliasEvent):
|
||||
raise ValueError("YAML aliases are not supported")
|
||||
return super().compose_node(parent, index)
|
||||
|
||||
@override
|
||||
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Hashable, Any]:
|
||||
result: dict[Hashable, Any] = {}
|
||||
for key_node, value_node in node.value:
|
||||
key = self.construct_object(key_node, deep=deep)
|
||||
if not isinstance(key, str):
|
||||
raise ValueError("YAML mapping keys must be strings")
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate YAML key: {key}")
|
||||
result[key] = self.construct_object(value_node, deep=deep)
|
||||
return result
|
||||
|
||||
|
||||
__all__ = ["RosterAgentPackageReader"]
|
||||
@@ -20,16 +20,14 @@ import io
|
||||
import posixpath
|
||||
import zipfile
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field, ValidationError, field_validator
|
||||
|
||||
from configs import dify_config
|
||||
from libs.zip_filename_recovery import open_zip_with_replacement_names
|
||||
|
||||
# Bounds — generous but finite so a hostile upload can't exhaust memory/disk.
|
||||
_MAX_UNCOMPRESSED_BYTES = 200 * 1024 * 1024
|
||||
_MAX_SKILL_MD_BYTES = 1 * 1024 * 1024
|
||||
_MAX_ENTRIES = 5000
|
||||
_ALLOWED_EXTENSIONS = (".zip", ".skill")
|
||||
_SKILL_MD_NAME = "SKILL.md"
|
||||
_SKILL_NAME_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
|
||||
@@ -50,15 +48,9 @@ class SkillPackageError(Exception):
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class SkillManifest(BaseModel):
|
||||
"""Validated metadata extracted from a Skill package."""
|
||||
|
||||
class _SkillMetadata(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=64, pattern=_SKILL_NAME_PATTERN)
|
||||
description: str = Field(min_length=1, max_length=_MAX_SKILL_DESCRIPTION_LENGTH)
|
||||
entry_path: str # path of SKILL.md inside the archive
|
||||
files: list[str] # all (safe) file paths inside the archive
|
||||
size: int # total uncompressed bytes
|
||||
hash: str # sha256 of the archive bytes
|
||||
|
||||
@field_validator("name", "description", mode="before")
|
||||
@classmethod
|
||||
@@ -66,18 +58,44 @@ class SkillManifest(BaseModel):
|
||||
return value.strip() if isinstance(value, str) else value
|
||||
|
||||
|
||||
class SkillPackageInspection(_SkillMetadata):
|
||||
"""Metadata read without rebuilding or fully extracting a Skill archive."""
|
||||
|
||||
uncompressed_size: int = Field(ge=0)
|
||||
|
||||
|
||||
class SkillManifest(_SkillMetadata):
|
||||
"""Validated metadata extracted from a Skill package."""
|
||||
|
||||
entry_path: str # path of SKILL.md inside the archive
|
||||
files: list[str] # all (safe) file paths inside the archive
|
||||
size: int # total uncompressed bytes
|
||||
hash: str # sha256 of the archive bytes
|
||||
|
||||
|
||||
class NormalizedSkillPackage(BaseModel):
|
||||
"""Canonical skill package bytes and metadata ready to store as Agent config."""
|
||||
|
||||
manifest: SkillManifest
|
||||
archive_bytes: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _InspectedSkillArchive:
|
||||
metadata: SkillPackageInspection
|
||||
normalized_members: dict[str, zipfile.ZipInfo]
|
||||
skill_md_bytes: bytes
|
||||
strip_prefix: str | None
|
||||
|
||||
|
||||
class SkillPackageService:
|
||||
"""Validate Skill archives and produce a normalized package."""
|
||||
|
||||
def inspect(self, *, content: bytes, filename: str) -> SkillPackageInspection:
|
||||
"""Validate archive metadata and SKILL.md without rebuilding the archive."""
|
||||
archive = self._open_archive(content=content, filename=filename)
|
||||
with archive:
|
||||
return self._inspect_archive(archive).metadata
|
||||
|
||||
def validate_and_normalize(self, *, content: bytes, filename: str) -> NormalizedSkillPackage:
|
||||
"""Return the canonical package for an uploaded skill archive.
|
||||
|
||||
@@ -89,43 +107,30 @@ class SkillPackageService:
|
||||
The returned manifest is normalized to archive-root ``SKILL.md`` and its
|
||||
hash describes the rebuilt archive bytes. Member read/decompression
|
||||
failures while consuming the archive are mapped to ``invalid_archive``.
|
||||
Malformed UTF-8 member names are decoded with replacement characters;
|
||||
recovered path collisions still fail the normal duplicate-path check.
|
||||
"""
|
||||
archive = self._open_archive(content=content, filename=filename)
|
||||
with archive:
|
||||
members = self._collect_file_members(archive)
|
||||
member_paths = [safe_path for _, safe_path in members]
|
||||
entry_path = self._find_skill_md(member_paths)
|
||||
strip_prefix = self._skill_root_prefix(entry_path)
|
||||
normalized_members = self._normalize_members(
|
||||
members=members,
|
||||
skill_root_prefix=strip_prefix,
|
||||
ignore_outside_selected_root=self._can_strip_single_top_level_folder(
|
||||
paths=member_paths, entry_path=entry_path
|
||||
),
|
||||
)
|
||||
skill_md_member = normalized_members[_SKILL_MD_NAME]
|
||||
self._validate_skill_md_size(skill_md_member)
|
||||
raw_skill_md_bytes = self._read_member_bytes_from_archive(archive, member_info=skill_md_member)
|
||||
skill_md = self._decode_skill_md(raw_skill_md_bytes)
|
||||
skill_md_bytes = skill_md.encode("utf-8")
|
||||
inspection = self._inspect_archive(archive)
|
||||
normalized_archive_bytes = self._build_normalized_archive(
|
||||
archive=archive,
|
||||
normalized_members=normalized_members,
|
||||
skill_md_bytes=skill_md_bytes,
|
||||
)
|
||||
normalized_size = sum(
|
||||
len(skill_md_bytes) if path == _SKILL_MD_NAME else max(info.file_size, 0)
|
||||
for path, info in normalized_members.items()
|
||||
normalized_members=inspection.normalized_members,
|
||||
skill_md_bytes=inspection.skill_md_bytes,
|
||||
)
|
||||
|
||||
name, description = self._parse_skill_md(skill_md)
|
||||
if len(normalized_archive_bytes) > dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024:
|
||||
raise SkillPackageError("archive_too_large", "normalized skill archive exceeds size limit", status_code=400)
|
||||
try:
|
||||
manifest = SkillManifest(
|
||||
name=name,
|
||||
description=description,
|
||||
name=inspection.metadata.name,
|
||||
description=inspection.metadata.description,
|
||||
entry_path=_SKILL_MD_NAME,
|
||||
files=sorted(normalized_members),
|
||||
size=normalized_size,
|
||||
files=sorted(inspection.normalized_members),
|
||||
size=sum(
|
||||
len(inspection.skill_md_bytes) if path == _SKILL_MD_NAME else info.file_size
|
||||
for path, info in inspection.normalized_members.items()
|
||||
),
|
||||
hash=hashlib.sha256(normalized_archive_bytes).hexdigest(),
|
||||
)
|
||||
except ValidationError as exc:
|
||||
@@ -133,8 +138,38 @@ class SkillPackageService:
|
||||
return NormalizedSkillPackage(
|
||||
manifest=manifest,
|
||||
archive_bytes=normalized_archive_bytes,
|
||||
)
|
||||
|
||||
def _inspect_archive(self, archive: zipfile.ZipFile) -> _InspectedSkillArchive:
|
||||
members = self._collect_file_members(archive)
|
||||
member_paths = [safe_path for _, safe_path in members]
|
||||
entry_path = self._find_skill_md(member_paths)
|
||||
strip_prefix = self._skill_root_prefix(entry_path)
|
||||
normalized_members = self._normalize_members(
|
||||
members=members,
|
||||
skill_root_prefix=strip_prefix,
|
||||
ignore_outside_selected_root=self._can_strip_single_top_level_folder(
|
||||
paths=member_paths, entry_path=entry_path
|
||||
),
|
||||
)
|
||||
skill_md_member = normalized_members[_SKILL_MD_NAME]
|
||||
self._validate_skill_md_size(skill_md_member)
|
||||
raw_skill_md_bytes = self._read_member_bytes_from_archive(archive, member_info=skill_md_member)
|
||||
skill_md = self._decode_skill_md(raw_skill_md_bytes)
|
||||
skill_md_bytes = skill_md.encode("utf-8")
|
||||
name, description = self._parse_skill_md(skill_md)
|
||||
try:
|
||||
metadata = SkillPackageInspection(
|
||||
name=name,
|
||||
description=description,
|
||||
uncompressed_size=sum(info.file_size for info, _ in members),
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise self._manifest_validation_error(exc) from exc
|
||||
return _InspectedSkillArchive(
|
||||
metadata=metadata,
|
||||
normalized_members=normalized_members,
|
||||
skill_md_bytes=skill_md_bytes,
|
||||
strip_prefix=strip_prefix,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -171,21 +206,24 @@ class SkillPackageService:
|
||||
raise SkillPackageError("archive_too_large", "skill archive exceeds size limit", status_code=400)
|
||||
|
||||
try:
|
||||
return zipfile.ZipFile(io.BytesIO(content))
|
||||
return open_zip_with_replacement_names(content)
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise SkillPackageError("invalid_archive", "skill archive is not a valid zip", status_code=400) from exc
|
||||
|
||||
def _collect_file_members(self, archive: zipfile.ZipFile) -> list[tuple[zipfile.ZipInfo, str]]:
|
||||
infos = [info for info in archive.infolist() if not info.is_dir()]
|
||||
if len(infos) > _MAX_ENTRIES:
|
||||
archive_infos = archive.infolist()
|
||||
if len(archive_infos) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
|
||||
raise SkillPackageError("too_many_entries", "skill archive has too many files", status_code=400)
|
||||
infos = [info for info in archive_infos if not info.is_dir()]
|
||||
|
||||
members: list[tuple[zipfile.ZipInfo, str]] = []
|
||||
total_uncompressed = 0
|
||||
for info in infos:
|
||||
members.append((info, self._safe_member_path(info.filename)))
|
||||
total_uncompressed += max(info.file_size, 0)
|
||||
if total_uncompressed > _MAX_UNCOMPRESSED_BYTES:
|
||||
total_uncompressed += info.file_size
|
||||
if info.file_size and info.compress_size == 0:
|
||||
raise SkillPackageError("invalid_archive", "skill package has invalid ZIP metadata", status_code=400)
|
||||
if total_uncompressed > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
|
||||
raise SkillPackageError(
|
||||
"archive_too_large",
|
||||
"skill archive uncompressed size exceeds limit",
|
||||
@@ -225,12 +263,8 @@ class SkillPackageService:
|
||||
else:
|
||||
normalized_path = safe_path
|
||||
|
||||
if (
|
||||
not normalized_path
|
||||
or normalized_path in {".", ".."}
|
||||
or normalized_path.startswith("/")
|
||||
or "\\" in normalized_path
|
||||
):
|
||||
# Paths are already normalized and safe; "." is not a file.
|
||||
if normalized_path == ".":
|
||||
raise SkillPackageError("unsafe_path", "skill archive contains an unsafe path", status_code=400)
|
||||
if normalized_path in normalized_members:
|
||||
raise SkillPackageError(
|
||||
@@ -306,7 +340,7 @@ class SkillPackageService:
|
||||
|
||||
@staticmethod
|
||||
def _validate_skill_md_size(member_info: zipfile.ZipInfo) -> None:
|
||||
if member_info.file_size > _MAX_SKILL_MD_BYTES:
|
||||
if member_info.file_size > dify_config.SKILL_PACKAGE_MAX_SKILL_MD_BYTES:
|
||||
raise SkillPackageError("skill_md_too_large", "SKILL.md exceeds size limit", status_code=400)
|
||||
|
||||
@staticmethod
|
||||
@@ -343,4 +377,10 @@ class SkillPackageService:
|
||||
return loaded if isinstance(loaded, dict) else {}
|
||||
|
||||
|
||||
__all__ = ["NormalizedSkillPackage", "SkillManifest", "SkillPackageError", "SkillPackageService"]
|
||||
__all__ = [
|
||||
"NormalizedSkillPackage",
|
||||
"SkillManifest",
|
||||
"SkillPackageError",
|
||||
"SkillPackageInspection",
|
||||
"SkillPackageService",
|
||||
]
|
||||
@@ -609,7 +609,7 @@ class AgentConfigService:
|
||||
)
|
||||
self._require_writable(target, surface=surface)
|
||||
try:
|
||||
skill_ref, _package = self._skill_normalizer.normalize(
|
||||
skill_ref = self._skill_normalizer.normalize(
|
||||
content=content,
|
||||
filename=filename,
|
||||
requested_name=None,
|
||||
@@ -1064,7 +1064,7 @@ class AgentConfigService:
|
||||
)
|
||||
archive_bytes = storage.load_once(tool_file.file_key)
|
||||
try:
|
||||
skill_ref, _package = self._skill_normalizer.normalize(
|
||||
skill_ref = self._skill_normalizer.normalize(
|
||||
content=archive_bytes,
|
||||
filename=tool_file.name,
|
||||
requested_name=name,
|
||||
|
||||
@@ -47,7 +47,8 @@ from models import Account, App, AppMode
|
||||
from models.agent import AgentScope
|
||||
from models.model import AppModelConfig, AppModelConfigDict, IconType, load_annotation_reply_config
|
||||
from models.workflow import Workflow
|
||||
from services.agent.dsl_service import AgentDslService, AgentPackage
|
||||
from services.agent.dsl_entities import AgentPackage, make_agent_app_dsl
|
||||
from services.agent.dsl_service import AgentDslService
|
||||
from services.agent.retirement_service import WorkflowAgentRetirementService
|
||||
from services.agent.workflow_publish_service import WorkflowAgentPublishService
|
||||
from services.dsl_content import DSL_MAX_SIZE, dsl_content_size
|
||||
@@ -60,6 +61,7 @@ from services.entities.dsl_entities import (
|
||||
ImportMode,
|
||||
ImportStatus,
|
||||
PendingImportOwner,
|
||||
make_app_dsl,
|
||||
)
|
||||
from services.errors.account import NoPermissionError
|
||||
from services.errors.app import WorkflowNotFoundError
|
||||
@@ -762,44 +764,29 @@ class AppDslService:
|
||||
"""
|
||||
app_mode = AppMode.value_of(app_model.mode)
|
||||
|
||||
export_data: dict[str, Any] = {
|
||||
"version": CURRENT_DSL_VERSION,
|
||||
"kind": "app",
|
||||
"app": {
|
||||
"name": app_model.name,
|
||||
"mode": app_model.mode.value if isinstance(app_model.mode, AppMode) else app_model.mode,
|
||||
"icon": app_model.icon,
|
||||
"icon_type": (
|
||||
app_model.icon_type.value if isinstance(app_model.icon_type, IconType) else app_model.icon_type
|
||||
),
|
||||
"icon_background": app_model.icon_background,
|
||||
"description": app_model.description,
|
||||
"use_icon_as_answer_icon": app_model.use_icon_as_answer_icon,
|
||||
},
|
||||
}
|
||||
|
||||
if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
|
||||
cls._append_workflow_export_data(
|
||||
export_data=export_data,
|
||||
app_model=app_model,
|
||||
include_secret=include_secret,
|
||||
workflow_id=workflow_id,
|
||||
session=session,
|
||||
)
|
||||
elif app_mode == AppMode.AGENT:
|
||||
if app_mode == AppMode.AGENT:
|
||||
package_ref, packages = AgentDslService(session).export_agent_app(app=app_model)
|
||||
export_data["agent"] = {"package_ref": package_ref}
|
||||
export_data["agent_packages"] = {key: package.model_dump(mode="json") for key, package in packages.items()}
|
||||
dependencies = AgentDslService(session).extract_package_dependencies(packages)
|
||||
export_data["dependencies"] = [
|
||||
jsonable_encoder(item.model_dump())
|
||||
for item in DependenciesAnalysisService.generate_dependencies(
|
||||
tenant_id=app_model.tenant_id,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
]
|
||||
export_data = make_agent_app_dsl(
|
||||
app_model,
|
||||
package_ref=package_ref,
|
||||
packages=packages,
|
||||
dependencies=DependenciesAnalysisService.generate_dependencies(
|
||||
tenant_id=app_model.tenant_id, dependencies=dependencies
|
||||
),
|
||||
).model_dump(mode="json")
|
||||
else:
|
||||
cls._append_model_config_export_data(export_data, app_model, session=session)
|
||||
export_data = make_app_dsl(app_model)
|
||||
if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
|
||||
cls._append_workflow_export_data(
|
||||
export_data=export_data,
|
||||
app_model=app_model,
|
||||
include_secret=include_secret,
|
||||
workflow_id=workflow_id,
|
||||
session=session,
|
||||
)
|
||||
else:
|
||||
cls._append_model_config_export_data(export_data, app_model, session=session)
|
||||
|
||||
return yaml.dump(export_data, allow_unicode=True)
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from constants.dsl_version import CURRENT_APP_DSL_VERSION
|
||||
from core.plugin.entities.plugin import PluginDependency
|
||||
from models.model import App, AppMode, IconType
|
||||
|
||||
|
||||
class ImportMode(StrEnum):
|
||||
@@ -39,3 +41,20 @@ class DslImportWarning(BaseModel):
|
||||
|
||||
class CheckDependenciesResult(BaseModel):
|
||||
leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
|
||||
|
||||
|
||||
def make_app_dsl(app: App) -> dict[str, Any]:
|
||||
"""Build the shared App DSL envelope for standalone and archive exports."""
|
||||
return {
|
||||
"version": CURRENT_APP_DSL_VERSION,
|
||||
"kind": "app",
|
||||
"app": {
|
||||
"name": app.name,
|
||||
"mode": app.mode.value if isinstance(app.mode, AppMode) else app.mode,
|
||||
"icon": app.icon,
|
||||
"icon_type": app.icon_type.value if isinstance(app.icon_type, IconType) else app.icon_type,
|
||||
"icon_background": app.icon_background,
|
||||
"description": app.description,
|
||||
"use_icon_as_answer_icon": app.use_icon_as_answer_icon,
|
||||
},
|
||||
}
|
||||
@@ -98,9 +98,6 @@ from services.file_service import FileService
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SKILL_MD = "SKILL.md"
|
||||
_MAX_SKILL_BYTES = 200 * 1024 * 1024
|
||||
_MAX_FILES_PER_SKILL = 5000
|
||||
_MAX_ZIP_COMPRESSION_RATIO = 1000
|
||||
_MAX_FILE_CHECK_ITEMS = 100
|
||||
_MAX_SKILLS_PER_WORKSPACE = 500
|
||||
_MAX_AGENT_SKILLS = 20
|
||||
@@ -509,6 +506,20 @@ class PublishedSkillArchive:
|
||||
payload: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RuntimeAgentSkillArchive:
|
||||
skill_id: str
|
||||
version_id: str
|
||||
tool_file_id: str
|
||||
storage_key: str
|
||||
name: str
|
||||
display_name: str
|
||||
description: str
|
||||
priority: int
|
||||
size: int
|
||||
hash: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SkillFileContent:
|
||||
filename: str
|
||||
@@ -2269,6 +2280,34 @@ class SkillManagementService:
|
||||
config_snapshot_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return workspace Skills from the Agent draft or a published snapshot."""
|
||||
return [
|
||||
{
|
||||
"id": item.skill_id,
|
||||
"name": item.name,
|
||||
"file_id": item.tool_file_id,
|
||||
"description": item.description,
|
||||
"size": item.size,
|
||||
"hash": item.hash,
|
||||
"mime_type": "application/zip",
|
||||
}
|
||||
for item in self.list_runtime_agent_skill_archives(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent_id,
|
||||
include_draft=include_draft,
|
||||
config_snapshot_id=config_snapshot_id,
|
||||
)
|
||||
]
|
||||
|
||||
def list_runtime_agent_skill_archives(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
agent_id: str,
|
||||
include_draft: bool = False,
|
||||
config_snapshot_id: str | None = None,
|
||||
) -> list[RuntimeAgentSkillArchive]:
|
||||
"""Return export-ready workspace Skill archives with published identity compatibility."""
|
||||
|
||||
with self._session_scope() as session:
|
||||
binding_model = AgentSkillBinding if include_draft else AgentSkillBindingSnapshot
|
||||
conditions = [
|
||||
@@ -2284,29 +2323,37 @@ class SkillManagementService:
|
||||
conditions.append(AgentSkillBindingSnapshot.config_snapshot_id == config_snapshot_id)
|
||||
rows = list(
|
||||
session.execute(
|
||||
select(binding_model, Skill, SkillVersion)
|
||||
select(binding_model, Skill, SkillVersion, ToolFile)
|
||||
.join(Skill, Skill.id == binding_model.skill_id)
|
||||
.join(SkillVersion, SkillVersion.id == Skill.latest_published_version_id)
|
||||
.join(Agent, Agent.id == binding_model.agent_id)
|
||||
.join(
|
||||
ToolFile,
|
||||
(ToolFile.id == SkillVersion.archive_tool_file_id) & (ToolFile.tenant_id == tenant_id),
|
||||
)
|
||||
.where(*conditions)
|
||||
.order_by(Skill.name)
|
||||
)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": skill.id,
|
||||
"name": published_name,
|
||||
"file_id": version.archive_tool_file_id,
|
||||
"description": published_description,
|
||||
"size": version.archive_size,
|
||||
"hash": version.hash_code,
|
||||
"mime_type": "application/zip",
|
||||
}
|
||||
for _binding, skill, version in rows
|
||||
for published_name, _published_display_name, published_description in [
|
||||
self._published_skill_identity(skill, version)
|
||||
]
|
||||
|
||||
return [
|
||||
RuntimeAgentSkillArchive(
|
||||
skill_id=skill.id,
|
||||
version_id=version.id,
|
||||
tool_file_id=version.archive_tool_file_id,
|
||||
storage_key=tool_file.file_key,
|
||||
name=published_name,
|
||||
display_name=published_display_name,
|
||||
description=published_description,
|
||||
priority=binding.priority,
|
||||
size=version.archive_size,
|
||||
hash=version.hash_code,
|
||||
)
|
||||
for binding, skill, version, tool_file in rows
|
||||
for published_name, published_display_name, published_description in [
|
||||
self._published_skill_identity(skill, version)
|
||||
]
|
||||
]
|
||||
|
||||
def pull_runtime_agent_skill(
|
||||
self,
|
||||
@@ -3899,7 +3946,7 @@ class SkillManagementService:
|
||||
def _validate_archive_limits(archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
"""Validate ZIP metadata before any member is decompressed or persisted."""
|
||||
infos = archive.infolist()
|
||||
if len(infos) > _MAX_FILES_PER_SKILL:
|
||||
if len(infos) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
|
||||
raise SkillManagementServiceError("too_many_files", "skill file count limit exceeded")
|
||||
|
||||
total_uncompressed = 0
|
||||
@@ -3908,16 +3955,13 @@ class SkillManagementService:
|
||||
raise SkillManagementServiceError("invalid_skill_package", "skill package has invalid ZIP metadata")
|
||||
|
||||
total_uncompressed += info.file_size
|
||||
if total_uncompressed > _MAX_SKILL_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
|
||||
if total_uncompressed > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
|
||||
|
||||
if info.is_dir() or info.file_size == 0:
|
||||
continue
|
||||
if info.compress_size == 0 or info.file_size / info.compress_size > _MAX_ZIP_COMPRESSION_RATIO:
|
||||
raise SkillManagementServiceError(
|
||||
"invalid_skill_package",
|
||||
"skill package compression ratio exceeds the allowed limit",
|
||||
)
|
||||
if info.compress_size == 0:
|
||||
raise SkillManagementServiceError("invalid_skill_package", "skill package has invalid ZIP metadata")
|
||||
return infos
|
||||
|
||||
def _draft_rows_from_archive_bytes(
|
||||
@@ -4107,7 +4151,7 @@ class SkillManagementService:
|
||||
)
|
||||
parent = posixpath.dirname(parent)
|
||||
|
||||
if len(entries_by_path) > _MAX_FILES_PER_SKILL:
|
||||
if len(entries_by_path) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
|
||||
raise SkillManagementServiceError("too_many_files", "skill file count limit exceeded")
|
||||
|
||||
rows: list[SkillDraftFile] = []
|
||||
@@ -4140,8 +4184,8 @@ class SkillManagementService:
|
||||
)
|
||||
)
|
||||
|
||||
if total_size > _MAX_SKILL_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
|
||||
if total_size > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
|
||||
return rows
|
||||
|
||||
def _sync_skill_metadata_from_draft_skill_md(self, *, skill: Skill, content: str) -> None:
|
||||
@@ -4401,8 +4445,8 @@ class SkillManagementService:
|
||||
@staticmethod
|
||||
def _enforce_total_size(files: list[SkillDraftFile]) -> None:
|
||||
total = sum(file.size or 0 for file in {file.path: file for file in files}.values())
|
||||
if total > _MAX_SKILL_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
|
||||
if total > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
|
||||
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
|
||||
|
||||
@staticmethod
|
||||
def _load_tool_file_bytes(*, tenant_id: str, file_id: str) -> bytes:
|
||||
@@ -4410,14 +4454,15 @@ class SkillManagementService:
|
||||
tool_file = session.scalar(select(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id == file_id))
|
||||
if tool_file is None:
|
||||
raise SkillManagementServiceError("skill_archive_missing", "skill archive is missing", status_code=404)
|
||||
try:
|
||||
return storage.load_once(tool_file.file_key)
|
||||
except (OSError, SQLAlchemyError) as exc:
|
||||
raise SkillManagementServiceError(
|
||||
"skill_archive_missing",
|
||||
"skill archive is missing",
|
||||
status_code=404,
|
||||
) from exc
|
||||
file_key = tool_file.file_key
|
||||
try:
|
||||
return storage.load_once(file_key)
|
||||
except (OSError, SQLAlchemyError) as exc:
|
||||
raise SkillManagementServiceError(
|
||||
"skill_archive_missing",
|
||||
"skill archive is missing",
|
||||
status_code=404,
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def _load_assistant_tool_file_bytes(*, tenant_id: str, file_id: str) -> bytes:
|
||||
|
||||
@@ -64,7 +64,12 @@ class _IsolatedDifyConfig(DifyConfig):
|
||||
|
||||
|
||||
def _make_config(**values: object) -> DifyConfig:
|
||||
return _IsolatedDifyConfig(**values)
|
||||
# Pydantic exposes a generated field-wise signature, while this helper intentionally forwards heterogeneous input.
|
||||
return _IsolatedDifyConfig(**values) # pyrefly: ignore[bad-argument-type]
|
||||
|
||||
|
||||
def test_skill_package_default_limit_matches_documentation() -> None:
|
||||
assert _make_config().UPLOAD_SKILL_FILE_SIZE_LIMIT == 50
|
||||
|
||||
|
||||
def test_dify_config_keeps_secret_key_empty_when_missing(tmp_path) -> None:
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import io
|
||||
import struct
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from libs.zip_filename_recovery import open_zip_with_replacement_names
|
||||
|
||||
|
||||
def _zip64(monkeypatch: pytest.MonkeyPatch, limit: int) -> bytes:
|
||||
output = io.BytesIO()
|
||||
with monkeypatch.context() as scoped:
|
||||
scoped.setattr(zipfile, "ZIP64_LIMIT", limit)
|
||||
with zipfile.ZipFile(output, "w", allowZip64=True) as archive:
|
||||
archive.writestr("padding.txt", b"p" * 200)
|
||||
archive.writestr("scripts/\u00e9.py", b"print('ok')\n")
|
||||
content = output.getvalue()
|
||||
assert len(content) < 2048
|
||||
assert b"PK\x06\x06" in content
|
||||
return content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit", [0, 100])
|
||||
@pytest.mark.parametrize("prefix", [b"", b"self-extracting-prefix"])
|
||||
def test_zip64_replaces_names_and_preserves_payload(monkeypatch: pytest.MonkeyPatch, limit: int, prefix: bytes) -> None:
|
||||
original = prefix + _zip64(monkeypatch, limit)
|
||||
with zipfile.ZipFile(io.BytesIO(original)) as archive:
|
||||
assert archive.read("scripts/\u00e9.py") == b"print('ok')\n"
|
||||
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
with open_zip_with_replacement_names(damaged) as archive:
|
||||
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
|
||||
assert archive.testzip() is None
|
||||
|
||||
|
||||
def test_zip64_uses_extended_values_when_legacy_fields_are_sentinels(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
content = bytearray(_zip64(monkeypatch, 0))
|
||||
end = content.rfind(b"PK\x05\x06")
|
||||
struct.pack_into("<HHII", content, end + 8, 0xFFFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFF)
|
||||
damaged = bytes(content).replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
with open_zip_with_replacement_names(damaged) as archive:
|
||||
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
|
||||
|
||||
|
||||
def test_zip64_extensible_data_is_not_part_of_central_directory(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
content = bytearray(_zip64(monkeypatch, 0))
|
||||
record = content.rfind(b"PK\x06\x06")
|
||||
locator = content.rfind(b"PK\x06\x07")
|
||||
extension = b"extension"
|
||||
struct.pack_into("<Q", content, record + 4, 44 + len(extension))
|
||||
content[locator:locator] = extension
|
||||
damaged = bytes(content).replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
with open_zip_with_replacement_names(damaged) as archive:
|
||||
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
|
||||
|
||||
|
||||
def test_zip64_recovery_preserves_crc_checks(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
content = _zip64(monkeypatch, 0)
|
||||
damaged = content.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"print('ok')", b"print('NO')")
|
||||
with open_zip_with_replacement_names(damaged) as archive:
|
||||
with pytest.raises(zipfile.BadZipFile, match="CRC"):
|
||||
archive.read("scripts/\ufffda.py")
|
||||
|
||||
|
||||
def test_zip64_recovery_rejects_mismatched_local_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
original = _zip64(monkeypatch, 0)
|
||||
directory = original.index(b"PK\x01\x02")
|
||||
damaged = original[:directory] + original[directory:].replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
with pytest.raises(zipfile.BadZipFile, match="names differ"):
|
||||
open_zip_with_replacement_names(damaged)
|
||||
|
||||
|
||||
def _damaged_utf8_zip() -> bytearray:
|
||||
output = io.BytesIO()
|
||||
with zipfile.ZipFile(output, "w") as archive:
|
||||
archive.writestr("scripts/\u00e9.py", b"print('ok')\n")
|
||||
return bytearray(output.getvalue().replace(b"\xc3\xa9.py", b"\xffa.py"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [0xFFFFFFFF, 0xFFFFFFFE])
|
||||
def test_recovery_rejects_unresolvable_local_header_offsets(offset: int) -> None:
|
||||
content = _damaged_utf8_zip()
|
||||
directory = content.index(b"PK\x01\x02")
|
||||
struct.pack_into("<I", content, directory + 42, offset)
|
||||
with pytest.raises(zipfile.BadZipFile, match="ZIP64 extra field|local header offset"):
|
||||
open_zip_with_replacement_names(bytes(content))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entries", [0, 2])
|
||||
def test_recovery_rejects_inconsistent_directory_entry_counts(entries: int) -> None:
|
||||
content = _damaged_utf8_zip()
|
||||
end = content.rfind(b"PK\x05\x06")
|
||||
struct.pack_into("<HH", content, end + 8, entries, entries)
|
||||
with pytest.raises(zipfile.BadZipFile, match="directory bounds|directory length mismatch"):
|
||||
open_zip_with_replacement_names(bytes(content))
|
||||
@@ -124,7 +124,8 @@ def _agent_node(node_id: str, binding: object | None = None) -> dict:
|
||||
return {"id": node_id, "data": data}
|
||||
|
||||
|
||||
def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -> None:
|
||||
@pytest.mark.parametrize("include_assets", [False, True])
|
||||
def test_make_portable_agent_package_strips_workspace_credentials_and_assets(include_assets: bool) -> None:
|
||||
soul = AgentSoulConfig.model_validate(
|
||||
{
|
||||
"model": {
|
||||
@@ -164,8 +165,10 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
|
||||
],
|
||||
},
|
||||
"env": {"secret_refs": [{"name": "GLOBAL_TOKEN", "value": "plain-secret", "id": "secret-1"}]},
|
||||
"config_skills": [{"name": "research", "file_kind": "tool_file", "file_id": "skill-file"}],
|
||||
"config_files": [{"name": "guide.md", "file_kind": "upload_file", "file_id": "config-file"}],
|
||||
"config_skills": [{"name": "research", "file_id": "skill-file"}, {"name": "missing", "is_missing": True}],
|
||||
"config_files": [
|
||||
{"name": "guide.md", "file_kind": "upload_file", "file_id": "config-file", "mime_type": "text/markdown"}
|
||||
],
|
||||
"human": {
|
||||
"contacts": [
|
||||
{
|
||||
@@ -179,7 +182,9 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
|
||||
}
|
||||
)
|
||||
|
||||
package = make_portable_agent_package(_agent(), soul)
|
||||
original = soul.model_dump(mode="json")
|
||||
package = make_portable_agent_package(_agent(), soul, include_assets=include_assets)
|
||||
assert soul.model_dump(mode="json") == original
|
||||
serialized = package.model_dump(mode="json")
|
||||
|
||||
assert package.soul.model is not None
|
||||
@@ -189,20 +194,32 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
|
||||
assert package.soul.tools.dify_tools[0].runtime_parameters["upload_file_id"] is None
|
||||
assert package.soul.tools.dify_tools[0].runtime_parameters["api_key"] is None
|
||||
assert package.soul.config_skills[0].name == "research"
|
||||
assert package.soul.config_skills[0].file_id == ""
|
||||
assert package.soul.config_skills[0].is_missing is True
|
||||
assert package.soul.config_skills[0].file_id == ("skill-file" if include_assets else "")
|
||||
assert package.soul.config_skills[0].is_missing is (not include_assets)
|
||||
assert package.soul.config_files[0].name == "guide.md"
|
||||
assert package.soul.config_files[0].file_id == ""
|
||||
assert package.soul.config_files[0].is_missing is True
|
||||
assert [asset.kind for asset in package.omitted_assets] == ["skill", "file"]
|
||||
assert package.soul.config_files[0].file_id == ("config-file" if include_assets else "")
|
||||
assert package.soul.config_files[0].is_missing is (not include_assets)
|
||||
assert package.soul.config_skills[1].is_missing
|
||||
assert package.soul.config_files[0].mime_type == "text/markdown"
|
||||
assert [asset.kind for asset in package.omitted_assets] == (
|
||||
["skill"] if include_assets else ["skill", "skill", "file"]
|
||||
)
|
||||
assert [asset.name for asset in package.omitted_assets] == (
|
||||
["missing"] if include_assets else ["research", "missing", "guide.md"]
|
||||
)
|
||||
assert "plain-secret" not in str(serialized)
|
||||
assert "model-secret" not in str(serialized)
|
||||
assert "tool-secret" not in str(serialized)
|
||||
assert "skill-file" not in str(serialized)
|
||||
assert "config-file" not in str(serialized)
|
||||
assert ("skill-file" in str(serialized)) is include_assets
|
||||
assert ("config-file" in str(serialized)) is include_assets
|
||||
assert package.soul.human.contacts[0].id is None
|
||||
assert package.soul.human.contacts[0].name == "Reviewer"
|
||||
|
||||
package.soul.config_skills[0].description = "Changed"
|
||||
package.soul.config_files[0].name = "changed.md"
|
||||
package.soul.tools.cli_tools[0].env.secret_refs[0].name = "CHANGED"
|
||||
assert soul.model_dump(mode="json") == original
|
||||
|
||||
|
||||
def test_agent_package_round_trips_as_strict_dsl_dto() -> None:
|
||||
package = make_portable_agent_package(_agent(), AgentSoulConfig())
|
||||
@@ -590,22 +607,28 @@ def test_clone_inline_binding_copies_soul(unbound_session: Session) -> None:
|
||||
assert create_kwargs["source"] == AgentSource.WORKFLOW
|
||||
|
||||
|
||||
def test_extract_package_dependencies_covers_model_tools_and_knowledge(
|
||||
def test_extract_package_dependencies_covers_models_features_tools_and_knowledge(
|
||||
monkeypatch: pytest.MonkeyPatch, unbound_session: Session
|
||||
) -> None:
|
||||
model_dependency = Mock(side_effect=lambda provider: f"model:{provider}")
|
||||
tool_dependency = Mock(side_effect=lambda provider: f"tool:{provider}")
|
||||
monkeypatch.setattr(
|
||||
"services.agent.dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
|
||||
"services.agent.dependency_service.DependenciesAnalysisService.analyze_model_provider_dependency",
|
||||
model_dependency,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.agent.dsl_service.DependenciesAnalysisService.analyze_tool_dependency",
|
||||
"services.agent.dependency_service.DependenciesAnalysisService.analyze_tool_dependency",
|
||||
tool_dependency,
|
||||
)
|
||||
soul = AgentSoulConfig.model_validate(
|
||||
{
|
||||
"model": {"plugin_id": "model-plugin", "model_provider": "provider/model", "model": "model"},
|
||||
"app_features": {
|
||||
"suggested_questions_after_answer": {
|
||||
"enabled": True,
|
||||
"model": {"provider": "provider/follow-up", "name": "follow-up-model"},
|
||||
}
|
||||
},
|
||||
"tools": {
|
||||
"dify_tools": [
|
||||
{
|
||||
@@ -631,9 +654,49 @@ def test_extract_package_dependencies_covers_model_tools_and_knowledge(
|
||||
"retrieval": {
|
||||
"mode": "single",
|
||||
"model": {"provider": "provider/retrieval", "name": "embed", "mode": "embedding"},
|
||||
},
|
||||
"metadata_filtering": {
|
||||
"mode": "automatic",
|
||||
"model_config": {
|
||||
"provider": "provider/metadata",
|
||||
"name": "metadata-model",
|
||||
"mode": "chat",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "set-2",
|
||||
"name": "Reranked Set",
|
||||
"datasets": [{"id": "dataset-2", "name": "Reranked Docs"}],
|
||||
"query": {"mode": "user_query", "value": "query"},
|
||||
"retrieval": {
|
||||
"mode": "multiple",
|
||||
"top_k": 5,
|
||||
"reranking_enable": True,
|
||||
"reranking_mode": "reranking_model",
|
||||
"reranking_model": {"provider": "provider/rerank", "model": "rerank"},
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "set-3",
|
||||
"name": "Weighted Set",
|
||||
"datasets": [{"id": "dataset-3", "name": "Weighted Docs"}],
|
||||
"query": {"mode": "user_query", "value": "query"},
|
||||
"retrieval": {
|
||||
"mode": "multiple",
|
||||
"top_k": 5,
|
||||
"reranking_enable": True,
|
||||
"reranking_mode": "weighted_score",
|
||||
"weights": {
|
||||
"vector_setting": {
|
||||
"embedding_provider_name": "provider/embedding",
|
||||
"embedding_model_name": "embedding-model",
|
||||
"vector_weight": 0.7,
|
||||
},
|
||||
"keyword_setting": {"keyword_weight": 0.3},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
@@ -645,13 +708,72 @@ def test_extract_package_dependencies_covers_model_tools_and_knowledge(
|
||||
|
||||
assert dependencies == [
|
||||
"model:provider/model",
|
||||
"model:provider/follow-up",
|
||||
"tool:provider/tool",
|
||||
"tool:plugin-id/fallback-provider",
|
||||
"model:provider/retrieval",
|
||||
"model:provider/metadata",
|
||||
"model:provider/rerank",
|
||||
"model:provider/embedding",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_package_dependencies_ignores_inactive_knowledge_models(
|
||||
monkeypatch: pytest.MonkeyPatch, unbound_session: Session
|
||||
) -> None:
|
||||
model_dependency = Mock(side_effect=lambda provider: f"model:{provider}")
|
||||
monkeypatch.setattr(
|
||||
"services.agent.dependency_service.DependenciesAnalysisService.analyze_model_provider_dependency",
|
||||
model_dependency,
|
||||
)
|
||||
soul = AgentSoulConfig.model_validate(
|
||||
{
|
||||
"knowledge": {
|
||||
"sets": [
|
||||
{
|
||||
"id": "set-1",
|
||||
"name": "Set",
|
||||
"datasets": [{"id": "dataset-1", "name": "Docs"}],
|
||||
"query": {"mode": "user_query", "value": "query"},
|
||||
"retrieval": {
|
||||
"mode": "multiple",
|
||||
"top_k": 5,
|
||||
"reranking_enable": False,
|
||||
"reranking_mode": "weighted_score",
|
||||
"reranking_model": {"provider": "provider/inactive-rerank", "model": "rerank"},
|
||||
"weights": {
|
||||
"vector_setting": {
|
||||
"embedding_provider_name": "provider/inactive-embedding",
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"provider": "provider/inactive-retrieval",
|
||||
"name": "retrieval-model",
|
||||
"mode": "chat",
|
||||
},
|
||||
},
|
||||
"metadata_filtering": {
|
||||
"mode": "disabled",
|
||||
"model_config": {
|
||||
"provider": "provider/inactive-metadata",
|
||||
"name": "metadata-model",
|
||||
"mode": "chat",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
dependencies = AgentDslService(unbound_session).extract_package_dependencies(
|
||||
{"agent_1": make_portable_agent_package(_agent(), soul)}
|
||||
)
|
||||
|
||||
assert dependencies == []
|
||||
model_dependency.assert_not_called()
|
||||
|
||||
|
||||
def test_create_imported_inline_agent_uses_import_provenance(unbound_session: Session) -> None:
|
||||
service = AgentDslService(unbound_session)
|
||||
soul = AgentSoulConfig(config_note="inline")
|
||||
|
||||
@@ -40,6 +40,7 @@ from models.agent_config_entities import (
|
||||
from models.enums import AppStatus, ConversationFromSource, ConversationStatus
|
||||
from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message
|
||||
from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest
|
||||
from models.tools import ToolFile
|
||||
from models.workflow import Workflow, WorkflowType
|
||||
from services.agent import composer_service, roster_service
|
||||
from services.agent.agent_soul_state import agent_soul_has_model
|
||||
@@ -5603,7 +5604,17 @@ class TestWorkflowAgentDraftBindingSync:
|
||||
skill_id="skill-1",
|
||||
priority=0,
|
||||
)
|
||||
session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding])
|
||||
archive_file = ToolFile(
|
||||
user_id="account-1",
|
||||
tenant_id="tenant-1",
|
||||
conversation_id=None,
|
||||
file_key="tools/research.zip",
|
||||
mimetype="application/zip",
|
||||
name="research.zip",
|
||||
size=1,
|
||||
)
|
||||
archive_file.id = "archive-1"
|
||||
session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding, archive_file])
|
||||
session.commit()
|
||||
|
||||
WorkflowAgentPublishService.validate_agent_nodes_for_publish(
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -9,8 +9,8 @@ import zlib
|
||||
|
||||
import pytest
|
||||
|
||||
from services.agent import skill_package_service as skill_package_service_module
|
||||
from services.agent.skill_package_service import NormalizedSkillPackage, SkillPackageError, SkillPackageService
|
||||
from tests.unit_tests.config_override import apply_config_overrides
|
||||
|
||||
_SKILL_MD = """---
|
||||
name: pdf-toolkit
|
||||
@@ -35,36 +35,110 @@ def _normalize(members: dict[str, bytes], *, filename: str = "skill.zip") -> Nor
|
||||
return SkillPackageService().validate_and_normalize(content=_zip(members), filename=filename)
|
||||
|
||||
|
||||
def _archive_members(content: bytes) -> list[str]:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
return sorted(info.filename for info in archive.infolist() if not info.is_dir())
|
||||
def test_normalize_replaces_invalid_utf8_member_names() -> None:
|
||||
original = _zip(
|
||||
{
|
||||
"SKILL.md": _SKILL_MD.encode(),
|
||||
"scripts/\u00e9.py": b"print('keep payload')\n",
|
||||
"assets/\u00f6.txt": b"valid unicode name",
|
||||
"assets/X.txt": b"legacy encoded name",
|
||||
}
|
||||
)
|
||||
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"assets/X.txt", b"assets/\x82.txt")
|
||||
service = SkillPackageService()
|
||||
assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
|
||||
normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
|
||||
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
|
||||
assert archive.read("scripts/\ufffda.py") == b"print('keep payload')\n"
|
||||
assert archive.read("assets/\u00f6.txt") == b"valid unicode name"
|
||||
assert archive.read("assets/\u00e9.txt") == b"legacy encoded name"
|
||||
assert archive.testzip() is None
|
||||
|
||||
|
||||
def test_valid_skill_normalizes_manifest():
|
||||
manifest = _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}).manifest
|
||||
|
||||
assert manifest.name == "pdf-toolkit"
|
||||
assert manifest.description == "Tools for working with PDF files."
|
||||
assert manifest.entry_path == "SKILL.md"
|
||||
assert set(manifest.files) == {"SKILL.md", "scripts/run.py"}
|
||||
assert manifest.size > 0
|
||||
assert len(manifest.hash) == 64
|
||||
def test_normalize_recovers_zip64_skill_names(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with monkeypatch.context() as scoped:
|
||||
scoped.setattr(zipfile, "ZIP64_LIMIT", 0)
|
||||
original = _zip({"SKILL.md": _SKILL_MD.encode(), "scripts/\u00e9.py": b"print('ok')\n"})
|
||||
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
service = SkillPackageService()
|
||||
assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
|
||||
normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
|
||||
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
|
||||
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
|
||||
|
||||
|
||||
def test_validate_and_normalize_accepts_crlf_skill_md():
|
||||
crlf_skill_md = _SKILL_MD.replace("\n", "\r\n")
|
||||
package = _normalize({"SKILL.md": crlf_skill_md.encode()})
|
||||
|
||||
assert package.manifest.name == "pdf-toolkit"
|
||||
assert package.manifest.description == "Tools for working with PDF files."
|
||||
assert b"\r" not in package.skill_md_bytes
|
||||
assert package.skill_md_bytes.decode() == _SKILL_MD
|
||||
def test_replacement_member_names_must_remain_unique() -> None:
|
||||
original = _zip(
|
||||
{
|
||||
"SKILL.md": _SKILL_MD.encode(),
|
||||
"scripts/\u00e9.py": b"one",
|
||||
"scripts/\u00f6.py": b"two",
|
||||
}
|
||||
)
|
||||
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"\xc3\xb6.py", b"\xfea.py")
|
||||
with pytest.raises(SkillPackageError) as error:
|
||||
SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
|
||||
assert error.value.code == "duplicate_member_path"
|
||||
|
||||
|
||||
def test_name_and_description_are_required_in_frontmatter():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": b"# heading-name\n\nbody"})
|
||||
assert exc_info.value.code == "missing_skill_name"
|
||||
def test_replacement_names_do_not_hide_local_header_mismatch() -> None:
|
||||
original = _zip({"SKILL.md": _SKILL_MD.encode(), "\u00e9.py": b"payload"})
|
||||
# Damage only the central name, leaving the local header unchanged.
|
||||
central = original.index(b"PK\x01\x02")
|
||||
damaged = original[:central] + original[central:].replace(b"\xc3\xa9.py", b"\xffa.py")
|
||||
with pytest.raises(SkillPackageError) as error:
|
||||
SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
|
||||
assert error.value.code == "invalid_archive"
|
||||
|
||||
|
||||
def test_normalized_highly_compressible_skill_passes_reinspection() -> None:
|
||||
payload = b"x" * (1024 * 1024)
|
||||
content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": payload}, compression=zipfile.ZIP_STORED)
|
||||
service = SkillPackageService()
|
||||
normalized = service.validate_and_normalize(content=content, filename="skill.zip")
|
||||
inspection = service.inspect(content=normalized.archive_bytes, filename="skill.zip")
|
||||
assert inspection.name == normalized.manifest.name
|
||||
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
|
||||
assert archive.getinfo("data.bin").compress_type == zipfile.ZIP_DEFLATED
|
||||
assert archive.read("data.bin") == payload
|
||||
|
||||
|
||||
def test_inspect_reports_uncompressed_size_without_rebuilding(monkeypatch: pytest.MonkeyPatch):
|
||||
members = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}
|
||||
|
||||
def fail_if_called(*_args: object, **_kwargs: object) -> bytes:
|
||||
raise AssertionError("inspection must not rebuild the archive")
|
||||
|
||||
monkeypatch.setattr(SkillPackageService, "_build_normalized_archive", fail_if_called)
|
||||
|
||||
inspection = SkillPackageService().inspect(content=_zip(members), filename="skill.zip")
|
||||
|
||||
assert inspection.name == "pdf-toolkit"
|
||||
assert inspection.uncompressed_size == sum(len(content) for content in members.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["", "pdf-toolkit/"])
|
||||
@pytest.mark.parametrize("crlf", [False, True])
|
||||
def test_inspection_and_normalization_keep_distinct_size_meanings(prefix: str, crlf: bool) -> None:
|
||||
skill_md = _SKILL_MD.replace("\n", "\r\n") if crlf else _SKILL_MD
|
||||
script = b"print('hi')\n"
|
||||
members = {f"{prefix}SKILL.md": skill_md.encode(), f"{prefix}scripts/run.py": script}
|
||||
if prefix:
|
||||
members["README.md"] = b"ignored outside the selected root"
|
||||
content = _zip(members)
|
||||
service = SkillPackageService()
|
||||
inspection = service.inspect(content=content, filename="skill.zip")
|
||||
normalized = service.validate_and_normalize(content=content, filename="skill.zip")
|
||||
assert inspection.uncompressed_size == sum(len(value) for value in members.values())
|
||||
expected_size = len(_SKILL_MD.encode()) + len(script)
|
||||
assert normalized.manifest.size == expected_size
|
||||
assert normalized.manifest.name == "pdf-toolkit"
|
||||
assert normalized.manifest.description == "Tools for working with PDF files."
|
||||
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
|
||||
assert sum(info.file_size for info in archive.infolist()) == expected_size
|
||||
assert archive.read("SKILL.md") == _SKILL_MD.encode()
|
||||
if prefix or crlf:
|
||||
assert inspection.uncompressed_size > normalized.manifest.size
|
||||
|
||||
|
||||
def test_shallowest_skill_md_preferred_during_normalization():
|
||||
@@ -73,96 +147,40 @@ def test_shallowest_skill_md_preferred_during_normalization():
|
||||
assert manifest.files == ["SKILL.md", "nested/SKILL.md"]
|
||||
|
||||
|
||||
def test_validate_and_normalize_keeps_root_skill_unchanged():
|
||||
package = _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"})
|
||||
@pytest.mark.parametrize(
|
||||
("prefix", "extra_members"),
|
||||
[
|
||||
pytest.param("", {}, id="root"),
|
||||
pytest.param("pdf-toolkit/", {}, id="single-folder"),
|
||||
pytest.param("pdf-toolkit/", {"README.md": b"bundle notes\n"}, id="root-outsider"),
|
||||
pytest.param("pdf-toolkit/", {".": b"x"}, id="dot-outsider"),
|
||||
pytest.param("pdf-toolkit/", {"bundle/other.txt": b"x"}, id="nested-outsider"),
|
||||
pytest.param("bundle/pdf-toolkit/", {}, id="deep-root"),
|
||||
],
|
||||
)
|
||||
def test_normalization_preserves_selected_skill_contents(prefix: str, extra_members: dict[str, bytes]) -> None:
|
||||
expected = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}
|
||||
original = _zip({**{prefix + path: content for path, content in expected.items()}, **extra_members})
|
||||
package = SkillPackageService().validate_and_normalize(content=original, filename="skill.zip")
|
||||
|
||||
assert package.manifest.name == "pdf-toolkit"
|
||||
assert package.manifest.description == "Tools for working with PDF files."
|
||||
assert package.manifest.size == sum(len(content) for content in expected.values())
|
||||
assert package.manifest.entry_path == "SKILL.md"
|
||||
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.skill_md_bytes == _SKILL_MD.encode()
|
||||
assert package.strip_prefix is None
|
||||
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
|
||||
assert len(package.manifest.hash) == 64
|
||||
|
||||
|
||||
def test_validate_and_normalize_strips_single_top_level_folder():
|
||||
package = _normalize(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
}
|
||||
)
|
||||
|
||||
assert package.manifest.entry_path == "SKILL.md"
|
||||
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.skill_md_bytes == _SKILL_MD.encode()
|
||||
assert package.strip_prefix == "pdf-toolkit/"
|
||||
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
|
||||
|
||||
|
||||
def test_validate_and_normalize_strips_single_top_level_folder_ignoring_other_root_entries():
|
||||
package = _normalize(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
"README.md": b"bundle notes\n",
|
||||
}
|
||||
)
|
||||
|
||||
assert package.manifest.entry_path == "SKILL.md"
|
||||
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.skill_md_bytes == _SKILL_MD.encode()
|
||||
assert package.strip_prefix == "pdf-toolkit/"
|
||||
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
|
||||
|
||||
|
||||
def test_validate_and_normalize_strips_single_top_level_folder_dropping_nested_foreign_paths():
|
||||
package = _normalize(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
"bundle/other.txt": b"x",
|
||||
}
|
||||
)
|
||||
|
||||
assert package.manifest.entry_path == "SKILL.md"
|
||||
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.skill_md_bytes == _SKILL_MD.encode()
|
||||
assert package.strip_prefix == "pdf-toolkit/"
|
||||
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_multiple_depth_2_skill_roots_with_sibling_skill_tree():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
"other-tool/SKILL.md": _SKILL_MD.encode(),
|
||||
}
|
||||
)
|
||||
assert exc_info.value.code == "files_outside_skill_root"
|
||||
|
||||
|
||||
def test_validate_and_normalize_strips_deeper_selected_skill_root():
|
||||
members = {
|
||||
"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"bundle/pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
}
|
||||
original_upload_bytes = _zip(members)
|
||||
package = SkillPackageService().validate_and_normalize(content=original_upload_bytes, filename="skill.zip")
|
||||
|
||||
assert package.manifest.entry_path == "SKILL.md"
|
||||
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.strip_prefix == "bundle/pdf-toolkit/"
|
||||
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
|
||||
assert package.manifest.files == sorted(expected)
|
||||
with zipfile.ZipFile(io.BytesIO(package.archive_bytes)) as archive:
|
||||
assert archive.namelist() == sorted(expected)
|
||||
assert {path: archive.read(path) for path in archive.namelist()} == expected
|
||||
assert package.manifest.hash == hashlib.sha256(package.archive_bytes).hexdigest()
|
||||
assert package.manifest.hash != hashlib.sha256(original_upload_bytes).hexdigest()
|
||||
if prefix:
|
||||
assert package.manifest.hash != hashlib.sha256(original).hexdigest()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("members", "filename", "code"),
|
||||
[
|
||||
({"README.md": b"x"}, "skill.zip", "missing_skill_md"),
|
||||
({"SKILL.md": _SKILL_MD.encode(), ".": b"x"}, "skill.zip", "unsafe_path"),
|
||||
({"SKILL.md": _SKILL_MD.encode()}, "skill.tar", "unsupported_extension"),
|
||||
({"SKILL.md": b""}, "skill.zip", "empty_skill_md"),
|
||||
({"SKILL.md": b"---\ndescription: valid\n---\n# no name here"}, "skill.zip", "missing_skill_name"),
|
||||
@@ -178,6 +196,43 @@ def test_validate_and_normalize_strips_deeper_selected_skill_root():
|
||||
"invalid_skill_description",
|
||||
),
|
||||
({"SKILL.md": b"\xff\xfenot utf8"}, "skill.zip", "skill_md_not_utf8"),
|
||||
pytest.param({"SKILL.md": b"# heading-name\n\nbody"}, "skill.zip", "missing_skill_name", id="no-frontmatter"),
|
||||
pytest.param(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
|
||||
"other-tool/SKILL.md": _SKILL_MD.encode(),
|
||||
},
|
||||
"skill.zip",
|
||||
"files_outside_skill_root",
|
||||
id="multiple-skill-roots",
|
||||
),
|
||||
pytest.param({"../evil.txt": b"x", "SKILL.md": _SKILL_MD.encode()}, "skill.zip", "unsafe_path", id="zip-slip"),
|
||||
pytest.param(
|
||||
{"SKILL.md": b"---\n: : : not yaml\n---\n# x\n"},
|
||||
"skill.zip",
|
||||
"invalid_frontmatter",
|
||||
id="invalid-frontmatter",
|
||||
),
|
||||
pytest.param(
|
||||
{"SKILL.md": b"---\n# heading-wins\nbody"}, "skill.zip", "missing_skill_name", id="unterminated-frontmatter"
|
||||
),
|
||||
pytest.param(
|
||||
{"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(), "README.md": b"x"},
|
||||
"skill.zip",
|
||||
"files_outside_skill_root",
|
||||
id="outside-deep-root",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('x')\n",
|
||||
"pdf-toolkit/scripts/./run.py": b"print('y')\n",
|
||||
},
|
||||
"skill.zip",
|
||||
"duplicate_member_path",
|
||||
id="duplicate-normalized-path",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_invalid_packages_rejected(members: dict[str, bytes], filename: str, code: str):
|
||||
@@ -193,13 +248,6 @@ def test_non_zip_content_rejected():
|
||||
assert exc_info.value.code == "invalid_archive"
|
||||
|
||||
|
||||
def test_zip_slip_member_rejected():
|
||||
payload = _zip({"../evil.txt": b"x", "SKILL.md": _SKILL_MD.encode()})
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
SkillPackageService().validate_and_normalize(content=payload, filename="skill.zip")
|
||||
assert exc_info.value.code == "unsafe_path"
|
||||
|
||||
|
||||
def test_empty_archive_rejected():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
SkillPackageService().validate_and_normalize(content=b"", filename="skill.zip")
|
||||
@@ -207,7 +255,7 @@ def test_empty_archive_rejected():
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_skill_md_too_large(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(skill_package_service_module, "_MAX_SKILL_MD_BYTES", 8)
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_SKILL_MD_BYTES=8)
|
||||
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": _SKILL_MD.encode()})
|
||||
@@ -215,15 +263,15 @@ def test_validate_and_normalize_rejects_skill_md_too_large(monkeypatch: pytest.M
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_too_many_entries(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(skill_package_service_module, "_MAX_ENTRIES", 1)
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_ENTRIES=1)
|
||||
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('x')\n"})
|
||||
_normalize({"SKILL.md": _SKILL_MD.encode(), "assets/": b""})
|
||||
assert exc_info.value.code == "too_many_entries"
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_archive_too_large_uncompressed(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(skill_package_service_module, "_MAX_UNCOMPRESSED_BYTES", 32)
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=32)
|
||||
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"x" * 33})
|
||||
@@ -231,8 +279,6 @@ def test_validate_and_normalize_rejects_archive_too_large_uncompressed(monkeypat
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_archive_too_large_uploaded_bytes(monkeypatch: pytest.MonkeyPatch):
|
||||
from tests.unit_tests.config_override import apply_config_overrides
|
||||
|
||||
apply_config_overrides(monkeypatch, UPLOAD_SKILL_FILE_SIZE_LIMIT=1)
|
||||
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
@@ -240,35 +286,14 @@ def test_validate_and_normalize_rejects_archive_too_large_uploaded_bytes(monkeyp
|
||||
assert exc_info.value.code == "archive_too_large"
|
||||
|
||||
|
||||
def test_bad_frontmatter_yaml_rejected():
|
||||
bad = b"---\n: : : not yaml\n---\n# x\n"
|
||||
def test_high_compression_ratio_uses_absolute_expansion_limit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": b"x" * (1024 * 1024)})
|
||||
service = SkillPackageService()
|
||||
assert service.inspect(content=content, filename="skill.zip").name == "pdf-toolkit"
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024)
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": bad})
|
||||
assert exc_info.value.code == "invalid_frontmatter"
|
||||
|
||||
|
||||
def test_unterminated_frontmatter_rejected():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"SKILL.md": b"---\n# heading-wins\nbody"})
|
||||
assert exc_info.value.code == "missing_skill_name"
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_files_outside_selected_skill_root():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize({"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(), "README.md": b"x"})
|
||||
assert exc_info.value.code == "files_outside_skill_root"
|
||||
|
||||
|
||||
def test_validate_and_normalize_rejects_duplicate_normalized_paths():
|
||||
with pytest.raises(SkillPackageError) as exc_info:
|
||||
_normalize(
|
||||
{
|
||||
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
|
||||
"pdf-toolkit/scripts/run.py": b"print('x')\n",
|
||||
"pdf-toolkit/scripts/./run.py": b"print('y')\n",
|
||||
}
|
||||
)
|
||||
assert exc_info.value.code == "duplicate_member_path"
|
||||
service.inspect(content=content, filename="skill.zip")
|
||||
assert exc_info.value.code == "archive_too_large"
|
||||
|
||||
|
||||
def test_validate_and_normalize_maps_member_decompression_failures_to_invalid_archive(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@@ -366,7 +366,7 @@ def test_push_accepts_tenant_scoped_tool_file_sources_from_different_upload_owne
|
||||
patch.object(
|
||||
service._skill_normalizer,
|
||||
"normalize",
|
||||
return_value=(skill_ref, object()),
|
||||
return_value=skill_ref,
|
||||
),
|
||||
):
|
||||
manifest = service.push(
|
||||
|
||||
@@ -9,11 +9,13 @@ import yaml
|
||||
from sqlalchemy import event, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from constants.dsl_version import CURRENT_APP_DSL_VERSION
|
||||
from core.rbac import RBACPermission, RBACResourceScope
|
||||
from core.workflow.llm_environment_variable import LLMEnvironmentVariable
|
||||
from models import Account, App, AppMode, Tenant
|
||||
from models.model import AppModelConfig, AppModelConfigDict, IconType
|
||||
from models.workflow import Workflow, WorkflowType
|
||||
from services.agent.dsl_entities import AgentPackage
|
||||
from services.app_dsl_service import AppDslService, Import, PendingData
|
||||
from services.enterprise.enterprise_service import EnterpriseService
|
||||
from services.entities.dsl_entities import ImportStatus
|
||||
@@ -795,3 +797,56 @@ def test_append_workflow_export_data_reports_missing_selected_workflow(
|
||||
session=unbound_session,
|
||||
workflow_id=workflow_id,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
[AppMode.AGENT, AppMode.WORKFLOW, AppMode.ADVANCED_CHAT, AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION],
|
||||
)
|
||||
def test_export_dsl_preserves_envelope_and_mode_specific_content(
|
||||
mode: AppMode, monkeypatch: pytest.MonkeyPatch, unbound_session: Session
|
||||
) -> None:
|
||||
app = _app(mode=mode)
|
||||
package = AgentPackage.model_validate({"metadata": {"name": app.name}, "soul": {}})
|
||||
monkeypatch.setattr(
|
||||
"services.app_dsl_service.AgentDslService.export_agent_app",
|
||||
Mock(return_value=("agent_1", {"agent_1": package})),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.app_dsl_service.DependenciesAnalysisService.generate_dependencies", Mock(return_value=[])
|
||||
)
|
||||
|
||||
def append_workflow(*, export_data: dict[str, object], **_kwargs: object) -> None:
|
||||
export_data["workflow"] = {"fixture": "workflow"}
|
||||
|
||||
def append_model(export_data: dict[str, object], *_args: object, **_kwargs: object) -> None:
|
||||
export_data["model_config"] = {"fixture": "model"}
|
||||
|
||||
monkeypatch.setattr(AppDslService, "_append_workflow_export_data", Mock(side_effect=append_workflow))
|
||||
monkeypatch.setattr(AppDslService, "_append_model_config_export_data", Mock(side_effect=append_model))
|
||||
data = yaml.safe_load(AppDslService.export_dsl(app, session=unbound_session))
|
||||
assert data["version"] == CURRENT_APP_DSL_VERSION
|
||||
assert data["kind"] == "app"
|
||||
assert data["app"] == {
|
||||
"name": "Existing app",
|
||||
"description": "",
|
||||
"mode": mode.value,
|
||||
"icon_type": "emoji",
|
||||
"icon": "robot",
|
||||
"icon_background": "#FFFFFF",
|
||||
"use_icon_as_answer_icon": False,
|
||||
}
|
||||
if mode == AppMode.AGENT:
|
||||
assert data["agent"] == {"package_ref": "agent_1"}
|
||||
assert AgentPackage.model_validate(data["agent_packages"]["agent_1"]) == package
|
||||
assert data["dependencies"] == []
|
||||
assert "workflow" not in data
|
||||
assert "model_config" not in data
|
||||
elif mode in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT}:
|
||||
assert data["workflow"] == {"fixture": "workflow"}
|
||||
assert "agent" not in data
|
||||
assert "model_config" not in data
|
||||
else:
|
||||
assert data["model_config"] == {"fixture": "model"}
|
||||
assert "agent" not in data
|
||||
assert "workflow" not in data
|
||||
@@ -2829,21 +2829,27 @@ def test_import_skill_package_rejects_archive_larger_than_upload_skill_limit(mon
|
||||
assert exc_info.value.code == "archive_too_large"
|
||||
|
||||
|
||||
def test_import_skill_package_rejects_zip_bomb_before_reading_members() -> None:
|
||||
def test_import_skill_package_rejects_excessive_expansion_before_reading_members(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024 * 1024)
|
||||
package = io.BytesIO()
|
||||
with zipfile.ZipFile(package, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("payload.bin", b"\x00" * (8 * 1024 * 1024))
|
||||
|
||||
service = SkillManagementService(tool_file_manager=_FakeToolFileManager())
|
||||
|
||||
with pytest.raises(SkillManagementServiceError) as exc_info:
|
||||
with (
|
||||
patch.object(zipfile.ZipFile, "read", side_effect=AssertionError("member read before size validation")),
|
||||
pytest.raises(SkillManagementServiceError) as exc_info,
|
||||
):
|
||||
service._draft_payload_from_zip(
|
||||
tenant_id=TENANT,
|
||||
user_id=USER,
|
||||
archive_bytes=package.getvalue(),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "invalid_skill_package"
|
||||
assert exc_info.value.code == "skill_too_large"
|
||||
|
||||
|
||||
def test_publish_and_export_include_binary_tool_files() -> None:
|
||||
@@ -3273,6 +3279,7 @@ def test_runtime_agent_skills_use_published_identity_when_draft_metadata_changed
|
||||
description="draft description",
|
||||
)
|
||||
version = SimpleNamespace(
|
||||
id="version-1",
|
||||
archive_tool_file_id="archive-1",
|
||||
archive_size=10,
|
||||
hash_code="hash-1",
|
||||
@@ -3283,8 +3290,10 @@ def test_runtime_agent_skills_use_published_identity_when_draft_metadata_changed
|
||||
description="published description",
|
||||
),
|
||||
)
|
||||
binding = SimpleNamespace(priority=0)
|
||||
tool_file = SimpleNamespace(file_key="tools/archive-1.zip")
|
||||
session = MagicMock()
|
||||
session.execute.return_value = [(SimpleNamespace(), skill, version)]
|
||||
session.execute.return_value = [(binding, skill, version, tool_file)]
|
||||
|
||||
with patch("services.skill_management_service.session_factory.create_session") as create_session:
|
||||
create_session.return_value.__enter__.return_value = session
|
||||
@@ -3358,6 +3367,29 @@ def test_agent_skill_binding_changes_require_agent_publish_before_runtime_load()
|
||||
|
||||
assert service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT)[0]["name"] == "finance-sop"
|
||||
|
||||
with session_factory.create_session() as session:
|
||||
agent = session.get(Agent, AGENT)
|
||||
assert agent is not None
|
||||
next_snapshot = AgentConfigSnapshot(
|
||||
tenant_id=TENANT,
|
||||
agent_id=AGENT,
|
||||
version=2,
|
||||
config_snapshot=AgentSoulConfig(),
|
||||
created_by=USER,
|
||||
)
|
||||
session.add(next_snapshot)
|
||||
session.flush()
|
||||
agent.active_config_snapshot_id = next_snapshot.id
|
||||
session.commit()
|
||||
|
||||
assert service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT) == []
|
||||
historical_skills = service.list_runtime_agent_skills(
|
||||
tenant_id=TENANT, agent_id=AGENT, config_snapshot_id=snapshot_id
|
||||
)
|
||||
assert [skill["name"] for skill in historical_skills] == ["finance-sop"]
|
||||
draft_skills = service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT, include_draft=True)
|
||||
assert [skill["name"] for skill in draft_skills] == ["finance-sop"]
|
||||
|
||||
|
||||
def test_runtime_agent_skill_pull_normalizes_archive_identity_to_published_metadata() -> None:
|
||||
archive_buffer = io.BytesIO()
|
||||
@@ -3378,3 +3410,14 @@ def test_runtime_agent_skill_pull_normalizes_archive_identity_to_published_metad
|
||||
assert "description: Published description" in skill_md
|
||||
assert "display-name: Published Name" in skill_md
|
||||
assert archive.read("references/example.md") == b"Example"
|
||||
|
||||
|
||||
def test_archive_limits_accept_high_compression_ratio_within_size_budget(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
output = io.BytesIO()
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("data.bin", b"x" * (1024 * 1024))
|
||||
with zipfile.ZipFile(io.BytesIO(output.getvalue())) as archive:
|
||||
assert SkillManagementService._validate_archive_limits(archive)[0].file_size == 1024 * 1024
|
||||
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024)
|
||||
with pytest.raises(SkillManagementServiceError, match="uncompressed size limit"):
|
||||
SkillManagementService._validate_archive_limits(archive)
|
||||
@@ -66,6 +66,12 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
|
||||
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
|
||||
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
|
||||
UPLOAD_SKILL_FILE_SIZE_LIMIT=50
|
||||
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=209715200
|
||||
SKILL_PACKAGE_MAX_ENTRIES=5000
|
||||
SKILL_PACKAGE_MAX_SKILL_MD_BYTES=1048576
|
||||
AGENT_PACKAGE_MAX_BYTES=536870912
|
||||
AGENT_PACKAGE_MAX_MANIFEST_BYTES=5242880
|
||||
AGENT_PACKAGE_MAX_ENTRIES=5000
|
||||
API_SENTRY_DSN=
|
||||
API_SENTRY_TRACES_SAMPLE_RATE=1.0
|
||||
API_SENTRY_PROFILES_SAMPLE_RATE=1.0
|
||||
|
||||
Reference in new issue
Block a user