feat(agent): add roster package export foundation (#42066)

This commit is contained in:
林玮 (Jade Lin) authored and GitHub committed 2026-09-17 07:05:02 +00:00
1 parent 5e9017dbb8
commit 93895fa7fb
26 files changed
+3549 -370

No files matched your search

+6
View File
@@ -460,6 +460,12 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
UPLOAD_SKILL_FILE_SIZE_LIMIT=50
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=209715200
SKILL_PACKAGE_MAX_ENTRIES=5000
SKILL_PACKAGE_MAX_SKILL_MD_BYTES=1048576
AGENT_PACKAGE_MAX_BYTES=536870912
AGENT_PACKAGE_MAX_MANIFEST_BYTES=5242880
AGENT_PACKAGE_MAX_ENTRIES=5000
# Comma-separated list of file extensions blocked from upload for security reasons.
# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll).
+30
View File
@@ -481,6 +481,36 @@ class FileUploadConfig(BaseSettings):
default=50,
)
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES: PositiveInt = Field(
description="Maximum total uncompressed size of a Skill package in bytes",
default=200 * 1024 * 1024,
)
SKILL_PACKAGE_MAX_ENTRIES: PositiveInt = Field(
description="Maximum number of entries in a Skill package",
default=5000,
)
SKILL_PACKAGE_MAX_SKILL_MD_BYTES: PositiveInt = Field(
description="Maximum allowed SKILL.md size in bytes",
default=1 * 1024 * 1024,
)
AGENT_PACKAGE_MAX_BYTES: PositiveInt = Field(
description="Maximum allowed Agent package size in bytes",
default=512 * 1024 * 1024,
)
AGENT_PACKAGE_MAX_MANIFEST_BYTES: PositiveInt = Field(
description="Maximum allowed size of each Agent package YAML document in bytes",
default=5 * 1024 * 1024,
)
AGENT_PACKAGE_MAX_ENTRIES: PositiveInt = Field(
description="Maximum number of entries in an Agent package",
default=5000,
)
BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
description="Maximum number of files allowed in a batch upload operation",
default=20,
+158
View File
@@ -0,0 +1,158 @@
"""Read malformed UTF-8 ZIP names without weakening payload integrity checks."""
from __future__ import annotations
import io
import struct
import zipfile
from dataclasses import dataclass
_UTF8_FLAG = 1 << 11
_END_RECORD = struct.Struct("<4s4H2IH")
_CENTRAL_HEADER = struct.Struct("<4s6H3I5H2I")
_LOCAL_HEADER = struct.Struct("<4s5H3I2H")
_ZIP64_LOCATOR = struct.Struct("<4sIQI")
_ZIP64_END_RECORD = struct.Struct("<4sQ2H2I4Q")
@dataclass(frozen=True)
class _Directory:
start: int
end: int
entries: int
prefix_size: int
def _locate_directory(content: bytes) -> _Directory:
end = content.rfind(b"PK\x05\x06", max(0, len(content) - 65535 - _END_RECORD.size))
if end < 0 or end + _END_RECORD.size > len(content):
raise zipfile.BadZipFile("Missing ZIP end record")
_, disk, directory_disk, disk_entries, entries, size, offset, comment_size = _END_RECORD.unpack_from(content, end)
if end + _END_RECORD.size + comment_size != len(content):
raise zipfile.BadZipFile("Invalid ZIP comment length")
locator_position = end - _ZIP64_LOCATOR.size
if locator_position >= 0 and content[locator_position : locator_position + 4] == b"PK\x06\x07":
_, record_disk, record_offset, disks = _ZIP64_LOCATOR.unpack_from(content, locator_position)
if record_disk != 0 or disks != 1:
raise zipfile.BadZipFile("Multi-disk ZIP64 archives are not supported")
record_position = record_offset
if record_position + _ZIP64_END_RECORD.size > locator_position:
raise zipfile.BadZipFile("Invalid ZIP64 end record offset")
if content[record_position : record_position + 4] != b"PK\x06\x06":
# Match zipfile's handling of prepended data: in this case it
# locates the fixed-size ZIP64 record immediately before the locator.
record_position = locator_position - _ZIP64_END_RECORD.size
if record_position < 0:
raise zipfile.BadZipFile("Missing ZIP64 end record")
signature, record_size, _, _, disk, directory_disk, disk_entries, entries, size, offset = (
_ZIP64_END_RECORD.unpack_from(content, record_position)
)
if (
signature != b"PK\x06\x06"
or record_size + 12 != locator_position - record_position
or offset + size != record_offset
):
raise zipfile.BadZipFile("Invalid ZIP64 end record")
end = record_position
elif entries == 0xFFFF or size == 0xFFFFFFFF or offset == 0xFFFFFFFF:
raise zipfile.BadZipFile("Missing ZIP64 directory information")
start = end - size
prefix_size = start - offset
if disk != 0 or directory_disk != 0 or disk_entries != entries:
raise zipfile.BadZipFile("Multi-disk ZIP archives are not supported")
if start < 0 or prefix_size < 0 or entries > size // _CENTRAL_HEADER.size:
raise zipfile.BadZipFile("Invalid ZIP directory bounds")
return _Directory(start=start, end=end, entries=entries, prefix_size=prefix_size)
def _local_header_offset(*, offset: int, file_size: int, compressed_size: int, extra: bytes) -> int:
if offset != 0xFFFFFFFF:
return offset
cursor = 0
while cursor + 4 <= len(extra):
tag, size = struct.unpack_from("<HH", extra, cursor)
cursor += 4
if cursor + size > len(extra):
raise zipfile.BadZipFile("Truncated ZIP extra field")
if tag == 1:
# ZIP64 includes only values whose corresponding 32-bit field
# contains the sentinel, in size/compressed-size/offset order.
skipped = 8 * (int(file_size == 0xFFFFFFFF) + int(compressed_size == 0xFFFFFFFF))
if skipped + 8 > size:
raise zipfile.BadZipFile("Missing ZIP64 local header offset")
return struct.unpack_from("<Q", extra, cursor + skipped)[0]
cursor += size
raise zipfile.BadZipFile("Missing ZIP64 extra field")
def open_zip_with_replacement_names(content: bytes) -> zipfile.ZipFile:
"""Open a bounded ZIP, replacing invalid UTF-8 name bytes with U+FFFD.
The normal path delegates entirely to zipfile. For a damaged directory,
temporarily read affected names as CP437 (a reversible byte mapping), and
expose their lossy UTF-8 form through ZipInfo.filename. orig_filename stays
byte-preserving so zipfile still checks local/central name consistency.
Callers must reject collisions and unsafe paths in the recovered names.
"""
try:
return zipfile.ZipFile(io.BytesIO(content))
except UnicodeDecodeError:
pass
directory = _locate_directory(content)
patched = bytearray(content)
recovered: dict[int, str] = {}
cursor = directory.start
for index in range(directory.entries):
if cursor + _CENTRAL_HEADER.size > directory.end:
raise zipfile.BadZipFile("Truncated ZIP directory")
header = _CENTRAL_HEADER.unpack_from(content, cursor)
if header[0] != b"PK\x01\x02":
raise zipfile.BadZipFile("Invalid ZIP directory header")
flags, name_size, extra_size, entry_comment_size = header[3], header[10], header[11], header[12]
name_start = cursor + _CENTRAL_HEADER.size
next_cursor = name_start + name_size + extra_size + entry_comment_size
if next_cursor > directory.end:
raise zipfile.BadZipFile("Truncated ZIP directory name")
raw_name = content[name_start : name_start + name_size]
if flags & _UTF8_FLAG:
try:
raw_name.decode("utf-8")
except UnicodeDecodeError:
extra_start = name_start + name_size
local_offset = (
_local_header_offset(
offset=header[16],
file_size=header[9],
compressed_size=header[8],
extra=content[extra_start : extra_start + extra_size],
)
+ directory.prefix_size
)
if local_offset < 0 or local_offset + _LOCAL_HEADER.size > directory.start:
raise zipfile.BadZipFile("Invalid ZIP local header offset")
local = _LOCAL_HEADER.unpack_from(content, local_offset)
local_name_start = local_offset + _LOCAL_HEADER.size
if (
local[0] != b"PK\x03\x04"
or local[9] != name_size
or content[local_name_start : local_name_start + local[9]] != raw_name
or not local[2] & _UTF8_FLAG
):
raise zipfile.BadZipFile("ZIP local and directory names differ")
struct.pack_into("<H", patched, cursor + 8, flags & ~_UTF8_FLAG)
struct.pack_into("<H", patched, local_offset + 6, local[2] & ~_UTF8_FLAG)
recovered[index] = raw_name.decode("utf-8", errors="replace")
cursor = next_cursor
if cursor != directory.end:
raise zipfile.BadZipFile("ZIP directory length mismatch")
archive = zipfile.ZipFile(io.BytesIO(patched))
for index, info in enumerate(archive.infolist()):
if index in recovered:
info.filename = recovered[index]
archive.NameToInfo = {info.filename: info for info in archive.infolist()}
return archive
@@ -9,7 +9,7 @@ from __future__ import annotations
from core.tools.tool_file_manager import ToolFileManager
from models.agent_config_entities import AgentConfigSkillRefConfig, validate_config_skill_name
from services.agent.skill_package_service import NormalizedSkillPackage, SkillPackageError, SkillPackageService
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
class ConfigSkillNormalizeService:
@@ -32,7 +32,7 @@ class ConfigSkillNormalizeService:
requested_name: str | None,
tenant_id: str,
user_id: str,
) -> tuple[AgentConfigSkillRefConfig, NormalizedSkillPackage]:
) -> AgentConfigSkillRefConfig:
package = self._package.validate_and_normalize(content=content, filename=filename)
normalized_name = validate_config_skill_name(requested_name or package.manifest.name)
if package.manifest.name != normalized_name:
@@ -50,16 +50,13 @@ class ConfigSkillNormalizeService:
mimetype="application/zip",
filename=f"{normalized_name}.zip",
)
return (
AgentConfigSkillRefConfig(
name=normalized_name,
description=package.manifest.description,
file_id=tool_file.id,
size=tool_file.size,
hash=package.manifest.hash,
mime_type=tool_file.mimetype,
),
package,
return AgentConfigSkillRefConfig(
name=normalized_name,
description=package.manifest.description,
file_id=tool_file.id,
size=tool_file.size,
hash=package.manifest.hash,
mime_type=tool_file.mimetype,
)
+65
View File
@@ -0,0 +1,65 @@
"""Plugin dependency discovery shared by portable Agent formats."""
from models.agent_config_entities import (
AgentKnowledgeMetadataFilteringConfig,
AgentKnowledgeRetrievalConfig,
AgentSoulConfig,
)
from services.plugin.dependencies_analysis import DependenciesAnalysisService
def _extract_knowledge_retrieval_dependencies(retrieval: AgentKnowledgeRetrievalConfig) -> list[str]:
if retrieval.mode == "single":
if retrieval.model is None:
return []
return [DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.model.provider)]
if not retrieval.reranking_enable:
return []
if retrieval.reranking_mode == "reranking_model":
if retrieval.reranking_model is None:
return []
return [DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.reranking_model.provider)]
if retrieval.reranking_mode == "weighted_score" and retrieval.weights is not None:
vector_setting = retrieval.weights.vector_setting
embedding_provider = vector_setting.get("embedding_provider_name") if vector_setting is not None else None
if embedding_provider:
return [DependenciesAnalysisService.analyze_model_provider_dependency(embedding_provider)]
return []
def _extract_metadata_filtering_dependencies(
metadata_filtering: AgentKnowledgeMetadataFilteringConfig,
) -> list[str]:
if metadata_filtering.mode != "automatic" or metadata_filtering.metadata_model_config is None:
return []
return [
DependenciesAnalysisService.analyze_model_provider_dependency(metadata_filtering.metadata_model_config.provider)
]
def extract_agent_soul_dependencies(soul: AgentSoulConfig) -> list[str]:
dependencies: list[str] = []
if soul.model is not None:
dependencies.append(DependenciesAnalysisService.analyze_model_provider_dependency(soul.model.model_provider))
suggested_questions = soul.app_features.suggested_questions_after_answer
if suggested_questions is not None and suggested_questions.model is not None:
dependencies.append(
DependenciesAnalysisService.analyze_model_provider_dependency(suggested_questions.model.provider)
)
for tool in soul.tools.dify_tools:
provider_id = tool.provider_id or (
f"{tool.plugin_id}/{tool.provider}" if tool.plugin_id and tool.provider else None
)
if provider_id:
dependencies.append(DependenciesAnalysisService.analyze_tool_dependency(provider_id))
for knowledge_set in soul.knowledge.sets:
dependencies.extend(_extract_knowledge_retrieval_dependencies(knowledge_set.retrieval))
dependencies.extend(_extract_metadata_filtering_dependencies(knowledge_set.metadata_filtering))
return dependencies
__all__ = ["extract_agent_soul_dependencies"]
+107 -35
View File
@@ -3,12 +3,15 @@
from __future__ import annotations
import hashlib
from typing import Any, Literal
from typing import Any, Literal, Self
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, model_validator
from core.plugin.entities.plugin import PluginDependency
from models.agent import Agent
from models.agent_config_entities import AgentSoulConfig
from models.model import App
from services.entities.dsl_entities import make_app_dsl
AGENT_PACKAGE_SCHEMA_VERSION = 1
AGENT_PACKAGE_REF_KEY = "package_ref"
@@ -59,6 +62,54 @@ class AgentPackage(BaseModel):
workspace_skills: list[AgentPackageWorkspaceSkill] = Field(default_factory=list)
class AgentAppReference(BaseModel):
model_config = ConfigDict(extra="forbid")
package_ref: str = Field(min_length=1)
class AgentAppDsl(BaseModel):
"""The existing standalone Agent App DSL, also stored as archive app.yaml."""
model_config = ConfigDict(extra="forbid")
version: str = Field(min_length=1)
kind: Literal["app"]
app: dict[str, Any]
agent: AgentAppReference
agent_packages: dict[str, AgentPackage]
dependencies: list[PluginDependency] = Field(default_factory=list)
@model_validator(mode="after")
def validate_agent_app(self) -> Self:
if self.app.get("mode") != "agent":
raise ValueError("Agent App DSL requires app.mode=agent")
if not isinstance(self.app.get("name"), str) or not self.app["name"]:
raise ValueError("Agent App DSL requires an app name")
if set(self.agent_packages) != {self.agent.package_ref}:
raise ValueError("Agent App DSL must contain exactly the referenced Agent package")
return self
@property
def package(self) -> AgentPackage:
return self.agent_packages[self.agent.package_ref]
def make_agent_app_dsl(
app: App,
*,
package_ref: str,
packages: dict[str, AgentPackage],
dependencies: list[PluginDependency],
) -> AgentAppDsl:
return AgentAppDsl(
**make_app_dsl(app),
agent=AgentAppReference(package_ref=package_ref),
agent_packages=packages,
dependencies=dependencies,
)
def portable_ref(prefix: str, value: str) -> str:
digest = hashlib.sha256(value.encode()).hexdigest()[:16]
return f"{prefix}-{digest}"
@@ -87,40 +138,15 @@ def _strip_sensitive_values(value: Any) -> Any:
return result
def make_portable_agent_package(
agent: Agent,
agent_soul: AgentSoulConfig,
workspace_skills: list[AgentPackageWorkspaceSkill] | None = None,
) -> AgentPackage:
"""Return a package safe to place in YAML or the system clipboard."""
def make_portable_agent_soul(agent_soul: AgentSoulConfig) -> AgentSoulConfig:
"""Return an Agent Soul without workspace-local credentials or identities.
Resource references are intentionally preserved. Container formats decide
whether to omit their payloads, as YAML DSL does, or remap them to
package-local identifiers, as Roster packages do.
"""
soul_data = agent_soul.model_dump(mode="json")
omitted_assets = [
AgentPackageOmittedAsset(
kind="skill",
name=item.name,
size=item.size,
hash=item.hash,
mime_type=item.mime_type,
)
for item in agent_soul.config_skills
]
omitted_assets.extend(
AgentPackageOmittedAsset(
kind="file",
name=item.name,
size=item.size,
hash=item.hash,
mime_type=item.mime_type,
)
for item in agent_soul.config_files
)
for item in soul_data.get("config_skills", []):
item["file_id"] = ""
item["is_missing"] = True
for item in soul_data.get("config_files", []):
item["file_id"] = ""
item["is_missing"] = True
if soul_data.get("model"):
soul_data["model"]["credential_ref"] = None
@@ -155,7 +181,53 @@ def make_portable_agent_package(
for key in ("id", "contact_id", "human_id", "tenant_id"):
contact[key] = None
portable_soul = AgentSoulConfig.model_validate(soul_data)
return AgentSoulConfig.model_validate(soul_data)
def make_portable_agent_package(
agent: Agent,
agent_soul: AgentSoulConfig,
workspace_skills: list[AgentPackageWorkspaceSkill] | None = None,
*,
include_assets: bool = False,
) -> AgentPackage:
"""Return a portable package for YAML, clipboard, or an asset-bearing archive.
Archives preserve package-local references to included assets; standalone
YAML and clipboard exports mark resources missing because they omit payloads.
"""
portable_soul = make_portable_agent_soul(agent_soul)
omitted_assets = [
AgentPackageOmittedAsset(
kind="skill",
name=item.name,
size=item.size,
hash=item.hash,
mime_type=item.mime_type,
)
for item in agent_soul.config_skills
if not include_assets or item.is_missing
]
omitted_assets.extend(
AgentPackageOmittedAsset(
kind="file",
name=item.name,
size=item.size,
hash=item.hash,
mime_type=item.mime_type,
)
for item in agent_soul.config_files
if not include_assets or item.is_missing
)
for skill_ref in portable_soul.config_skills:
if not include_assets or skill_ref.is_missing:
skill_ref.file_id = ""
skill_ref.is_missing = True
for file_ref in portable_soul.config_files:
if not include_assets or file_ref.is_missing:
file_ref.file_id = ""
file_ref.is_missing = True
icon_type = agent.icon_type.value if agent.icon_type is not None else None
return AgentPackage(
metadata=AgentPackageMetadata(
+2 -24
View File
@@ -41,6 +41,7 @@ from models.model import App, AppModelConfig
from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot, Skill
from models.workflow import Workflow
from services.agent.agent_soul_state import agent_soul_has_model
from services.agent.dependency_service import extract_agent_soul_dependencies
from services.agent.dsl_entities import (
AGENT_NODE_JOB_DSL_KEY,
AGENT_PACKAGE_REF_KEY,
@@ -53,7 +54,6 @@ from services.agent.dsl_entities import (
from services.agent.knowledge_datasets import get_tenant_knowledge_dataset_rows
from services.agent.roster_service import AgentRosterService
from services.entities.dsl_entities import DslImportWarning
from services.plugin.dependencies_analysis import DependenciesAnalysisService
class AgentPackageImportResult(BaseModel):
@@ -375,29 +375,7 @@ class AgentDslService:
def extract_package_dependencies(self, packages: Mapping[str, AgentPackage]) -> list[str]:
dependencies: list[str] = []
for package in packages.values():
soul = package.soul
if soul.model is not None:
dependencies.append(
DependenciesAnalysisService.analyze_model_provider_dependency(soul.model.model_provider)
)
for tool in soul.tools.dify_tools:
provider_id = tool.provider_id or (
f"{tool.plugin_id}/{tool.provider}" if tool.plugin_id and tool.provider else None
)
if provider_id:
dependencies.append(DependenciesAnalysisService.analyze_tool_dependency(provider_id))
for knowledge_set in soul.knowledge.sets:
retrieval = knowledge_set.retrieval
if retrieval.model is not None:
dependencies.append(
DependenciesAnalysisService.analyze_model_provider_dependency(retrieval.model.provider)
)
if retrieval.reranking_model is not None:
dependencies.append(
DependenciesAnalysisService.analyze_model_provider_dependency(
retrieval.reranking_model.provider
)
)
dependencies.extend(extract_agent_soul_dependencies(package.soul))
return dependencies
def _workspace_skills_for_export(
+18
View File
@@ -53,6 +53,24 @@ class AgentHomeSnapshotTooLargeError(BaseHTTPException):
code = 413
class InvalidRosterAgentPackageError(BaseHTTPException):
error_code = "invalid_roster_agent_package"
description = "The Roster Agent package is invalid."
code = 400
class RosterAgentPackageTooLargeError(BaseHTTPException):
error_code = "roster_agent_package_too_large"
description = "The Roster Agent package exceeds the size limit."
code = 413
class RosterAgentPackageExportFailedError(BaseHTTPException):
error_code = "roster_agent_package_export_failed"
description = "The Roster Agent package could not be exported."
code = 500
class AgentSoulLockedError(BadRequest):
description = "Agent Soul is locked for this workflow node."
@@ -0,0 +1,202 @@
"""Strict data contract for portable Roster Agent archives."""
from __future__ import annotations
import re
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import BinaryIO, Final, Literal, Self
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from services.agent.dsl_entities import AgentAppDsl
ROSTER_AGENT_PACKAGE_FORMAT: Final[Literal["dify.roster-agent"]] = "dify.roster-agent"
ROSTER_AGENT_PACKAGE_FORMAT_VERSION: Final[Literal[1]] = 1
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES = 64 * 1024
_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
@dataclass(frozen=True)
class RosterAgentPackageMember:
size: int
sha256: str
class RosterAgentPackageAudit(BaseModel):
model_config = ConfigDict(extra="forbid")
ref: str = Field(min_length=1, max_length=255)
class _RosterAgentPackageResource(BaseModel):
model_config = ConfigDict(extra="forbid")
path: str = Field(min_length=1, max_length=255)
size: int = Field(ge=0)
sha256: str
audit: RosterAgentPackageAudit | None = None
@field_validator("sha256")
@classmethod
def validate_sha256(cls, value: str) -> str:
if _SHA256_PATTERN.fullmatch(value) is None:
raise ValueError("sha256 must be a lowercase hexadecimal SHA-256 digest")
return value
class RosterAgentPackageApp(_RosterAgentPackageResource):
@field_validator("path")
@classmethod
def validate_app_path(cls, value: str) -> str:
if "/" in value or "\\" in value or not value.endswith((".yaml", ".yml")):
raise ValueError("app path must be a root-level YAML member")
if value.casefold() == "manifest.yaml":
raise ValueError("app path must not use the package manifest name")
return value
class RosterAgentPackageSkill(_RosterAgentPackageResource):
id: str = Field(pattern=r"^s_[0-9]{6}$")
scope: Literal["agent_config", "workspace"]
name: str = Field(min_length=1, max_length=64)
@model_validator(mode="after")
def validate_skill_path(self) -> Self:
if self.path != f"{self.id}.zip":
raise ValueError("skill path must be the resource id with a .zip extension")
return self
class RosterAgentPackageFile(_RosterAgentPackageResource):
id: str = Field(pattern=r"^f_[0-9]{6}$")
@model_validator(mode="after")
def validate_file_metadata(self) -> Self:
if self.path == self.id or not self.path.startswith(f"{self.id}."):
raise ValueError("file path must start with the resource id and include an extension")
if "/" in self.path or "\\" in self.path:
raise ValueError("file path must be a root-level archive member")
return self
class RosterAgentPackageManifest(BaseModel):
"""Package metadata and resource index stored in ``manifest.yaml``."""
model_config = ConfigDict(extra="forbid")
format: Literal["dify.roster-agent"]
format_version: Literal[1]
audit: RosterAgentPackageAudit | None = None
apps: list[RosterAgentPackageApp] = Field(min_length=1)
skills: list[RosterAgentPackageSkill] = Field(default_factory=list)
files: list[RosterAgentPackageFile] = Field(default_factory=list)
@model_validator(mode="after")
def validate_resource_index(self) -> Self:
ids = [item.id for item in self.skills] + [item.id for item in self.files]
if len(ids) != len(set(ids)):
raise ValueError("resource ids must be unique")
paths = [item.path.casefold() for item in [*self.apps, *self.skills, *self.files]]
if len(paths) != len(set(paths)):
raise ValueError("resource paths must be unique")
skill_names = [item.name for item in self.skills]
if len(skill_names) != len(set(skill_names)):
raise ValueError("skill names must be unique after workspace Skills are localized")
return self
def validate_apps(self, apps: Mapping[str, AgentAppDsl]) -> None:
"""Resolve all application references against the shared resource index."""
if set(apps) != {item.path for item in self.apps}:
raise ValueError("app members must match the package app index")
skill_by_id = {item.id: item for item in self.skills}
file_by_id = {item.id: item for item in self.files}
workspace_names = {item.name for item in self.skills if item.scope == "workspace"}
referenced_skill_ids: set[str] = set()
referenced_file_ids: set[str] = set()
referenced_workspace_names: set[str] = set()
for app in apps.values():
if app.package.soul.schema_version != 1:
raise ValueError("unsupported Agent Soul schema version")
for skill_ref in app.package.soul.config_skills:
if skill_ref.is_missing:
continue
skill_resource = skill_by_id.get(skill_ref.file_id)
if skill_resource is None or skill_resource.scope != "agent_config":
raise ValueError("config skill reference must resolve to an agent_config skill")
if skill_resource.name != skill_ref.name:
raise ValueError("config skill name must match its resource metadata")
referenced_skill_ids.add(skill_resource.id)
app_workspace_names = [item.name for item in app.package.workspace_skills]
if len(app_workspace_names) != len(set(app_workspace_names)):
raise ValueError("workspace skill names must be unique in the Agent DSL")
if set(app_workspace_names) - workspace_names:
raise ValueError("workspace skill references must match the package resource index")
referenced_workspace_names.update(app_workspace_names)
for file_ref in app.package.soul.config_files:
if file_ref.is_missing:
continue
if file_ref.file_id not in file_by_id:
raise ValueError("config file reference must resolve to a package file")
referenced_file_ids.add(file_ref.file_id)
if {item.id for item in self.skills if item.scope == "agent_config"} - referenced_skill_ids:
raise ValueError("agent_config skill resources must be referenced by the Agent Soul")
if workspace_names != referenced_workspace_names:
raise ValueError("workspace skill references must match the package resource index")
if set(file_by_id) - referenced_file_ids:
raise ValueError("file resources must be referenced by the Agent Soul")
@dataclass
class PreparedRosterAgentPackage:
"""Validated archive retained in a bounded spool for later materialization."""
archive: BinaryIO
manifest: RosterAgentPackageManifest
apps: dict[str, AgentAppDsl]
members: dict[str, RosterAgentPackageMember]
invalid_skills: dict[str, str] = field(default_factory=dict)
def close(self) -> None:
self.archive.close()
def __enter__(self) -> PreparedRosterAgentPackage:
return self
def __exit__(self, *_args: object) -> None:
self.close()
@dataclass
class RosterAgentPackageExport:
"""Download artifact owning its archive until the response closes."""
archive: BinaryIO
filename: str
size: int
def close(self) -> None:
self.archive.close()
def __enter__(self) -> RosterAgentPackageExport:
return self
def __exit__(self, *_args: object) -> None:
self.close()
__all__ = [
"ROSTER_AGENT_PACKAGE_FORMAT",
"ROSTER_AGENT_PACKAGE_FORMAT_VERSION",
"ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES",
"PreparedRosterAgentPackage",
"RosterAgentPackageApp",
"RosterAgentPackageAudit",
"RosterAgentPackageExport",
"RosterAgentPackageFile",
"RosterAgentPackageManifest",
"RosterAgentPackageMember",
"RosterAgentPackageSkill",
]
@@ -0,0 +1,468 @@
"""Export active Roster Agents as portable ``.ifpkg`` archives."""
from __future__ import annotations
import hashlib
import os
import re
import tempfile
import zipfile
from collections.abc import Callable, Generator, Sequence
from dataclasses import dataclass
from typing import BinaryIO, Literal, Protocol, cast
import yaml
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from configs import dify_config
from core.db.session_factory import session_factory
from core.plugin.entities.plugin import PluginDependency
from extensions.ext_storage import storage
from models.agent import (
APP_BACKED_AGENT_SOURCES,
Agent,
AgentConfigDraft,
AgentConfigDraftType,
AgentConfigSnapshot,
AgentScope,
AgentStatus,
)
from models.agent_config_entities import AgentSoulConfig
from models.enums import AppStatus
from models.model import App, AppMode, UploadFile
from models.tools import ToolFile
from services.agent.dependency_service import extract_agent_soul_dependencies
from services.agent.dsl_entities import (
AgentAppDsl,
AgentPackageWorkspaceSkill,
make_agent_app_dsl,
make_portable_agent_package,
)
from services.agent.errors import (
AgentNotFoundError,
AgentVersionNotFoundError,
RosterAgentPackageExportFailedError,
RosterAgentPackageTooLargeError,
)
from services.agent.roster_package_entities import (
ROSTER_AGENT_PACKAGE_FORMAT,
ROSTER_AGENT_PACKAGE_FORMAT_VERSION,
RosterAgentPackageApp,
RosterAgentPackageAudit,
RosterAgentPackageExport,
RosterAgentPackageFile,
RosterAgentPackageManifest,
RosterAgentPackageSkill,
)
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
from services.plugin.dependencies_analysis import DependenciesAnalysisService
from services.skill_management_service import RuntimeAgentSkillArchive, SkillManagementService
class _Storage(Protocol):
def load_stream(self, filename: str) -> Generator[bytes, None, None]: ...
@dataclass(frozen=True)
class _SkillSource:
path: str
storage_key: str
id: str
scope: Literal["agent_config", "workspace"]
name: str
display_name: str | None
description: str
priority: int | None
audit_ref: str
@dataclass(frozen=True)
class _FileSource:
path: str
storage_key: str
id: str
audit_ref: str
class RosterAgentPackageExporter:
"""Collect a Roster Agent through a dedicated read Session and build its archive."""
def __init__(
self,
*,
storage_backend: _Storage = storage,
dependency_provider: Callable[[str, list[str]], list[PluginDependency]] | None = None,
) -> None:
self._storage = storage_backend
self._dependency_provider = dependency_provider or DependenciesAnalysisService.generate_dependencies
def export(self, *, tenant_id: str, agent_id: str) -> RosterAgentPackageExport:
with session_factory.create_session() as session:
row = session.execute(
select(Agent, App)
.join(
App,
(App.id == Agent.app_id) & (App.tenant_id == Agent.tenant_id),
)
.where(
Agent.id == agent_id,
Agent.tenant_id == tenant_id,
Agent.scope == AgentScope.ROSTER,
Agent.source.in_(APP_BACKED_AGENT_SOURCES),
Agent.status == AgentStatus.ACTIVE,
Agent.app_id.is_not(None),
or_(Agent.backing_app_id == Agent.app_id, Agent.backing_app_id.is_(None)),
App.mode == AppMode.AGENT,
App.status == AppStatus.NORMAL,
)
.limit(1)
).one_or_none()
if row is None:
raise AgentNotFoundError()
agent, app_model = row
draft = session.scalar(
select(AgentConfigDraft)
.where(
AgentConfigDraft.tenant_id == tenant_id,
AgentConfigDraft.agent_id == agent.id,
AgentConfigDraft.draft_type == AgentConfigDraftType.DRAFT,
AgentConfigDraft.draft_owner_key == "",
)
.limit(1)
)
if draft is not None:
soul = AgentSoulConfig.model_validate(draft.config_snapshot_dict)
else:
snapshot = session.scalar(
select(AgentConfigSnapshot)
.where(
AgentConfigSnapshot.id == agent.active_config_snapshot_id,
AgentConfigSnapshot.tenant_id == tenant_id,
AgentConfigSnapshot.agent_id == agent.id,
)
.limit(1)
)
if snapshot is None:
raise AgentVersionNotFoundError()
soul = AgentSoulConfig.model_validate(snapshot.config_snapshot_dict)
resource_soul, skill_sources, file_sources = self._collect_payloads(
session=session,
tenant_id=tenant_id,
soul=soul,
)
package = make_portable_agent_package(agent, resource_soul, include_assets=True)
app = make_agent_app_dsl(app_model, package_ref="agent_1", packages={"agent_1": package}, dependencies=[])
audit = RosterAgentPackageAudit(ref=agent.id)
dependency_ids = extract_agent_soul_dependencies(package.soul)
workspace_skills = SkillManagementService().list_runtime_agent_skill_archives(
tenant_id=tenant_id,
agent_id=agent_id,
include_draft=draft is not None,
)
skill_sources.extend(
self._workspace_skill_sources(
soul=package.soul,
archives=workspace_skills,
start_index=len(skill_sources),
)
)
package.workspace_skills = [
AgentPackageWorkspaceSkill(
name=item.name,
display_name=item.display_name or "",
description=item.description,
priority=item.priority,
)
for item in skill_sources
if item.scope == "workspace" and item.priority is not None
]
app.dependencies = self._dependency_provider(tenant_id, dependency_ids)
return self._build_archive(
app=app,
audit=audit,
skill_sources=skill_sources,
file_sources=file_sources,
)
def _build_archive(
self,
*,
app: AgentAppDsl,
skill_sources: Sequence[_SkillSource],
file_sources: Sequence[_FileSource],
audit: RosterAgentPackageAudit | None = None,
) -> RosterAgentPackageExport:
required_entries = len(skill_sources) + len(file_sources) + 2
if required_entries > dify_config.AGENT_PACKAGE_MAX_ENTRIES:
raise RosterAgentPackageTooLargeError("Roster Agent package has too many members")
# Ownership is transferred to RosterAgentPackageExport.
output = cast(
BinaryIO,
tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b"), # noqa: SIM115
)
try:
skills: list[RosterAgentPackageSkill] = []
files: list[RosterAgentPackageFile] = []
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
total_size = 0
nested_uncompressed_size = 0
skill_packages = SkillPackageService()
sources: list[_SkillSource | _FileSource] = [*skill_sources, *file_sources]
for source in sources:
remaining_bytes = dify_config.AGENT_PACKAGE_MAX_BYTES - total_size
if isinstance(source, _SkillSource):
remaining_bytes = min(remaining_bytes, dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024)
member_size, member_digest = self._write_storage_member(
archive, path=source.path, storage_key=source.storage_key, max_bytes=remaining_bytes
)
total_size += member_size
if isinstance(source, _SkillSource):
# Inspect only the embedded Skill; our writer already owns
# the outer container, resource sizes, and digests.
try:
inspection = skill_packages.inspect(content=archive.read(source.path), filename=source.path)
except SkillPackageError as exc:
raise RosterAgentPackageExportFailedError(
f"Roster Agent package contains unusable Skill {source.name!r}"
) from exc
if inspection.name != source.name:
raise RosterAgentPackageExportFailedError(
f"Roster Agent package contains unusable Skill {source.name!r}: name mismatch"
)
nested_uncompressed_size += inspection.uncompressed_size
if nested_uncompressed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError(
"Roster Agent package nested Skill contents exceed the size limit"
)
skills.append(
RosterAgentPackageSkill(
id=source.id,
scope=source.scope,
name=source.name,
path=source.path,
size=member_size,
sha256=member_digest,
audit=RosterAgentPackageAudit(ref=source.audit_ref),
)
)
else:
files.append(
RosterAgentPackageFile(
id=source.id,
path=source.path,
size=member_size,
sha256=member_digest,
audit=RosterAgentPackageAudit(ref=source.audit_ref),
)
)
app_bytes = yaml.safe_dump(
app.model_dump(mode="json", exclude_none=True), allow_unicode=True, sort_keys=False
).encode("utf-8")
manifest = RosterAgentPackageManifest(
format=ROSTER_AGENT_PACKAGE_FORMAT,
format_version=ROSTER_AGENT_PACKAGE_FORMAT_VERSION,
audit=audit,
apps=[
RosterAgentPackageApp(
path="app.yaml", size=len(app_bytes), sha256=hashlib.sha256(app_bytes).hexdigest()
)
],
skills=skills,
files=files,
)
manifest.validate_apps({"app.yaml": app})
manifest_bytes = yaml.safe_dump(
manifest.model_dump(mode="json", exclude_none=True), allow_unicode=True, sort_keys=False
).encode("utf-8")
for path, document_bytes in (("app.yaml", app_bytes), ("manifest.yaml", manifest_bytes)):
if len(document_bytes) > dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES:
raise RosterAgentPackageTooLargeError(f"Roster Agent package {path} exceeds the size limit")
total_size += len(document_bytes)
if total_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the size limit")
archive.writestr(path, document_bytes)
size = output.tell()
if size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the archive size limit")
output.seek(0)
return RosterAgentPackageExport(
archive=output,
filename=f"{self._safe_slug(app.package.metadata.name)}.ifpkg",
size=size,
)
except Exception:
output.close()
raise
def _collect_payloads(
self,
*,
session: Session,
tenant_id: str,
soul: AgentSoulConfig,
) -> tuple[AgentSoulConfig, list[_SkillSource], list[_FileSource]]:
resource_data = soul.model_dump(mode="json")
skill_sources: list[_SkillSource] = []
file_sources: list[_FileSource] = []
skill_file_ids = [item.file_id for item in soul.config_skills if not item.is_missing]
tool_files = self._tool_files(session=session, tenant_id=tenant_id, file_ids=skill_file_ids)
for skill_ref, portable_skill_ref in zip(soul.config_skills, resource_data["config_skills"]):
if skill_ref.is_missing:
continue
resource_id = f"s_{len(skill_sources) + 1:06d}"
tool_file = tool_files.get(skill_ref.file_id)
if tool_file is None:
raise RosterAgentPackageExportFailedError(f"Config skill {skill_ref.name!r} payload is unavailable")
path = f"{resource_id}.zip"
portable_skill_ref["file_id"] = resource_id
portable_skill_ref["is_missing"] = False
skill_sources.append(
_SkillSource(
path=path,
storage_key=tool_file.file_key,
id=resource_id,
scope="agent_config",
name=skill_ref.name,
display_name=None,
description=skill_ref.description,
priority=None,
audit_ref=tool_file.id,
)
)
tool_file_refs = [
item.file_id for item in soul.config_files if not item.is_missing and item.file_kind == "tool_file"
]
upload_file_refs = [
item.file_id for item in soul.config_files if not item.is_missing and item.file_kind == "upload_file"
]
file_tool_files = self._tool_files(session=session, tenant_id=tenant_id, file_ids=tool_file_refs)
upload_files = self._upload_files(session=session, tenant_id=tenant_id, file_ids=upload_file_refs)
for file_ref, portable_file_ref in zip(soul.config_files, resource_data["config_files"]):
if file_ref.is_missing:
continue
resource_id = f"f_{len(file_sources) + 1:06d}"
extension = self._safe_extension(file_ref.name)
path = f"{resource_id}{extension}"
if file_ref.file_kind == "tool_file":
tool_file = file_tool_files.get(file_ref.file_id)
if tool_file is None:
raise RosterAgentPackageExportFailedError(f"Config file {file_ref.name!r} payload is unavailable")
storage_key = tool_file.file_key
mime_type = file_ref.mime_type or tool_file.mimetype
audit_ref = tool_file.id
else:
upload_file = upload_files.get(file_ref.file_id)
if upload_file is None:
raise RosterAgentPackageExportFailedError(f"Config file {file_ref.name!r} payload is unavailable")
storage_key = upload_file.key
mime_type = file_ref.mime_type or upload_file.mime_type
audit_ref = upload_file.id
portable_file_ref["file_id"] = resource_id
portable_file_ref["is_missing"] = False
portable_file_ref["mime_type"] = mime_type or "application/octet-stream"
file_sources.append(
_FileSource(
path=path,
storage_key=storage_key,
id=resource_id,
audit_ref=audit_ref,
)
)
return AgentSoulConfig.model_validate(resource_data), skill_sources, file_sources
@staticmethod
def _workspace_skill_sources(
*, soul: AgentSoulConfig, archives: Sequence[RuntimeAgentSkillArchive], start_index: int
) -> list[_SkillSource]:
sources: list[_SkillSource] = []
effective_skill_names = {item.name for item in soul.config_skills if not item.is_missing}
for archive in sorted(archives, key=lambda item: item.priority):
if archive.name in effective_skill_names:
continue
effective_skill_names.add(archive.name)
resource_id = f"s_{start_index + len(sources) + 1:06d}"
path = f"{resource_id}.zip"
sources.append(
_SkillSource(
path=path,
storage_key=archive.storage_key,
id=resource_id,
scope="workspace",
name=archive.name,
display_name=archive.display_name,
description=archive.description,
priority=archive.priority,
audit_ref=archive.version_id,
)
)
return sources
@staticmethod
def _tool_files(*, session: Session, tenant_id: str, file_ids: Sequence[str]) -> dict[str, ToolFile]:
if not file_ids:
return {}
rows = session.scalars(
select(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id.in_(set(file_ids)))
).all()
return {item.id: item for item in rows}
@staticmethod
def _upload_files(*, session: Session, tenant_id: str, file_ids: Sequence[str]) -> dict[str, UploadFile]:
if not file_ids:
return {}
rows = session.scalars(
select(UploadFile).where(UploadFile.tenant_id == tenant_id, UploadFile.id.in_(set(file_ids)))
).all()
return {item.id: item for item in rows}
def _write_storage_member(
self,
archive: zipfile.ZipFile,
*,
path: str,
storage_key: str,
max_bytes: int,
) -> tuple[int, str]:
digest = hashlib.sha256()
size = 0
try:
with archive.open(path, "w", force_zip64=True) as target:
for chunk in self._storage.load_stream(storage_key):
if not isinstance(chunk, bytes):
raise TypeError("storage stream returned a non-bytes chunk")
if size + len(chunk) > max_bytes:
raise RosterAgentPackageTooLargeError("Roster Agent package payload exceeds the size limit")
size += len(chunk)
digest.update(chunk)
target.write(chunk)
except RosterAgentPackageTooLargeError:
raise
except Exception as exc:
raise RosterAgentPackageExportFailedError(f"Unable to read package resource {path!r}") from exc
return size, digest.hexdigest()
@staticmethod
def _safe_extension(filename: str) -> str:
extension = os.path.splitext(filename)[1].lower()
if not extension or not re.fullmatch(r"\.[a-z0-9]{1,16}", extension):
return ".bin"
return extension
@staticmethod
def _safe_slug(name: str) -> str:
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
return slug[:80] or "agent"
__all__ = ["RosterAgentPackageExporter"]
+346
View File
@@ -0,0 +1,346 @@
"""Read and validate portable Roster Agent ``.ifpkg`` archives."""
from __future__ import annotations
import hashlib
import io
import posixpath
import stat
import tempfile
import zipfile
import zlib
from collections.abc import Hashable
from typing import Any, BinaryIO, cast, override
import yaml
from pydantic import ValidationError
from configs import dify_config
from constants.dsl_version import CURRENT_APP_DSL_VERSION
from services.agent.dsl_entities import AgentAppDsl
from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError
from services.agent.roster_package_entities import (
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES,
PreparedRosterAgentPackage,
RosterAgentPackageApp,
RosterAgentPackageManifest,
RosterAgentPackageMember,
RosterAgentPackageSkill,
)
from services.agent.skill_package_service import SkillPackageError, SkillPackageService
from services.dsl_version import check_version_compatibility
from services.entities.dsl_entities import ImportStatus
_COPY_CHUNK_SIZE = 1024 * 1024
_ALLOWED_COMPRESSIONS = {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}
class RosterAgentPackageReader:
"""Produce a validated, bounded package without external side effects."""
def __init__(self) -> None:
self._skill_packages = SkillPackageService()
def read(self, source: BinaryIO) -> PreparedRosterAgentPackage:
"""Validate the container and record unusable Skill payloads for import warnings."""
# Ownership is transferred to PreparedRosterAgentPackage.
spool = cast(
BinaryIO,
tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b"), # noqa: SIM115
)
try:
self._copy_bounded(source, spool)
spool.seek(0)
invalid_skills: dict[str, str] = {}
manifest, apps, members = self._validate_archive(spool, invalid_skills=invalid_skills)
spool.seek(0)
return PreparedRosterAgentPackage(
archive=spool, manifest=manifest, apps=apps, members=members, invalid_skills=invalid_skills
)
except Exception:
spool.close()
raise
def read_member_bytes(self, package: PreparedRosterAgentPackage, path: str, *, max_bytes: int) -> bytes:
"""Read one already-validated member while rechecking its size and digest."""
member = package.members.get(path)
if member is None:
raise InvalidRosterAgentPackageError("Roster Agent package member is unavailable")
try:
package.archive.seek(0)
with zipfile.ZipFile(package.archive) as archive:
info = archive.getinfo(path)
payload, digest, _ = self._read_member(
archive,
info,
collect=True,
max_bytes=max_bytes,
expected_size=member.size,
)
except (KeyError, OSError, zipfile.BadZipFile, EOFError, RuntimeError, ValueError, zlib.error) as exc:
raise InvalidRosterAgentPackageError("Roster Agent package member is unavailable") from exc
if digest != member.sha256:
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
return payload
def _validate_archive(
self, archive_file: BinaryIO, *, invalid_skills: dict[str, str]
) -> tuple[RosterAgentPackageManifest, dict[str, AgentAppDsl], dict[str, RosterAgentPackageMember]]:
try:
with zipfile.ZipFile(archive_file) as archive:
infos = archive.infolist()
if not infos:
raise InvalidRosterAgentPackageError("Roster Agent package is empty")
if len(infos) > dify_config.AGENT_PACKAGE_MAX_ENTRIES:
raise InvalidRosterAgentPackageError("Roster Agent package has too many members")
info_by_path: dict[str, zipfile.ZipInfo] = {}
casefold_paths: set[str] = set()
total_uncompressed = 0
for info in infos:
path = self._validate_member_info(info)
if path in info_by_path or path.casefold() in casefold_paths:
raise InvalidRosterAgentPackageError("Roster Agent package contains duplicate member paths")
info_by_path[path] = info
casefold_paths.add(path.casefold())
total_uncompressed += info.file_size
if total_uncompressed > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError("Roster Agent package uncompressed size exceeds the limit")
manifest_data, manifest_size = self._read_yaml_document(archive, info_by_path, "manifest.yaml")
try:
manifest = RosterAgentPackageManifest.model_validate(manifest_data)
except ValidationError as exc:
raise InvalidRosterAgentPackageError("Roster Agent package manifest is invalid") from exc
apps: dict[str, AgentAppDsl] = {}
members: dict[str, RosterAgentPackageMember] = {}
streamed_size = manifest_size
for app_resource in manifest.apps:
app_data, app_size = self._read_yaml_document(
archive, info_by_path, app_resource.path, resource=app_resource
)
streamed_size += app_size
if streamed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError(
"Roster Agent package uncompressed size exceeds the limit"
)
try:
app = AgentAppDsl.model_validate(app_data)
if check_version_compatibility(app.version, CURRENT_APP_DSL_VERSION) in {
ImportStatus.FAILED,
ImportStatus.PENDING,
}:
raise ValueError("unsupported App DSL version")
except ValueError as exc:
raise InvalidRosterAgentPackageError("Roster Agent package app is invalid") from exc
apps[app_resource.path] = app
members[app_resource.path] = RosterAgentPackageMember(size=app_size, sha256=app_resource.sha256)
try:
manifest.validate_apps(apps)
except ValueError as exc:
raise InvalidRosterAgentPackageError("Roster Agent package app is invalid") from exc
expected_paths = {
"manifest.yaml",
*(item.path for item in manifest.apps),
*(item.path for item in manifest.skills),
*(item.path for item in manifest.files),
}
actual_paths = set(info_by_path)
if "signature.sig" in actual_paths:
if info_by_path["signature.sig"].file_size > ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES:
raise RosterAgentPackageTooLargeError("Roster Agent package signature is too large")
actual_paths.remove("signature.sig")
if actual_paths != expected_paths:
raise InvalidRosterAgentPackageError("Roster Agent package members do not match the manifest")
nested_uncompressed_size = 0
signature_info = info_by_path.get("signature.sig")
if signature_info is not None:
_, _, signature_size = self._read_member(
archive,
signature_info,
collect=False,
max_bytes=min(
ROSTER_AGENT_PACKAGE_MAX_SIGNATURE_BYTES,
dify_config.AGENT_PACKAGE_MAX_BYTES - streamed_size,
),
expected_size=signature_info.file_size,
)
streamed_size += signature_size
for resource in [*manifest.skills, *manifest.files]:
info = info_by_path[resource.path]
payload = b""
remaining_package_bytes = dify_config.AGENT_PACKAGE_MAX_BYTES - streamed_size
if isinstance(resource, RosterAgentPackageSkill):
max_skill_bytes = dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024
if info.file_size > max_skill_bytes:
raise RosterAgentPackageTooLargeError(
f"Roster Agent package Skill {resource.name!r} exceeds the size limit"
)
try:
payload, digest, actual_size = self._read_member(
archive,
info,
collect=True,
max_bytes=min(max_skill_bytes, remaining_package_bytes),
expected_size=resource.size,
)
except (InvalidRosterAgentPackageError, zipfile.BadZipFile, EOFError, zlib.error):
invalid_skills[resource.id] = "archive_integrity_failed"
streamed_size += info.file_size
continue
else:
_, digest, actual_size = self._read_member(
archive,
info,
collect=False,
max_bytes=remaining_package_bytes,
expected_size=resource.size,
)
streamed_size += actual_size
if digest != resource.sha256:
if isinstance(resource, RosterAgentPackageSkill):
invalid_skills[resource.id] = "checksum_mismatch"
continue
raise InvalidRosterAgentPackageError(
f"Roster Agent package resource {resource.path!r} failed integrity checks",
)
members[resource.path] = RosterAgentPackageMember(
size=resource.size,
sha256=digest,
)
if isinstance(resource, RosterAgentPackageSkill):
try:
inspection = self._skill_packages.inspect(content=payload, filename=resource.path)
except SkillPackageError as exc:
invalid_skills[resource.id] = exc.code
continue
nested_uncompressed_size += inspection.uncompressed_size
if nested_uncompressed_size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError(
"Roster Agent package nested Skill contents exceed the size limit"
)
if inspection.name != resource.name:
invalid_skills[resource.id] = "skill_name_mismatch"
return manifest, apps, members
except (InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError):
raise
except (OSError, zipfile.BadZipFile, EOFError, RuntimeError, ValueError, zlib.error) as exc:
raise InvalidRosterAgentPackageError("Roster Agent package is not a valid ZIP") from exc
@staticmethod
def _copy_bounded(source: BinaryIO, target: BinaryIO) -> None:
size = 0
while True:
chunk = source.read(_COPY_CHUNK_SIZE)
if not chunk:
break
if not isinstance(chunk, bytes):
raise InvalidRosterAgentPackageError("Roster Agent package must be binary")
size += len(chunk)
if size > dify_config.AGENT_PACKAGE_MAX_BYTES:
raise RosterAgentPackageTooLargeError("Roster Agent package exceeds the archive size limit")
target.write(chunk)
@staticmethod
def _validate_member_info(info: zipfile.ZipInfo) -> str:
name = info.filename
if info.is_dir():
raise InvalidRosterAgentPackageError("Roster Agent package must use root-level files")
if "\x00" in name or "\\" in name or any(ord(char) < 0x20 for char in name):
raise InvalidRosterAgentPackageError("Roster Agent package contains an unsafe path")
normalized = posixpath.normpath(name)
if normalized != name or normalized in {"", ".", ".."} or normalized.startswith("/") or "/" in normalized:
raise InvalidRosterAgentPackageError("Roster Agent package contains an unsafe path")
if stat.S_ISLNK(info.external_attr >> 16):
raise InvalidRosterAgentPackageError("Roster Agent package must not contain symbolic links")
if info.flag_bits & 0x1:
raise InvalidRosterAgentPackageError("Roster Agent package must not contain encrypted members")
if info.compress_type not in _ALLOWED_COMPRESSIONS:
raise InvalidRosterAgentPackageError("Roster Agent package uses unsupported compression")
if info.file_size and info.compress_size == 0:
raise InvalidRosterAgentPackageError("Roster Agent package contains invalid ZIP metadata")
return normalized
@staticmethod
def _read_member(
archive: zipfile.ZipFile,
info: zipfile.ZipInfo,
*,
collect: bool,
max_bytes: int,
expected_size: int,
) -> tuple[bytes, str, int]:
digest = hashlib.sha256()
output = io.BytesIO() if collect else None
size = 0
with archive.open(info) as member:
while chunk := member.read(_COPY_CHUNK_SIZE):
size += len(chunk)
if size > max_bytes:
raise RosterAgentPackageTooLargeError("Roster Agent package member exceeds the size limit")
if size > expected_size:
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
digest.update(chunk)
if output is not None:
output.write(chunk)
if size != expected_size:
raise InvalidRosterAgentPackageError("Roster Agent package member failed integrity checks")
return output.getvalue() if output is not None else b"", digest.hexdigest(), size
def _read_yaml_document(
self,
archive: zipfile.ZipFile,
infos: dict[str, zipfile.ZipInfo],
path: str,
*,
resource: RosterAgentPackageApp | None = None,
) -> tuple[Any, int]:
info = infos.get(path)
if info is None:
raise InvalidRosterAgentPackageError(f"Roster Agent package is missing {path}")
if info.file_size > dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES:
raise RosterAgentPackageTooLargeError(f"Roster Agent package {path} exceeds the size limit")
payload, digest, size = self._read_member(
archive,
info,
collect=True,
max_bytes=dify_config.AGENT_PACKAGE_MAX_MANIFEST_BYTES,
expected_size=resource.size if resource is not None else info.file_size,
)
if resource is not None and digest != resource.sha256:
raise InvalidRosterAgentPackageError(f"Roster Agent package {path} failed integrity checks")
try:
# The custom SafeLoader also rejects duplicate keys and aliases.
return yaml.load(payload, Loader=_PackageYamlLoader), size # noqa: S506
except (UnicodeDecodeError, yaml.YAMLError, ValueError, RecursionError) as exc:
raise InvalidRosterAgentPackageError(
f"Roster Agent package {path.removesuffix('.yaml')} is invalid"
) from exc
class _PackageYamlLoader(yaml.SafeLoader):
"""Reject ambiguous mappings and aliases in untrusted package documents."""
@override
def compose_node(self, parent: yaml.Node | None, index: Any) -> yaml.Node | None:
if self.check_event(yaml.AliasEvent):
raise ValueError("YAML aliases are not supported")
return super().compose_node(parent, index)
@override
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Hashable, Any]:
result: dict[Hashable, Any] = {}
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
if not isinstance(key, str):
raise ValueError("YAML mapping keys must be strings")
if key in result:
raise ValueError(f"duplicate YAML key: {key}")
result[key] = self.construct_object(value_node, deep=deep)
return result
__all__ = ["RosterAgentPackageReader"]
+93 -53
View File
@@ -20,16 +20,14 @@ import io
import posixpath
import zipfile
import zlib
from dataclasses import dataclass
import yaml
from pydantic import BaseModel, Field, ValidationError, field_validator
from configs import dify_config
from libs.zip_filename_recovery import open_zip_with_replacement_names
# Bounds — generous but finite so a hostile upload can't exhaust memory/disk.
_MAX_UNCOMPRESSED_BYTES = 200 * 1024 * 1024
_MAX_SKILL_MD_BYTES = 1 * 1024 * 1024
_MAX_ENTRIES = 5000
_ALLOWED_EXTENSIONS = (".zip", ".skill")
_SKILL_MD_NAME = "SKILL.md"
_SKILL_NAME_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
@@ -50,15 +48,9 @@ class SkillPackageError(Exception):
self.status_code = status_code
class SkillManifest(BaseModel):
"""Validated metadata extracted from a Skill package."""
class _SkillMetadata(BaseModel):
name: str = Field(min_length=1, max_length=64, pattern=_SKILL_NAME_PATTERN)
description: str = Field(min_length=1, max_length=_MAX_SKILL_DESCRIPTION_LENGTH)
entry_path: str # path of SKILL.md inside the archive
files: list[str] # all (safe) file paths inside the archive
size: int # total uncompressed bytes
hash: str # sha256 of the archive bytes
@field_validator("name", "description", mode="before")
@classmethod
@@ -66,18 +58,44 @@ class SkillManifest(BaseModel):
return value.strip() if isinstance(value, str) else value
class SkillPackageInspection(_SkillMetadata):
"""Metadata read without rebuilding or fully extracting a Skill archive."""
uncompressed_size: int = Field(ge=0)
class SkillManifest(_SkillMetadata):
"""Validated metadata extracted from a Skill package."""
entry_path: str # path of SKILL.md inside the archive
files: list[str] # all (safe) file paths inside the archive
size: int # total uncompressed bytes
hash: str # sha256 of the archive bytes
class NormalizedSkillPackage(BaseModel):
"""Canonical skill package bytes and metadata ready to store as Agent config."""
manifest: SkillManifest
archive_bytes: bytes
@dataclass(frozen=True)
class _InspectedSkillArchive:
metadata: SkillPackageInspection
normalized_members: dict[str, zipfile.ZipInfo]
skill_md_bytes: bytes
strip_prefix: str | None
class SkillPackageService:
"""Validate Skill archives and produce a normalized package."""
def inspect(self, *, content: bytes, filename: str) -> SkillPackageInspection:
"""Validate archive metadata and SKILL.md without rebuilding the archive."""
archive = self._open_archive(content=content, filename=filename)
with archive:
return self._inspect_archive(archive).metadata
def validate_and_normalize(self, *, content: bytes, filename: str) -> NormalizedSkillPackage:
"""Return the canonical package for an uploaded skill archive.
@@ -89,43 +107,30 @@ class SkillPackageService:
The returned manifest is normalized to archive-root ``SKILL.md`` and its
hash describes the rebuilt archive bytes. Member read/decompression
failures while consuming the archive are mapped to ``invalid_archive``.
Malformed UTF-8 member names are decoded with replacement characters;
recovered path collisions still fail the normal duplicate-path check.
"""
archive = self._open_archive(content=content, filename=filename)
with archive:
members = self._collect_file_members(archive)
member_paths = [safe_path for _, safe_path in members]
entry_path = self._find_skill_md(member_paths)
strip_prefix = self._skill_root_prefix(entry_path)
normalized_members = self._normalize_members(
members=members,
skill_root_prefix=strip_prefix,
ignore_outside_selected_root=self._can_strip_single_top_level_folder(
paths=member_paths, entry_path=entry_path
),
)
skill_md_member = normalized_members[_SKILL_MD_NAME]
self._validate_skill_md_size(skill_md_member)
raw_skill_md_bytes = self._read_member_bytes_from_archive(archive, member_info=skill_md_member)
skill_md = self._decode_skill_md(raw_skill_md_bytes)
skill_md_bytes = skill_md.encode("utf-8")
inspection = self._inspect_archive(archive)
normalized_archive_bytes = self._build_normalized_archive(
archive=archive,
normalized_members=normalized_members,
skill_md_bytes=skill_md_bytes,
)
normalized_size = sum(
len(skill_md_bytes) if path == _SKILL_MD_NAME else max(info.file_size, 0)
for path, info in normalized_members.items()
normalized_members=inspection.normalized_members,
skill_md_bytes=inspection.skill_md_bytes,
)
name, description = self._parse_skill_md(skill_md)
if len(normalized_archive_bytes) > dify_config.UPLOAD_SKILL_FILE_SIZE_LIMIT * 1024 * 1024:
raise SkillPackageError("archive_too_large", "normalized skill archive exceeds size limit", status_code=400)
try:
manifest = SkillManifest(
name=name,
description=description,
name=inspection.metadata.name,
description=inspection.metadata.description,
entry_path=_SKILL_MD_NAME,
files=sorted(normalized_members),
size=normalized_size,
files=sorted(inspection.normalized_members),
size=sum(
len(inspection.skill_md_bytes) if path == _SKILL_MD_NAME else info.file_size
for path, info in inspection.normalized_members.items()
),
hash=hashlib.sha256(normalized_archive_bytes).hexdigest(),
)
except ValidationError as exc:
@@ -133,8 +138,38 @@ class SkillPackageService:
return NormalizedSkillPackage(
manifest=manifest,
archive_bytes=normalized_archive_bytes,
)
def _inspect_archive(self, archive: zipfile.ZipFile) -> _InspectedSkillArchive:
members = self._collect_file_members(archive)
member_paths = [safe_path for _, safe_path in members]
entry_path = self._find_skill_md(member_paths)
strip_prefix = self._skill_root_prefix(entry_path)
normalized_members = self._normalize_members(
members=members,
skill_root_prefix=strip_prefix,
ignore_outside_selected_root=self._can_strip_single_top_level_folder(
paths=member_paths, entry_path=entry_path
),
)
skill_md_member = normalized_members[_SKILL_MD_NAME]
self._validate_skill_md_size(skill_md_member)
raw_skill_md_bytes = self._read_member_bytes_from_archive(archive, member_info=skill_md_member)
skill_md = self._decode_skill_md(raw_skill_md_bytes)
skill_md_bytes = skill_md.encode("utf-8")
name, description = self._parse_skill_md(skill_md)
try:
metadata = SkillPackageInspection(
name=name,
description=description,
uncompressed_size=sum(info.file_size for info, _ in members),
)
except ValidationError as exc:
raise self._manifest_validation_error(exc) from exc
return _InspectedSkillArchive(
metadata=metadata,
normalized_members=normalized_members,
skill_md_bytes=skill_md_bytes,
strip_prefix=strip_prefix,
)
@staticmethod
@@ -171,21 +206,24 @@ class SkillPackageService:
raise SkillPackageError("archive_too_large", "skill archive exceeds size limit", status_code=400)
try:
return zipfile.ZipFile(io.BytesIO(content))
return open_zip_with_replacement_names(content)
except zipfile.BadZipFile as exc:
raise SkillPackageError("invalid_archive", "skill archive is not a valid zip", status_code=400) from exc
def _collect_file_members(self, archive: zipfile.ZipFile) -> list[tuple[zipfile.ZipInfo, str]]:
infos = [info for info in archive.infolist() if not info.is_dir()]
if len(infos) > _MAX_ENTRIES:
archive_infos = archive.infolist()
if len(archive_infos) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
raise SkillPackageError("too_many_entries", "skill archive has too many files", status_code=400)
infos = [info for info in archive_infos if not info.is_dir()]
members: list[tuple[zipfile.ZipInfo, str]] = []
total_uncompressed = 0
for info in infos:
members.append((info, self._safe_member_path(info.filename)))
total_uncompressed += max(info.file_size, 0)
if total_uncompressed > _MAX_UNCOMPRESSED_BYTES:
total_uncompressed += info.file_size
if info.file_size and info.compress_size == 0:
raise SkillPackageError("invalid_archive", "skill package has invalid ZIP metadata", status_code=400)
if total_uncompressed > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
raise SkillPackageError(
"archive_too_large",
"skill archive uncompressed size exceeds limit",
@@ -225,12 +263,8 @@ class SkillPackageService:
else:
normalized_path = safe_path
if (
not normalized_path
or normalized_path in {".", ".."}
or normalized_path.startswith("/")
or "\\" in normalized_path
):
# Paths are already normalized and safe; "." is not a file.
if normalized_path == ".":
raise SkillPackageError("unsafe_path", "skill archive contains an unsafe path", status_code=400)
if normalized_path in normalized_members:
raise SkillPackageError(
@@ -306,7 +340,7 @@ class SkillPackageService:
@staticmethod
def _validate_skill_md_size(member_info: zipfile.ZipInfo) -> None:
if member_info.file_size > _MAX_SKILL_MD_BYTES:
if member_info.file_size > dify_config.SKILL_PACKAGE_MAX_SKILL_MD_BYTES:
raise SkillPackageError("skill_md_too_large", "SKILL.md exceeds size limit", status_code=400)
@staticmethod
@@ -343,4 +377,10 @@ class SkillPackageService:
return loaded if isinstance(loaded, dict) else {}
__all__ = ["NormalizedSkillPackage", "SkillManifest", "SkillPackageError", "SkillPackageService"]
__all__ = [
"NormalizedSkillPackage",
"SkillManifest",
"SkillPackageError",
"SkillPackageInspection",
"SkillPackageService",
]
+2 -2
View File
@@ -609,7 +609,7 @@ class AgentConfigService:
)
self._require_writable(target, surface=surface)
try:
skill_ref, _package = self._skill_normalizer.normalize(
skill_ref = self._skill_normalizer.normalize(
content=content,
filename=filename,
requested_name=None,
@@ -1064,7 +1064,7 @@ class AgentConfigService:
)
archive_bytes = storage.load_once(tool_file.file_key)
try:
skill_ref, _package = self._skill_normalizer.normalize(
skill_ref = self._skill_normalizer.normalize(
content=archive_bytes,
filename=tool_file.name,
requested_name=name,
+23 -36
View File
@@ -47,7 +47,8 @@ from models import Account, App, AppMode
from models.agent import AgentScope
from models.model import AppModelConfig, AppModelConfigDict, IconType, load_annotation_reply_config
from models.workflow import Workflow
from services.agent.dsl_service import AgentDslService, AgentPackage
from services.agent.dsl_entities import AgentPackage, make_agent_app_dsl
from services.agent.dsl_service import AgentDslService
from services.agent.retirement_service import WorkflowAgentRetirementService
from services.agent.workflow_publish_service import WorkflowAgentPublishService
from services.dsl_content import DSL_MAX_SIZE, dsl_content_size
@@ -60,6 +61,7 @@ from services.entities.dsl_entities import (
ImportMode,
ImportStatus,
PendingImportOwner,
make_app_dsl,
)
from services.errors.account import NoPermissionError
from services.errors.app import WorkflowNotFoundError
@@ -762,44 +764,29 @@ class AppDslService:
"""
app_mode = AppMode.value_of(app_model.mode)
export_data: dict[str, Any] = {
"version": CURRENT_DSL_VERSION,
"kind": "app",
"app": {
"name": app_model.name,
"mode": app_model.mode.value if isinstance(app_model.mode, AppMode) else app_model.mode,
"icon": app_model.icon,
"icon_type": (
app_model.icon_type.value if isinstance(app_model.icon_type, IconType) else app_model.icon_type
),
"icon_background": app_model.icon_background,
"description": app_model.description,
"use_icon_as_answer_icon": app_model.use_icon_as_answer_icon,
},
}
if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
cls._append_workflow_export_data(
export_data=export_data,
app_model=app_model,
include_secret=include_secret,
workflow_id=workflow_id,
session=session,
)
elif app_mode == AppMode.AGENT:
if app_mode == AppMode.AGENT:
package_ref, packages = AgentDslService(session).export_agent_app(app=app_model)
export_data["agent"] = {"package_ref": package_ref}
export_data["agent_packages"] = {key: package.model_dump(mode="json") for key, package in packages.items()}
dependencies = AgentDslService(session).extract_package_dependencies(packages)
export_data["dependencies"] = [
jsonable_encoder(item.model_dump())
for item in DependenciesAnalysisService.generate_dependencies(
tenant_id=app_model.tenant_id,
dependencies=dependencies,
)
]
export_data = make_agent_app_dsl(
app_model,
package_ref=package_ref,
packages=packages,
dependencies=DependenciesAnalysisService.generate_dependencies(
tenant_id=app_model.tenant_id, dependencies=dependencies
),
).model_dump(mode="json")
else:
cls._append_model_config_export_data(export_data, app_model, session=session)
export_data = make_app_dsl(app_model)
if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
cls._append_workflow_export_data(
export_data=export_data,
app_model=app_model,
include_secret=include_secret,
workflow_id=workflow_id,
session=session,
)
else:
cls._append_model_config_export_data(export_data, app_model, session=session)
return yaml.dump(export_data, allow_unicode=True)
+19
View File
@@ -3,7 +3,9 @@ from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from constants.dsl_version import CURRENT_APP_DSL_VERSION
from core.plugin.entities.plugin import PluginDependency
from models.model import App, AppMode, IconType
class ImportMode(StrEnum):
@@ -39,3 +41,20 @@ class DslImportWarning(BaseModel):
class CheckDependenciesResult(BaseModel):
leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
def make_app_dsl(app: App) -> dict[str, Any]:
"""Build the shared App DSL envelope for standalone and archive exports."""
return {
"version": CURRENT_APP_DSL_VERSION,
"kind": "app",
"app": {
"name": app.name,
"mode": app.mode.value if isinstance(app.mode, AppMode) else app.mode,
"icon": app.icon,
"icon_type": app.icon_type.value if isinstance(app.icon_type, IconType) else app.icon_type,
"icon_background": app.icon_background,
"description": app.description,
"use_icon_as_answer_icon": app.use_icon_as_answer_icon,
},
}
+84 -39
View File
@@ -98,9 +98,6 @@ from services.file_service import FileService
logger = logging.getLogger(__name__)
_SKILL_MD = "SKILL.md"
_MAX_SKILL_BYTES = 200 * 1024 * 1024
_MAX_FILES_PER_SKILL = 5000
_MAX_ZIP_COMPRESSION_RATIO = 1000
_MAX_FILE_CHECK_ITEMS = 100
_MAX_SKILLS_PER_WORKSPACE = 500
_MAX_AGENT_SKILLS = 20
@@ -509,6 +506,20 @@ class PublishedSkillArchive:
payload: bytes
@dataclass(frozen=True, slots=True)
class RuntimeAgentSkillArchive:
skill_id: str
version_id: str
tool_file_id: str
storage_key: str
name: str
display_name: str
description: str
priority: int
size: int
hash: str
@dataclass(frozen=True, slots=True)
class SkillFileContent:
filename: str
@@ -2269,6 +2280,34 @@ class SkillManagementService:
config_snapshot_id: str | None = None,
) -> list[dict[str, Any]]:
"""Return workspace Skills from the Agent draft or a published snapshot."""
return [
{
"id": item.skill_id,
"name": item.name,
"file_id": item.tool_file_id,
"description": item.description,
"size": item.size,
"hash": item.hash,
"mime_type": "application/zip",
}
for item in self.list_runtime_agent_skill_archives(
tenant_id=tenant_id,
agent_id=agent_id,
include_draft=include_draft,
config_snapshot_id=config_snapshot_id,
)
]
def list_runtime_agent_skill_archives(
self,
*,
tenant_id: str,
agent_id: str,
include_draft: bool = False,
config_snapshot_id: str | None = None,
) -> list[RuntimeAgentSkillArchive]:
"""Return export-ready workspace Skill archives with published identity compatibility."""
with self._session_scope() as session:
binding_model = AgentSkillBinding if include_draft else AgentSkillBindingSnapshot
conditions = [
@@ -2284,29 +2323,37 @@ class SkillManagementService:
conditions.append(AgentSkillBindingSnapshot.config_snapshot_id == config_snapshot_id)
rows = list(
session.execute(
select(binding_model, Skill, SkillVersion)
select(binding_model, Skill, SkillVersion, ToolFile)
.join(Skill, Skill.id == binding_model.skill_id)
.join(SkillVersion, SkillVersion.id == Skill.latest_published_version_id)
.join(Agent, Agent.id == binding_model.agent_id)
.join(
ToolFile,
(ToolFile.id == SkillVersion.archive_tool_file_id) & (ToolFile.tenant_id == tenant_id),
)
.where(*conditions)
.order_by(Skill.name)
)
)
return [
{
"id": skill.id,
"name": published_name,
"file_id": version.archive_tool_file_id,
"description": published_description,
"size": version.archive_size,
"hash": version.hash_code,
"mime_type": "application/zip",
}
for _binding, skill, version in rows
for published_name, _published_display_name, published_description in [
self._published_skill_identity(skill, version)
]
return [
RuntimeAgentSkillArchive(
skill_id=skill.id,
version_id=version.id,
tool_file_id=version.archive_tool_file_id,
storage_key=tool_file.file_key,
name=published_name,
display_name=published_display_name,
description=published_description,
priority=binding.priority,
size=version.archive_size,
hash=version.hash_code,
)
for binding, skill, version, tool_file in rows
for published_name, published_display_name, published_description in [
self._published_skill_identity(skill, version)
]
]
def pull_runtime_agent_skill(
self,
@@ -3899,7 +3946,7 @@ class SkillManagementService:
def _validate_archive_limits(archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
"""Validate ZIP metadata before any member is decompressed or persisted."""
infos = archive.infolist()
if len(infos) > _MAX_FILES_PER_SKILL:
if len(infos) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
raise SkillManagementServiceError("too_many_files", "skill file count limit exceeded")
total_uncompressed = 0
@@ -3908,16 +3955,13 @@ class SkillManagementService:
raise SkillManagementServiceError("invalid_skill_package", "skill package has invalid ZIP metadata")
total_uncompressed += info.file_size
if total_uncompressed > _MAX_SKILL_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
if total_uncompressed > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
if info.is_dir() or info.file_size == 0:
continue
if info.compress_size == 0 or info.file_size / info.compress_size > _MAX_ZIP_COMPRESSION_RATIO:
raise SkillManagementServiceError(
"invalid_skill_package",
"skill package compression ratio exceeds the allowed limit",
)
if info.compress_size == 0:
raise SkillManagementServiceError("invalid_skill_package", "skill package has invalid ZIP metadata")
return infos
def _draft_rows_from_archive_bytes(
@@ -4107,7 +4151,7 @@ class SkillManagementService:
)
parent = posixpath.dirname(parent)
if len(entries_by_path) > _MAX_FILES_PER_SKILL:
if len(entries_by_path) > dify_config.SKILL_PACKAGE_MAX_ENTRIES:
raise SkillManagementServiceError("too_many_files", "skill file count limit exceeded")
rows: list[SkillDraftFile] = []
@@ -4140,8 +4184,8 @@ class SkillManagementService:
)
)
if total_size > _MAX_SKILL_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
if total_size > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
return rows
def _sync_skill_metadata_from_draft_skill_md(self, *, skill: Skill, content: str) -> None:
@@ -4401,8 +4445,8 @@ class SkillManagementService:
@staticmethod
def _enforce_total_size(files: list[SkillDraftFile]) -> None:
total = sum(file.size or 0 for file in {file.path: file for file in files}.values())
if total > _MAX_SKILL_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds 200MB limit")
if total > dify_config.SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES:
raise SkillManagementServiceError("skill_too_large", "skill exceeds uncompressed size limit")
@staticmethod
def _load_tool_file_bytes(*, tenant_id: str, file_id: str) -> bytes:
@@ -4410,14 +4454,15 @@ class SkillManagementService:
tool_file = session.scalar(select(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id == file_id))
if tool_file is None:
raise SkillManagementServiceError("skill_archive_missing", "skill archive is missing", status_code=404)
try:
return storage.load_once(tool_file.file_key)
except (OSError, SQLAlchemyError) as exc:
raise SkillManagementServiceError(
"skill_archive_missing",
"skill archive is missing",
status_code=404,
) from exc
file_key = tool_file.file_key
try:
return storage.load_once(file_key)
except (OSError, SQLAlchemyError) as exc:
raise SkillManagementServiceError(
"skill_archive_missing",
"skill archive is missing",
status_code=404,
) from exc
@staticmethod
def _load_assistant_tool_file_bytes(*, tenant_id: str, file_id: str) -> bytes:
@@ -64,7 +64,12 @@ class _IsolatedDifyConfig(DifyConfig):
def _make_config(**values: object) -> DifyConfig:
return _IsolatedDifyConfig(**values)
# Pydantic exposes a generated field-wise signature, while this helper intentionally forwards heterogeneous input.
return _IsolatedDifyConfig(**values) # pyrefly: ignore[bad-argument-type]
def test_skill_package_default_limit_matches_documentation() -> None:
assert _make_config().UPLOAD_SKILL_FILE_SIZE_LIMIT == 50
def test_dify_config_keeps_secret_key_empty_when_missing(tmp_path) -> None:
@@ -0,0 +1,94 @@
import io
import struct
import zipfile
import pytest
from libs.zip_filename_recovery import open_zip_with_replacement_names
def _zip64(monkeypatch: pytest.MonkeyPatch, limit: int) -> bytes:
output = io.BytesIO()
with monkeypatch.context() as scoped:
scoped.setattr(zipfile, "ZIP64_LIMIT", limit)
with zipfile.ZipFile(output, "w", allowZip64=True) as archive:
archive.writestr("padding.txt", b"p" * 200)
archive.writestr("scripts/\u00e9.py", b"print('ok')\n")
content = output.getvalue()
assert len(content) < 2048
assert b"PK\x06\x06" in content
return content
@pytest.mark.parametrize("limit", [0, 100])
@pytest.mark.parametrize("prefix", [b"", b"self-extracting-prefix"])
def test_zip64_replaces_names_and_preserves_payload(monkeypatch: pytest.MonkeyPatch, limit: int, prefix: bytes) -> None:
original = prefix + _zip64(monkeypatch, limit)
with zipfile.ZipFile(io.BytesIO(original)) as archive:
assert archive.read("scripts/\u00e9.py") == b"print('ok')\n"
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py")
with open_zip_with_replacement_names(damaged) as archive:
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
assert archive.testzip() is None
def test_zip64_uses_extended_values_when_legacy_fields_are_sentinels(monkeypatch: pytest.MonkeyPatch) -> None:
content = bytearray(_zip64(monkeypatch, 0))
end = content.rfind(b"PK\x05\x06")
struct.pack_into("<HHII", content, end + 8, 0xFFFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFF)
damaged = bytes(content).replace(b"\xc3\xa9.py", b"\xffa.py")
with open_zip_with_replacement_names(damaged) as archive:
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
def test_zip64_extensible_data_is_not_part_of_central_directory(monkeypatch: pytest.MonkeyPatch) -> None:
content = bytearray(_zip64(monkeypatch, 0))
record = content.rfind(b"PK\x06\x06")
locator = content.rfind(b"PK\x06\x07")
extension = b"extension"
struct.pack_into("<Q", content, record + 4, 44 + len(extension))
content[locator:locator] = extension
damaged = bytes(content).replace(b"\xc3\xa9.py", b"\xffa.py")
with open_zip_with_replacement_names(damaged) as archive:
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
def test_zip64_recovery_preserves_crc_checks(monkeypatch: pytest.MonkeyPatch) -> None:
content = _zip64(monkeypatch, 0)
damaged = content.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"print('ok')", b"print('NO')")
with open_zip_with_replacement_names(damaged) as archive:
with pytest.raises(zipfile.BadZipFile, match="CRC"):
archive.read("scripts/\ufffda.py")
def test_zip64_recovery_rejects_mismatched_local_name(monkeypatch: pytest.MonkeyPatch) -> None:
original = _zip64(monkeypatch, 0)
directory = original.index(b"PK\x01\x02")
damaged = original[:directory] + original[directory:].replace(b"\xc3\xa9.py", b"\xffa.py")
with pytest.raises(zipfile.BadZipFile, match="names differ"):
open_zip_with_replacement_names(damaged)
def _damaged_utf8_zip() -> bytearray:
output = io.BytesIO()
with zipfile.ZipFile(output, "w") as archive:
archive.writestr("scripts/\u00e9.py", b"print('ok')\n")
return bytearray(output.getvalue().replace(b"\xc3\xa9.py", b"\xffa.py"))
@pytest.mark.parametrize("offset", [0xFFFFFFFF, 0xFFFFFFFE])
def test_recovery_rejects_unresolvable_local_header_offsets(offset: int) -> None:
content = _damaged_utf8_zip()
directory = content.index(b"PK\x01\x02")
struct.pack_into("<I", content, directory + 42, offset)
with pytest.raises(zipfile.BadZipFile, match="ZIP64 extra field|local header offset"):
open_zip_with_replacement_names(bytes(content))
@pytest.mark.parametrize("entries", [0, 2])
def test_recovery_rejects_inconsistent_directory_entry_counts(entries: int) -> None:
content = _damaged_utf8_zip()
end = content.rfind(b"PK\x05\x06")
struct.pack_into("<HH", content, end + 8, entries, entries)
with pytest.raises(zipfile.BadZipFile, match="directory bounds|directory length mismatch"):
open_zip_with_replacement_names(bytes(content))
@@ -124,7 +124,8 @@ def _agent_node(node_id: str, binding: object | None = None) -> dict:
return {"id": node_id, "data": data}
def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -> None:
@pytest.mark.parametrize("include_assets", [False, True])
def test_make_portable_agent_package_strips_workspace_credentials_and_assets(include_assets: bool) -> None:
soul = AgentSoulConfig.model_validate(
{
"model": {
@@ -164,8 +165,10 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
],
},
"env": {"secret_refs": [{"name": "GLOBAL_TOKEN", "value": "plain-secret", "id": "secret-1"}]},
"config_skills": [{"name": "research", "file_kind": "tool_file", "file_id": "skill-file"}],
"config_files": [{"name": "guide.md", "file_kind": "upload_file", "file_id": "config-file"}],
"config_skills": [{"name": "research", "file_id": "skill-file"}, {"name": "missing", "is_missing": True}],
"config_files": [
{"name": "guide.md", "file_kind": "upload_file", "file_id": "config-file", "mime_type": "text/markdown"}
],
"human": {
"contacts": [
{
@@ -179,7 +182,9 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
}
)
package = make_portable_agent_package(_agent(), soul)
original = soul.model_dump(mode="json")
package = make_portable_agent_package(_agent(), soul, include_assets=include_assets)
assert soul.model_dump(mode="json") == original
serialized = package.model_dump(mode="json")
assert package.soul.model is not None
@@ -189,20 +194,32 @@ def test_make_portable_agent_package_strips_workspace_credentials_and_assets() -
assert package.soul.tools.dify_tools[0].runtime_parameters["upload_file_id"] is None
assert package.soul.tools.dify_tools[0].runtime_parameters["api_key"] is None
assert package.soul.config_skills[0].name == "research"
assert package.soul.config_skills[0].file_id == ""
assert package.soul.config_skills[0].is_missing is True
assert package.soul.config_skills[0].file_id == ("skill-file" if include_assets else "")
assert package.soul.config_skills[0].is_missing is (not include_assets)
assert package.soul.config_files[0].name == "guide.md"
assert package.soul.config_files[0].file_id == ""
assert package.soul.config_files[0].is_missing is True
assert [asset.kind for asset in package.omitted_assets] == ["skill", "file"]
assert package.soul.config_files[0].file_id == ("config-file" if include_assets else "")
assert package.soul.config_files[0].is_missing is (not include_assets)
assert package.soul.config_skills[1].is_missing
assert package.soul.config_files[0].mime_type == "text/markdown"
assert [asset.kind for asset in package.omitted_assets] == (
["skill"] if include_assets else ["skill", "skill", "file"]
)
assert [asset.name for asset in package.omitted_assets] == (
["missing"] if include_assets else ["research", "missing", "guide.md"]
)
assert "plain-secret" not in str(serialized)
assert "model-secret" not in str(serialized)
assert "tool-secret" not in str(serialized)
assert "skill-file" not in str(serialized)
assert "config-file" not in str(serialized)
assert ("skill-file" in str(serialized)) is include_assets
assert ("config-file" in str(serialized)) is include_assets
assert package.soul.human.contacts[0].id is None
assert package.soul.human.contacts[0].name == "Reviewer"
package.soul.config_skills[0].description = "Changed"
package.soul.config_files[0].name = "changed.md"
package.soul.tools.cli_tools[0].env.secret_refs[0].name = "CHANGED"
assert soul.model_dump(mode="json") == original
def test_agent_package_round_trips_as_strict_dsl_dto() -> None:
package = make_portable_agent_package(_agent(), AgentSoulConfig())
@@ -590,22 +607,28 @@ def test_clone_inline_binding_copies_soul(unbound_session: Session) -> None:
assert create_kwargs["source"] == AgentSource.WORKFLOW
def test_extract_package_dependencies_covers_model_tools_and_knowledge(
def test_extract_package_dependencies_covers_models_features_tools_and_knowledge(
monkeypatch: pytest.MonkeyPatch, unbound_session: Session
) -> None:
model_dependency = Mock(side_effect=lambda provider: f"model:{provider}")
tool_dependency = Mock(side_effect=lambda provider: f"tool:{provider}")
monkeypatch.setattr(
"services.agent.dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
"services.agent.dependency_service.DependenciesAnalysisService.analyze_model_provider_dependency",
model_dependency,
)
monkeypatch.setattr(
"services.agent.dsl_service.DependenciesAnalysisService.analyze_tool_dependency",
"services.agent.dependency_service.DependenciesAnalysisService.analyze_tool_dependency",
tool_dependency,
)
soul = AgentSoulConfig.model_validate(
{
"model": {"plugin_id": "model-plugin", "model_provider": "provider/model", "model": "model"},
"app_features": {
"suggested_questions_after_answer": {
"enabled": True,
"model": {"provider": "provider/follow-up", "name": "follow-up-model"},
}
},
"tools": {
"dify_tools": [
{
@@ -631,9 +654,49 @@ def test_extract_package_dependencies_covers_model_tools_and_knowledge(
"retrieval": {
"mode": "single",
"model": {"provider": "provider/retrieval", "name": "embed", "mode": "embedding"},
},
"metadata_filtering": {
"mode": "automatic",
"model_config": {
"provider": "provider/metadata",
"name": "metadata-model",
"mode": "chat",
},
},
},
{
"id": "set-2",
"name": "Reranked Set",
"datasets": [{"id": "dataset-2", "name": "Reranked Docs"}],
"query": {"mode": "user_query", "value": "query"},
"retrieval": {
"mode": "multiple",
"top_k": 5,
"reranking_enable": True,
"reranking_mode": "reranking_model",
"reranking_model": {"provider": "provider/rerank", "model": "rerank"},
},
}
},
{
"id": "set-3",
"name": "Weighted Set",
"datasets": [{"id": "dataset-3", "name": "Weighted Docs"}],
"query": {"mode": "user_query", "value": "query"},
"retrieval": {
"mode": "multiple",
"top_k": 5,
"reranking_enable": True,
"reranking_mode": "weighted_score",
"weights": {
"vector_setting": {
"embedding_provider_name": "provider/embedding",
"embedding_model_name": "embedding-model",
"vector_weight": 0.7,
},
"keyword_setting": {"keyword_weight": 0.3},
},
},
},
]
},
}
@@ -645,13 +708,72 @@ def test_extract_package_dependencies_covers_model_tools_and_knowledge(
assert dependencies == [
"model:provider/model",
"model:provider/follow-up",
"tool:provider/tool",
"tool:plugin-id/fallback-provider",
"model:provider/retrieval",
"model:provider/metadata",
"model:provider/rerank",
"model:provider/embedding",
]
def test_extract_package_dependencies_ignores_inactive_knowledge_models(
monkeypatch: pytest.MonkeyPatch, unbound_session: Session
) -> None:
model_dependency = Mock(side_effect=lambda provider: f"model:{provider}")
monkeypatch.setattr(
"services.agent.dependency_service.DependenciesAnalysisService.analyze_model_provider_dependency",
model_dependency,
)
soul = AgentSoulConfig.model_validate(
{
"knowledge": {
"sets": [
{
"id": "set-1",
"name": "Set",
"datasets": [{"id": "dataset-1", "name": "Docs"}],
"query": {"mode": "user_query", "value": "query"},
"retrieval": {
"mode": "multiple",
"top_k": 5,
"reranking_enable": False,
"reranking_mode": "weighted_score",
"reranking_model": {"provider": "provider/inactive-rerank", "model": "rerank"},
"weights": {
"vector_setting": {
"embedding_provider_name": "provider/inactive-embedding",
}
},
"model": {
"provider": "provider/inactive-retrieval",
"name": "retrieval-model",
"mode": "chat",
},
},
"metadata_filtering": {
"mode": "disabled",
"model_config": {
"provider": "provider/inactive-metadata",
"name": "metadata-model",
"mode": "chat",
},
},
}
]
}
}
)
dependencies = AgentDslService(unbound_session).extract_package_dependencies(
{"agent_1": make_portable_agent_package(_agent(), soul)}
)
assert dependencies == []
model_dependency.assert_not_called()
def test_create_imported_inline_agent_uses_import_provenance(unbound_session: Session) -> None:
service = AgentDslService(unbound_session)
soul = AgentSoulConfig(config_note="inline")
@@ -40,6 +40,7 @@ from models.agent_config_entities import (
from models.enums import AppStatus, ConversationFromSource, ConversationStatus
from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message
from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest
from models.tools import ToolFile
from models.workflow import Workflow, WorkflowType
from services.agent import composer_service, roster_service
from services.agent.agent_soul_state import agent_soul_has_model
@@ -5603,7 +5604,17 @@ class TestWorkflowAgentDraftBindingSync:
skill_id="skill-1",
priority=0,
)
session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding])
archive_file = ToolFile(
user_id="account-1",
tenant_id="tenant-1",
conversation_id=None,
file_key="tools/research.zip",
mimetype="application/zip",
name="research.zip",
size=1,
)
archive_file.id = "archive-1"
session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding, archive_file])
session.commit()
WorkflowAgentPublishService.validate_agent_nodes_for_publish(
File diff suppressed because it is too large. Load diff
@@ -9,8 +9,8 @@ import zlib
import pytest
from services.agent import skill_package_service as skill_package_service_module
from services.agent.skill_package_service import NormalizedSkillPackage, SkillPackageError, SkillPackageService
from tests.unit_tests.config_override import apply_config_overrides
_SKILL_MD = """---
name: pdf-toolkit
@@ -35,36 +35,110 @@ def _normalize(members: dict[str, bytes], *, filename: str = "skill.zip") -> Nor
return SkillPackageService().validate_and_normalize(content=_zip(members), filename=filename)
def _archive_members(content: bytes) -> list[str]:
with zipfile.ZipFile(io.BytesIO(content)) as archive:
return sorted(info.filename for info in archive.infolist() if not info.is_dir())
def test_normalize_replaces_invalid_utf8_member_names() -> None:
original = _zip(
{
"SKILL.md": _SKILL_MD.encode(),
"scripts/\u00e9.py": b"print('keep payload')\n",
"assets/\u00f6.txt": b"valid unicode name",
"assets/X.txt": b"legacy encoded name",
}
)
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"assets/X.txt", b"assets/\x82.txt")
service = SkillPackageService()
assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
assert archive.read("scripts/\ufffda.py") == b"print('keep payload')\n"
assert archive.read("assets/\u00f6.txt") == b"valid unicode name"
assert archive.read("assets/\u00e9.txt") == b"legacy encoded name"
assert archive.testzip() is None
def test_valid_skill_normalizes_manifest():
manifest = _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}).manifest
assert manifest.name == "pdf-toolkit"
assert manifest.description == "Tools for working with PDF files."
assert manifest.entry_path == "SKILL.md"
assert set(manifest.files) == {"SKILL.md", "scripts/run.py"}
assert manifest.size > 0
assert len(manifest.hash) == 64
def test_normalize_recovers_zip64_skill_names(monkeypatch: pytest.MonkeyPatch) -> None:
with monkeypatch.context() as scoped:
scoped.setattr(zipfile, "ZIP64_LIMIT", 0)
original = _zip({"SKILL.md": _SKILL_MD.encode(), "scripts/\u00e9.py": b"print('ok')\n"})
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py")
service = SkillPackageService()
assert service.inspect(content=damaged, filename="skill.zip").name == "pdf-toolkit"
normalized = service.validate_and_normalize(content=damaged, filename="skill.zip")
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
assert archive.read("scripts/\ufffda.py") == b"print('ok')\n"
def test_validate_and_normalize_accepts_crlf_skill_md():
crlf_skill_md = _SKILL_MD.replace("\n", "\r\n")
package = _normalize({"SKILL.md": crlf_skill_md.encode()})
assert package.manifest.name == "pdf-toolkit"
assert package.manifest.description == "Tools for working with PDF files."
assert b"\r" not in package.skill_md_bytes
assert package.skill_md_bytes.decode() == _SKILL_MD
def test_replacement_member_names_must_remain_unique() -> None:
original = _zip(
{
"SKILL.md": _SKILL_MD.encode(),
"scripts/\u00e9.py": b"one",
"scripts/\u00f6.py": b"two",
}
)
damaged = original.replace(b"\xc3\xa9.py", b"\xffa.py").replace(b"\xc3\xb6.py", b"\xfea.py")
with pytest.raises(SkillPackageError) as error:
SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
assert error.value.code == "duplicate_member_path"
def test_name_and_description_are_required_in_frontmatter():
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": b"# heading-name\n\nbody"})
assert exc_info.value.code == "missing_skill_name"
def test_replacement_names_do_not_hide_local_header_mismatch() -> None:
original = _zip({"SKILL.md": _SKILL_MD.encode(), "\u00e9.py": b"payload"})
# Damage only the central name, leaving the local header unchanged.
central = original.index(b"PK\x01\x02")
damaged = original[:central] + original[central:].replace(b"\xc3\xa9.py", b"\xffa.py")
with pytest.raises(SkillPackageError) as error:
SkillPackageService().validate_and_normalize(content=damaged, filename="skill.zip")
assert error.value.code == "invalid_archive"
def test_normalized_highly_compressible_skill_passes_reinspection() -> None:
payload = b"x" * (1024 * 1024)
content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": payload}, compression=zipfile.ZIP_STORED)
service = SkillPackageService()
normalized = service.validate_and_normalize(content=content, filename="skill.zip")
inspection = service.inspect(content=normalized.archive_bytes, filename="skill.zip")
assert inspection.name == normalized.manifest.name
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
assert archive.getinfo("data.bin").compress_type == zipfile.ZIP_DEFLATED
assert archive.read("data.bin") == payload
def test_inspect_reports_uncompressed_size_without_rebuilding(monkeypatch: pytest.MonkeyPatch):
members = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}
def fail_if_called(*_args: object, **_kwargs: object) -> bytes:
raise AssertionError("inspection must not rebuild the archive")
monkeypatch.setattr(SkillPackageService, "_build_normalized_archive", fail_if_called)
inspection = SkillPackageService().inspect(content=_zip(members), filename="skill.zip")
assert inspection.name == "pdf-toolkit"
assert inspection.uncompressed_size == sum(len(content) for content in members.values())
@pytest.mark.parametrize("prefix", ["", "pdf-toolkit/"])
@pytest.mark.parametrize("crlf", [False, True])
def test_inspection_and_normalization_keep_distinct_size_meanings(prefix: str, crlf: bool) -> None:
skill_md = _SKILL_MD.replace("\n", "\r\n") if crlf else _SKILL_MD
script = b"print('hi')\n"
members = {f"{prefix}SKILL.md": skill_md.encode(), f"{prefix}scripts/run.py": script}
if prefix:
members["README.md"] = b"ignored outside the selected root"
content = _zip(members)
service = SkillPackageService()
inspection = service.inspect(content=content, filename="skill.zip")
normalized = service.validate_and_normalize(content=content, filename="skill.zip")
assert inspection.uncompressed_size == sum(len(value) for value in members.values())
expected_size = len(_SKILL_MD.encode()) + len(script)
assert normalized.manifest.size == expected_size
assert normalized.manifest.name == "pdf-toolkit"
assert normalized.manifest.description == "Tools for working with PDF files."
with zipfile.ZipFile(io.BytesIO(normalized.archive_bytes)) as archive:
assert sum(info.file_size for info in archive.infolist()) == expected_size
assert archive.read("SKILL.md") == _SKILL_MD.encode()
if prefix or crlf:
assert inspection.uncompressed_size > normalized.manifest.size
def test_shallowest_skill_md_preferred_during_normalization():
@@ -73,96 +147,40 @@ def test_shallowest_skill_md_preferred_during_normalization():
assert manifest.files == ["SKILL.md", "nested/SKILL.md"]
def test_validate_and_normalize_keeps_root_skill_unchanged():
package = _normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"})
@pytest.mark.parametrize(
("prefix", "extra_members"),
[
pytest.param("", {}, id="root"),
pytest.param("pdf-toolkit/", {}, id="single-folder"),
pytest.param("pdf-toolkit/", {"README.md": b"bundle notes\n"}, id="root-outsider"),
pytest.param("pdf-toolkit/", {".": b"x"}, id="dot-outsider"),
pytest.param("pdf-toolkit/", {"bundle/other.txt": b"x"}, id="nested-outsider"),
pytest.param("bundle/pdf-toolkit/", {}, id="deep-root"),
],
)
def test_normalization_preserves_selected_skill_contents(prefix: str, extra_members: dict[str, bytes]) -> None:
expected = {"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('hi')\n"}
original = _zip({**{prefix + path: content for path, content in expected.items()}, **extra_members})
package = SkillPackageService().validate_and_normalize(content=original, filename="skill.zip")
assert package.manifest.name == "pdf-toolkit"
assert package.manifest.description == "Tools for working with PDF files."
assert package.manifest.size == sum(len(content) for content in expected.values())
assert package.manifest.entry_path == "SKILL.md"
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
assert package.skill_md_bytes == _SKILL_MD.encode()
assert package.strip_prefix is None
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
assert len(package.manifest.hash) == 64
def test_validate_and_normalize_strips_single_top_level_folder():
package = _normalize(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
}
)
assert package.manifest.entry_path == "SKILL.md"
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
assert package.skill_md_bytes == _SKILL_MD.encode()
assert package.strip_prefix == "pdf-toolkit/"
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
def test_validate_and_normalize_strips_single_top_level_folder_ignoring_other_root_entries():
package = _normalize(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
"README.md": b"bundle notes\n",
}
)
assert package.manifest.entry_path == "SKILL.md"
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
assert package.skill_md_bytes == _SKILL_MD.encode()
assert package.strip_prefix == "pdf-toolkit/"
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
def test_validate_and_normalize_strips_single_top_level_folder_dropping_nested_foreign_paths():
package = _normalize(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
"bundle/other.txt": b"x",
}
)
assert package.manifest.entry_path == "SKILL.md"
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
assert package.skill_md_bytes == _SKILL_MD.encode()
assert package.strip_prefix == "pdf-toolkit/"
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
def test_validate_and_normalize_rejects_multiple_depth_2_skill_roots_with_sibling_skill_tree():
with pytest.raises(SkillPackageError) as exc_info:
_normalize(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
"other-tool/SKILL.md": _SKILL_MD.encode(),
}
)
assert exc_info.value.code == "files_outside_skill_root"
def test_validate_and_normalize_strips_deeper_selected_skill_root():
members = {
"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"bundle/pdf-toolkit/scripts/run.py": b"print('hi')\n",
}
original_upload_bytes = _zip(members)
package = SkillPackageService().validate_and_normalize(content=original_upload_bytes, filename="skill.zip")
assert package.manifest.entry_path == "SKILL.md"
assert package.manifest.files == ["SKILL.md", "scripts/run.py"]
assert package.strip_prefix == "bundle/pdf-toolkit/"
assert _archive_members(package.archive_bytes) == ["SKILL.md", "scripts/run.py"]
assert package.manifest.files == sorted(expected)
with zipfile.ZipFile(io.BytesIO(package.archive_bytes)) as archive:
assert archive.namelist() == sorted(expected)
assert {path: archive.read(path) for path in archive.namelist()} == expected
assert package.manifest.hash == hashlib.sha256(package.archive_bytes).hexdigest()
assert package.manifest.hash != hashlib.sha256(original_upload_bytes).hexdigest()
if prefix:
assert package.manifest.hash != hashlib.sha256(original).hexdigest()
@pytest.mark.parametrize(
("members", "filename", "code"),
[
({"README.md": b"x"}, "skill.zip", "missing_skill_md"),
({"SKILL.md": _SKILL_MD.encode(), ".": b"x"}, "skill.zip", "unsafe_path"),
({"SKILL.md": _SKILL_MD.encode()}, "skill.tar", "unsupported_extension"),
({"SKILL.md": b""}, "skill.zip", "empty_skill_md"),
({"SKILL.md": b"---\ndescription: valid\n---\n# no name here"}, "skill.zip", "missing_skill_name"),
@@ -178,6 +196,43 @@ def test_validate_and_normalize_strips_deeper_selected_skill_root():
"invalid_skill_description",
),
({"SKILL.md": b"\xff\xfenot utf8"}, "skill.zip", "skill_md_not_utf8"),
pytest.param({"SKILL.md": b"# heading-name\n\nbody"}, "skill.zip", "missing_skill_name", id="no-frontmatter"),
pytest.param(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('hi')\n",
"other-tool/SKILL.md": _SKILL_MD.encode(),
},
"skill.zip",
"files_outside_skill_root",
id="multiple-skill-roots",
),
pytest.param({"../evil.txt": b"x", "SKILL.md": _SKILL_MD.encode()}, "skill.zip", "unsafe_path", id="zip-slip"),
pytest.param(
{"SKILL.md": b"---\n: : : not yaml\n---\n# x\n"},
"skill.zip",
"invalid_frontmatter",
id="invalid-frontmatter",
),
pytest.param(
{"SKILL.md": b"---\n# heading-wins\nbody"}, "skill.zip", "missing_skill_name", id="unterminated-frontmatter"
),
pytest.param(
{"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(), "README.md": b"x"},
"skill.zip",
"files_outside_skill_root",
id="outside-deep-root",
),
pytest.param(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('x')\n",
"pdf-toolkit/scripts/./run.py": b"print('y')\n",
},
"skill.zip",
"duplicate_member_path",
id="duplicate-normalized-path",
),
],
)
def test_invalid_packages_rejected(members: dict[str, bytes], filename: str, code: str):
@@ -193,13 +248,6 @@ def test_non_zip_content_rejected():
assert exc_info.value.code == "invalid_archive"
def test_zip_slip_member_rejected():
payload = _zip({"../evil.txt": b"x", "SKILL.md": _SKILL_MD.encode()})
with pytest.raises(SkillPackageError) as exc_info:
SkillPackageService().validate_and_normalize(content=payload, filename="skill.zip")
assert exc_info.value.code == "unsafe_path"
def test_empty_archive_rejected():
with pytest.raises(SkillPackageError) as exc_info:
SkillPackageService().validate_and_normalize(content=b"", filename="skill.zip")
@@ -207,7 +255,7 @@ def test_empty_archive_rejected():
def test_validate_and_normalize_rejects_skill_md_too_large(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(skill_package_service_module, "_MAX_SKILL_MD_BYTES", 8)
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_SKILL_MD_BYTES=8)
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": _SKILL_MD.encode()})
@@ -215,15 +263,15 @@ def test_validate_and_normalize_rejects_skill_md_too_large(monkeypatch: pytest.M
def test_validate_and_normalize_rejects_too_many_entries(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(skill_package_service_module, "_MAX_ENTRIES", 1)
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_ENTRIES=1)
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"print('x')\n"})
_normalize({"SKILL.md": _SKILL_MD.encode(), "assets/": b""})
assert exc_info.value.code == "too_many_entries"
def test_validate_and_normalize_rejects_archive_too_large_uncompressed(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(skill_package_service_module, "_MAX_UNCOMPRESSED_BYTES", 32)
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=32)
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": _SKILL_MD.encode(), "scripts/run.py": b"x" * 33})
@@ -231,8 +279,6 @@ def test_validate_and_normalize_rejects_archive_too_large_uncompressed(monkeypat
def test_validate_and_normalize_rejects_archive_too_large_uploaded_bytes(monkeypatch: pytest.MonkeyPatch):
from tests.unit_tests.config_override import apply_config_overrides
apply_config_overrides(monkeypatch, UPLOAD_SKILL_FILE_SIZE_LIMIT=1)
with pytest.raises(SkillPackageError) as exc_info:
@@ -240,35 +286,14 @@ def test_validate_and_normalize_rejects_archive_too_large_uploaded_bytes(monkeyp
assert exc_info.value.code == "archive_too_large"
def test_bad_frontmatter_yaml_rejected():
bad = b"---\n: : : not yaml\n---\n# x\n"
def test_high_compression_ratio_uses_absolute_expansion_limit(monkeypatch: pytest.MonkeyPatch) -> None:
content = _zip({"SKILL.md": _SKILL_MD.encode(), "data.bin": b"x" * (1024 * 1024)})
service = SkillPackageService()
assert service.inspect(content=content, filename="skill.zip").name == "pdf-toolkit"
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024)
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": bad})
assert exc_info.value.code == "invalid_frontmatter"
def test_unterminated_frontmatter_rejected():
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"SKILL.md": b"---\n# heading-wins\nbody"})
assert exc_info.value.code == "missing_skill_name"
def test_validate_and_normalize_rejects_files_outside_selected_skill_root():
with pytest.raises(SkillPackageError) as exc_info:
_normalize({"bundle/pdf-toolkit/SKILL.md": _SKILL_MD.encode(), "README.md": b"x"})
assert exc_info.value.code == "files_outside_skill_root"
def test_validate_and_normalize_rejects_duplicate_normalized_paths():
with pytest.raises(SkillPackageError) as exc_info:
_normalize(
{
"pdf-toolkit/SKILL.md": _SKILL_MD.encode(),
"pdf-toolkit/scripts/run.py": b"print('x')\n",
"pdf-toolkit/scripts/./run.py": b"print('y')\n",
}
)
assert exc_info.value.code == "duplicate_member_path"
service.inspect(content=content, filename="skill.zip")
assert exc_info.value.code == "archive_too_large"
def test_validate_and_normalize_maps_member_decompression_failures_to_invalid_archive(monkeypatch: pytest.MonkeyPatch):
@@ -366,7 +366,7 @@ def test_push_accepts_tenant_scoped_tool_file_sources_from_different_upload_owne
patch.object(
service._skill_normalizer,
"normalize",
return_value=(skill_ref, object()),
return_value=skill_ref,
),
):
manifest = service.push(
@@ -9,11 +9,13 @@ import yaml
from sqlalchemy import event, select
from sqlalchemy.orm import Session, sessionmaker
from constants.dsl_version import CURRENT_APP_DSL_VERSION
from core.rbac import RBACPermission, RBACResourceScope
from core.workflow.llm_environment_variable import LLMEnvironmentVariable
from models import Account, App, AppMode, Tenant
from models.model import AppModelConfig, AppModelConfigDict, IconType
from models.workflow import Workflow, WorkflowType
from services.agent.dsl_entities import AgentPackage
from services.app_dsl_service import AppDslService, Import, PendingData
from services.enterprise.enterprise_service import EnterpriseService
from services.entities.dsl_entities import ImportStatus
@@ -795,3 +797,56 @@ def test_append_workflow_export_data_reports_missing_selected_workflow(
session=unbound_session,
workflow_id=workflow_id,
)
@pytest.mark.parametrize(
"mode",
[AppMode.AGENT, AppMode.WORKFLOW, AppMode.ADVANCED_CHAT, AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION],
)
def test_export_dsl_preserves_envelope_and_mode_specific_content(
mode: AppMode, monkeypatch: pytest.MonkeyPatch, unbound_session: Session
) -> None:
app = _app(mode=mode)
package = AgentPackage.model_validate({"metadata": {"name": app.name}, "soul": {}})
monkeypatch.setattr(
"services.app_dsl_service.AgentDslService.export_agent_app",
Mock(return_value=("agent_1", {"agent_1": package})),
)
monkeypatch.setattr(
"services.app_dsl_service.DependenciesAnalysisService.generate_dependencies", Mock(return_value=[])
)
def append_workflow(*, export_data: dict[str, object], **_kwargs: object) -> None:
export_data["workflow"] = {"fixture": "workflow"}
def append_model(export_data: dict[str, object], *_args: object, **_kwargs: object) -> None:
export_data["model_config"] = {"fixture": "model"}
monkeypatch.setattr(AppDslService, "_append_workflow_export_data", Mock(side_effect=append_workflow))
monkeypatch.setattr(AppDslService, "_append_model_config_export_data", Mock(side_effect=append_model))
data = yaml.safe_load(AppDslService.export_dsl(app, session=unbound_session))
assert data["version"] == CURRENT_APP_DSL_VERSION
assert data["kind"] == "app"
assert data["app"] == {
"name": "Existing app",
"description": "",
"mode": mode.value,
"icon_type": "emoji",
"icon": "robot",
"icon_background": "#FFFFFF",
"use_icon_as_answer_icon": False,
}
if mode == AppMode.AGENT:
assert data["agent"] == {"package_ref": "agent_1"}
assert AgentPackage.model_validate(data["agent_packages"]["agent_1"]) == package
assert data["dependencies"] == []
assert "workflow" not in data
assert "model_config" not in data
elif mode in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT}:
assert data["workflow"] == {"fixture": "workflow"}
assert "agent" not in data
assert "model_config" not in data
else:
assert data["model_config"] == {"fixture": "model"}
assert "agent" not in data
assert "workflow" not in data
@@ -2829,21 +2829,27 @@ def test_import_skill_package_rejects_archive_larger_than_upload_skill_limit(mon
assert exc_info.value.code == "archive_too_large"
def test_import_skill_package_rejects_zip_bomb_before_reading_members() -> None:
def test_import_skill_package_rejects_excessive_expansion_before_reading_members(
monkeypatch: pytest.MonkeyPatch,
) -> None:
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024 * 1024)
package = io.BytesIO()
with zipfile.ZipFile(package, "w", compression=zipfile.ZIP_DEFLATED) as archive:
archive.writestr("payload.bin", b"\x00" * (8 * 1024 * 1024))
service = SkillManagementService(tool_file_manager=_FakeToolFileManager())
with pytest.raises(SkillManagementServiceError) as exc_info:
with (
patch.object(zipfile.ZipFile, "read", side_effect=AssertionError("member read before size validation")),
pytest.raises(SkillManagementServiceError) as exc_info,
):
service._draft_payload_from_zip(
tenant_id=TENANT,
user_id=USER,
archive_bytes=package.getvalue(),
)
assert exc_info.value.code == "invalid_skill_package"
assert exc_info.value.code == "skill_too_large"
def test_publish_and_export_include_binary_tool_files() -> None:
@@ -3273,6 +3279,7 @@ def test_runtime_agent_skills_use_published_identity_when_draft_metadata_changed
description="draft description",
)
version = SimpleNamespace(
id="version-1",
archive_tool_file_id="archive-1",
archive_size=10,
hash_code="hash-1",
@@ -3283,8 +3290,10 @@ def test_runtime_agent_skills_use_published_identity_when_draft_metadata_changed
description="published description",
),
)
binding = SimpleNamespace(priority=0)
tool_file = SimpleNamespace(file_key="tools/archive-1.zip")
session = MagicMock()
session.execute.return_value = [(SimpleNamespace(), skill, version)]
session.execute.return_value = [(binding, skill, version, tool_file)]
with patch("services.skill_management_service.session_factory.create_session") as create_session:
create_session.return_value.__enter__.return_value = session
@@ -3358,6 +3367,29 @@ def test_agent_skill_binding_changes_require_agent_publish_before_runtime_load()
assert service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT)[0]["name"] == "finance-sop"
with session_factory.create_session() as session:
agent = session.get(Agent, AGENT)
assert agent is not None
next_snapshot = AgentConfigSnapshot(
tenant_id=TENANT,
agent_id=AGENT,
version=2,
config_snapshot=AgentSoulConfig(),
created_by=USER,
)
session.add(next_snapshot)
session.flush()
agent.active_config_snapshot_id = next_snapshot.id
session.commit()
assert service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT) == []
historical_skills = service.list_runtime_agent_skills(
tenant_id=TENANT, agent_id=AGENT, config_snapshot_id=snapshot_id
)
assert [skill["name"] for skill in historical_skills] == ["finance-sop"]
draft_skills = service.list_runtime_agent_skills(tenant_id=TENANT, agent_id=AGENT, include_draft=True)
assert [skill["name"] for skill in draft_skills] == ["finance-sop"]
def test_runtime_agent_skill_pull_normalizes_archive_identity_to_published_metadata() -> None:
archive_buffer = io.BytesIO()
@@ -3378,3 +3410,14 @@ def test_runtime_agent_skill_pull_normalizes_archive_identity_to_published_metad
assert "description: Published description" in skill_md
assert "display-name: Published Name" in skill_md
assert archive.read("references/example.md") == b"Example"
def test_archive_limits_accept_high_compression_ratio_within_size_budget(monkeypatch: pytest.MonkeyPatch) -> None:
output = io.BytesIO()
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
archive.writestr("data.bin", b"x" * (1024 * 1024))
with zipfile.ZipFile(io.BytesIO(output.getvalue())) as archive:
assert SkillManagementService._validate_archive_limits(archive)[0].file_size == 1024 * 1024
apply_config_overrides(monkeypatch, SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=1024)
with pytest.raises(SkillManagementServiceError, match="uncompressed size limit"):
SkillManagementService._validate_archive_limits(archive)
@@ -66,6 +66,12 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
UPLOAD_SKILL_FILE_SIZE_LIMIT=50
SKILL_PACKAGE_MAX_UNCOMPRESSED_BYTES=209715200
SKILL_PACKAGE_MAX_ENTRIES=5000
SKILL_PACKAGE_MAX_SKILL_MD_BYTES=1048576
AGENT_PACKAGE_MAX_BYTES=536870912
AGENT_PACKAGE_MAX_MANIFEST_BYTES=5242880
AGENT_PACKAGE_MAX_ENTRIES=5000
API_SENTRY_DSN=
API_SENTRY_TRACES_SAMPLE_RATE=1.0
API_SENTRY_PROFILES_SAMPLE_RATE=1.0