mirror of
https://github.com/langgenius/dify.git
synced 2026-10-04 13:47:55 +08:00
feat(agent): record E2B sandbox runtime usage (#42554)
This commit is contained in:
1 parent
1fa3245e83
commit
af4f19840c
29 files changed
+3501
-5
No files matched your search
@@ -5,7 +5,9 @@ instances for plugin-daemon and Dify API inner calls, route wiring, and a
|
||||
process-local scheduler. Run execution happens in background ``asyncio`` tasks
|
||||
rather than request handlers, so client disconnects do not cancel the agent
|
||||
runtime. Redis persists run records and per-run event streams with configured
|
||||
retention only; it is not used as a job queue. Agenton layers and providers
|
||||
retention. Optional provider accounting is exposed as a separate one-shot HTTP
|
||||
endpoint driven by Celery Beat; it starts no tasks during lifespan startup and
|
||||
adds no reporting to business operations. Redis is not used as a run job queue. Agenton layers and providers
|
||||
stay state-only: they borrow the lifespan-owned clients through the runner and
|
||||
receive runtime-backend and Shell settings through provider construction rather
|
||||
than reading environment variables themselves. The standard server mounts the
|
||||
@@ -29,6 +31,7 @@ from dify_agent.runtime.compositor_factory import create_default_layer_providers
|
||||
from dify_agent.runtime.run_scheduler import RunScheduler
|
||||
from dify_agent.server.auth import create_bearer_token_dependency
|
||||
from dify_agent.server.observability import configure_server_observability
|
||||
from dify_agent.server.routes.e2b_usage import create_e2b_usage_router
|
||||
from dify_agent.server.routes.runs import create_runs_router
|
||||
from dify_agent.server.routes.execution_bindings import create_execution_bindings_router
|
||||
from dify_agent.server.routes.home_snapshots import create_home_snapshots_router
|
||||
@@ -146,6 +149,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
|
||||
control_plane_router.include_router(create_home_snapshots_router(lambda: home_snapshot_service))
|
||||
control_plane_router.include_router(create_binding_files_router(lambda: binding_file_service))
|
||||
app.include_router(control_plane_router)
|
||||
app.include_router(create_e2b_usage_router(resolved_settings))
|
||||
app.include_router(
|
||||
create_agent_stub_router(
|
||||
token_codec=agent_stub_token_codec,
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Provider-specific client for E2B usage facts and completed-scan checkpoints.
|
||||
|
||||
Only the scheduled collector uses this client. Business operations do not report
|
||||
accounting events or borrow this connection pool.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_MAX_BATCH_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
class E2BUsageState(BaseModel):
|
||||
enabled: bool
|
||||
project_id: str
|
||||
started_at: datetime | None = None
|
||||
checkpoint_at: datetime | None = None
|
||||
full_scan_at: datetime | None = None
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class _IngestResult(BaseModel):
|
||||
accepted: int = Field(ge=0)
|
||||
duplicates: int = Field(ge=0)
|
||||
conflicts: int = Field(ge=0)
|
||||
ignored: int = Field(ge=0)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class E2BUsageApiClient:
|
||||
"""Use a collection-scoped inner client; credentials never reach E2B."""
|
||||
|
||||
client: httpx.AsyncClient
|
||||
base_url: str
|
||||
api_key: str
|
||||
project_id: str
|
||||
|
||||
async def get_state(self) -> E2BUsageState:
|
||||
response = await self.client.get(
|
||||
f"{self.base_url.rstrip('/')}/inner/api/agent/sandbox-usage/state",
|
||||
params={"project_id": self.project_id},
|
||||
headers={"X-Inner-Api-Key": self.api_key},
|
||||
timeout=15.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
state = E2BUsageState.model_validate(response.json())
|
||||
if state.project_id != self.project_id:
|
||||
raise ValueError("sandbox usage state project mismatch")
|
||||
for value in (state.started_at, state.checkpoint_at, state.full_scan_at):
|
||||
if value is not None and value.utcoffset() is None:
|
||||
raise ValueError("sandbox usage state requires timezone-aware timestamps")
|
||||
if state.enabled and state.started_at is None:
|
||||
raise ValueError("enabled sandbox usage state requires a fixed activation time")
|
||||
return state
|
||||
|
||||
async def post_events(self, events: list[dict[str, Any]]) -> None:
|
||||
if not events or len(events) > 100:
|
||||
raise ValueError("sandbox usage batches require 1 to 100 events")
|
||||
# The API limits the whole HTTP body, not just its event count. Splitting
|
||||
# here covers provider pages. Repeated scheduled scans use stable
|
||||
# provider IDs, so partial success cannot multiply execution usage.
|
||||
batch: list[dict[str, Any]] = []
|
||||
for event in events:
|
||||
candidate = [*batch, event]
|
||||
body = self._encode_batch(candidate)
|
||||
if len(body) > _MAX_BATCH_BYTES:
|
||||
if not batch:
|
||||
raise ValueError("sandbox usage event exceeds API request size")
|
||||
await self._post_batch(self._encode_batch(batch), len(batch))
|
||||
batch = [event]
|
||||
if len(self._encode_batch(batch)) > _MAX_BATCH_BYTES:
|
||||
raise ValueError("sandbox usage event exceeds API request size")
|
||||
else:
|
||||
batch = candidate
|
||||
if batch:
|
||||
await self._post_batch(self._encode_batch(batch), len(batch))
|
||||
|
||||
def _encode_batch(self, events: list[dict[str, Any]]) -> bytes:
|
||||
return json.dumps(
|
||||
{"project_id": self.project_id, "events": events},
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
|
||||
async def _post_batch(self, body: bytes, count: int) -> None:
|
||||
response = await self.client.post(
|
||||
f"{self.base_url.rstrip('/')}/inner/api/agent/sandbox-usage/events",
|
||||
headers={"X-Inner-Api-Key": self.api_key, "Content-Type": "application/json"},
|
||||
content=body,
|
||||
timeout=15.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = _IngestResult.model_validate(response.json())
|
||||
if result.accepted + result.duplicates + result.conflicts + result.ignored != count:
|
||||
raise ValueError("sandbox usage acknowledgement does not cover the entire batch")
|
||||
if result.conflicts:
|
||||
logger.error("sandbox usage ingestion reported conflicts", extra={"conflicts": result.conflicts})
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""A single bounded E2B event scan, invoked by the existing Celery Beat schedule.
|
||||
|
||||
Offsets are used within one bounded scan only. Every scan starts at zero, with
|
||||
an overlap window and periodic rescan of the provider retention window. Neither
|
||||
redelivery nor overlapping scheduled scans can inflate the API's idempotent
|
||||
execution ledger. A partial scan never advances coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
from typing import Any, Callable
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from dify_agent.server.e2b_usage_client import E2BUsageApiClient, utc_now
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_PROVIDER_RETENTION = timedelta(days=7)
|
||||
_EVENTS_URL = "https://api.e2b.app/events/sandboxes"
|
||||
|
||||
|
||||
def _field(event: dict[str, Any], camel: str, snake: str) -> Any:
|
||||
return event.get(camel, event.get(snake))
|
||||
|
||||
|
||||
def _timestamp(value: Any) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("provider event timestamp is invalid")
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.utcoffset() is None:
|
||||
raise ValueError("provider event timestamp must have a timezone")
|
||||
return parsed
|
||||
|
||||
|
||||
def provider_event(raw: dict[str, Any], *, project_id: str) -> dict[str, Any]:
|
||||
"""Preserve the raw provider payload; API owns normalization and T0 filtering."""
|
||||
project = _field(raw, "sandboxTeamId", "sandbox_team_id")
|
||||
if project != project_id:
|
||||
raise ValueError("E2B event project mismatch or missing project")
|
||||
event_id = raw.get("id")
|
||||
event_type = raw.get("type")
|
||||
if not isinstance(event_id, str) or not event_id or not isinstance(event_type, str):
|
||||
raise ValueError("E2B event missing stable identity")
|
||||
_timestamp(raw.get("timestamp"))
|
||||
return {
|
||||
"id": event_id,
|
||||
"source": "provider",
|
||||
"type": event_type,
|
||||
"timestamp": raw.get("timestamp"),
|
||||
"sandbox_id": _field(raw, "sandboxId", "sandbox_id"),
|
||||
"execution_id": _field(raw, "sandboxExecutionId", "sandbox_execution_id"),
|
||||
"payload": raw,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class E2BUsageCollector:
|
||||
provider_client: httpx.AsyncClient
|
||||
usage_client: E2BUsageApiClient
|
||||
api_key: str
|
||||
project_id: str
|
||||
overlap_seconds: int = 900
|
||||
full_scan_interval_seconds: int = 3600
|
||||
max_pages: int = 1000
|
||||
clock: Callable[[], datetime] = utc_now
|
||||
|
||||
async def collect_once(self) -> bool:
|
||||
state = await self.usage_client.get_state()
|
||||
if not state.enabled:
|
||||
return False
|
||||
if state.started_at is None or state.project_id != self.project_id:
|
||||
raise ValueError("sandbox usage activation/project state is invalid")
|
||||
now = self.clock()
|
||||
if state.started_at > now:
|
||||
return False
|
||||
retention_floor = now - _PROVIDER_RETENTION
|
||||
full = (
|
||||
state.full_scan_at is None or (now - state.full_scan_at).total_seconds() >= self.full_scan_interval_seconds
|
||||
)
|
||||
anchor = state.started_at if full else (state.checkpoint_at or state.started_at)
|
||||
lower = max(state.started_at, retention_floor, anchor - timedelta(seconds=self.overlap_seconds))
|
||||
if (state.checkpoint_at or state.started_at) < retention_floor:
|
||||
# The API checkpoint also records window_start, so a retention gap is
|
||||
# visible instead of being misrepresented as complete T0 coverage.
|
||||
logger.error("sandbox usage collection gap exceeds provider retention")
|
||||
await self.usage_client.post_events(
|
||||
[
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"source": "application",
|
||||
"type": "collector_retention_gap",
|
||||
"timestamp": now.isoformat(),
|
||||
"payload": {
|
||||
"uncovered_start": (state.checkpoint_at or state.started_at).isoformat(),
|
||||
"uncovered_end": retention_floor.isoformat(),
|
||||
"assumed_retention_seconds": int(_PROVIDER_RETENTION.total_seconds()),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
count = 0
|
||||
complete = False
|
||||
pages = 0
|
||||
for page in range(self.max_pages):
|
||||
response = await self.provider_client.get(
|
||||
_EVENTS_URL,
|
||||
headers={"X-API-Key": self.api_key},
|
||||
params={"limit": 100, "offset": page * 100, "orderAsc": "false"},
|
||||
timeout=15.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
raw_events = response.json()
|
||||
if not isinstance(raw_events, list) or len(raw_events) > 100:
|
||||
raise ValueError("E2B events response must be a page of up to 100 events")
|
||||
pages += 1
|
||||
events: list[dict[str, Any]] = []
|
||||
all_older = bool(raw_events)
|
||||
for raw in raw_events:
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("invalid E2B event")
|
||||
event = provider_event(raw, project_id=self.project_id)
|
||||
occurred = _timestamp(raw.get("timestamp"))
|
||||
if occurred is None or occurred >= lower:
|
||||
all_older = False
|
||||
events.append(event)
|
||||
if events:
|
||||
await self.usage_client.post_events(events)
|
||||
count += len(events)
|
||||
if len(raw_events) < 100 or all_older:
|
||||
complete = True
|
||||
break
|
||||
if not complete:
|
||||
logger.warning("sandbox usage scan reached page bound; checkpoint unchanged", extra={"pages": pages})
|
||||
return False
|
||||
await self.usage_client.post_events(
|
||||
[
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"source": "application",
|
||||
"type": "collector_checkpoint",
|
||||
"timestamp": now.isoformat(),
|
||||
"payload": {
|
||||
"scan_started_at": now.isoformat(),
|
||||
"window_start": lower.isoformat(),
|
||||
"window_end": now.isoformat(),
|
||||
"completed": True,
|
||||
"mode": "full" if full else "incremental",
|
||||
"pages": pages,
|
||||
"events": count,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,99 @@
|
||||
"""One-shot provider accounting endpoint, invoked by the API's Celery task.
|
||||
|
||||
The E2B credential stays in its existing service. Collection owns separate HTTP
|
||||
clients, has no Redis leader or recurring task, and never touches runtime leases.
|
||||
Invalid optional accounting configuration fails only this endpoint.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from dify_agent.server.auth import create_bearer_token_dependency
|
||||
from dify_agent.server.e2b_usage_client import E2BUsageApiClient
|
||||
from dify_agent.server.e2b_usage_collector import E2BUsageCollector
|
||||
from dify_agent.server.settings import ServerSettings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_COLLECTION_TIMEOUT_SECONDS = 240.0
|
||||
|
||||
|
||||
class E2BUsageCollectRequest(BaseModel):
|
||||
project_id: str = Field(min_length=1, max_length=128)
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class E2BUsageCollectResponse(BaseModel):
|
||||
completed: bool
|
||||
|
||||
|
||||
class _CollectionOptions(BaseModel):
|
||||
overlap_seconds: int = Field(ge=1)
|
||||
full_scan_interval_seconds: int = Field(ge=1)
|
||||
max_pages: int = Field(ge=1, le=10000)
|
||||
|
||||
|
||||
def create_e2b_usage_router(settings: ServerSettings) -> APIRouter:
|
||||
async def require_configured_auth() -> None:
|
||||
# Other legacy control-plane routes may permit a missing token. New
|
||||
# collection requests must never expose the project key without auth.
|
||||
if not settings.api_token:
|
||||
raise HTTPException(status_code=503, detail="e2b_usage_collection_auth_unconfigured")
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/internal/e2b/usage",
|
||||
dependencies=[Depends(require_configured_auth), create_bearer_token_dependency(settings.api_token)],
|
||||
)
|
||||
|
||||
@router.post("/collect", response_model=E2BUsageCollectResponse)
|
||||
async def collect(request: E2BUsageCollectRequest) -> E2BUsageCollectResponse:
|
||||
project_id = settings.e2b_project_id.strip()
|
||||
if (
|
||||
not settings.sandbox_metering_enabled
|
||||
or settings.runtime_backend != "e2b"
|
||||
or not settings.e2b_api_key
|
||||
or not project_id
|
||||
or not settings.inner_api_key
|
||||
):
|
||||
raise HTTPException(status_code=503, detail="e2b_usage_collection_unavailable")
|
||||
if request.project_id != project_id:
|
||||
raise HTTPException(status_code=403, detail="e2b_usage_project_mismatch")
|
||||
try:
|
||||
options = _CollectionOptions.model_validate(
|
||||
{
|
||||
"overlap_seconds": settings.sandbox_metering_overlap_seconds,
|
||||
"full_scan_interval_seconds": settings.sandbox_metering_full_scan_interval_seconds,
|
||||
"max_pages": settings.sandbox_metering_max_pages,
|
||||
}
|
||||
)
|
||||
async with (
|
||||
asyncio.timeout(_COLLECTION_TIMEOUT_SECONDS),
|
||||
httpx.AsyncClient(timeout=15.0, trust_env=False) as provider_client,
|
||||
httpx.AsyncClient(timeout=15.0, trust_env=False) as inner_client,
|
||||
):
|
||||
collector = E2BUsageCollector(
|
||||
provider_client=provider_client,
|
||||
usage_client=E2BUsageApiClient(
|
||||
client=inner_client,
|
||||
base_url=settings.inner_api_url,
|
||||
api_key=settings.inner_api_key,
|
||||
project_id=project_id,
|
||||
),
|
||||
api_key=settings.e2b_api_key,
|
||||
project_id=project_id,
|
||||
overlap_seconds=options.overlap_seconds,
|
||||
full_scan_interval_seconds=options.full_scan_interval_seconds,
|
||||
max_pages=options.max_pages,
|
||||
)
|
||||
return E2BUsageCollectResponse(completed=await collector.collect_once())
|
||||
except TimeoutError as exc:
|
||||
logger.warning("E2B usage collection exceeded its deadline")
|
||||
raise HTTPException(status_code=504, detail="e2b_usage_collection_timed_out") from exc
|
||||
except Exception as exc:
|
||||
logger.warning("E2B usage collection failed", extra={"error_type": type(exc).__name__})
|
||||
raise HTTPException(status_code=503, detail="e2b_usage_collection_failed") from exc
|
||||
|
||||
return router
|
||||
@@ -88,6 +88,13 @@ class ServerSettings(BaseSettings):
|
||||
le=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
|
||||
)
|
||||
e2b_shellctl_port: int = Field(default=5004, ge=1, le=65535)
|
||||
sandbox_metering_enabled: bool = False
|
||||
e2b_project_id: str = ""
|
||||
# Optional accounting validates its numeric configuration at invocation,
|
||||
# so a bad metering value cannot prevent ordinary runtime startup.
|
||||
sandbox_metering_overlap_seconds: int | str = 900
|
||||
sandbox_metering_full_scan_interval_seconds: int | str = 3600
|
||||
sandbox_metering_max_pages: int | str = 1000
|
||||
openshell_gateway_endpoint: str | None = None
|
||||
openshell_workspace: str = "default"
|
||||
openshell_bearer_token: str | None = None
|
||||
|
||||
Reference in new issue
Block a user