feat(agent): record E2B sandbox runtime usage (#42554)

This commit is contained in:
zyssyz123 authored and GitHub committed 2026-09-21 07:11:12 +00:00
1 parent 1fa3245e83
commit af4f19840c
29 files changed
+3501 -5

No files matched your search

+5 -1
View File
@@ -5,7 +5,9 @@ instances for plugin-daemon and Dify API inner calls, route wiring, and a
process-local scheduler. Run execution happens in background ``asyncio`` tasks
rather than request handlers, so client disconnects do not cancel the agent
runtime. Redis persists run records and per-run event streams with configured
retention only; it is not used as a job queue. Agenton layers and providers
retention. Optional provider accounting is exposed as a separate one-shot HTTP
endpoint driven by Celery Beat; it starts no tasks during lifespan startup and
adds no reporting to business operations. Redis is not used as a run job queue. Agenton layers and providers
stay state-only: they borrow the lifespan-owned clients through the runner and
receive runtime-backend and Shell settings through provider construction rather
than reading environment variables themselves. The standard server mounts the
@@ -29,6 +31,7 @@ from dify_agent.runtime.compositor_factory import create_default_layer_providers
from dify_agent.runtime.run_scheduler import RunScheduler
from dify_agent.server.auth import create_bearer_token_dependency
from dify_agent.server.observability import configure_server_observability
from dify_agent.server.routes.e2b_usage import create_e2b_usage_router
from dify_agent.server.routes.runs import create_runs_router
from dify_agent.server.routes.execution_bindings import create_execution_bindings_router
from dify_agent.server.routes.home_snapshots import create_home_snapshots_router
@@ -146,6 +149,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
control_plane_router.include_router(create_home_snapshots_router(lambda: home_snapshot_service))
control_plane_router.include_router(create_binding_files_router(lambda: binding_file_service))
app.include_router(control_plane_router)
app.include_router(create_e2b_usage_router(resolved_settings))
app.include_router(
create_agent_stub_router(
token_codec=agent_stub_token_codec,
@@ -0,0 +1,112 @@
"""Provider-specific client for E2B usage facts and completed-scan checkpoints.
Only the scheduled collector uses this client. Business operations do not report
accounting events or borrow this connection pool.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import json
import logging
from typing import Any
import httpx
from pydantic import BaseModel, ConfigDict, Field
logger = logging.getLogger(__name__)
_MAX_BATCH_BYTES = 1024 * 1024
class E2BUsageState(BaseModel):
enabled: bool
project_id: str
started_at: datetime | None = None
checkpoint_at: datetime | None = None
full_scan_at: datetime | None = None
model_config = ConfigDict(extra="ignore")
class _IngestResult(BaseModel):
accepted: int = Field(ge=0)
duplicates: int = Field(ge=0)
conflicts: int = Field(ge=0)
ignored: int = Field(ge=0)
@dataclass(slots=True)
class E2BUsageApiClient:
"""Use a collection-scoped inner client; credentials never reach E2B."""
client: httpx.AsyncClient
base_url: str
api_key: str
project_id: str
async def get_state(self) -> E2BUsageState:
response = await self.client.get(
f"{self.base_url.rstrip('/')}/inner/api/agent/sandbox-usage/state",
params={"project_id": self.project_id},
headers={"X-Inner-Api-Key": self.api_key},
timeout=15.0,
)
response.raise_for_status()
state = E2BUsageState.model_validate(response.json())
if state.project_id != self.project_id:
raise ValueError("sandbox usage state project mismatch")
for value in (state.started_at, state.checkpoint_at, state.full_scan_at):
if value is not None and value.utcoffset() is None:
raise ValueError("sandbox usage state requires timezone-aware timestamps")
if state.enabled and state.started_at is None:
raise ValueError("enabled sandbox usage state requires a fixed activation time")
return state
async def post_events(self, events: list[dict[str, Any]]) -> None:
if not events or len(events) > 100:
raise ValueError("sandbox usage batches require 1 to 100 events")
# The API limits the whole HTTP body, not just its event count. Splitting
# here covers provider pages. Repeated scheduled scans use stable
# provider IDs, so partial success cannot multiply execution usage.
batch: list[dict[str, Any]] = []
for event in events:
candidate = [*batch, event]
body = self._encode_batch(candidate)
if len(body) > _MAX_BATCH_BYTES:
if not batch:
raise ValueError("sandbox usage event exceeds API request size")
await self._post_batch(self._encode_batch(batch), len(batch))
batch = [event]
if len(self._encode_batch(batch)) > _MAX_BATCH_BYTES:
raise ValueError("sandbox usage event exceeds API request size")
else:
batch = candidate
if batch:
await self._post_batch(self._encode_batch(batch), len(batch))
def _encode_batch(self, events: list[dict[str, Any]]) -> bytes:
return json.dumps(
{"project_id": self.project_id, "events": events},
ensure_ascii=False,
allow_nan=False,
separators=(",", ":"),
).encode()
async def _post_batch(self, body: bytes, count: int) -> None:
response = await self.client.post(
f"{self.base_url.rstrip('/')}/inner/api/agent/sandbox-usage/events",
headers={"X-Inner-Api-Key": self.api_key, "Content-Type": "application/json"},
content=body,
timeout=15.0,
)
response.raise_for_status()
result = _IngestResult.model_validate(response.json())
if result.accepted + result.duplicates + result.conflicts + result.ignored != count:
raise ValueError("sandbox usage acknowledgement does not cover the entire batch")
if result.conflicts:
logger.error("sandbox usage ingestion reported conflicts", extra={"conflicts": result.conflicts})
def utc_now() -> datetime:
return datetime.now(UTC)
@@ -0,0 +1,160 @@
"""A single bounded E2B event scan, invoked by the existing Celery Beat schedule.
Offsets are used within one bounded scan only. Every scan starts at zero, with
an overlap window and periodic rescan of the provider retention window. Neither
redelivery nor overlapping scheduled scans can inflate the API's idempotent
execution ledger. A partial scan never advances coverage.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta
import logging
from typing import Any, Callable
from uuid import uuid4
import httpx
from dify_agent.server.e2b_usage_client import E2BUsageApiClient, utc_now
logger = logging.getLogger(__name__)
_PROVIDER_RETENTION = timedelta(days=7)
_EVENTS_URL = "https://api.e2b.app/events/sandboxes"
def _field(event: dict[str, Any], camel: str, snake: str) -> Any:
return event.get(camel, event.get(snake))
def _timestamp(value: Any) -> datetime | None:
if value is None:
return None
if not isinstance(value, str):
raise ValueError("provider event timestamp is invalid")
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.utcoffset() is None:
raise ValueError("provider event timestamp must have a timezone")
return parsed
def provider_event(raw: dict[str, Any], *, project_id: str) -> dict[str, Any]:
"""Preserve the raw provider payload; API owns normalization and T0 filtering."""
project = _field(raw, "sandboxTeamId", "sandbox_team_id")
if project != project_id:
raise ValueError("E2B event project mismatch or missing project")
event_id = raw.get("id")
event_type = raw.get("type")
if not isinstance(event_id, str) or not event_id or not isinstance(event_type, str):
raise ValueError("E2B event missing stable identity")
_timestamp(raw.get("timestamp"))
return {
"id": event_id,
"source": "provider",
"type": event_type,
"timestamp": raw.get("timestamp"),
"sandbox_id": _field(raw, "sandboxId", "sandbox_id"),
"execution_id": _field(raw, "sandboxExecutionId", "sandbox_execution_id"),
"payload": raw,
}
@dataclass(slots=True)
class E2BUsageCollector:
provider_client: httpx.AsyncClient
usage_client: E2BUsageApiClient
api_key: str
project_id: str
overlap_seconds: int = 900
full_scan_interval_seconds: int = 3600
max_pages: int = 1000
clock: Callable[[], datetime] = utc_now
async def collect_once(self) -> bool:
state = await self.usage_client.get_state()
if not state.enabled:
return False
if state.started_at is None or state.project_id != self.project_id:
raise ValueError("sandbox usage activation/project state is invalid")
now = self.clock()
if state.started_at > now:
return False
retention_floor = now - _PROVIDER_RETENTION
full = (
state.full_scan_at is None or (now - state.full_scan_at).total_seconds() >= self.full_scan_interval_seconds
)
anchor = state.started_at if full else (state.checkpoint_at or state.started_at)
lower = max(state.started_at, retention_floor, anchor - timedelta(seconds=self.overlap_seconds))
if (state.checkpoint_at or state.started_at) < retention_floor:
# The API checkpoint also records window_start, so a retention gap is
# visible instead of being misrepresented as complete T0 coverage.
logger.error("sandbox usage collection gap exceeds provider retention")
await self.usage_client.post_events(
[
{
"id": str(uuid4()),
"source": "application",
"type": "collector_retention_gap",
"timestamp": now.isoformat(),
"payload": {
"uncovered_start": (state.checkpoint_at or state.started_at).isoformat(),
"uncovered_end": retention_floor.isoformat(),
"assumed_retention_seconds": int(_PROVIDER_RETENTION.total_seconds()),
},
}
]
)
count = 0
complete = False
pages = 0
for page in range(self.max_pages):
response = await self.provider_client.get(
_EVENTS_URL,
headers={"X-API-Key": self.api_key},
params={"limit": 100, "offset": page * 100, "orderAsc": "false"},
timeout=15.0,
)
response.raise_for_status()
raw_events = response.json()
if not isinstance(raw_events, list) or len(raw_events) > 100:
raise ValueError("E2B events response must be a page of up to 100 events")
pages += 1
events: list[dict[str, Any]] = []
all_older = bool(raw_events)
for raw in raw_events:
if not isinstance(raw, dict):
raise ValueError("invalid E2B event")
event = provider_event(raw, project_id=self.project_id)
occurred = _timestamp(raw.get("timestamp"))
if occurred is None or occurred >= lower:
all_older = False
events.append(event)
if events:
await self.usage_client.post_events(events)
count += len(events)
if len(raw_events) < 100 or all_older:
complete = True
break
if not complete:
logger.warning("sandbox usage scan reached page bound; checkpoint unchanged", extra={"pages": pages})
return False
await self.usage_client.post_events(
[
{
"id": str(uuid4()),
"source": "application",
"type": "collector_checkpoint",
"timestamp": now.isoformat(),
"payload": {
"scan_started_at": now.isoformat(),
"window_start": lower.isoformat(),
"window_end": now.isoformat(),
"completed": True,
"mode": "full" if full else "incremental",
"pages": pages,
"events": count,
},
}
]
)
return True
@@ -0,0 +1,99 @@
"""One-shot provider accounting endpoint, invoked by the API's Celery task.
The E2B credential stays in its existing service. Collection owns separate HTTP
clients, has no Redis leader or recurring task, and never touches runtime leases.
Invalid optional accounting configuration fails only this endpoint.
"""
import asyncio
import logging
import httpx
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, Field
from dify_agent.server.auth import create_bearer_token_dependency
from dify_agent.server.e2b_usage_client import E2BUsageApiClient
from dify_agent.server.e2b_usage_collector import E2BUsageCollector
from dify_agent.server.settings import ServerSettings
logger = logging.getLogger(__name__)
_COLLECTION_TIMEOUT_SECONDS = 240.0
class E2BUsageCollectRequest(BaseModel):
project_id: str = Field(min_length=1, max_length=128)
model_config = ConfigDict(extra="forbid")
class E2BUsageCollectResponse(BaseModel):
completed: bool
class _CollectionOptions(BaseModel):
overlap_seconds: int = Field(ge=1)
full_scan_interval_seconds: int = Field(ge=1)
max_pages: int = Field(ge=1, le=10000)
def create_e2b_usage_router(settings: ServerSettings) -> APIRouter:
async def require_configured_auth() -> None:
# Other legacy control-plane routes may permit a missing token. New
# collection requests must never expose the project key without auth.
if not settings.api_token:
raise HTTPException(status_code=503, detail="e2b_usage_collection_auth_unconfigured")
router = APIRouter(
prefix="/internal/e2b/usage",
dependencies=[Depends(require_configured_auth), create_bearer_token_dependency(settings.api_token)],
)
@router.post("/collect", response_model=E2BUsageCollectResponse)
async def collect(request: E2BUsageCollectRequest) -> E2BUsageCollectResponse:
project_id = settings.e2b_project_id.strip()
if (
not settings.sandbox_metering_enabled
or settings.runtime_backend != "e2b"
or not settings.e2b_api_key
or not project_id
or not settings.inner_api_key
):
raise HTTPException(status_code=503, detail="e2b_usage_collection_unavailable")
if request.project_id != project_id:
raise HTTPException(status_code=403, detail="e2b_usage_project_mismatch")
try:
options = _CollectionOptions.model_validate(
{
"overlap_seconds": settings.sandbox_metering_overlap_seconds,
"full_scan_interval_seconds": settings.sandbox_metering_full_scan_interval_seconds,
"max_pages": settings.sandbox_metering_max_pages,
}
)
async with (
asyncio.timeout(_COLLECTION_TIMEOUT_SECONDS),
httpx.AsyncClient(timeout=15.0, trust_env=False) as provider_client,
httpx.AsyncClient(timeout=15.0, trust_env=False) as inner_client,
):
collector = E2BUsageCollector(
provider_client=provider_client,
usage_client=E2BUsageApiClient(
client=inner_client,
base_url=settings.inner_api_url,
api_key=settings.inner_api_key,
project_id=project_id,
),
api_key=settings.e2b_api_key,
project_id=project_id,
overlap_seconds=options.overlap_seconds,
full_scan_interval_seconds=options.full_scan_interval_seconds,
max_pages=options.max_pages,
)
return E2BUsageCollectResponse(completed=await collector.collect_once())
except TimeoutError as exc:
logger.warning("E2B usage collection exceeded its deadline")
raise HTTPException(status_code=504, detail="e2b_usage_collection_timed_out") from exc
except Exception as exc:
logger.warning("E2B usage collection failed", extra={"error_type": type(exc).__name__})
raise HTTPException(status_code=503, detail="e2b_usage_collection_failed") from exc
return router
@@ -88,6 +88,13 @@ class ServerSettings(BaseSettings):
le=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
)
e2b_shellctl_port: int = Field(default=5004, ge=1, le=65535)
sandbox_metering_enabled: bool = False
e2b_project_id: str = ""
# Optional accounting validates its numeric configuration at invocation,
# so a bad metering value cannot prevent ordinary runtime startup.
sandbox_metering_overlap_seconds: int | str = 900
sandbox_metering_full_scan_interval_seconds: int | str = 3600
sandbox_metering_max_pages: int | str = 1000
openshell_gateway_endpoint: str | None = None
openshell_workspace: str = "default"
openshell_bearer_token: str | None = None