Merge branch 'main' into feat/dify-builder

This commit is contained in:
twwu committed 2026-09-21 15:23:36 +08:00
commit d9991a11e1
468 files changed
+10532 -3406

No files matched your search

+148 -32
View File
@@ -14,10 +14,56 @@ concurrency:
cancel-in-progress: true
jobs:
api-unit-plan:
name: Plan API unit shards
runs-on: depot-ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
with:
python-version: '3.12'
enable-cache: false
- name: Download shared timing baseline
# Public statistics are independent of PR refs and require no credentials.
run: |
if ! curl --fail --silent --show-error --location \
--connect-timeout 5 --max-time 15 --retry 2 --retry-max-time 30 \
https://raw.githubusercontent.com/langgenius/dify-test-infra/generated-stats/stats/test-times.json \
--output unit-test-durations.json; then
rm -f unit-test-durations.json
echo '::warning::Timing baseline unavailable; using round-robin file allocation.'
fi
- name: Freeze file assignments for every shard
run: |
uv run --project api --no-sync python api/dev/pytest_sharding.py \
--shard-index 1 --shard-total 2 \
--durations unit-test-durations.json \
--write-plan unit-test-plan.json \
--split-threshold 60 \
--ignore api/tests/unit_tests/controllers \
api/tests/unit_tests \
api/providers/vdb/*/tests/unit_tests \
api/providers/trace/*/tests/unit_tests
- name: Upload shared plan
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-unit-plan
path: |
unit-test-plan.json
unit-test-durations.json
retention-days: 7
if-no-files-found: error
api-unit:
needs: api-unit-plan
name: API Unit Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
runs-on: depot-ubuntu-24.04
env:
PYTHONUNBUFFERED: '1'
COVERAGE_FILE: coverage-unit-${{ matrix.python-version }}-${{ matrix.shardIndex }}
defaults:
run:
@@ -30,9 +76,8 @@ jobs:
shardIndex:
- 1
- 2
- 3
shardTotal:
- 3
- 2
steps:
- name: Checkout code
@@ -53,29 +98,50 @@ jobs:
- name: Install dependencies
run: uv sync --project api --dev
- name: Download shared plan
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: api-unit-plan
- name: Run Unit Tests
run: |
uv run --project api python api/dev/pytest_sharding.py \
--plan unit-test-plan.json \
--shard-index "${{ matrix.shardIndex }}" \
--shard-total "${{ matrix.shardTotal }}" \
--ignore api/tests/unit_tests/controllers \
api/tests/unit_tests \
api/providers/vdb/*/tests/unit_tests \
api/providers/trace/*/tests/unit_tests > "$RUNNER_TEMP/unit-test-files.txt"
uv run --project api pytest \
-p no:benchmark \
--timeout "${PYTEST_TIMEOUT:-20}" \
-n auto \
--shard-index "${{ matrix.shardIndex }}" \
--shard-total "${{ matrix.shardTotal }}" \
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
--cov-report= \
api/tests/unit_tests \
api/providers/vdb/*/tests/unit_tests \
api/providers/trace/*/tests/unit_tests \
--ignore=api/tests/unit_tests/controllers
# Controller tests register Flask routes at import time, so keep them out of xdist.
uv run --project api pytest \
--timeout "${PYTEST_TIMEOUT:-20}" \
--file-shard-plan unit-test-plan.json \
--shard-index "${{ matrix.shardIndex }}" \
--shard-total "${{ matrix.shardTotal }}" \
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
--cov-report= \
--cov-append \
api/tests/unit_tests/controllers
--write-test-durations "test-results/durations-${{ matrix.shardIndex }}.json" \
--junitxml=test-results/unit.xml \
"@$RUNNER_TEMP/unit-test-files.txt"
- name: Upload unit test reports
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-test-results-unit-${{ matrix.python-version }}-${{ matrix.shardIndex }}
path: test-results/*.xml
if-no-files-found: warn
retention-days: 7
- name: Upload successful timing observations
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-unit-durations-${{ matrix.shardIndex }}
path: test-results/durations-*.json
retention-days: 7
if-no-files-found: error
- name: Upload unit coverage data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -85,11 +151,63 @@ jobs:
if-no-files-found: error
retention-days: 1
api-controllers:
name: API Controller Tests (3.12)
runs-on: depot-ubuntu-24.04
env:
PYTHONUNBUFFERED: '1'
COVERAGE_FILE: coverage-controllers-3.12
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup UV and Python
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
with:
enable-cache: true
python-version: '3.12'
cache-dependency-glob: api/uv.lock
- name: Check UV lockfile
run: uv lock --project api --check
- name: Install dependencies
run: uv sync --project api --dev
- name: Run Controller Tests
# Controller tests register Flask routes at import time, so keep them out of xdist.
run: |
uv run --project api pytest \
--timeout "${PYTEST_TIMEOUT:-20}" \
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
--cov-report= \
--junitxml=test-results/controllers.xml \
api/tests/unit_tests/controllers
- name: Upload controller test reports
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-test-results-controllers-3.12
path: test-results/*.xml
if-no-files-found: warn
retention-days: 7
- name: Upload controller coverage data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-coverage-controllers-3.12
path: ${{ env.COVERAGE_FILE }}
if-no-files-found: error
retention-days: 1
api-integration:
name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
name: API Integration Tests (${{ matrix.python-version }})
runs-on: depot-ubuntu-24.04-4
env:
COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}
STORAGE_TYPE: opendal
OPENDAL_SCHEME: fs
OPENDAL_FS_ROOT: /tmp/dify-storage
@@ -101,13 +219,6 @@ jobs:
matrix:
python-version:
- '3.12'
shardIndex:
- 1
- 2
- 3
- 4
shardTotal:
- 4
steps:
- name: Checkout code
@@ -133,9 +244,8 @@ jobs:
uv run --project api pytest \
-p no:benchmark \
--start-middleware \
--middleware-stop-timeout 2 \
-n auto \
--shard-index "${{ matrix.shardIndex }}" \
--shard-total "${{ matrix.shardTotal }}" \
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
--cov-report= \
--timeout "${PYTEST_TIMEOUT:-180}" \
@@ -146,7 +256,7 @@ jobs:
- name: Upload integration coverage data
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
name: api-coverage-integration-${{ matrix.python-version }}
path: ${{ env.COVERAGE_FILE }}
if-no-files-found: error
retention-days: 1
@@ -156,6 +266,7 @@ jobs:
runs-on: depot-ubuntu-24.04
needs:
- api-unit
- api-controllers
- api-integration
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -195,18 +306,23 @@ jobs:
} >> "$GITHUB_STEP_SUMMARY"
mapfile -t unit_coverage < <(find coverage-data -type f -name 'coverage-unit-*' -print | sort)
mapfile -t controller_coverage < <(find coverage-data -type f -name 'coverage-controllers-*' -print | sort)
mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort)
if [[ "${#unit_coverage[@]}" -ne 3 ]]; then
echo "expected 3 unit coverage artifacts, found ${#unit_coverage[@]}" >&2
if [[ "${#unit_coverage[@]}" -ne 2 ]]; then
echo "expected 2 unit coverage artifacts, found ${#unit_coverage[@]}" >&2
exit 1
fi
if [[ "${#integration_coverage[@]}" -ne 4 ]]; then
echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2
if [[ "${#controller_coverage[@]}" -ne 1 ]]; then
echo "expected 1 controller coverage artifact, found ${#controller_coverage[@]}" >&2
exit 1
fi
if [[ "${#integration_coverage[@]}" -ne 1 ]]; then
echo "expected 1 integration coverage artifact, found ${#integration_coverage[@]}" >&2
exit 1
fi
report_file="$(mktemp)"
uvx --from "coverage==$coverage_version" coverage combine "${unit_coverage[@]}" "${integration_coverage[@]}"
uvx --from "coverage==$coverage_version" coverage combine "${unit_coverage[@]}" "${controller_coverage[@]}" "${integration_coverage[@]}"
uvx --from "coverage==$coverage_version" coverage report --show-missing | tee "$report_file"
echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"
{
+2 -2
View File
@@ -40,7 +40,7 @@ jobs:
cp envs/middleware.env.example middleware.env
- name: Set up Middlewares
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
uses: hoverkraft-tech/compose-action@b659c744ee9bed25aea2307c017ad849bd104433 # v3.2.0
with:
compose-file: |
docker/docker-compose.middleware.yaml
@@ -94,7 +94,7 @@ jobs:
sed -i 's/DB_USERNAME=postgres/DB_USERNAME=mysql/' middleware.env
- name: Set up Middlewares
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
uses: hoverkraft-tech/compose-action@b659c744ee9bed25aea2307c017ad849bd104433 # v3.2.0
with:
compose-file: |
docker/docker-compose.middleware.yaml
+39
View File
@@ -0,0 +1,39 @@
name: Image optimization
on:
pull_request:
permissions:
contents: read
jobs:
image-resources:
name: Image optimization
runs-on: depot-ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
fetch-depth: 0
- name: Set up workspace toolchain
uses: ./.github/actions/setup-web
with:
run-install: 'false'
- name: Install image checker dependencies
run: pnpm --filter @dify/image-resources... install --frozen-lockfile --ignore-scripts
- name: Type-check image checker
run: pnpm --filter @dify/image-resources type-check
- name: Test image checker
run: pnpm --filter @dify/image-resources test
- name: Check image optimization
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
pnpm --filter @dify/image-resources check:images --base "$BASE_SHA"
+1 -1
View File
@@ -162,7 +162,7 @@ jobs:
- name: Run Claude Code for Translation Sync
if: steps.context.outputs.CHANGED_FILES != ''
uses: anthropics/claude-code-action@3b8197d3d486006dd4af54613517f21ac6ac625e # v1.0.227
uses: anthropics/claude-code-action@4036a180cf690f49529f5d8c79c998855287f590 # v1.0.230
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
- name: Restore Next.js build cache
id: next-cache
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: web/.next/cache
key: e2e-next-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml') }}-${{ steps.build-source.outputs.hash }}
@@ -81,7 +81,7 @@ jobs:
- name: Save Next.js build cache
if: github.event_name != 'merge_group' && steps.next-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: web/.next/cache
key: ${{ steps.next-cache.outputs.cache-primary-key }}