mirror of
https://github.com/langgenius/dify.git
synced 2026-10-04 13:47:55 +08:00
Merge branch 'main' into feat/dify-builder
This commit is contained in:
468 files changed
+10532
-3406
No files matched your search
+148
-32
@@ -14,10 +14,56 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
api-unit-plan:
|
||||
name: Plan API unit shards
|
||||
runs-on: depot-ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup UV and Python
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
enable-cache: false
|
||||
- name: Download shared timing baseline
|
||||
# Public statistics are independent of PR refs and require no credentials.
|
||||
run: |
|
||||
if ! curl --fail --silent --show-error --location \
|
||||
--connect-timeout 5 --max-time 15 --retry 2 --retry-max-time 30 \
|
||||
https://raw.githubusercontent.com/langgenius/dify-test-infra/generated-stats/stats/test-times.json \
|
||||
--output unit-test-durations.json; then
|
||||
rm -f unit-test-durations.json
|
||||
echo '::warning::Timing baseline unavailable; using round-robin file allocation.'
|
||||
fi
|
||||
- name: Freeze file assignments for every shard
|
||||
run: |
|
||||
uv run --project api --no-sync python api/dev/pytest_sharding.py \
|
||||
--shard-index 1 --shard-total 2 \
|
||||
--durations unit-test-durations.json \
|
||||
--write-plan unit-test-plan.json \
|
||||
--split-threshold 60 \
|
||||
--ignore api/tests/unit_tests/controllers \
|
||||
api/tests/unit_tests \
|
||||
api/providers/vdb/*/tests/unit_tests \
|
||||
api/providers/trace/*/tests/unit_tests
|
||||
- name: Upload shared plan
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-unit-plan
|
||||
path: |
|
||||
unit-test-plan.json
|
||||
unit-test-durations.json
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
api-unit:
|
||||
needs: api-unit-plan
|
||||
name: API Unit Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
|
||||
runs-on: depot-ubuntu-24.04
|
||||
env:
|
||||
PYTHONUNBUFFERED: '1'
|
||||
COVERAGE_FILE: coverage-unit-${{ matrix.python-version }}-${{ matrix.shardIndex }}
|
||||
defaults:
|
||||
run:
|
||||
@@ -30,9 +76,8 @@ jobs:
|
||||
shardIndex:
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
shardTotal:
|
||||
- 3
|
||||
- 2
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -53,29 +98,50 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Download shared plan
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: api-unit-plan
|
||||
|
||||
- name: Run Unit Tests
|
||||
run: |
|
||||
uv run --project api python api/dev/pytest_sharding.py \
|
||||
--plan unit-test-plan.json \
|
||||
--shard-index "${{ matrix.shardIndex }}" \
|
||||
--shard-total "${{ matrix.shardTotal }}" \
|
||||
--ignore api/tests/unit_tests/controllers \
|
||||
api/tests/unit_tests \
|
||||
api/providers/vdb/*/tests/unit_tests \
|
||||
api/providers/trace/*/tests/unit_tests > "$RUNNER_TEMP/unit-test-files.txt"
|
||||
uv run --project api pytest \
|
||||
-p no:benchmark \
|
||||
--timeout "${PYTEST_TIMEOUT:-20}" \
|
||||
-n auto \
|
||||
--shard-index "${{ matrix.shardIndex }}" \
|
||||
--shard-total "${{ matrix.shardTotal }}" \
|
||||
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
|
||||
--cov-report= \
|
||||
api/tests/unit_tests \
|
||||
api/providers/vdb/*/tests/unit_tests \
|
||||
api/providers/trace/*/tests/unit_tests \
|
||||
--ignore=api/tests/unit_tests/controllers
|
||||
# Controller tests register Flask routes at import time, so keep them out of xdist.
|
||||
uv run --project api pytest \
|
||||
--timeout "${PYTEST_TIMEOUT:-20}" \
|
||||
--file-shard-plan unit-test-plan.json \
|
||||
--shard-index "${{ matrix.shardIndex }}" \
|
||||
--shard-total "${{ matrix.shardTotal }}" \
|
||||
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
|
||||
--cov-report= \
|
||||
--cov-append \
|
||||
api/tests/unit_tests/controllers
|
||||
--write-test-durations "test-results/durations-${{ matrix.shardIndex }}.json" \
|
||||
--junitxml=test-results/unit.xml \
|
||||
"@$RUNNER_TEMP/unit-test-files.txt"
|
||||
|
||||
- name: Upload unit test reports
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-test-results-unit-${{ matrix.python-version }}-${{ matrix.shardIndex }}
|
||||
path: test-results/*.xml
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload successful timing observations
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-unit-durations-${{ matrix.shardIndex }}
|
||||
path: test-results/durations-*.json
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload unit coverage data
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
@@ -85,11 +151,63 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
api-controllers:
|
||||
name: API Controller Tests (3.12)
|
||||
runs-on: depot-ubuntu-24.04
|
||||
env:
|
||||
PYTHONUNBUFFERED: '1'
|
||||
COVERAGE_FILE: coverage-controllers-3.12
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: '3.12'
|
||||
cache-dependency-glob: api/uv.lock
|
||||
|
||||
- name: Check UV lockfile
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Run Controller Tests
|
||||
# Controller tests register Flask routes at import time, so keep them out of xdist.
|
||||
run: |
|
||||
uv run --project api pytest \
|
||||
--timeout "${PYTEST_TIMEOUT:-20}" \
|
||||
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
|
||||
--cov-report= \
|
||||
--junitxml=test-results/controllers.xml \
|
||||
api/tests/unit_tests/controllers
|
||||
|
||||
- name: Upload controller test reports
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-test-results-controllers-3.12
|
||||
path: test-results/*.xml
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload controller coverage data
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-coverage-controllers-3.12
|
||||
path: ${{ env.COVERAGE_FILE }}
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
api-integration:
|
||||
name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
|
||||
name: API Integration Tests (${{ matrix.python-version }})
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
env:
|
||||
COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
|
||||
COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}
|
||||
STORAGE_TYPE: opendal
|
||||
OPENDAL_SCHEME: fs
|
||||
OPENDAL_FS_ROOT: /tmp/dify-storage
|
||||
@@ -101,13 +219,6 @@ jobs:
|
||||
matrix:
|
||||
python-version:
|
||||
- '3.12'
|
||||
shardIndex:
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
shardTotal:
|
||||
- 4
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -133,9 +244,8 @@ jobs:
|
||||
uv run --project api pytest \
|
||||
-p no:benchmark \
|
||||
--start-middleware \
|
||||
--middleware-stop-timeout 2 \
|
||||
-n auto \
|
||||
--shard-index "${{ matrix.shardIndex }}" \
|
||||
--shard-total "${{ matrix.shardTotal }}" \
|
||||
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
|
||||
--cov-report= \
|
||||
--timeout "${PYTEST_TIMEOUT:-180}" \
|
||||
@@ -146,7 +256,7 @@ jobs:
|
||||
- name: Upload integration coverage data
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
|
||||
name: api-coverage-integration-${{ matrix.python-version }}
|
||||
path: ${{ env.COVERAGE_FILE }}
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
@@ -156,6 +266,7 @@ jobs:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
needs:
|
||||
- api-unit
|
||||
- api-controllers
|
||||
- api-integration
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
@@ -195,18 +306,23 @@ jobs:
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
mapfile -t unit_coverage < <(find coverage-data -type f -name 'coverage-unit-*' -print | sort)
|
||||
mapfile -t controller_coverage < <(find coverage-data -type f -name 'coverage-controllers-*' -print | sort)
|
||||
mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort)
|
||||
if [[ "${#unit_coverage[@]}" -ne 3 ]]; then
|
||||
echo "expected 3 unit coverage artifacts, found ${#unit_coverage[@]}" >&2
|
||||
if [[ "${#unit_coverage[@]}" -ne 2 ]]; then
|
||||
echo "expected 2 unit coverage artifacts, found ${#unit_coverage[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${#integration_coverage[@]}" -ne 4 ]]; then
|
||||
echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2
|
||||
if [[ "${#controller_coverage[@]}" -ne 1 ]]; then
|
||||
echo "expected 1 controller coverage artifact, found ${#controller_coverage[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${#integration_coverage[@]}" -ne 1 ]]; then
|
||||
echo "expected 1 integration coverage artifact, found ${#integration_coverage[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
report_file="$(mktemp)"
|
||||
uvx --from "coverage==$coverage_version" coverage combine "${unit_coverage[@]}" "${integration_coverage[@]}"
|
||||
uvx --from "coverage==$coverage_version" coverage combine "${unit_coverage[@]}" "${controller_coverage[@]}" "${integration_coverage[@]}"
|
||||
uvx --from "coverage==$coverage_version" coverage report --show-missing | tee "$report_file"
|
||||
echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"
|
||||
{
|
||||
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
cp envs/middleware.env.example middleware.env
|
||||
|
||||
- name: Set up Middlewares
|
||||
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
|
||||
uses: hoverkraft-tech/compose-action@b659c744ee9bed25aea2307c017ad849bd104433 # v3.2.0
|
||||
with:
|
||||
compose-file: |
|
||||
docker/docker-compose.middleware.yaml
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
sed -i 's/DB_USERNAME=postgres/DB_USERNAME=mysql/' middleware.env
|
||||
|
||||
- name: Set up Middlewares
|
||||
uses: hoverkraft-tech/compose-action@ee6af68587292d72db67743171809c19787df4c9 # v3.1.0
|
||||
uses: hoverkraft-tech/compose-action@b659c744ee9bed25aea2307c017ad849bd104433 # v3.2.0
|
||||
with:
|
||||
compose-file: |
|
||||
docker/docker-compose.middleware.yaml
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Image optimization
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
image-resources:
|
||||
name: Image optimization
|
||||
runs-on: depot-ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up workspace toolchain
|
||||
uses: ./.github/actions/setup-web
|
||||
with:
|
||||
run-install: 'false'
|
||||
|
||||
- name: Install image checker dependencies
|
||||
run: pnpm --filter @dify/image-resources... install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Type-check image checker
|
||||
run: pnpm --filter @dify/image-resources type-check
|
||||
|
||||
- name: Test image checker
|
||||
run: pnpm --filter @dify/image-resources test
|
||||
|
||||
- name: Check image optimization
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
pnpm --filter @dify/image-resources check:images --base "$BASE_SHA"
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Run Claude Code for Translation Sync
|
||||
if: steps.context.outputs.CHANGED_FILES != ''
|
||||
uses: anthropics/claude-code-action@3b8197d3d486006dd4af54613517f21ac6ac625e # v1.0.227
|
||||
uses: anthropics/claude-code-action@4036a180cf690f49529f5d8c79c998855287f590 # v1.0.230
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
- name: Restore Next.js build cache
|
||||
id: next-cache
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: web/.next/cache
|
||||
key: e2e-next-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml') }}-${{ steps.build-source.outputs.hash }}
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
|
||||
- name: Save Next.js build cache
|
||||
if: github.event_name != 'merge_group' && steps.next-cache.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: web/.next/cache
|
||||
key: ${{ steps.next-cache.outputs.cache-primary-key }}
|
||||
|
||||
Reference in new issue
Block a user