SunnyQjm/dify
1
0 Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-10-04 13:47:55 +08:00
Code Issues Projects Releases Wiki Activity

Fix/langchain document schema (#2539)

Browse Source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
Jyongandjyong authored and GitHub committed 2024-02-23 14:16:44 +08:00
1 parent 769be13189
commit 91ea6fe4ee
15 files changed
+983 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/core/splitter/fixed_text_splitter.py
+4 -5
View File
@@ -3,7 +3,10 @@ from __future__ import annotations
from typing import Any, Optional, cast
from langchain.text_splitter import (
from core.model_manager import ModelInstance
from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
from core.splitter.text_splitter import (
TS,
AbstractSet,
Collection,
@@ -14,10 +17,6 @@ from langchain.text_splitter import (
Union,
)
from core.model_manager import ModelInstance
from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
class EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):
"""