mirror of
https://github.com/langflow-ai/langflow.git
synced 2026-07-25 04:10:28 +08:00
75 lines
26 KiB
HTML
75 lines
26 KiB
HTML
<!doctype html>
|
||
<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-components/text-splitters">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="generator" content="Docusaurus v2.4.1">
|
||
<title data-rh="true">Text Splitters | Langflow Documentation</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://logspace-ai.github.io/components/text-splitters"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Text Splitters | Langflow Documentation"><meta data-rh="true" name="description" content="We appreciate your understanding as we polish our documentation – it may"><meta data-rh="true" property="og:description" content="We appreciate your understanding as we polish our documentation – it may"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://logspace-ai.github.io/components/text-splitters"><link data-rh="true" rel="alternate" href="https://logspace-ai.github.io/components/text-splitters" hreflang="en"><link data-rh="true" rel="alternate" href="https://logspace-ai.github.io/components/text-splitters" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.20a7d952.css">
|
||
<link rel="preload" href="/assets/js/runtime~main.7b74dc3b.js" as="script">
|
||
<link rel="preload" href="/assets/js/main.847862f8.js" as="script">
|
||
</head>
|
||
<body class="navigation-with-keyboard">
|
||
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
|
||
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:#E8EBF1;color:#1C1E21" role="banner"><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like ⛓️Langflow, star it on <a target="_blank" rel="noopener noreferrer" href="https://github.com/logspace-ai/langflow">GitHub</a>! ⭐️</div></div><nav aria-label="Main" class="navbar navbar--fixed-top navbarHideable_m1mJ"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/chain.png" alt="Langflow" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/chain.png" alt="Langflow" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Langflow</b></a></div><div class="navbar__items navbar__items--right"><a href="https://github.com/logspace-ai/langflow" target="_blank" class="navbar__item navbar__link header-github-link"></a><a href="https://twitter.com/logspace_ai" target="_blank" class="navbar__item navbar__link header-twitter-link"></a><a href="https://discord.gg/EqksyE2EX9" target="_blank" class="navbar__item navbar__link header-discord-link"></a><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd sidebarWithHideableNavbar_wUlq"><a tabindex="-1" class="sidebarLogo_isFc" href="/"><img src="/img/chain.png" alt="Langflow" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/chain.png" alt="Langflow" class="themedImage_ToTc themedImage--dark_i4oU"><b>Langflow</b></a><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/">Getting Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/">👋 Welcome to Langflow</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/getting-started/installation">📦 How to install?</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/getting-started/hugging-face-spaces">🤗 HuggingFace Spaces</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/getting-started/creating-flows">🎨 Creating Flows</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/guidelines/login">Guidelines</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/login">Sign up and Sign in</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/api">API Keys</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/async-api">Asynchronous Processing</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/components">Component</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/features">Features</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/collection">Collection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/prompt-customization">Prompt Customization</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/chat-interface">Chat Interface</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/chat-widget">Chat Widget</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guidelines/custom-component">Custom Components</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/components/agents">Component Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/agents">Agents</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/chains">Chains</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/custom">Custom Components</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/embeddings">Embeddings</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/llms">LLMs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/loaders">Loaders</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/memories">Memories</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/prompts">Prompts</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/retrievers">Retrievers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/components/text-splitters">Text Splitters</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/toolkits">Toolkits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/tools">Tools</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/utilities">Utilities</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/vector-stores">Vector Stores</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/components/wrappers">Wrappers</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/guides/async-tasks">Step-by-Step Guides</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guides/async-tasks">Async API</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guides/loading_document">Integrating documents with prompt variables</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guides/chatprompttemplate_guide">Building chatbots with System Message</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/guides/langfuse_integration">Integrating Langfuse with Langflow</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/examples/flow-runner">Examples</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/flow-runner">FlowRunner Component</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/conversation-chain">Conversation Chain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/buffer-memory">Buffer Memory</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/midjourney-prompt-chain">MidJourney Prompt Chain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/csv-loader">CSV Loader</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/serp-api-tool">Serp API Tool</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/multiple-vectorstores">Multiple Vector Stores</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/python-function">Python Function</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/examples/how-upload-examples">📚 How to Upload Examples?</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/deployment/gcp-deployment">Deployment</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/deployment/gcp-deployment">Deploy on Google Cloud Platform</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="true" href="/contributing/how-contribute">Contributing</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/contributing/how-contribute">How to contribute?</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/contributing/github-issues">GitHub Issues</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/contributing/community">Community</a></li></ul></li></ul></nav><button type="button" title="Collapse sidebar" aria-label="Collapse sidebar" class="button button--secondary button--outline collapseSidebarButton_PEFL"><svg width="20" height="20" aria-hidden="true" class="collapseSidebarButtonIcon_kv0_"><g fill="#7a7a7a"><path d="M9.992 10.023c0 .2-.062.399-.172.547l-4.996 7.492a.982.982 0 01-.828.454H1c-.55 0-1-.453-1-1 0-.2.059-.403.168-.551l4.629-6.942L.168 3.078A.939.939 0 010 2.528c0-.548.45-.997 1-.997h2.996c.352 0 .649.18.828.45L9.82 9.472c.11.148.172.347.172.55zm0 0"></path><path d="M19.98 10.023c0 .2-.058.399-.168.547l-4.996 7.492a.987.987 0 01-.828.454h-3c-.547 0-.996-.453-.996-1 0-.2.059-.403.168-.551l4.625-6.942-4.625-6.945a.939.939 0 01-.168-.55 1 1 0 01.996-.997h3c.348 0 .649.18.828.45l4.996 7.492c.11.148.168.347.168.55zm0 0"></path></g></svg></button></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Component Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Splitters</span><meta itemprop="position" content="2"></li></ul></nav><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Text Splitters</h1>
|
||
<div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy">🚧</span>ZONE UNDER CONSTRUCTION</div><div class="admonitionContent_S0QG"><p></p><p>We appreciate your understanding as we polish our documentation – it may
|
||
contain some rough edges. Share your feedback or report issues to help us
|
||
improve! 🛠️📝</p><p></p></div></div>
|
||
<p>A text splitter is a tool that divides a document or text into smaller chunks or segments. It is used to break down large texts into more manageable pieces for analysis or processing.</p>
|
||
<hr>
|
||
<h3 class="anchor anchorWithHideOnScrollNavbar_WYt5" id="charactertextsplitter">CharacterTextSplitter<a href="#charactertextsplitter" class="hash-link" aria-label="Direct link to CharacterTextSplitter" title="Direct link to CharacterTextSplitter"></a></h3>
|
||
<p>The <code>CharacterTextSplitter</code> is used to split a long text into smaller chunks based on a specified character. It splits the text by trying to keep paragraphs, sentences, and words together as long as possible, as these are semantically related pieces of text.</p>
|
||
<p><strong>Params</strong></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong>Documents:</strong> Input documents to split.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_overlap:</strong> Determines the number of characters that overlap between consecutive chunks when splitting text. It specifies how much of the previous chunk should be included in the next chunk.</p>
|
||
<p>For example, if the <code>chunk_overlap</code> is set to 20 and the <code>chunk_size</code> is set to 100, the splitter will create chunks of 100 characters each, but the last 20 characters of each chunk will overlap with the first 20 characters of the next chunk. This allows for a smoother transition between chunks and ensures that no information is lost – defaults to <code>200</code>.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_size:</strong> Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.</p>
|
||
<p>For example, if the chunk_size is set to 100, the splitter will create chunks of 100 characters each. If the text is longer than 100 characters, it will be divided into multiple chunks of equal size, except for the last chunk, which may be smaller if there are remaining characters –defaults to <code>1000</code>.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>separator:</strong> Specifies the character that will be used to split the text into chunks – defaults to <code>.</code></p>
|
||
</li>
|
||
</ul>
|
||
<hr>
|
||
<h3 class="anchor anchorWithHideOnScrollNavbar_WYt5" id="recursivecharactertextsplitter">RecursiveCharacterTextSplitter<a href="#recursivecharactertextsplitter" class="hash-link" aria-label="Direct link to RecursiveCharacterTextSplitter" title="Direct link to RecursiveCharacterTextSplitter"></a></h3>
|
||
<p>The <code>RecursiveCharacterTextSplitter</code> splits the text by trying to keep paragraphs, sentences, and words together as long as possible, similar to the <code>CharacterTextSplitter</code>. However, it also recursively splits the text into smaller chunks if the chunk size exceeds a specified threshold.</p>
|
||
<p><strong>Params</strong></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong>Documents:</strong> Input documents to split.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_overlap:</strong> Determines the number of characters that overlap between consecutive chunks when splitting text. It specifies how much of the previous chunk should be included in the next chunk.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_size:</strong> Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>separators:</strong> The <code>separators</code> in RecursiveCharacterTextSplitter are the characters used to split the text into chunks. The text splitter tries to create chunks based on splitting on the first character in the list of <code>separators</code>. If any chunks are too large, it moves on to the next character in the list and continues splitting. Defaults to ["\n\n", "\n", " ", ""].</p>
|
||
</li>
|
||
</ul>
|
||
<h3 class="anchor anchorWithHideOnScrollNavbar_WYt5" id="languagerecursivetextsplitter">LanguageRecursiveTextSplitter<a href="#languagerecursivetextsplitter" class="hash-link" aria-label="Direct link to LanguageRecursiveTextSplitter" title="Direct link to LanguageRecursiveTextSplitter"></a></h3>
|
||
<p>The <code>LanguageRecursiveTextSplitter</code> is a text splitter that splits the text into smaller chunks based on the (programming) language of the text.</p>
|
||
<p><strong>Params</strong></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong>Documents:</strong> Input documents to split.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_overlap:</strong> Determines the number of characters that overlap between consecutive chunks when splitting text. It specifies how much of the previous chunk should be included in the next chunk.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>chunk_size:</strong> Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>separator_type:</strong> The parameter allows the user to split the code with multiple language support. It supports various languages such as Ruby, Python, Solidity, Java, and more. Defaults to <code>Python</code>.</p>
|
||
</li>
|
||
</ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/components/retrievers"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Retrievers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/components/toolkits"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Toolkits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#charactertextsplitter" class="table-of-contents__link toc-highlight">CharacterTextSplitter</a></li><li><a href="#recursivecharactertextsplitter" class="table-of-contents__link toc-highlight">RecursiveCharacterTextSplitter</a></li><li><a href="#languagerecursivetextsplitter" class="table-of-contents__link toc-highlight">LanguageRecursiveTextSplitter</a></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2023 Logspace.</div></div></div></footer><div class="ms-fixed ms-bottom-4 ms-right-8 ms-z-[100] ms-flex ms-flex-col ms-items-end hover:ms-cursor-pointer"><div class="ms-relative ms-mb-4"><div style="background-color:#f6f6f6" class="ms-absolute -ms-bottom-1 ms-right-6 -ms-z-10 ms-h-4 ms-w-4 ms-rotate-45"></div><div style="background-color:#f6f6f6" class="ms-z-10 ms-flex ms-items-center ms-justify-center ms-rounded-lg ms-px-3 ms-py-2"><p style="color:#000000;margin:0" class="ms-text-sm">Hi, how can I help you?</p></div></div><div style="background-color:#f6f6f6" class="ms-flex ms-h-16 ms-w-16 ms-flex-col ms-items-center ms-justify-center ms-self-end ms-rounded-full ms-p-2 ms-transition"><div class="ms-rotate-0 ms-flex ms-h-full ms-w-full ms-transform ms-cursor-pointer ms-items-center ms-justify-center ms-rounded-full ms-transition-all ms-delay-75 ms-duration-100 ms-ease-in-out"><div style="color:#000000;font-size:22px;width:48px;height:48px;margin:0px;padding:0px;display:flex;align-items:center;justify-content:center;text-align:center"><img src="/img/chain.png" style="width:40px"></div></div></div><div></div></div></div>
|
||
<script src="/assets/js/runtime~main.7b74dc3b.js"></script>
|
||
<script src="/assets/js/main.847862f8.js"></script>
|
||
</body>
|
||
</html> |