TokenTextSplitter
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:14
Extends
Section titled “Extends”TokenTextSplitter<
Options>(nodes,options?):TextNode<Metadata>[]
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:14
Type Parameters
Section titled “Type Parameters”Options
Section titled “Options”Options extends Record<string, unknown>
Parameters
Section titled “Parameters”options?
Section titled “options?”Options
Returns
Section titled “Returns”Constructors
Section titled “Constructors”Constructor
Section titled “Constructor”new TokenTextSplitter(
params?):TokenTextSplitter
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:23
Parameters
Section titled “Parameters”params?
Section titled “params?”SplitterParams & PartialWithUndefined<{ chunkSize: number; chunkOverlap: number; separator: string; backupSeparators: string[]; }> & object
Returns
Section titled “Returns”TokenTextSplitter
Overrides
Section titled “Overrides”MetadataAwareTextSplitter.constructor
Properties
Section titled “Properties”includeMetadata
Section titled “includeMetadata”includeMetadata:
boolean=true
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:17
Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.includeMetadata
includePrevNextRel
Section titled “includePrevNextRel”includePrevNextRel:
boolean=true
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:18
Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.includePrevNextRel
chunkSize
Section titled “chunkSize”chunkSize:
number=DEFAULT_CHUNK_SIZE
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:15
chunkOverlap
Section titled “chunkOverlap”chunkOverlap:
number=DEFAULT_CHUNK_OVERLAP
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:16
separator
Section titled “separator”separator:
string=" "
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:17
backupSeparators
Section titled “backupSeparators”backupSeparators:
string[]
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:18
id:
string
Defined in: .build/typescript/packages/core/src/schema/type.ts:22
Inherited from
Section titled “Inherited from”Methods
Section titled “Methods”postProcessParsedNodes()
Section titled “postProcessParsedNodes()”
protectedpostProcessParsedNodes(nodes,parentDocMap):TextNode<Metadata>[]
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:27
Parameters
Section titled “Parameters”parentDocMap
Section titled “parentDocMap”Map<string, TextNode<Metadata>>
Returns
Section titled “Returns”Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.postProcessParsedNodes
getNodesFromDocuments()
Section titled “getNodesFromDocuments()”Defined in: .build/typescript/packages/core/src/node-parser/base.ts:84
Parameters
Section titled “Parameters”documents
Section titled “documents”Returns
Section titled “Returns”Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.getNodesFromDocuments
splitTexts()
Section titled “splitTexts()”splitTexts(
texts):string[]
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:126
Parameters
Section titled “Parameters”string[]
Returns
Section titled “Returns”string[]
Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.splitTexts
splitTextsMetadataAware()
Section titled “splitTextsMetadataAware()”splitTextsMetadataAware(
texts,metadata):string[]
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:142
Parameters
Section titled “Parameters”string[]
metadata
Section titled “metadata”string[]
Returns
Section titled “Returns”string[]
Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.splitTextsMetadataAware
getMetadataString()
Section titled “getMetadataString()”
protectedgetMetadataString(node):string
Defined in: .build/typescript/packages/core/src/node-parser/base.ts:151
Parameters
Section titled “Parameters”Returns
Section titled “Returns”string
Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.getMetadataString
parseNodes()
Section titled “parseNodes()”Defined in: .build/typescript/packages/core/src/node-parser/base.ts:161
Parameters
Section titled “Parameters”Returns
Section titled “Returns”Inherited from
Section titled “Inherited from”MetadataAwareTextSplitter.parseNodes
splitTextMetadataAware()
Section titled “splitTextMetadataAware()”splitTextMetadataAware(
text,metadata):string[]
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:57
Split text into chunks, reserving space required for metadata string.
Parameters
Section titled “Parameters”string
The text to split.
metadata
Section titled “metadata”string
The metadata string.
Returns
Section titled “Returns”string[]
An array of text chunks.
Overrides
Section titled “Overrides”MetadataAwareTextSplitter.splitTextMetadataAware
splitText()
Section titled “splitText()”splitText(
text):string[]
Defined in: .build/typescript/packages/core/src/node-parser/token-text-splitter.ts:82
Split text into chunks.
Parameters
Section titled “Parameters”string
The text to split.
Returns
Section titled “Returns”string[]
An array of text chunks.