Skip to main content

Class: TokenTextSplitter()

Extends​

TokenTextSplitter<Options>(nodes, options?): BaseNode<Metadata>[]

Type Parameters​

• Options extends Record<string, unknown>

Parameters​

• nodes: BaseNode<Metadata>[]

• options?: Options

Returns​

BaseNode<Metadata>[]

Defined in​

packages/core/node-parser/dist/index.d.ts:106

Constructors​

new TokenTextSplitter()​

new TokenTextSplitter(params?): TokenTextSplitter

Parameters​

• params?: SplitterParams & Partial<object>

Returns​

TokenTextSplitter

Overrides​

MetadataAwareTextSplitter.constructor

Defined in​

packages/core/node-parser/dist/index.d.ts:112

Properties​

backupSeparators​

backupSeparators: string[]

Defined in​

packages/core/node-parser/dist/index.d.ts:111

chunkOverlap​

chunkOverlap: number

Defined in​

packages/core/node-parser/dist/index.d.ts:109

chunkSize​

chunkSize: number

Defined in​

packages/core/node-parser/dist/index.d.ts:108

id​

id: string

Inherited from​

MetadataAwareTextSplitter.id

Defined in​

packages/core/schema/dist/index.d.ts:172

includeMetadata​

includeMetadata: boolean

Inherited from​

MetadataAwareTextSplitter.includeMetadata

Defined in​

packages/core/node-parser/dist/index.d.ts:6

includePrevNextRel​

includePrevNextRel: boolean

Inherited from​

MetadataAwareTextSplitter.includePrevNextRel

Defined in​

packages/core/node-parser/dist/index.d.ts:7

separator​

separator: string

Defined in​

packages/core/node-parser/dist/index.d.ts:110

Methods​

getMetadataString()​

protected getMetadataString(node): string

Parameters​

• node: TextNode<Metadata>

Returns​

string

Inherited from​

MetadataAwareTextSplitter.getMetadataString

Defined in​

packages/core/node-parser/dist/index.d.ts:21

getNodesFromDocuments()​

getNodesFromDocuments(documents): TextNode<Metadata>[]

Parameters​

• documents: TextNode<Metadata>[]

Returns​

TextNode<Metadata>[]

Inherited from​

MetadataAwareTextSplitter.getNodesFromDocuments

Defined in​

packages/core/node-parser/dist/index.d.ts:11

parseNodes()​

protected parseNodes(nodes): TextNode<Metadata>[]

Parameters​

• nodes: TextNode<Metadata>[]

Returns​

TextNode<Metadata>[]

Inherited from​

MetadataAwareTextSplitter.parseNodes

Defined in​

packages/core/node-parser/dist/index.d.ts:22

postProcessParsedNodes()​

protected postProcessParsedNodes(nodes, parentDocMap): TextNode<Metadata>[]

Parameters​

• nodes: TextNode<Metadata>[]

• parentDocMap: Map<string, TextNode<Metadata>>

Returns​

TextNode<Metadata>[]

Inherited from​

MetadataAwareTextSplitter.postProcessParsedNodes

Defined in​

packages/core/node-parser/dist/index.d.ts:9

splitText()​

splitText(text): string[]

Split text into chunks.

Parameters​

• text: string

The text to split.

Returns​

string[]

An array of text chunks.

Overrides​

MetadataAwareTextSplitter.splitText

Defined in​

packages/core/node-parser/dist/index.d.ts:125

splitTextMetadataAware()​

splitTextMetadataAware(text, metadata): string[]

Split text into chunks, reserving space required for metadata string.

Parameters​

• text: string

The text to split.

• metadata: string

The metadata string.

Returns​

string[]

An array of text chunks.

Overrides​

MetadataAwareTextSplitter.splitTextMetadataAware

Defined in​

packages/core/node-parser/dist/index.d.ts:119

splitTexts()​

splitTexts(texts): string[]

Parameters​

• texts: string[]

Returns​

string[]

Inherited from​

MetadataAwareTextSplitter.splitTexts

Defined in​

packages/core/node-parser/dist/index.d.ts:15

splitTextsMetadataAware()​

splitTextsMetadataAware(texts, metadata): string[]

Parameters​

• texts: string[]

• metadata: string[]

Returns​

string[]

Inherited from​

MetadataAwareTextSplitter.splitTextsMetadataAware

Defined in​

packages/core/node-parser/dist/index.d.ts:20