splitText method

List<AIDocument> splitText(
  1. String text, {
  2. Map<String, dynamic> metadata = const {},
})

Splits a single string of text into multiple AIDocument chunks.

Metadata is injected into each generated chunk.

Implementation

List<AIDocument> splitText(String text,
    {Map<String, dynamic> metadata = const {}}) {
  final docs = <AIDocument>[];

  if (text.isEmpty) return docs;

  // Simple character-based sliding window splitting
  for (int i = 0; i < text.length; i += (chunkSize - chunkOverlap)) {
    final end = (i + chunkSize < text.length) ? i + chunkSize : text.length;
    final chunk = text.substring(i, end).trim();

    if (chunk.isNotEmpty) {
      docs.add(AIDocument(
        content: chunk,
        metadata: {
          ...metadata,
          'start_index': i,
          'end_index': end,
        },
      ));
    }

    // Prevent infinite loop if overlap >= size
    if (chunkSize - chunkOverlap <= 0) break;
    if (end >= text.length) break;
  }

  return docs;
}