1
import re
2
from langchain_text_splitters import RecursiveCharacterTextSplitter
3

4
def clean_markdown(content):
5
    content = re.sub(r'---.*?---', '', content, flags=re.DOTALL)
6
    return content.strip()
7

8
def process_file(file_path):
9
    with open(file_path, 'r') as f:
10
        clean_text = clean_markdown(f.read())
11

12
    splitter = RecursiveCharacterTextSplitter(
13
        chunk_size=1000, chunk_overlap=100,
14
        separators=["\n## ", "\n### ", "\n\n", "\n", " "]
15
    )
16
    return splitter.split_text(clean_text)

1
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
2
import fs from "fs/promises";
3

4
async function ingestFile(path) {
5
    const raw = await fs.readFile(path, 'utf-8');
6
    // Strip YAML frontmatter
7
    const clean = raw.replace(/---[\s\S]*?---/, '').trim();
8

9
    const splitter = new RecursiveCharacterTextSplitter({
10
        chunkSize: 1000,
11
        chunkOverlap: 100,
12
        separators: ["\n## ", "\n### ", "\n\n", "\n", " "]
13
    });
14

15
    const chunks = await splitter.splitText(clean);
16
    console.log(`Ingestion Complete: ${chunks.length} chunks.`);
17
    return chunks;
18
}

1
import dev.langchain4j.data.document.Document;
2
import dev.langchain4j.data.document.splitter.DocumentByParagraphSplitter;
3
// Using LangChain4j for production-grade Java RAG
4

5
public class Ingestor {
6
    public String cleanMarkdown(String raw) {
7
        return raw.replaceAll("(?s)---.*?---", "").trim();
8
    }
9

10
    public List<String> processFile(String content) {
11
        String clean = cleanMarkdown(content);
12
        // LangChain4j splits by paragraphs/sections to maintain context
13
        DocumentByParagraphSplitter splitter = new DocumentByParagraphSplitter(1000, 100);
14
        return splitter.split(Document.from(clean))
15
                    .stream().map(d -> d.text()).toList();
16
    }
17
}