80a4be1d21
* refactor(memory): rename package second-brain to memory * refactor(memory): use OpenAI env naming and fix trailing slash * feat(memory): add chunking with env-configurable size and overlap * feat(memory): dedup search results by source in top-K * test(memory): add chunking, batching, dedup, live tests * docs(memory): update README and project map after rename * docs(handoff): add handoff and ADR-032 for memory refactor * docs(handoff): set PR number * fix(ci): reduce index.py complexity to rank A --------- Co-authored-by: opencode-agent <agent@opencode.local>
47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
import os
|
|
|
|
os.environ.setdefault("OPENAI_BASE_URL", "http://test/v1")
|
|
|
|
from src.memory.index import _chunk_text
|
|
|
|
|
|
class TestChunkText:
|
|
def test_empty(self) -> None:
|
|
assert _chunk_text("", 512, 64) == []
|
|
|
|
def test_single_char(self) -> None:
|
|
assert _chunk_text("x", 512, 64) == [("x", 0)]
|
|
|
|
def test_exactly_size(self) -> None:
|
|
text = "a" * 512
|
|
assert _chunk_text(text, 512, 64) == [(text, 0)]
|
|
|
|
def test_size_plus_one(self) -> None:
|
|
text = "a" * 513
|
|
chunks = _chunk_text(text, 512, 64)
|
|
assert len(chunks) == 2
|
|
assert chunks[0] == (text[:512], 0)
|
|
assert chunks[1][1] == 512 - 64
|
|
assert chunks[1][0] == text[448 : 448 + 512]
|
|
|
|
def test_unicode_emoji(self) -> None:
|
|
text = "😀" * 100
|
|
chunks = _chunk_text(text, 10, 2)
|
|
assert len(chunks) >= 2
|
|
offsets = [off for _, off in chunks]
|
|
assert offsets == sorted(offsets)
|
|
assert offsets[0] == 0
|
|
for chunk, _ in chunks:
|
|
assert chunk in text
|
|
|
|
def test_size_less_than_overlap(self) -> None:
|
|
text = "a" * 100
|
|
chunks = _chunk_text(text, 10, 20)
|
|
assert len(chunks) == 1
|
|
assert chunks[0] == (text, 0)
|
|
|
|
def test_size_equals_overlap(self) -> None:
|
|
text = "a" * 100
|
|
chunks = _chunk_text(text, 10, 10)
|
|
assert len(chunks) == 1
|
|
assert chunks[0] == (text, 0)
|