Test first batch matches (#53)

Summary: Test Plan:
2025-09-15 08:39:44 +00:00 · 2025-02-13 10:05:08 -08:00 · 2025-02-13 10:05:08 -08:00 · 85c2f28f26
commit 85c2f28f26
parent 9d907fed1c
4 changed files with 53 additions and 0 deletions
--- a/bytelatent/test_entropy_model.py
+++ b/bytelatent/test_entropy_model.py
@ -25,6 +25,7 @@ def test_entropy_model():
        row_num=0,
        arrow_batch_size=100,
        s3_profile=None,
+        file_format="arrow",
    )
    arrow_file = initial_state.build()
    tokenizer_args = TokenizerArgs(