vikarti.anatra/blt
1
0
Fork 0
mirror of https://github.com/facebookresearch/blt.git synced 2025-02-23 05:22:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge bd461af91a into sapling-pr-archive-EntilZha
Some checks are pending
Lint with Black / lint (push) Waiting to run
Details
Lint with isort / lint (push) Waiting to run
Details

Browse source
This commit is contained in:
Pedro Rodriguez 2025-01-24 10:56:34 -08:00 committed by GitHub
parent cbcc3e1868 bd461af91a
commit 17b727465f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bytelatent/args.py
View file

@ -150,11 +150,13 @@ class DataloaderArgs(BaseModel):
enable_byte_ngrams=self.enable_byte_ngrams,
)
packing_iterator = PackingIterator(sampling_iterator, packing_args=packing_args)
mp_iterator = MultiprocessIterator(
packing_iterator, n_batches_to_prefetch=self.prefetch_size
)
return mp_iterator
if self.load_async:
mp_iterator = MultiprocessIterator(
packing_iterator, n_batches_to_prefetch=self.prefetch_size
)
return mp_iterator
else:
return packing_iterator
class TrainArgs(BaseModel):