vikarti.anatra/blt
1
0
Fork 0
mirror of https://github.com/facebookresearch/blt.git synced 2025-02-22 21:12:15 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Use load_async flag to not start MP iterator

Browse source 
Summary:

Test Plan:
This commit is contained in:
Pedro Rodriguez 2025-01-24 18:56:18 +00:00
parent bc42cebd7d
commit bd461af91a
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bytelatent/args.py
View file

@ -150,11 +150,13 @@ class DataloaderArgs(BaseModel):
enable_byte_ngrams=self.enable_byte_ngrams,
)
packing_iterator = PackingIterator(sampling_iterator, packing_args=packing_args)
mp_iterator = MultiprocessIterator(
packing_iterator, n_batches_to_prefetch=self.prefetch_size
)
return mp_iterator
if self.load_async:
mp_iterator = MultiprocessIterator(
packing_iterator, n_batches_to_prefetch=self.prefetch_size
)
return mp_iterator
else:
return packing_iterator
class TrainArgs(BaseModel):