vikarti.anatra/blt
1
0
Fork 0
mirror of https://github.com/facebookresearch/blt.git synced 2025-02-23 13:32:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use load_async flag to not start MP iterator

Browse source 
Summary:

Test Plan:
This commit is contained in:
Pedro Rodriguez 2025-01-24 18:56:18 +00:00
parent bc42cebd7d
commit bd461af91a
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bytelatent/args.py
View file

@ -150,11 +150,13 @@ class DataloaderArgs(BaseModel):
enable_byte_ngrams=self.enable_byte_ngrams, enable_byte_ngrams=self.enable_byte_ngrams,
) )
packing_iterator = PackingIterator(sampling_iterator, packing_args=packing_args) packing_iterator = PackingIterator(sampling_iterator, packing_args=packing_args)
mp_iterator = MultiprocessIterator( if self.load_async:
packing_iterator, n_batches_to_prefetch=self.prefetch_size mp_iterator = MultiprocessIterator(
) packing_iterator, n_batches_to_prefetch=self.prefetch_size
)
return mp_iterator return mp_iterator
else:
return packing_iterator
class TrainArgs(BaseModel): class TrainArgs(BaseModel):