From a5ceaaa226873e659634a6eefa99789961e432a6 Mon Sep 17 00:00:00 2001
From: Srinivasan Iyer <sviyer@fb.com>
Date: Tue, 11 Mar 2025 11:03:24 -0700
Subject: [PATCH] When merging configs, do not merge data sources (#79)
* When merging configs, do not merge data sources
* Add todo
---------
Co-authored-by: Srini Iyer <sviyer@meta.com>
---
bytelatent/config_parser.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/bytelatent/config_parser.py b/bytelatent/config_parser.py
index 2e310a2..4c2e2b3 100644
--- a/bytelatent/config_parser.py
+++ b/bytelatent/config_parser.py
@@ -58,6 +58,11 @@ def parse_args_with_default(
if default_cfg is not None:
ordered_cfgs.insert(0, default_cfg)
cfg = OmegaConf.merge(*ordered_cfgs)
+ # TODO: Change sources to list[tuple,str, float]] so that this special case isn't needed
+ for c in reversed(ordered_cfgs):
+ if "data" in c and "sources" in c["data"]:
+ cfg["data"]["sources"] = c["data"]["sources"]
+ break
return OmegaConf.to_container(cfg, resolve=True, throw_on_missing=True)