From 662262a528e44389b4599aaa8ee013913973c090 Mon Sep 17 00:00:00 2001 From: Srini Iyer Date: Tue, 11 Mar 2025 17:44:37 +0000 Subject: [PATCH] When merging configs, do not merge data sources --- bytelatent/config_parser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bytelatent/config_parser.py b/bytelatent/config_parser.py index 2e310a2..a11d825 100644 --- a/bytelatent/config_parser.py +++ b/bytelatent/config_parser.py @@ -58,6 +58,10 @@ def parse_args_with_default( if default_cfg is not None: ordered_cfgs.insert(0, default_cfg) cfg = OmegaConf.merge(*ordered_cfgs) + for c in reversed(ordered_cfgs): + if "data" in c and "sources" in c["data"]: + cfg["data"]["sources"] = c["data"]["sources"] + break return OmegaConf.to_container(cfg, resolve=True, throw_on_missing=True)