diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 887add245..cc68e9ca7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1145,6 +1145,47 @@ static void print_test_result_locked(printer * output_printer, const test_result output_printer->print_test_result(result); } +// Splits the -o filter into comma separated entries. Commas inside parentheses +// (i.e. inside a full test case string) are not treated as separators. +static std::vector op_filter_entries(const char * op_names_filter) { + std::vector entries; + if (op_names_filter == nullptr) { + return entries; + } + std::string_view filter(op_names_filter); + while (!filter.empty()) { + auto comma_pos = filter.find_first_of(','); + const auto lparen_pos = filter.find_first_of('('); + if (lparen_pos < comma_pos) { + const auto rparen_pos = filter.find_first_of(')'); + comma_pos = filter.find_first_of(',', rparen_pos); + } + entries.push_back(filter.substr(0, comma_pos)); + filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; + } + return entries; +} + +// An entry from the -o filter matches an op if it is either +// * an exact op name as given by ggml_op_desc() (e.g. "ADD"), or +// * a regex that matches the op name (e.g. "DSV4.*") +static bool op_filter_entry_matches(std::string_view entry, std::string_view op_name) { + if (entry == op_name) { + return true; + } + // plain op names are matched exactly, anything else is treated as a regex + if (std::regex_match(std::string(entry), std::regex("[A-Z0-9_]+"))) { + return false; + } + std::regex re; + try { + re = std::regex(std::string(entry)); + } catch (const std::regex_error &) { + return false; + } + return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re); +} + struct test_case { virtual ~test_case() {} @@ -1308,34 +1349,24 @@ struct test_case { return t; } - // Checks an op against the test filter, which is a comma separated list of OP names or specific variations + // Checks an op against the test filter, which is a comma separated list of OP names, regexes, or specific variations bool matches_filter(ggml_tensor * op, const char * op_names_filter) { - if (op_names_filter) { - const auto op_name = op_desc(op); - const auto op_full_name = op_name + "(" + vars() + ")"; - std::string_view filter(op_names_filter); - while (!filter.empty()) { - auto comma_pos = filter.find_first_of(','); - const auto lparen_pos = filter.find_first_of('('); - if (lparen_pos < comma_pos) { - auto rparen_pos = filter.find_first_of(')'); - comma_pos = filter.find_first_of(',', rparen_pos); - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_full_name) { - return true; - } - } else { - const auto op_filter = filter.substr(0, comma_pos); - if (op_filter == op_name) { - return true; - } - } - filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; - } - return false; - } else { + if (op_names_filter == nullptr) { return true; } + const auto op_name = op_desc(op); + const auto op_full_name = op_name + "(" + vars() + ")"; + for (const auto & entry : op_filter_entries(op_names_filter)) { + if (entry.find_first_of('(') != std::string_view::npos) { + // a full test case string, matched exactly + if (entry == op_full_name) { + return true; + } + } else if (op_filter_entry_matches(entry, op_name)) { + return true; + } + } + return false; } test_status_t eval(ggml_backend_t backend1, @@ -11597,25 +11628,16 @@ static std::vector fa_vec_legal_ne(int dk, int dv) { } static bool op_names_filter_selects(const char * op_names_filter, const char * op_name) { - if (!op_names_filter) { + if (op_names_filter == nullptr) { return true; } - std::string_view filter(op_names_filter); - while (!filter.empty()) { - auto comma_pos = filter.find_first_of(','); - const auto lparen_pos = filter.find_first_of('('); - std::string_view entry; - if (lparen_pos < comma_pos) { - const auto rparen_pos = filter.find_first_of(')'); - comma_pos = filter.find_first_of(',', rparen_pos); - entry = filter.substr(0, lparen_pos); - } else { - entry = filter.substr(0, comma_pos); - } - if (entry == op_name) { + for (const auto & entry : op_filter_entries(op_names_filter)) { + // a full test case string is matched by its op name prefix + const auto lparen_pos = entry.find_first_of('('); + const auto op_entry = lparen_pos != std::string_view::npos ? entry.substr(0, lparen_pos) : entry; + if (op_filter_entry_matches(op_entry, op_name)) { return true; } - filter = comma_pos != std::string_view::npos ? filter.substr(comma_pos + 1) : ""; } return false; } @@ -11632,8 +11654,6 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, return true; } - printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n"); - auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); auto set_ov = (set_fa_vec_override_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_metal_tuning_set_fa_vec_override"); @@ -11642,13 +11662,16 @@ static bool run_fa_vec_slice(ggml_backend_t backend, ggml_backend_t backend_cpu, return true; // not the Metal backend: nothing to force } + printf("Running FA vec slice tests (env LLAMA_TEST_FA_VEC_DISABLE=1 to skip)\n"); + struct shape_t { int dk, dv; }; const shape_t shapes[] = { { 128, 128 }, { 576, 512 } }; // mainstream head size + MLA shared K/V view const int ne01_pts[] = { 1, 3 }; // decode, and padded rows for Q=2 and Q=4 const int ne11_pts[] = { 512, 4097 }; // nsg=1, and nsg>=2 together with kvpad const ggml_type types[] = { GGML_TYPE_F16, GGML_TYPE_Q4_0 }; - int n_run = 0, n_fail = 0; + int n_run = 0; + int n_fail = 0; for (auto s : shapes) { for (int ne : fa_vec_legal_ne(s.dk, s.dv)) { for (int Q : { 1, 2, 4 }) { @@ -11968,20 +11991,29 @@ static void show_test_coverage() { } static void usage(char ** argv) { - printf("Usage: %s [mode] [-o ] [-b ] [-p ] [--output ] [--list-ops]", argv[0]); - printf(" [--show-coverage] [--test-file ] [-j ]\n"); - printf(" valid modes:\n"); - printf(" - test (default, compare with CPU backend for correctness)\n"); - printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); - printf(" - perf (performance evaluation)\n"); - printf(" - support (probe backend operation support)\n"); - printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n"); - printf(" optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n"); - printf(" --output specifies output format (default: console, options: console, sql, csv)\n"); - printf(" --list-ops lists all available GGML operations\n"); - printf(" --show-coverage shows test coverage\n"); - printf(" --test-file reads test operators from a test file generated by test-export-graph-ops\n"); - printf(" -j runs tests using parallel worker threads (default: 1, test mode only)\n"); + printf("Usage: %s [mode] [options]\n\n", argv[0]); + printf("Valid modes:\n"); + printf(" test (default) compare with CPU backend for correctness\n"); + printf(" grad compare gradients from backpropagation with method of finite differences\n"); + printf(" perf performance evaluation\n"); + printf(" support probe backend operation support\n\n"); + printf("Options:\n"); + printf(" -o comma separated list of exact op names (as given by ggml_op_desc()),\n"); + printf(" full test case strings, and/or regexes matched against the op name\n"); + printf(" -b run tests on the given backend (e.g. CPU, MTL0, CUDA0)\n"); + printf(" -p filter test cases by a regex matched against their params\n"); + printf(" --output output format (default: console)\n"); + printf(" --list-ops list all available GGML operations\n"); + printf(" --show-coverage show test coverage\n"); + printf(" --test-file read test operators from a test file generated by test-export-graph-ops\n"); + printf(" -j run tests using parallel worker threads (default: 1, test mode only)\n\n"); + printf("Examples:\n"); + printf(" %s -j 8\n", argv[0]); + printf(" %s -o ADD,MUL_MAT\n", argv[0]); + printf(" %s -o ADD -p 'type=f16.*perm1=0'\n", argv[0]); + printf(" %s -b MTL0 -o 'DSV4.*'\n", argv[0]); + printf(" %s -b CUDA0 -o 'ADD(type=f16,ne=[1,1,1,1],nr=[32,1,1,1],nf=1,perm1=0,src_overlap=0)'\n", argv[0]); + printf(" %s perf -o 'MUL_MAT.*'\n", argv[0]); } int main(int argc, char ** argv) {