mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-27 09:13:34 +00:00
common : gracefully fallback on unsupported regex patterns in JSON schema (#26939)
This commit is contained in:
Aldehir Rojas
GitHub
parent
70aff25250
commit
dc64a1620e
2 changed files with 171 additions and 35 deletions
|
|
@ -278,7 +278,9 @@ static std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
|
|||
{'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
|
||||
};
|
||||
|
||||
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
|
||||
static const int MAX_PATTERN_DEPTH = 100;
|
||||
|
||||
static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'};
|
||||
static std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
|
||||
|
||||
static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {
|
||||
|
|
@ -309,6 +311,32 @@ static std::string format_literal(const std::string & literal) {
|
|||
|
||||
std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
|
||||
|
||||
static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
|
||||
if (pos + 1 >= pattern.length() || pattern[pos] != '\\') {
|
||||
return 0;
|
||||
}
|
||||
size_t n_hex = 0;
|
||||
switch (pattern[pos + 1]) {
|
||||
case 'x': n_hex = 2; break;
|
||||
case 'u': n_hex = 4; break;
|
||||
case 'U': n_hex = 8; break;
|
||||
case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']':
|
||||
return 2;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
if (pos + 2 + n_hex > pattern.length()) {
|
||||
return 0;
|
||||
}
|
||||
for (size_t i = pos + 2; i < pos + 2 + n_hex; i++) {
|
||||
char h = pattern[i];
|
||||
if (!((h >= '0' && h <= '9') || (h >= 'a' && h <= 'f') || (h >= 'A' && h <= 'F'))) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 2 + n_hex;
|
||||
}
|
||||
|
||||
class common_schema_converter {
|
||||
private:
|
||||
friend class common_schema_info;
|
||||
|
|
@ -345,16 +373,42 @@ private:
|
|||
return string_join(rules, " | ");
|
||||
}
|
||||
|
||||
// thrown when the pattern is a valid regex with no grammar equivalent
|
||||
struct unsupported_pattern : public std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
// thrown when the pattern is not a valid regex
|
||||
struct invalid_pattern : public std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
std::string _visit_pattern(const std::string & pattern, const std::string & name) {
|
||||
if (!(pattern.front() == '^' && pattern.back() == '$')) {
|
||||
_errors.push_back("Pattern must start with '^' and end with '$'");
|
||||
auto rules_snapshot = _rules;
|
||||
try {
|
||||
return _pattern_to_rule(pattern, name);
|
||||
} catch (const unsupported_pattern & err) {
|
||||
// revert rules
|
||||
_rules = std::move(rules_snapshot);
|
||||
_warnings.push_back("pattern " + pattern + " is not supported (" + err.what() + "), accepting any string");
|
||||
return _add_rule(name, _add_primitive("string", PRIMITIVE_RULES.at("string")));
|
||||
} catch (const invalid_pattern & err) {
|
||||
_rules = std::move(rules_snapshot);
|
||||
_errors.push_back("Invalid pattern " + pattern + ": " + err.what());
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
std::string _pattern_to_rule(const std::string & pattern, const std::string & name) {
|
||||
if (pattern.length() < 2 || pattern.front() != '^' || pattern.back() != '$') {
|
||||
throw unsupported_pattern("not anchored with '^' and '$'");
|
||||
}
|
||||
std::string sub_pattern = pattern.substr(1, pattern.length() - 2);
|
||||
std::unordered_map<std::string, std::string> sub_rule_ids;
|
||||
|
||||
size_t i = 0;
|
||||
size_t length = sub_pattern.length();
|
||||
int paren_depth = 0;
|
||||
|
||||
using literal_or_rule = std::pair<std::string, bool>;
|
||||
auto to_rule = [&](const literal_or_rule & ls) {
|
||||
|
|
@ -363,7 +417,6 @@ private:
|
|||
return is_literal ? "\"" + s + "\"" : s;
|
||||
};
|
||||
std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {
|
||||
size_t start = i;
|
||||
std::vector<literal_or_rule> seq;
|
||||
|
||||
auto get_dot = [&]() {
|
||||
|
|
@ -420,43 +473,42 @@ private:
|
|||
if (i + 1 < length && sub_pattern[i + 1] == ':') {
|
||||
i += 2; // skip "?:" for non-capturing group, treat as regular group
|
||||
} else {
|
||||
// lookahead/lookbehind (?=, ?!, ?<=, ?<!) - not supported
|
||||
_warnings.push_back("Unsupported pattern syntax");
|
||||
// skip to matching ')' to avoid UB on empty seq
|
||||
int depth = 1;
|
||||
while (i < length && depth > 0) {
|
||||
if (sub_pattern[i] == '\\' && i + 1 < length) {
|
||||
i += 2; // skip escaped character
|
||||
} else {
|
||||
if (sub_pattern[i] == '(') depth++;
|
||||
else if (sub_pattern[i] == ')') depth--;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
// lookaround, named group, inline flags, ...
|
||||
throw unsupported_pattern("unsupported group syntax");
|
||||
}
|
||||
}
|
||||
paren_depth++;
|
||||
if (paren_depth > MAX_PATTERN_DEPTH) {
|
||||
throw unsupported_pattern("pattern nesting too deep");
|
||||
}
|
||||
seq.emplace_back("(" + to_rule(transform()) + ")", false);
|
||||
} else if (c == ')') {
|
||||
i++;
|
||||
if (start > 0 && sub_pattern[start - 1] != '(' && (start < 2 || sub_pattern[start - 2] != '?' || sub_pattern[start - 1] != ':')) {
|
||||
_errors.push_back("Unbalanced parentheses");
|
||||
if (paren_depth == 0) {
|
||||
throw invalid_pattern("unbalanced parentheses");
|
||||
}
|
||||
paren_depth--;
|
||||
return join_seq();
|
||||
} else if (c == '^' || c == '$') {
|
||||
throw unsupported_pattern("anchor inside the pattern");
|
||||
} else if (c == '[') {
|
||||
std::string square_brackets = std::string(1, c);
|
||||
i++;
|
||||
while (i < length && sub_pattern[i] != ']') {
|
||||
if (sub_pattern[i] == '\\') {
|
||||
square_brackets += sub_pattern.substr(i, 2);
|
||||
i += 2;
|
||||
auto escape_length = gbnf_escape_length(sub_pattern, i);
|
||||
if (escape_length == 0) {
|
||||
throw unsupported_pattern("unsupported escape in character class: " + sub_pattern.substr(i, 2));
|
||||
}
|
||||
square_brackets += sub_pattern.substr(i, escape_length);
|
||||
i += escape_length;
|
||||
} else {
|
||||
square_brackets += sub_pattern[i];
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (i >= length) {
|
||||
_errors.push_back("Unbalanced square brackets");
|
||||
throw invalid_pattern("unterminated character class");
|
||||
}
|
||||
square_brackets += ']';
|
||||
i++;
|
||||
|
|
@ -465,6 +517,9 @@ private:
|
|||
seq.emplace_back("|", false);
|
||||
i++;
|
||||
} else if (c == '*' || c == '+' || c == '?') {
|
||||
if (seq.empty()) {
|
||||
throw invalid_pattern("nothing to repeat");
|
||||
}
|
||||
seq.back() = std::make_pair(to_rule(seq.back()) + c, false);
|
||||
i++;
|
||||
} else if (c == '{') {
|
||||
|
|
@ -475,18 +530,19 @@ private:
|
|||
i++;
|
||||
}
|
||||
if (i >= length) {
|
||||
_errors.push_back("Unbalanced curly brackets");
|
||||
throw unsupported_pattern("unterminated curly brackets");
|
||||
}
|
||||
curly_brackets += '}';
|
||||
i++;
|
||||
auto nums = string_split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");
|
||||
int min_times = 0;
|
||||
int max_times = std::numeric_limits<int>::max();
|
||||
if (nums.size() != 1 && nums.size() != 2) {
|
||||
throw unsupported_pattern("wrong number of values in curly brackets");
|
||||
}
|
||||
try {
|
||||
if (nums.size() == 1) {
|
||||
min_times = max_times = std::stoi(nums[0]);
|
||||
} else if (nums.size() != 2) {
|
||||
_errors.push_back("Wrong number of values in curly brackets");
|
||||
} else {
|
||||
if (!nums[0].empty()) {
|
||||
min_times = std::stoi(nums[0]);
|
||||
|
|
@ -495,9 +551,11 @@ private:
|
|||
max_times = std::stoi(nums[1]);
|
||||
}
|
||||
}
|
||||
} catch (const std::invalid_argument & e) {
|
||||
_errors.push_back("Invalid number in curly brackets");
|
||||
return std::make_pair("", false);
|
||||
} catch (const std::logic_error &) {
|
||||
throw unsupported_pattern("invalid number in curly brackets");
|
||||
}
|
||||
if (seq.empty()) {
|
||||
throw invalid_pattern("nothing to repeat");
|
||||
}
|
||||
auto &last = seq.back();
|
||||
auto &sub = last.first;
|
||||
|
|
@ -523,15 +581,22 @@ private:
|
|||
return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();
|
||||
};
|
||||
while (i < length) {
|
||||
if (sub_pattern[i] == '\\' && i < length - 1) {
|
||||
if (sub_pattern[i] == '\\') {
|
||||
if (i == length - 1) {
|
||||
throw invalid_pattern("trailing backslash");
|
||||
}
|
||||
char next = sub_pattern[i + 1];
|
||||
if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {
|
||||
i++;
|
||||
literal += sub_pattern[i];
|
||||
i++;
|
||||
} else {
|
||||
literal += sub_pattern.substr(i, 2);
|
||||
i += 2;
|
||||
auto escape_length = gbnf_escape_length(sub_pattern, i);
|
||||
if (escape_length == 0) {
|
||||
throw unsupported_pattern("unsupported escape: " + sub_pattern.substr(i, 2));
|
||||
}
|
||||
literal += sub_pattern.substr(i, escape_length);
|
||||
i += escape_length;
|
||||
}
|
||||
} else if (sub_pattern[i] == '"') {
|
||||
literal += "\\\"";
|
||||
|
|
@ -544,14 +609,21 @@ private:
|
|||
break;
|
||||
}
|
||||
}
|
||||
if (!literal.empty()) {
|
||||
seq.emplace_back(literal, true);
|
||||
if (literal.empty()) { // nothing was consumed, ex. a stray ']' or '}'
|
||||
throw unsupported_pattern(std::string("unsupported character: ") + c);
|
||||
}
|
||||
seq.emplace_back(literal, true);
|
||||
}
|
||||
}
|
||||
return join_seq();
|
||||
};
|
||||
return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"");
|
||||
|
||||
auto rule = to_rule(transform());
|
||||
if (paren_depth != 0) {
|
||||
throw invalid_pattern("unbalanced parentheses");
|
||||
}
|
||||
|
||||
return _add_rule(name, "\"\\\"\" (" + rule + ") \"\\\"\"");
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1564,6 +1564,70 @@ int main() {
|
|||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""",
|
||||
});
|
||||
|
||||
run({
|
||||
SUCCESS,
|
||||
"unanchored regexp",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "[0-9]+"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
|
||||
// the rules of the partial conversion (here "root-0") must not leak into the grammar
|
||||
run({
|
||||
SUCCESS,
|
||||
"regexp with unsupported shorthand",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]{3}\\w$"
|
||||
})""",
|
||||
R"""(
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= string
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
|
||||
// a regexp that is invalid under any flavor is still an error
|
||||
run({
|
||||
FAILURE,
|
||||
"regexp with unbalanced parentheses",
|
||||
R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(a$"
|
||||
})""",
|
||||
""
|
||||
});
|
||||
|
||||
// only the property with the bad pattern degrades
|
||||
run({
|
||||
SUCCESS,
|
||||
"unsupported regexp in a property",
|
||||
R"""({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": { "type": "string", "pattern": "^[a-z\\-]+$" }
|
||||
},
|
||||
"required": ["a"],
|
||||
"additionalProperties": false
|
||||
})""",
|
||||
R"""(
|
||||
a ::= string
|
||||
a-kv ::= "\"a\"" space ":" space a
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
root ::= "{" space a-kv space "}"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\""
|
||||
)""",
|
||||
});
|
||||
}
|
||||
|
||||
if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue