mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-11 09:34:37 +00:00
resync and updated sdcpp for flux and sd3 support
This commit is contained in:
Concedo
parent
33721615b5
commit
f32a874966
30 changed files with 2434248 additions and 1729 deletions
|
|
@ -31,16 +31,6 @@ std::pair<std::unordered_map<std::string, float>, std::string> extract_and_remov
|
|||
return std::make_pair(filename2multiplier, text);
|
||||
}
|
||||
|
||||
const std::string UNK_TOKEN = "<|endoftext|>";
|
||||
const std::string BOS_TOKEN = "<|startoftext|>";
|
||||
const std::string EOS_TOKEN = "<|endoftext|>";
|
||||
const std::string PAD_TOEKN = "<|endoftext|>";
|
||||
|
||||
const int UNK_TOKEN_ID = 49407;
|
||||
const int BOS_TOKEN_ID = 49406;
|
||||
const int EOS_TOKEN_ID = 49407;
|
||||
const int PAD_TOKEN_ID = 49407;
|
||||
|
||||
std::vector<std::pair<int, std::u32string>> bytes_to_unicode() {
|
||||
std::vector<std::pair<int, std::u32string>> byte_unicode_pairs;
|
||||
std::set<int> byte_set;
|
||||
|
|
@ -73,12 +63,27 @@ typedef std::function<bool(std::string&, std::vector<int32_t>&)> on_new_token_cb
|
|||
|
||||
class CLIPTokenizer {
|
||||
private:
|
||||
SDVersion version = VERSION_1_x;
|
||||
std::map<int, std::u32string> byte_encoder;
|
||||
std::map<std::u32string, int> byte_decoder;
|
||||
std::map<std::u32string, int> encoder;
|
||||
std::map<int, std::u32string> decoder;
|
||||
std::map<std::pair<std::u32string, std::u32string>, int> bpe_ranks;
|
||||
std::regex pat;
|
||||
int encoder_len;
|
||||
int bpe_len;
|
||||
|
||||
public:
|
||||
const std::string UNK_TOKEN = "<|endoftext|>";
|
||||
const std::string BOS_TOKEN = "<|startoftext|>";
|
||||
const std::string EOS_TOKEN = "<|endoftext|>";
|
||||
const std::string PAD_TOKEN = "<|endoftext|>";
|
||||
|
||||
const int UNK_TOKEN_ID = 49407;
|
||||
const int BOS_TOKEN_ID = 49406;
|
||||
const int EOS_TOKEN_ID = 49407;
|
||||
const int PAD_TOKEN_ID = 49407;
|
||||
|
||||
private:
|
||||
static std::string strip(const std::string& str) {
|
||||
std::string::size_type start = str.find_first_not_of(" \t\n\r\v\f");
|
||||
std::string::size_type end = str.find_last_not_of(" \t\n\r\v\f");
|
||||
|
|
@ -113,12 +118,22 @@ private:
|
|||
}
|
||||
|
||||
public:
|
||||
CLIPTokenizer(SDVersion version = VERSION_1_x)
|
||||
: version(version) {}
|
||||
CLIPTokenizer(int pad_token_id = 49407, const std::string& merges_utf8_str = "")
|
||||
: PAD_TOKEN_ID(pad_token_id) {
|
||||
if (merges_utf8_str.size() > 0) {
|
||||
load_from_merges(merges_utf8_str);
|
||||
} else {
|
||||
load_from_merges(ModelLoader::load_merges());
|
||||
}
|
||||
}
|
||||
|
||||
void load_from_merges(const std::string& merges_utf8_str) {
|
||||
auto byte_unicode_pairs = bytes_to_unicode();
|
||||
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
|
||||
// printf("byte_unicode_pairs have %lu pairs \n", byte_unicode_pairs.size());
|
||||
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
|
||||
for (auto& pair : byte_unicode_pairs) {
|
||||
byte_decoder[pair.second] = pair.first;
|
||||
}
|
||||
// for (auto & pair: byte_unicode_pairs) {
|
||||
// std::cout << pair.first << ": " << pair.second << std::endl;
|
||||
// }
|
||||
|
|
@ -138,6 +153,8 @@ public:
|
|||
size_t space_pos = merge.find(' ');
|
||||
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
||||
// LOG_DEBUG("%s", utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
|
||||
// printf("%s :: %s | %s \n", utf32_to_utf8(merge).c_str(), utf32_to_utf8(merge.substr(0, space_pos)).c_str(),
|
||||
// utf32_to_utf8(merge.substr(space_pos + 1)).c_str());
|
||||
}
|
||||
std::vector<std::u32string> vocab;
|
||||
for (const auto& pair : byte_unicode_pairs) {
|
||||
|
|
@ -154,15 +171,36 @@ public:
|
|||
LOG_DEBUG("vocab size: %llu", vocab.size());
|
||||
int i = 0;
|
||||
for (const auto& token : vocab) {
|
||||
encoder[token] = i++;
|
||||
encoder[token] = i;
|
||||
decoder[i] = token;
|
||||
i++;
|
||||
}
|
||||
encoder_len = i;
|
||||
|
||||
auto it = encoder.find(utf8_to_utf32("img</w>"));
|
||||
if (it != encoder.end()) {
|
||||
LOG_DEBUG(" trigger word img already in vocab");
|
||||
} else {
|
||||
LOG_DEBUG(" trigger word img not in vocab yet");
|
||||
}
|
||||
|
||||
int rank = 0;
|
||||
for (const auto& merge : merge_pairs) {
|
||||
bpe_ranks[merge] = rank++;
|
||||
}
|
||||
bpe_len = rank;
|
||||
};
|
||||
|
||||
void add_token(const std::string& text) {
|
||||
std::u32string token = utf8_to_utf32(text);
|
||||
auto it = encoder.find(token);
|
||||
if (it != encoder.end()) {
|
||||
encoder[token] = encoder_len;
|
||||
decoder[encoder_len] = token;
|
||||
encoder_len++;
|
||||
}
|
||||
}
|
||||
|
||||
std::u32string bpe(const std::u32string& token) {
|
||||
std::vector<std::u32string> word;
|
||||
|
||||
|
|
@ -243,6 +281,7 @@ public:
|
|||
size_t max_length = 0,
|
||||
bool padding = false) {
|
||||
std::vector<int32_t> tokens = encode(text, on_new_token_cb);
|
||||
|
||||
tokens.insert(tokens.begin(), BOS_TOKEN_ID);
|
||||
if (max_length > 0) {
|
||||
if (tokens.size() > max_length - 1) {
|
||||
|
|
@ -251,17 +290,83 @@ public:
|
|||
} else {
|
||||
tokens.push_back(EOS_TOKEN_ID);
|
||||
if (padding) {
|
||||
int pad_token_id = PAD_TOKEN_ID;
|
||||
if (version == VERSION_2_x) {
|
||||
pad_token_id = 0;
|
||||
}
|
||||
tokens.insert(tokens.end(), max_length - tokens.size(), pad_token_id);
|
||||
tokens.insert(tokens.end(), max_length - tokens.size(), PAD_TOKEN_ID);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
void pad_tokens(std::vector<int>& tokens,
|
||||
std::vector<float>& weights,
|
||||
size_t max_length = 0,
|
||||
bool padding = false) {
|
||||
if (max_length > 0 && padding) {
|
||||
size_t n = std::ceil(tokens.size() * 1.0 / (max_length - 2));
|
||||
if (n == 0) {
|
||||
n = 1;
|
||||
}
|
||||
size_t length = max_length * n;
|
||||
LOG_DEBUG("token length: %llu", length);
|
||||
std::vector<int> new_tokens;
|
||||
std::vector<float> new_weights;
|
||||
new_tokens.push_back(BOS_TOKEN_ID);
|
||||
new_weights.push_back(1.0);
|
||||
int token_idx = 0;
|
||||
for (int i = 1; i < length; i++) {
|
||||
if (token_idx >= tokens.size()) {
|
||||
break;
|
||||
}
|
||||
if (i % max_length == 0) {
|
||||
new_tokens.push_back(BOS_TOKEN_ID);
|
||||
new_weights.push_back(1.0);
|
||||
} else if (i % max_length == max_length - 1) {
|
||||
new_tokens.push_back(EOS_TOKEN_ID);
|
||||
new_weights.push_back(1.0);
|
||||
} else {
|
||||
new_tokens.push_back(tokens[token_idx]);
|
||||
new_weights.push_back(weights[token_idx]);
|
||||
token_idx++;
|
||||
}
|
||||
}
|
||||
|
||||
new_tokens.push_back(EOS_TOKEN_ID);
|
||||
new_weights.push_back(1.0);
|
||||
tokens = new_tokens;
|
||||
weights = new_weights;
|
||||
|
||||
if (padding) {
|
||||
tokens.insert(tokens.end(), length - tokens.size(), PAD_TOKEN_ID);
|
||||
weights.insert(weights.end(), length - weights.size(), 1.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string decode(const std::vector<int>& tokens) {
|
||||
std::string text = "";
|
||||
for (int t : tokens) {
|
||||
if (t == 49406 || t == 49407)
|
||||
continue;
|
||||
std::u32string ts = decoder[t];
|
||||
// printf("%d, %s \n", t, utf32_to_utf8(ts).c_str());
|
||||
std::string s = utf32_to_utf8(ts);
|
||||
if (s.length() >= 4 && ends_with(s, "</w>")) {
|
||||
text += " " + s.replace(s.length() - 4, s.length() - 1, "");
|
||||
} else {
|
||||
text += " " + s;
|
||||
}
|
||||
}
|
||||
// std::vector<unsigned char> bytes;
|
||||
// for (auto c : text){
|
||||
// bytes.push_back(byte_decoder[c]);
|
||||
// }
|
||||
|
||||
// std::string s((char *)bytes.data());
|
||||
// std::string s = "";
|
||||
return trim(text);
|
||||
}
|
||||
|
||||
std::vector<int> encode(std::string text, on_new_token_cb_t on_new_token_cb) {
|
||||
std::string original_text = text;
|
||||
std::vector<int32_t> bpe_tokens;
|
||||
|
|
@ -283,7 +388,7 @@ public:
|
|||
std::string token_str = token.str();
|
||||
std::u32string utf32_token;
|
||||
for (int i = 0; i < token_str.length(); i++) {
|
||||
char b = token_str[i];
|
||||
unsigned char b = token_str[i];
|
||||
utf32_token += byte_encoder[b];
|
||||
}
|
||||
auto bpe_strs = bpe(utf32_token);
|
||||
|
|
@ -308,118 +413,12 @@ public:
|
|||
ss << "\"" << token << "\", ";
|
||||
}
|
||||
ss << "]";
|
||||
LOG_DEBUG("split prompt \"%s\" to tokens %s", original_text.c_str(), ss.str().c_str());
|
||||
// LOG_DEBUG("split prompt \"%s\" to tokens %s", original_text.c_str(), ss.str().c_str());
|
||||
// printf("split prompt \"%s\" to tokens %s \n", original_text.c_str(), ss.str().c_str());
|
||||
return bpe_tokens;
|
||||
}
|
||||
};
|
||||
|
||||
// Ref: https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/cad87bf4e3e0b0a759afa94e933527c3123d59bc/modules/prompt_parser.py#L345
|
||||
//
|
||||
// Parses a string with attention tokens and returns a list of pairs: text and its associated weight.
|
||||
// Accepted tokens are:
|
||||
// (abc) - increases attention to abc by a multiplier of 1.1
|
||||
// (abc:3.12) - increases attention to abc by a multiplier of 3.12
|
||||
// [abc] - decreases attention to abc by a multiplier of 1.1
|
||||
// \( - literal character '('
|
||||
// \[ - literal character '['
|
||||
// \) - literal character ')'
|
||||
// \] - literal character ']'
|
||||
// \\ - literal character '\'
|
||||
// anything else - just text
|
||||
//
|
||||
// >>> parse_prompt_attention('normal text')
|
||||
// [['normal text', 1.0]]
|
||||
// >>> parse_prompt_attention('an (important) word')
|
||||
// [['an ', 1.0], ['important', 1.1], [' word', 1.0]]
|
||||
// >>> parse_prompt_attention('(unbalanced')
|
||||
// [['unbalanced', 1.1]]
|
||||
// >>> parse_prompt_attention('\(literal\]')
|
||||
// [['(literal]', 1.0]]
|
||||
// >>> parse_prompt_attention('(unnecessary)(parens)')
|
||||
// [['unnecessaryparens', 1.1]]
|
||||
// >>> parse_prompt_attention('a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).')
|
||||
// [['a ', 1.0],
|
||||
// ['house', 1.5730000000000004],
|
||||
// [' ', 1.1],
|
||||
// ['on', 1.0],
|
||||
// [' a ', 1.1],
|
||||
// ['hill', 0.55],
|
||||
// [', sun, ', 1.1],
|
||||
// ['sky', 1.4641000000000006],
|
||||
// ['.', 1.1]]
|
||||
std::vector<std::pair<std::string, float>> parse_prompt_attention(const std::string& text) {
|
||||
std::vector<std::pair<std::string, float>> res;
|
||||
std::vector<int> round_brackets;
|
||||
std::vector<int> square_brackets;
|
||||
|
||||
float round_bracket_multiplier = 1.1f;
|
||||
float square_bracket_multiplier = 1 / 1.1f;
|
||||
|
||||
std::regex re_attention(R"(\\\(|\\\)|\\\[|\\\]|\\\\|\\|\(|\[|:([+-]?[.\d]+)\)|\)|\]|[^\\()\[\]:]+|:)");
|
||||
std::regex re_break(R"(\s*\bBREAK\b\s*)");
|
||||
|
||||
auto multiply_range = [&](int start_position, float multiplier) {
|
||||
for (int p = start_position; p < res.size(); ++p) {
|
||||
res[p].second *= multiplier;
|
||||
}
|
||||
};
|
||||
|
||||
std::smatch m;
|
||||
std::string remaining_text = text;
|
||||
|
||||
while (std::regex_search(remaining_text, m, re_attention)) {
|
||||
std::string text = m[0];
|
||||
std::string weight = m[1];
|
||||
|
||||
if (text == "(") {
|
||||
round_brackets.push_back((int)res.size());
|
||||
} else if (text == "[") {
|
||||
square_brackets.push_back((int)res.size());
|
||||
} else if (!weight.empty()) {
|
||||
if (!round_brackets.empty()) {
|
||||
multiply_range(round_brackets.back(), std::stof(weight));
|
||||
round_brackets.pop_back();
|
||||
}
|
||||
} else if (text == ")" && !round_brackets.empty()) {
|
||||
multiply_range(round_brackets.back(), round_bracket_multiplier);
|
||||
round_brackets.pop_back();
|
||||
} else if (text == "]" && !square_brackets.empty()) {
|
||||
multiply_range(square_brackets.back(), square_bracket_multiplier);
|
||||
square_brackets.pop_back();
|
||||
} else if (text == "\\(") {
|
||||
res.push_back({text.substr(1), 1.0f});
|
||||
} else {
|
||||
res.push_back({text, 1.0f});
|
||||
}
|
||||
|
||||
remaining_text = m.suffix();
|
||||
}
|
||||
|
||||
for (int pos : round_brackets) {
|
||||
multiply_range(pos, round_bracket_multiplier);
|
||||
}
|
||||
|
||||
for (int pos : square_brackets) {
|
||||
multiply_range(pos, square_bracket_multiplier);
|
||||
}
|
||||
|
||||
if (res.empty()) {
|
||||
res.push_back({"", 1.0f});
|
||||
}
|
||||
|
||||
int i = 0;
|
||||
while (i + 1 < res.size()) {
|
||||
if (res[i].second == res[i + 1].second) {
|
||||
res[i].first += res[i + 1].first;
|
||||
res.erase(res.begin() + i + 1);
|
||||
} else {
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/*================================================ FrozenCLIPEmbedder ================================================*/
|
||||
|
||||
// Ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/modeling_clip.py
|
||||
|
|
@ -469,7 +468,8 @@ public:
|
|||
: d_model(d_model),
|
||||
n_head(n_head),
|
||||
intermediate_size(intermediate_size) {
|
||||
blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head));
|
||||
blocks["self_attn"] = std::shared_ptr<GGMLBlock>(new MultiheadAttention(d_model, n_head, true, true));
|
||||
|
||||
blocks["layer_norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
|
||||
blocks["layer_norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(d_model));
|
||||
|
||||
|
|
@ -508,7 +508,7 @@ public:
|
|||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x, int clip_skip = -1, bool mask = true) {
|
||||
// x: [N, n_token, d_model]
|
||||
int layer_idx = n_layer - 1;
|
||||
LOG_DEBUG("clip_skip %d", clip_skip);
|
||||
// LOG_DEBUG("clip_skip %d", clip_skip);
|
||||
if (clip_skip > 0) {
|
||||
layer_idx = n_layer - clip_skip;
|
||||
}
|
||||
|
|
@ -520,7 +520,7 @@ public:
|
|||
}
|
||||
std::string name = "layers." + std::to_string(i);
|
||||
auto layer = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
|
||||
x = layer->forward(ctx, x); // [N, n_token, d_model]
|
||||
x = layer->forward(ctx, x, mask); // [N, n_token, d_model]
|
||||
// LOG_DEBUG("layer %d", i);
|
||||
}
|
||||
return x;
|
||||
|
|
@ -558,11 +558,14 @@ public:
|
|||
auto token_embed_weight = params["token_embedding.weight"];
|
||||
auto position_embed_weight = params["position_embedding.weight"];
|
||||
|
||||
GGML_ASSERT(input_ids->ne[0] <= position_embed_weight->ne[0]);
|
||||
GGML_ASSERT(input_ids->ne[0] == position_embed_weight->ne[1]);
|
||||
input_ids = ggml_reshape_3d(ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
|
||||
auto token_embedding = ggml_get_rows(ctx, custom_embed_weight != NULL ? custom_embed_weight : token_embed_weight, input_ids);
|
||||
token_embedding = ggml_reshape_3d(ctx, token_embedding, token_embedding->ne[0], token_embedding->ne[1], token_embedding->ne[3]);
|
||||
|
||||
// token_embedding + position_embedding
|
||||
auto x = ggml_add(ctx,
|
||||
ggml_get_rows(ctx, custom_embed_weight != NULL ? custom_embed_weight : token_embed_weight, input_ids),
|
||||
token_embedding,
|
||||
position_embed_weight); // [N, n_token, embed_dim]
|
||||
return x;
|
||||
}
|
||||
|
|
@ -617,8 +620,8 @@ public:
|
|||
class_embedding = ggml_repeat(ctx, class_embed_weight, class_embedding); // [N, embed_dim]
|
||||
class_embedding = ggml_reshape_4d(ctx, class_embedding, 1, embed_dim, 1, N); // [N, 1, embed_dim, 1]
|
||||
|
||||
struct ggml_tensor* x = ggml_concat(ctx, class_embedding, patch_embedding, 2); // [N, num_positions, embed_dim, 1]
|
||||
x = ggml_reshape_3d(ctx, x, embed_dim, num_positions, N); // [N, num_positions, embed_dim]
|
||||
struct ggml_tensor* x = ggml_concat(ctx, class_embedding, patch_embedding, 2); // [N, num_positions, embed_dim, 1]
|
||||
x = ggml_reshape_3d(ctx, x, embed_dim, num_positions, N); // [N, num_positions, embed_dim]
|
||||
x = ggml_add(ctx, x, position_embed_weight);
|
||||
return x; // [N, num_positions, embed_dim]
|
||||
}
|
||||
|
|
@ -717,11 +720,6 @@ public:
|
|||
};
|
||||
|
||||
class CLIPVisionModel : public GGMLBlock {
|
||||
protected:
|
||||
void init_params(struct ggml_context* ctx, ggml_type wtype) {
|
||||
params["visual_projection"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, projection_dim, hidden_size);
|
||||
}
|
||||
|
||||
public:
|
||||
// network hparams
|
||||
int32_t num_channels = 3;
|
||||
|
|
@ -732,16 +730,14 @@ public:
|
|||
int32_t intermediate_size = 4096;
|
||||
int32_t n_head = 16;
|
||||
int32_t n_layer = 24;
|
||||
int32_t projection_dim = 768;
|
||||
|
||||
public:
|
||||
CLIPVisionModel(CLIPVersion version = OPEN_CLIP_VIT_H_14) {
|
||||
CLIPVisionModel(CLIPVersion version = OPENAI_CLIP_VIT_L_14) {
|
||||
if (version == OPEN_CLIP_VIT_H_14) {
|
||||
hidden_size = 1280;
|
||||
intermediate_size = 5120;
|
||||
n_head = 16;
|
||||
n_layer = 32;
|
||||
projection_dim = 1024;
|
||||
} else if (version == OPEN_CLIP_VIT_BIGG_14) {
|
||||
hidden_size = 1664;
|
||||
intermediate_size = 8192;
|
||||
|
|
@ -755,9 +751,8 @@ public:
|
|||
blocks["post_layernorm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values) {
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values, bool return_pooled = true) {
|
||||
// pixel_values: [N, num_channels, image_size, image_size]
|
||||
// return: // [N, projection_dim]
|
||||
auto embeddings = std::dynamic_pointer_cast<CLIPVisionEmbeddings>(blocks["embeddings"]);
|
||||
auto pre_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["pre_layernorm"]);
|
||||
auto encoder = std::dynamic_pointer_cast<CLIPEncoder>(blocks["encoder"]);
|
||||
|
|
@ -765,26 +760,60 @@ public:
|
|||
|
||||
auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim]
|
||||
x = pre_layernorm->forward(ctx, x);
|
||||
x = encoder->forward(ctx, x, -1, true);
|
||||
x = encoder->forward(ctx, x, -1, false);
|
||||
x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size]
|
||||
|
||||
GGML_ASSERT(x->ne[2] == 1);
|
||||
int64_t max_token_idx = 0;
|
||||
ggml_tensor* pooled = ggml_view_1d(ctx, x, x->ne[0], x->nb[1] * max_token_idx); // assert N == 1
|
||||
auto visual_projection = params["visual_projection"];
|
||||
pooled = ggml_mul_mat(ctx, ggml_cont(ctx, ggml_transpose(ctx, visual_projection)), pooled);
|
||||
return pooled; // [N, projection_dim]
|
||||
GGML_ASSERT(x->ne[3] == 1);
|
||||
if (return_pooled) {
|
||||
ggml_tensor* pooled = ggml_cont(ctx, ggml_view_2d(ctx, x, x->ne[0], x->ne[2], x->nb[2], 0));
|
||||
return pooled; // [N, hidden_size]
|
||||
} else {
|
||||
return x; // [N, n_token, hidden_size]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class CLIPProjection : public UnaryBlock {
|
||||
protected:
|
||||
int64_t in_features;
|
||||
int64_t out_features;
|
||||
bool transpose_weight;
|
||||
|
||||
void init_params(struct ggml_context* ctx, ggml_type wtype) {
|
||||
if (transpose_weight) {
|
||||
LOG_ERROR("transpose_weight");
|
||||
params["weight"] = ggml_new_tensor_2d(ctx, wtype, out_features, in_features);
|
||||
} else {
|
||||
params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
CLIPProjection(int64_t in_features,
|
||||
int64_t out_features,
|
||||
bool transpose_weight = false)
|
||||
: in_features(in_features),
|
||||
out_features(out_features),
|
||||
transpose_weight(transpose_weight) {}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* x) {
|
||||
struct ggml_tensor* w = params["weight"];
|
||||
if (transpose_weight) {
|
||||
w = ggml_cont(ctx, ggml_transpose(ctx, w));
|
||||
}
|
||||
return ggml_nn_linear(ctx, x, w, NULL);
|
||||
}
|
||||
};
|
||||
|
||||
class CLIPVisionModelProjection : public GGMLBlock {
|
||||
public:
|
||||
int32_t hidden_size = 1024;
|
||||
int32_t projection_dim = 1024;
|
||||
int32_t projection_dim = 768;
|
||||
int32_t image_size = 224;
|
||||
|
||||
public:
|
||||
CLIPVisionModelProjection(CLIPVersion version = OPEN_CLIP_VIT_H_14) {
|
||||
CLIPVisionModelProjection(CLIPVersion version = OPENAI_CLIP_VIT_L_14,
|
||||
bool transpose_proj_w = false) {
|
||||
if (version == OPEN_CLIP_VIT_H_14) {
|
||||
hidden_size = 1280;
|
||||
projection_dim = 1024;
|
||||
|
|
@ -792,199 +821,86 @@ public:
|
|||
hidden_size = 1664;
|
||||
}
|
||||
|
||||
blocks["visual_model"] = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version));
|
||||
blocks["visual_projection"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, projection_dim, false));
|
||||
blocks["vision_model"] = std::shared_ptr<GGMLBlock>(new CLIPVisionModel(version));
|
||||
blocks["visual_projection"] = std::shared_ptr<GGMLBlock>(new CLIPProjection(hidden_size, projection_dim, transpose_proj_w));
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx, struct ggml_tensor* pixel_values) {
|
||||
// pixel_values: [N, num_channels, image_size, image_size]
|
||||
// return: [N, num_positions, projection_dim]
|
||||
auto visual_model = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["visual_model"]);
|
||||
auto visual_projection = std::dynamic_pointer_cast<Linear>(blocks["visual_projection"]);
|
||||
// return: [N, projection_dim]
|
||||
auto vision_model = std::dynamic_pointer_cast<CLIPVisionModel>(blocks["vision_model"]);
|
||||
auto visual_projection = std::dynamic_pointer_cast<CLIPProjection>(blocks["visual_projection"]);
|
||||
|
||||
auto x = visual_model->forward(ctx, pixel_values); // [N, embed_dim]
|
||||
auto x = vision_model->forward(ctx, pixel_values); // [N, hidden_size]
|
||||
x = visual_projection->forward(ctx, x); // [N, projection_dim]
|
||||
|
||||
return x; // [N, projection_dim]
|
||||
}
|
||||
};
|
||||
|
||||
// ldm.modules.encoders.modules.FrozenCLIPEmbedder
|
||||
// Ref: https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/cad87bf4e3e0b0a759afa94e933527c3123d59bc/modules/sd_hijack_clip.py#L283
|
||||
struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
|
||||
SDVersion version = VERSION_1_x;
|
||||
CLIPTokenizer tokenizer;
|
||||
CLIPTextModel text_model;
|
||||
CLIPTextModel text_model2;
|
||||
struct CLIPTextModelRunner : public GGMLRunner {
|
||||
CLIPTextModel model;
|
||||
|
||||
std::string embd_dir;
|
||||
int32_t num_custom_embeddings = 0;
|
||||
std::vector<uint8_t> token_embed_custom;
|
||||
std::vector<std::string> readed_embeddings;
|
||||
|
||||
FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend,
|
||||
ggml_type wtype,
|
||||
SDVersion version = VERSION_1_x,
|
||||
int clip_skip = -1)
|
||||
: GGMLModule(backend, wtype), version(version), tokenizer(version) {
|
||||
if (clip_skip <= 0) {
|
||||
clip_skip = 1;
|
||||
if (version == VERSION_2_x || version == VERSION_XL) {
|
||||
clip_skip = 2;
|
||||
}
|
||||
}
|
||||
if (version == VERSION_1_x) {
|
||||
text_model = CLIPTextModel(OPENAI_CLIP_VIT_L_14, clip_skip);
|
||||
text_model.init(params_ctx, wtype);
|
||||
} else if (version == VERSION_2_x) {
|
||||
text_model = CLIPTextModel(OPEN_CLIP_VIT_H_14, clip_skip);
|
||||
text_model.init(params_ctx, wtype);
|
||||
} else if (version == VERSION_XL) {
|
||||
text_model = CLIPTextModel(OPENAI_CLIP_VIT_L_14, clip_skip, false);
|
||||
text_model2 = CLIPTextModel(OPEN_CLIP_VIT_BIGG_14, clip_skip, false);
|
||||
text_model.init(params_ctx, wtype);
|
||||
text_model2.init(params_ctx, wtype);
|
||||
}
|
||||
CLIPTextModelRunner(ggml_backend_t backend,
|
||||
ggml_type wtype,
|
||||
CLIPVersion version = OPENAI_CLIP_VIT_L_14,
|
||||
int clip_skip_value = 1,
|
||||
bool with_final_ln = true)
|
||||
: GGMLRunner(backend, wtype), model(version, clip_skip_value, with_final_ln) {
|
||||
model.init(params_ctx, wtype);
|
||||
}
|
||||
|
||||
std::string get_desc() {
|
||||
return "clip";
|
||||
}
|
||||
|
||||
size_t get_params_mem_size() {
|
||||
size_t params_mem_size = text_model.get_params_mem_size();
|
||||
if (version == VERSION_XL) {
|
||||
params_mem_size += text_model2.get_params_mem_size();
|
||||
}
|
||||
return params_mem_size;
|
||||
}
|
||||
|
||||
size_t get_params_num() {
|
||||
size_t params_num = text_model.get_params_num();
|
||||
if (version == VERSION_XL) {
|
||||
params_num += text_model2.get_params_num();
|
||||
}
|
||||
return params_num;
|
||||
}
|
||||
|
||||
void set_clip_skip(int clip_skip) {
|
||||
text_model.set_clip_skip(clip_skip);
|
||||
if (version == VERSION_XL) {
|
||||
text_model2.set_clip_skip(clip_skip);
|
||||
}
|
||||
model.set_clip_skip(clip_skip);
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
|
||||
text_model.get_param_tensors(tensors, prefix + "transformer.text_model");
|
||||
if (version == VERSION_XL) {
|
||||
text_model2.get_param_tensors(tensors, prefix + "1.transformer.text_model");
|
||||
}
|
||||
}
|
||||
|
||||
bool load_embedding(std::string embd_name, std::string embd_path, std::vector<int32_t>& bpe_tokens) {
|
||||
// the order matters
|
||||
ModelLoader model_loader;
|
||||
if (!model_loader.init_from_file(embd_path)) {
|
||||
LOG_ERROR("embedding '%s' failed", embd_name.c_str());
|
||||
return false;
|
||||
}
|
||||
struct ggml_init_params params;
|
||||
params.mem_size = 32 * 1024; // max for custom embeddings 32 KB
|
||||
params.mem_buffer = NULL;
|
||||
params.no_alloc = false;
|
||||
struct ggml_context* embd_ctx = ggml_init(params);
|
||||
struct ggml_tensor* embd = NULL;
|
||||
auto on_load = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) {
|
||||
if (tensor_storage.ne[0] != text_model.hidden_size) {
|
||||
LOG_DEBUG("embedding wrong hidden size, got %i, expected %i", tensor_storage.ne[0], text_model.hidden_size);
|
||||
return false;
|
||||
}
|
||||
embd = ggml_new_tensor_2d(embd_ctx, wtype, text_model.hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
|
||||
*dst_tensor = embd;
|
||||
return true;
|
||||
};
|
||||
model_loader.load_tensors(on_load, NULL);
|
||||
readed_embeddings.push_back(embd_name);
|
||||
token_embed_custom.resize(token_embed_custom.size() + ggml_nbytes(embd));
|
||||
memcpy((void*)(token_embed_custom.data() + num_custom_embeddings * text_model.hidden_size * ggml_type_size(wtype)),
|
||||
embd->data,
|
||||
ggml_nbytes(embd));
|
||||
for (int i = 0; i < embd->ne[1]; i++) {
|
||||
bpe_tokens.push_back(text_model.vocab_size + num_custom_embeddings);
|
||||
// LOG_DEBUG("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
|
||||
num_custom_embeddings++;
|
||||
}
|
||||
LOG_DEBUG("embedding '%s' applied, custom embeddings: %i", embd_name.c_str(), num_custom_embeddings);
|
||||
return true;
|
||||
model.get_param_tensors(tensors, prefix);
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_ids2,
|
||||
struct ggml_tensor* embeddings,
|
||||
size_t max_token_idx = 0,
|
||||
bool return_pooled = false) {
|
||||
if (return_pooled) {
|
||||
return text_model2.forward(ctx, input_ids2, NULL, max_token_idx, return_pooled);
|
||||
size_t N = input_ids->ne[1];
|
||||
size_t n_token = input_ids->ne[0];
|
||||
if (input_ids->ne[0] > model.n_token) {
|
||||
GGML_ASSERT(input_ids->ne[0] % model.n_token == 0);
|
||||
input_ids = ggml_reshape_2d(ctx, input_ids, model.n_token, input_ids->ne[0] / model.n_token);
|
||||
}
|
||||
auto hidden_states = text_model.forward(ctx, input_ids, embeddings); // [N, n_token, hidden_size]
|
||||
// LOG_DEBUG("hidden_states: %d %d %d %d", hidden_states->ne[0], hidden_states->ne[1], hidden_states->ne[2], hidden_states->ne[3]);
|
||||
if (version == VERSION_XL) {
|
||||
hidden_states = ggml_reshape_4d(ctx,
|
||||
hidden_states,
|
||||
hidden_states->ne[0],
|
||||
hidden_states->ne[1],
|
||||
hidden_states->ne[2],
|
||||
hidden_states->ne[3]);
|
||||
hidden_states = ggml_cont(ctx, ggml_permute(ctx, hidden_states, 2, 0, 1, 3));
|
||||
|
||||
auto hidden_states2 = text_model2.forward(ctx, input_ids2, NULL); // [N, n_token, hidden_size2]
|
||||
// LOG_DEBUG("hidden_states: %d %d %d %d", hidden_states->ne[0], hidden_states->ne[1], hidden_states->ne[2], hidden_states->ne[3]);
|
||||
hidden_states2 = ggml_reshape_4d(ctx,
|
||||
hidden_states2,
|
||||
hidden_states2->ne[0],
|
||||
hidden_states2->ne[1],
|
||||
hidden_states2->ne[2],
|
||||
hidden_states2->ne[3]);
|
||||
hidden_states2 = ggml_cont(ctx, ggml_permute(ctx, hidden_states2, 2, 0, 1, 3));
|
||||
|
||||
hidden_states = ggml_concat(ctx, hidden_states, hidden_states2, 2); // [N, n_token, hidden_size + hidden_size2]
|
||||
|
||||
hidden_states = ggml_cont(ctx, ggml_permute(ctx, hidden_states, 1, 2, 0, 3));
|
||||
}
|
||||
// LOG_DEBUG("hidden_states: %d %d %d %d", hidden_states->ne[0], hidden_states->ne[1], hidden_states->ne[2], hidden_states->ne[3]);
|
||||
return hidden_states;
|
||||
return model.forward(ctx, input_ids, embeddings, max_token_idx, return_pooled);
|
||||
}
|
||||
|
||||
struct ggml_cgraph* build_graph(struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_ids2 = NULL,
|
||||
size_t max_token_idx = 0,
|
||||
bool return_pooled = false) {
|
||||
int num_custom_embeddings = 0,
|
||||
void* custom_embeddings_data = NULL,
|
||||
size_t max_token_idx = 0,
|
||||
bool return_pooled = false) {
|
||||
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
|
||||
|
||||
input_ids2 = to_backend(input_ids2);
|
||||
if (!return_pooled) {
|
||||
input_ids = to_backend(input_ids);
|
||||
}
|
||||
input_ids = to_backend(input_ids);
|
||||
|
||||
struct ggml_tensor* embeddings = NULL;
|
||||
|
||||
if (num_custom_embeddings > 0 && version != VERSION_XL) {
|
||||
auto custom_embeddings = ggml_new_tensor_3d(compute_ctx,
|
||||
if (num_custom_embeddings > 0 && custom_embeddings_data != NULL) {
|
||||
auto custom_embeddings = ggml_new_tensor_2d(compute_ctx,
|
||||
wtype,
|
||||
text_model.hidden_size,
|
||||
1,
|
||||
model.hidden_size,
|
||||
num_custom_embeddings);
|
||||
set_backend_tensor_data(custom_embeddings, token_embed_custom.data());
|
||||
set_backend_tensor_data(custom_embeddings, custom_embeddings_data);
|
||||
|
||||
auto token_embed_weight = text_model.get_token_embed_weight();
|
||||
token_embed_weight = ggml_reshape_3d(compute_ctx, token_embed_weight, token_embed_weight->ne[0], 1, token_embed_weight->ne[1]);
|
||||
auto token_embed_weight = model.get_token_embed_weight();
|
||||
// concatenate custom embeddings
|
||||
embeddings = ggml_concat(compute_ctx, token_embed_weight, custom_embeddings, 2);
|
||||
embeddings = ggml_reshape_2d(compute_ctx, embeddings, embeddings->ne[0], embeddings->ne[2]);
|
||||
embeddings = ggml_concat(compute_ctx, token_embed_weight, custom_embeddings, 1);
|
||||
}
|
||||
|
||||
struct ggml_tensor* hidden_states = forward(compute_ctx, input_ids, input_ids2, embeddings, max_token_idx, return_pooled);
|
||||
struct ggml_tensor* hidden_states = forward(compute_ctx, input_ids, embeddings, max_token_idx, return_pooled);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
|
|
@ -993,147 +909,17 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
|
|||
|
||||
void compute(const int n_threads,
|
||||
struct ggml_tensor* input_ids,
|
||||
struct ggml_tensor* input_ids2,
|
||||
int num_custom_embeddings,
|
||||
void* custom_embeddings_data,
|
||||
size_t max_token_idx,
|
||||
bool return_pooled,
|
||||
ggml_tensor** output,
|
||||
ggml_context* output_ctx = NULL) {
|
||||
auto get_graph = [&]() -> struct ggml_cgraph* {
|
||||
return build_graph(input_ids, input_ids2, max_token_idx, return_pooled);
|
||||
return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled);
|
||||
};
|
||||
GGMLModule::compute(get_graph, n_threads, true, output, output_ctx);
|
||||
}
|
||||
|
||||
std::pair<std::vector<int>, std::vector<float>> tokenize(std::string text,
|
||||
bool padding = false) {
|
||||
return tokenize(text, text_model.n_token, padding);
|
||||
}
|
||||
|
||||
std::pair<std::vector<int>, std::vector<float>> tokenize(std::string text,
|
||||
size_t max_length = 0,
|
||||
bool padding = false) {
|
||||
auto parsed_attention = parse_prompt_attention(text);
|
||||
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "[";
|
||||
for (const auto& item : parsed_attention) {
|
||||
ss << "['" << item.first << "', " << item.second << "], ";
|
||||
}
|
||||
ss << "]";
|
||||
LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
|
||||
}
|
||||
|
||||
auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
|
||||
size_t word_end = str.find(",");
|
||||
std::string embd_name = word_end == std::string::npos ? str : str.substr(0, word_end);
|
||||
embd_name = trim(embd_name);
|
||||
std::string embd_path = get_full_path(embd_dir, embd_name + ".pt");
|
||||
if (embd_path.size() == 0) {
|
||||
embd_path = get_full_path(embd_dir, embd_name + ".ckpt");
|
||||
}
|
||||
if (embd_path.size() == 0) {
|
||||
embd_path = get_full_path(embd_dir, embd_name + ".safetensors");
|
||||
}
|
||||
if (embd_path.size() > 0) {
|
||||
if (load_embedding(embd_name, embd_path, bpe_tokens)) {
|
||||
if (word_end != std::string::npos) {
|
||||
str = str.substr(word_end);
|
||||
} else {
|
||||
str = "";
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
std::vector<int> tokens;
|
||||
std::vector<float> weights;
|
||||
for (const auto& item : parsed_attention) {
|
||||
const std::string& curr_text = item.first;
|
||||
float curr_weight = item.second;
|
||||
std::vector<int> curr_tokens = tokenizer.encode(curr_text, on_new_token_cb);
|
||||
tokens.insert(tokens.end(), curr_tokens.begin(), curr_tokens.end());
|
||||
weights.insert(weights.end(), curr_tokens.size(), curr_weight);
|
||||
}
|
||||
tokens.insert(tokens.begin(), BOS_TOKEN_ID);
|
||||
weights.insert(weights.begin(), 1.0);
|
||||
|
||||
if (max_length > 0) {
|
||||
if (tokens.size() > max_length - 1) {
|
||||
tokens.resize(max_length - 1);
|
||||
weights.resize(max_length - 1);
|
||||
tokens.push_back(EOS_TOKEN_ID);
|
||||
weights.push_back(1.0);
|
||||
} else {
|
||||
tokens.push_back(EOS_TOKEN_ID);
|
||||
weights.push_back(1.0);
|
||||
if (padding) {
|
||||
int pad_token_id = PAD_TOKEN_ID;
|
||||
if (version == VERSION_2_x) {
|
||||
pad_token_id = 0;
|
||||
}
|
||||
tokens.insert(tokens.end(), max_length - tokens.size(), pad_token_id);
|
||||
weights.insert(weights.end(), max_length - weights.size(), 1.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// for (int i = 0; i < tokens.size(); i++) {
|
||||
// std::cout << tokens[i] << ":" << weights[i] << ", ";
|
||||
// }
|
||||
// std::cout << std::endl;
|
||||
|
||||
return {tokens, weights};
|
||||
GGMLRunner::compute(get_graph, n_threads, true, output, output_ctx);
|
||||
}
|
||||
};
|
||||
|
||||
struct FrozenCLIPVisionEmbedder : public GGMLModule {
|
||||
CLIPVisionModel vision_model;
|
||||
|
||||
FrozenCLIPVisionEmbedder(ggml_backend_t backend, ggml_type wtype)
|
||||
: GGMLModule(backend, wtype) {
|
||||
vision_model.init(params_ctx, wtype);
|
||||
}
|
||||
|
||||
std::string get_desc() {
|
||||
return "clip_vision";
|
||||
}
|
||||
|
||||
size_t get_params_mem_size() {
|
||||
return vision_model.get_params_mem_size();
|
||||
}
|
||||
|
||||
size_t get_params_num() {
|
||||
return vision_model.get_params_num();
|
||||
}
|
||||
|
||||
void get_param_tensors(std::map<std::string, struct ggml_tensor*>& tensors, const std::string prefix) {
|
||||
vision_model.get_param_tensors(tensors, prefix + "transformer.visual_model");
|
||||
}
|
||||
|
||||
struct ggml_cgraph* build_graph(struct ggml_tensor* pixel_values) {
|
||||
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
|
||||
|
||||
pixel_values = to_backend(pixel_values);
|
||||
|
||||
struct ggml_tensor* hidden_states = vision_model.forward(compute_ctx, pixel_values);
|
||||
|
||||
ggml_build_forward_expand(gf, hidden_states);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
void compute(const int n_threads,
|
||||
ggml_tensor* pixel_values,
|
||||
ggml_tensor** output,
|
||||
ggml_context* output_ctx) {
|
||||
auto get_graph = [&]() -> struct ggml_cgraph* {
|
||||
return build_graph(pixel_values);
|
||||
};
|
||||
GGMLModule::compute(get_graph, n_threads, true, output, output_ctx);
|
||||
}
|
||||
};
|
||||
|
||||
#endif // __CLIP_HPP__
|
||||
#endif // __CLIP_HPP__
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue