mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-11 17:44:38 +00:00
merged leejet changes
This commit is contained in:
Concedo
parent
b6edb79648
commit
a1175cf34f
5 changed files with 296 additions and 176 deletions
|
|
@ -13,10 +13,10 @@ struct DiffusionModel {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
int num_video_frames = -1,
|
||||
std::vector<struct ggml_tensor*> controls = {},
|
||||
float control_strength = 0.f,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
struct ggml_tensor** output = NULL,
|
||||
struct ggml_context* output_ctx = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) = 0;
|
||||
|
|
@ -69,10 +69,10 @@ struct UNetModel : public DiffusionModel {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
int num_video_frames = -1,
|
||||
std::vector<struct ggml_tensor*> controls = {},
|
||||
float control_strength = 0.f,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
struct ggml_tensor** output = NULL,
|
||||
struct ggml_context* output_ctx = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
|
|
@ -120,10 +120,10 @@ struct MMDiTModel : public DiffusionModel {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
int num_video_frames = -1,
|
||||
std::vector<struct ggml_tensor*> controls = {},
|
||||
float control_strength = 0.f,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
struct ggml_tensor** output = NULL,
|
||||
struct ggml_context* output_ctx = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
|
|
@ -172,14 +172,14 @@ struct FluxModel : public DiffusionModel {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
int num_video_frames = -1,
|
||||
std::vector<struct ggml_tensor*> controls = {},
|
||||
float control_strength = 0.f,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
struct ggml_tensor** output = NULL,
|
||||
struct ggml_context* output_ctx = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
return flux.compute(n_threads, x, timesteps, context, c_concat, y, guidance, kontext_imgs, output, output_ctx, skip_layers);
|
||||
return flux.compute(n_threads, x, timesteps, context, c_concat, y, guidance, ref_latents, output, output_ctx, skip_layers);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -672,51 +672,81 @@ namespace Flux {
|
|||
}
|
||||
|
||||
// Generate IDs for image patches and text
|
||||
std::vector<std::vector<float>> gen_ids(int h, int w, int patch_size, int index = 0) {
|
||||
std::vector<std::vector<float>> gen_txt_ids(int bs, int context_len) {
|
||||
return std::vector<std::vector<float>>(bs * context_len, std::vector<float>(3, 0.0));
|
||||
}
|
||||
|
||||
std::vector<std::vector<float>> gen_img_ids(int h, int w, int patch_size, int bs, int index = 0, int h_offset = 0, int w_offset = 0) {
|
||||
int h_len = (h + (patch_size / 2)) / patch_size;
|
||||
int w_len = (w + (patch_size / 2)) / patch_size;
|
||||
|
||||
std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(3, (float)index));
|
||||
std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(3, 0.0));
|
||||
|
||||
std::vector<float> row_ids = linspace(0, h_len - 1, h_len);
|
||||
std::vector<float> col_ids = linspace(0, w_len - 1, w_len);
|
||||
std::vector<float> row_ids = linspace(h_offset, h_len - 1 + h_offset, h_len);
|
||||
std::vector<float> col_ids = linspace(w_offset, w_len - 1 + w_offset, w_len);
|
||||
|
||||
for (int i = 0; i < h_len; ++i) {
|
||||
for (int j = 0; j < w_len; ++j) {
|
||||
img_ids[i * w_len + j][0] = index;
|
||||
img_ids[i * w_len + j][1] = row_ids[i];
|
||||
img_ids[i * w_len + j][2] = col_ids[j];
|
||||
}
|
||||
}
|
||||
|
||||
return img_ids;
|
||||
std::vector<std::vector<float>> img_ids_repeated(bs * img_ids.size(), std::vector<float>(3));
|
||||
for (int i = 0; i < bs; ++i) {
|
||||
for (int j = 0; j < img_ids.size(); ++j) {
|
||||
img_ids_repeated[i * img_ids.size() + j] = img_ids[j];
|
||||
}
|
||||
}
|
||||
return img_ids_repeated;
|
||||
}
|
||||
|
||||
std::vector<std::vector<float>> concat_ids(const std::vector<std::vector<float>>& a,
|
||||
const std::vector<std::vector<float>>& b,
|
||||
int bs) {
|
||||
size_t a_len = a.size() / bs;
|
||||
size_t b_len = b.size() / bs;
|
||||
std::vector<std::vector<float>> ids(a.size() + b.size(), std::vector<float>(3));
|
||||
for (int i = 0; i < bs; ++i) {
|
||||
for (int j = 0; j < a_len; ++j) {
|
||||
ids[i * (a_len + b_len) + j] = a[i * a_len + j];
|
||||
}
|
||||
for (int j = 0; j < b_len; ++j) {
|
||||
ids[i * (a_len + b_len) + a_len + j] = b[i * b_len + j];
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
std::vector<std::vector<float>> gen_ids(int h, int w, int patch_size, int bs, int context_len, std::vector<ggml_tensor*> ref_latents) {
|
||||
auto txt_ids = gen_txt_ids(bs, context_len);
|
||||
auto img_ids = gen_img_ids(h, w, patch_size, bs);
|
||||
|
||||
auto ids = concat_ids(txt_ids, img_ids, bs);
|
||||
uint64_t curr_h_offset = 0;
|
||||
uint64_t curr_w_offset = 0;
|
||||
for (ggml_tensor* ref : ref_latents) {
|
||||
uint64_t h_offset = 0;
|
||||
uint64_t w_offset = 0;
|
||||
if (ref->ne[1] + curr_h_offset > ref->ne[0] + curr_w_offset) {
|
||||
w_offset = curr_w_offset;
|
||||
} else {
|
||||
h_offset = curr_h_offset;
|
||||
}
|
||||
|
||||
auto ref_ids = gen_img_ids(ref->ne[1], ref->ne[0], patch_size, bs, 1, h_offset, w_offset);
|
||||
ids = concat_ids(ids, ref_ids, bs);
|
||||
|
||||
curr_h_offset = std::max(curr_h_offset, ref->ne[1] + h_offset);
|
||||
curr_w_offset = std::max(curr_w_offset, ref->ne[0] + w_offset);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
// Generate positional embeddings
|
||||
std::vector<float> gen_pe(std::vector<struct ggml_tensor*> imgs, struct ggml_tensor* context, int patch_size, int theta, const std::vector<int>& axes_dim) {
|
||||
int context_len = context->ne[1];
|
||||
int bs = imgs[0]->ne[3];
|
||||
|
||||
std::vector<std::vector<float>> img_ids;
|
||||
for (int i = 0; i < imgs.size(); i++) {
|
||||
auto x = imgs[i];
|
||||
if (x) {
|
||||
int h = x->ne[1];
|
||||
int w = x->ne[0];
|
||||
std::vector<std::vector<float>> img_ids_i = gen_ids(h, w, patch_size, i);
|
||||
img_ids.insert(img_ids.end(), img_ids_i.begin(), img_ids_i.end());
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::vector<float>> txt_ids(bs * context_len, std::vector<float>(3, 0.0));
|
||||
std::vector<std::vector<float>> ids(bs * (context_len + img_ids.size()), std::vector<float>(3));
|
||||
for (int i = 0; i < bs; ++i) {
|
||||
for (int j = 0; j < context_len; ++j) {
|
||||
ids[i * (context_len + img_ids.size()) + j] = txt_ids[j];
|
||||
}
|
||||
for (int j = 0; j < img_ids.size(); ++j) {
|
||||
ids[i * (context_len + img_ids.size()) + context_len + j] = img_ids[j];
|
||||
}
|
||||
}
|
||||
std::vector<float> gen_pe(int h, int w, int patch_size, int bs, int context_len, std::vector<ggml_tensor*> ref_latents, int theta, const std::vector<int>& axes_dim) {
|
||||
std::vector<std::vector<float>> ids = gen_ids(h, w, patch_size, bs, context_len, ref_latents);
|
||||
|
||||
std::vector<std::vector<float>> trans_ids = transpose(ids);
|
||||
size_t pos_len = ids.size();
|
||||
|
|
@ -843,7 +873,7 @@ namespace Flux {
|
|||
struct ggml_tensor* guidance,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* arange = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
std::vector<int> skip_layers = {}) {
|
||||
auto img_in = std::dynamic_pointer_cast<Linear>(blocks["img_in"]);
|
||||
auto txt_in = std::dynamic_pointer_cast<Linear>(blocks["txt_in"]);
|
||||
auto final_layer = std::dynamic_pointer_cast<LastLayer>(blocks["final_layer"]);
|
||||
|
|
@ -929,8 +959,23 @@ namespace Flux {
|
|||
return img;
|
||||
}
|
||||
|
||||
struct ggml_tensor* process_img(struct ggml_context* ctx,
|
||||
struct ggml_tensor* x) {
|
||||
|
||||
int64_t W = x->ne[0];
|
||||
int64_t H = x->ne[1];
|
||||
int64_t patch_size = 2;
|
||||
int pad_h = (patch_size - H % patch_size) % patch_size;
|
||||
int pad_w = (patch_size - W % patch_size) % patch_size;
|
||||
x = ggml_pad(ctx, x, pad_w, pad_h, 0, 0); // [N, C, H + pad_h, W + pad_w]
|
||||
|
||||
// img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size)
|
||||
auto img = patchify(ctx, x, patch_size); // [N, h*w, C * patch_size * patch_size]
|
||||
return img;
|
||||
}
|
||||
|
||||
struct ggml_tensor* forward(struct ggml_context* ctx,
|
||||
std::vector<struct ggml_tensor*> imgs,
|
||||
struct ggml_tensor* x,
|
||||
struct ggml_tensor* timestep,
|
||||
struct ggml_tensor* context,
|
||||
struct ggml_tensor* c_concat,
|
||||
|
|
@ -938,8 +983,8 @@ namespace Flux {
|
|||
struct ggml_tensor* guidance,
|
||||
struct ggml_tensor* pe,
|
||||
struct ggml_tensor* arange = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>(),
|
||||
SDVersion version = VERSION_FLUX) {
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
std::vector<int> skip_layers = {}) {
|
||||
// Forward pass of DiT.
|
||||
// x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
|
||||
// timestep: (N,) tensor of diffusion timesteps
|
||||
|
|
@ -950,47 +995,41 @@ namespace Flux {
|
|||
// pe: (L, d_head/2, 2, 2)
|
||||
// return: (N, C, H, W)
|
||||
|
||||
auto x = imgs[0];
|
||||
GGML_ASSERT(x->ne[3] == 1);
|
||||
|
||||
int64_t W = x->ne[0];
|
||||
int64_t H = x->ne[1];
|
||||
int64_t C = x->ne[2];
|
||||
int64_t patch_size = 2;
|
||||
int pad_h = (patch_size - x->ne[0] % patch_size) % patch_size;
|
||||
int pad_w = (patch_size - x->ne[1] % patch_size) % patch_size;
|
||||
int pad_h = (patch_size - H % patch_size) % patch_size;
|
||||
int pad_w = (patch_size - W % patch_size) % patch_size;
|
||||
|
||||
// img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size)
|
||||
ggml_tensor* img = NULL; // [N, h*w, C * patch_size * patch_size]
|
||||
int64_t patchified_img_size;
|
||||
for (auto& x : imgs) {
|
||||
int pad_h = (patch_size - x->ne[0] % patch_size) % patch_size;
|
||||
int pad_w = (patch_size - x->ne[1] % patch_size) % patch_size;
|
||||
ggml_tensor* pad_x = ggml_pad(ctx, x, pad_w, pad_h, 0, 0);
|
||||
pad_x = patchify(ctx, pad_x, patch_size);
|
||||
if (img) {
|
||||
img = ggml_concat(ctx, img, pad_x, 1);
|
||||
} else {
|
||||
img = pad_x;
|
||||
patchified_img_size = img->ne[1];
|
||||
}
|
||||
}
|
||||
auto img = process_img(ctx, x);
|
||||
uint64_t img_tokens = img->ne[1];
|
||||
|
||||
if (c_concat != NULL) {
|
||||
ggml_tensor* masked = ggml_view_4d(ctx, c_concat, c_concat->ne[0], c_concat->ne[1], C, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], 0);
|
||||
ggml_tensor* mask = ggml_view_4d(ctx, c_concat, c_concat->ne[0], c_concat->ne[1], 8 * 8, 1, c_concat->nb[1], c_concat->nb[2], c_concat->nb[3], c_concat->nb[2] * C);
|
||||
|
||||
masked = ggml_pad(ctx, masked, pad_w, pad_h, 0, 0);
|
||||
mask = ggml_pad(ctx, mask, pad_w, pad_h, 0, 0);
|
||||
|
||||
masked = patchify(ctx, masked, patch_size);
|
||||
mask = patchify(ctx, mask, patch_size);
|
||||
masked = process_img(ctx, masked);
|
||||
mask = process_img(ctx, mask);
|
||||
|
||||
img = ggml_concat(ctx, img, ggml_concat(ctx, masked, mask, 0), 0);
|
||||
}
|
||||
|
||||
auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, arange, skip_layers); // [N, h*w, C * patch_size * patch_size]
|
||||
out = ggml_cont(ctx, ggml_view_2d(ctx, out, out->ne[0], patchified_img_size, out->nb[1], 0));
|
||||
if (ref_latents.size() > 0) {
|
||||
for (ggml_tensor* ref : ref_latents) {
|
||||
ref = process_img(ctx, ref);
|
||||
img = ggml_concat(ctx, img, ref, 1);
|
||||
}
|
||||
}
|
||||
|
||||
auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, arange, skip_layers); // [N, num_tokens, C * patch_size * patch_size]
|
||||
if (out->ne[1] > img_tokens) {
|
||||
out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); // [num_tokens, N, C * patch_size * patch_size]
|
||||
out = ggml_view_3d(ctx, out, out->ne[0], out->ne[1], img_tokens, out->nb[1], out->nb[2], 0);
|
||||
out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); // [N, h*w, C * patch_size * patch_size]
|
||||
}
|
||||
|
||||
// rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)
|
||||
out = unpatchify(ctx, out, (H + pad_h) / patch_size, (W + pad_w) / patch_size, patch_size); // [N, C, H + pad_h, W + pad_w]
|
||||
|
|
@ -1076,7 +1115,7 @@ namespace Flux {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
GGML_ASSERT(x->ne[3] == 1);
|
||||
struct ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, FLUX_GRAPH_SIZE, false);
|
||||
|
|
@ -1088,9 +1127,7 @@ namespace Flux {
|
|||
if (c_concat != NULL) {
|
||||
c_concat = to_backend(c_concat);
|
||||
}
|
||||
for (auto &img : kontext_imgs){
|
||||
img = to_backend(img);
|
||||
}
|
||||
|
||||
if (flux_params.is_chroma) {
|
||||
const char* SD_CHROMA_ENABLE_GUIDANCE = getenv("SD_CHROMA_ENABLE_GUIDANCE");
|
||||
bool disable_guidance = true;
|
||||
|
|
@ -1131,10 +1168,11 @@ namespace Flux {
|
|||
if (flux_params.guidance_embed || flux_params.is_chroma) {
|
||||
guidance = to_backend(guidance);
|
||||
}
|
||||
auto imgs = kontext_imgs;
|
||||
imgs.insert(imgs.begin(), x);
|
||||
for (int i = 0; i < ref_latents.size(); i++) {
|
||||
ref_latents[i] = to_backend(ref_latents[i]);
|
||||
}
|
||||
|
||||
pe_vec = flux.gen_pe(imgs, context, 2, flux_params.theta, flux_params.axes_dim);
|
||||
pe_vec = flux.gen_pe(x->ne[1], x->ne[0], 2, x->ne[3], context->ne[1], ref_latents, flux_params.theta, flux_params.axes_dim);
|
||||
int pos_len = pe_vec.size() / flux_params.axes_dim_sum / 2;
|
||||
// LOG_DEBUG("pos_len %d", pos_len);
|
||||
auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, flux_params.axes_dim_sum / 2, pos_len);
|
||||
|
|
@ -1144,7 +1182,7 @@ namespace Flux {
|
|||
set_backend_tensor_data(pe, pe_vec.data());
|
||||
|
||||
struct ggml_tensor* out = flux.forward(compute_ctx,
|
||||
imgs,
|
||||
x,
|
||||
timesteps,
|
||||
context,
|
||||
c_concat,
|
||||
|
|
@ -1152,6 +1190,7 @@ namespace Flux {
|
|||
guidance,
|
||||
pe,
|
||||
precompute_arange,
|
||||
ref_latents,
|
||||
skip_layers);
|
||||
|
||||
ggml_build_forward_expand(gf, out);
|
||||
|
|
@ -1166,7 +1205,7 @@ namespace Flux {
|
|||
struct ggml_tensor* c_concat,
|
||||
struct ggml_tensor* y,
|
||||
struct ggml_tensor* guidance,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
struct ggml_tensor** output = NULL,
|
||||
struct ggml_context* output_ctx = NULL,
|
||||
std::vector<int> skip_layers = std::vector<int>()) {
|
||||
|
|
@ -1176,7 +1215,7 @@ namespace Flux {
|
|||
// y: [N, adm_in_channels] or [1, adm_in_channels]
|
||||
// guidance: [N, ]
|
||||
auto get_graph = [&]() -> struct ggml_cgraph* {
|
||||
return build_graph(x, timesteps, context, c_concat, y, guidance, kontext_imgs, skip_layers);
|
||||
return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, skip_layers);
|
||||
};
|
||||
|
||||
GGMLRunner::compute(get_graph, n_threads, false, output, output_ctx);
|
||||
|
|
@ -1216,7 +1255,7 @@ namespace Flux {
|
|||
struct ggml_tensor* out = NULL;
|
||||
|
||||
int t0 = ggml_time_ms();
|
||||
compute(8, x, timesteps, context, NULL, y, guidance, std::vector<struct ggml_tensor*>(), &out, work_ctx);
|
||||
compute(8, x, timesteps, context, NULL, y, guidance, {}, &out, work_ctx);
|
||||
int t1 = ggml_time_ms();
|
||||
|
||||
print_ggml_tensor(out);
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ const char* modes_str[] = {
|
|||
"txt2img",
|
||||
"img2img",
|
||||
"img2vid",
|
||||
"edit",
|
||||
"convert",
|
||||
};
|
||||
|
||||
|
|
@ -61,6 +62,7 @@ enum SDMode {
|
|||
TXT2IMG,
|
||||
IMG2IMG,
|
||||
IMG2VID,
|
||||
EDIT,
|
||||
CONVERT,
|
||||
MODE_COUNT
|
||||
};
|
||||
|
|
@ -86,8 +88,7 @@ struct SDParams {
|
|||
std::string input_path;
|
||||
std::string mask_path;
|
||||
std::string control_image_path;
|
||||
|
||||
std::vector<std::string> kontext_image_paths;
|
||||
std::vector<std::string> ref_image_paths;
|
||||
|
||||
std::string prompt;
|
||||
std::string negative_prompt;
|
||||
|
|
@ -153,6 +154,10 @@ void print_params(SDParams params) {
|
|||
printf(" init_img: %s\n", params.input_path.c_str());
|
||||
printf(" mask_img: %s\n", params.mask_path.c_str());
|
||||
printf(" control_image: %s\n", params.control_image_path.c_str());
|
||||
printf(" ref_images_paths:\n");
|
||||
for (auto& path : params.ref_image_paths) {
|
||||
printf(" %s\n", path.c_str());
|
||||
};
|
||||
printf(" clip on cpu: %s\n", params.clip_on_cpu ? "true" : "false");
|
||||
printf(" controlnet cpu: %s\n", params.control_net_cpu ? "true" : "false");
|
||||
printf(" vae decoder on cpu:%s\n", params.vae_on_cpu ? "true" : "false");
|
||||
|
|
@ -207,6 +212,7 @@ void print_usage(int argc, const char* argv[]) {
|
|||
printf(" -i, --init-img [IMAGE] path to the input image, required by img2img\n");
|
||||
printf(" --mask [MASK] path to the mask image, required by img2img with mask\n");
|
||||
printf(" --control-image [IMAGE] path to image condition, control net\n");
|
||||
printf(" -r, --ref_image [PATH] reference image for Flux Kontext models (can be used multiple times) \n");
|
||||
printf(" -o, --output OUTPUT path to write result image to (default: ./output.png)\n");
|
||||
printf(" -p, --prompt [PROMPT] the prompt to render\n");
|
||||
printf(" -n, --negative-prompt PROMPT the negative prompt (default: \"\")\n");
|
||||
|
|
@ -242,9 +248,8 @@ void print_usage(int argc, const char* argv[]) {
|
|||
printf(" This might crash if it is not supported by the backend.\n");
|
||||
printf(" --control-net-cpu keep controlnet in cpu (for low vram)\n");
|
||||
printf(" --canny apply canny preprocessor (edge detection)\n");
|
||||
printf(" --color Colors the logging tags according to level\n");
|
||||
printf(" --color colors the logging tags according to level\n");
|
||||
printf(" -v, --verbose print extra info\n");
|
||||
printf(" -ki, --kontext_img [PATH] Reference image for Flux Kontext models (can be used multiple times) \n");
|
||||
}
|
||||
|
||||
void parse_args(int argc, const char** argv, SDParams& params) {
|
||||
|
|
@ -629,12 +634,12 @@ void parse_args(int argc, const char** argv, SDParams& params) {
|
|||
break;
|
||||
}
|
||||
params.skip_layer_end = std::stof(argv[i]);
|
||||
} else if (arg == "-ki" || arg == "--kontext-img") {
|
||||
} else if (arg == "-r" || arg == "--ref-image") {
|
||||
if (++i >= argc) {
|
||||
invalid_arg = true;
|
||||
break;
|
||||
}
|
||||
params.kontext_image_paths.push_back(argv[i]);
|
||||
params.ref_image_paths.push_back(argv[i]);
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
print_usage(argc, argv);
|
||||
|
|
@ -663,7 +668,13 @@ void parse_args(int argc, const char** argv, SDParams& params) {
|
|||
}
|
||||
|
||||
if ((params.mode == IMG2IMG || params.mode == IMG2VID) && params.input_path.length() == 0) {
|
||||
fprintf(stderr, "error: when using the img2img mode, the following arguments are required: init-img\n");
|
||||
fprintf(stderr, "error: when using the img2img/img2vid mode, the following arguments are required: init-img\n");
|
||||
print_usage(argc, argv);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (params.mode == EDIT && params.ref_image_paths.size() == 0) {
|
||||
fprintf(stderr, "error: when using the edit mode, the following arguments are required: ref-image\n");
|
||||
print_usage(argc, argv);
|
||||
exit(1);
|
||||
}
|
||||
|
|
@ -827,43 +838,12 @@ int main(int argc, const char* argv[]) {
|
|||
fprintf(stderr, "SVD support is broken, do not use it!!!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool vae_decode_only = true;
|
||||
|
||||
std::vector<sd_image_t> kontext_imgs;
|
||||
for (auto& path : params.kontext_image_paths) {
|
||||
vae_decode_only = false;
|
||||
int c = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
uint8_t* image_buffer = stbi_load(path.c_str(), &width, &height, &c, 3);
|
||||
if (image_buffer == NULL) {
|
||||
fprintf(stderr, "load image from '%s' failed\n", path.c_str());
|
||||
return 1;
|
||||
}
|
||||
if (c < 3) {
|
||||
fprintf(stderr, "the number of channels for the input image must be >= 3, but got %d channels\n", c);
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
if (width <= 0) {
|
||||
fprintf(stderr, "error: the width of image must be greater than 0\n");
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
if (height <= 0) {
|
||||
fprintf(stderr, "error: the height of image must be greater than 0\n");
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
kontext_imgs.push_back({(uint32_t)width,
|
||||
(uint32_t)height,
|
||||
3,
|
||||
image_buffer});
|
||||
}
|
||||
|
||||
uint8_t* input_image_buffer = NULL;
|
||||
uint8_t* control_image_buffer = NULL;
|
||||
uint8_t* mask_image_buffer = NULL;
|
||||
std::vector<sd_image_t> ref_images;
|
||||
|
||||
if (params.mode == IMG2IMG || params.mode == IMG2VID) {
|
||||
vae_decode_only = false;
|
||||
|
|
@ -915,6 +895,37 @@ int main(int argc, const char* argv[]) {
|
|||
free(input_image_buffer);
|
||||
input_image_buffer = resized_image_buffer;
|
||||
}
|
||||
} else if (params.mode == EDIT) {
|
||||
vae_decode_only = false;
|
||||
for (auto& path : params.ref_image_paths) {
|
||||
int c = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
uint8_t* image_buffer = stbi_load(path.c_str(), &width, &height, &c, 3);
|
||||
if (image_buffer == NULL) {
|
||||
fprintf(stderr, "load image from '%s' failed\n", path.c_str());
|
||||
return 1;
|
||||
}
|
||||
if (c < 3) {
|
||||
fprintf(stderr, "the number of channels for the input image must be >= 3, but got %d channels\n", c);
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
if (width <= 0) {
|
||||
fprintf(stderr, "error: the width of image must be greater than 0\n");
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
if (height <= 0) {
|
||||
fprintf(stderr, "error: the height of image must be greater than 0\n");
|
||||
free(image_buffer);
|
||||
return 1;
|
||||
}
|
||||
ref_images.push_back({(uint32_t)width,
|
||||
(uint32_t)height,
|
||||
3,
|
||||
image_buffer});
|
||||
}
|
||||
}
|
||||
|
||||
sd_ctx_t* sd_ctx = new_sd_ctx(params.model_path.c_str(),
|
||||
|
|
@ -1001,14 +1012,12 @@ int main(int argc, const char* argv[]) {
|
|||
params.style_ratio,
|
||||
params.normalize_input,
|
||||
params.input_id_images_path.c_str(),
|
||||
kontext_imgs.data(), kontext_imgs.size(),
|
||||
params.skip_layers.data(),
|
||||
params.skip_layers.size(),
|
||||
params.slg_scale,
|
||||
params.skip_layer_start,
|
||||
params.skip_layer_end,
|
||||
nullptr);
|
||||
} else {
|
||||
params.skip_layer_end);
|
||||
} else if (params.mode == IMG2IMG || params.mode == IMG2VID) {
|
||||
sd_image_t input_image = {(uint32_t)params.width,
|
||||
(uint32_t)params.height,
|
||||
3,
|
||||
|
|
@ -1072,14 +1081,38 @@ int main(int argc, const char* argv[]) {
|
|||
params.style_ratio,
|
||||
params.normalize_input,
|
||||
params.input_id_images_path.c_str(),
|
||||
kontext_imgs.data(), kontext_imgs.size(),
|
||||
params.skip_layers.data(),
|
||||
params.skip_layers.size(),
|
||||
params.slg_scale,
|
||||
params.skip_layer_start,
|
||||
params.skip_layer_end,
|
||||
nullptr);
|
||||
params.skip_layer_end);
|
||||
}
|
||||
} else { // EDIT
|
||||
results = edit(sd_ctx,
|
||||
ref_images.data(),
|
||||
ref_images.size(),
|
||||
params.prompt.c_str(),
|
||||
params.negative_prompt.c_str(),
|
||||
params.clip_skip,
|
||||
params.cfg_scale,
|
||||
params.guidance,
|
||||
params.eta,
|
||||
params.width,
|
||||
params.height,
|
||||
params.sample_method,
|
||||
params.sample_steps,
|
||||
params.strength,
|
||||
params.seed,
|
||||
params.batch_count,
|
||||
control_image,
|
||||
params.control_strength,
|
||||
params.style_ratio,
|
||||
params.normalize_input,
|
||||
params.skip_layers.data(),
|
||||
params.skip_layers.size(),
|
||||
params.slg_scale,
|
||||
params.skip_layer_start,
|
||||
params.skip_layer_end);
|
||||
}
|
||||
|
||||
if (results == NULL) {
|
||||
|
|
@ -1141,7 +1174,7 @@ int main(int argc, const char* argv[]) {
|
|||
continue;
|
||||
}
|
||||
std::string final_image_path = i > 0 ? dummy_name + "_" + std::to_string(i + 1) + ext : dummy_name + ext;
|
||||
if (is_jpg) {
|
||||
if(is_jpg) {
|
||||
stbi_write_jpg(final_image_path.c_str(), results[i].width, results[i].height, results[i].channel,
|
||||
results[i].data, 90);
|
||||
printf("save result JPEG image to '%s'\n", final_image_path.c_str());
|
||||
|
|
|
|||
|
|
@ -587,7 +587,47 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
|
|||
extraimage_buffers.push_back(kcpp_base64_decode(extra_image_data[i]));
|
||||
input_extraimage_buffers.push_back(stbi_load_from_memory(extraimage_buffers[i].data(), extraimage_buffers[i].size(), &nx2, &ny2, &nc2, desiredchannels));
|
||||
// Resize the image
|
||||
int resok = stbir_resize_uint8(input_extraimage_buffers[i], nx2, ny2, 0, resized_extraimage_bufs[i].data(), img2imgW, img2imgH, 0, desiredchannels);
|
||||
int desiredWidth = nx2;
|
||||
int desiredHeight = ny2;
|
||||
float aspect_ratio = static_cast<float>(nx2) / ny2;
|
||||
int maxsize = 1024; // no image can exceed this
|
||||
int minsize = 256;
|
||||
|
||||
if (desiredWidth > maxsize || desiredHeight > maxsize) { // Enforce maxsize first
|
||||
if (aspect_ratio > 1.0f) { // wider than tall
|
||||
desiredWidth = maxsize;
|
||||
desiredHeight = static_cast<int>(maxsize / aspect_ratio);
|
||||
} else { // taller than wide or square
|
||||
desiredHeight = maxsize;
|
||||
desiredWidth = static_cast<int>(maxsize * aspect_ratio);
|
||||
}
|
||||
}
|
||||
|
||||
if (desiredWidth < minsize || desiredHeight < minsize) { // Enforce minsize only if it won't exceed maxsize
|
||||
if (aspect_ratio > 1.0f) { // wider than tall
|
||||
// Try to scale width up to max of (minsize, maxsize)
|
||||
desiredWidth = std::min(maxsize, std::max(minsize, desiredWidth));
|
||||
desiredHeight = static_cast<int>(desiredWidth / aspect_ratio);
|
||||
if (desiredHeight > maxsize) { // If height now exceeds maxsize, clamp based on height instead
|
||||
desiredHeight = maxsize;
|
||||
desiredWidth = static_cast<int>(maxsize * aspect_ratio);
|
||||
}
|
||||
} else {
|
||||
// Taller than wide or square
|
||||
desiredHeight = std::min(maxsize, std::max(minsize, desiredHeight));
|
||||
desiredWidth = static_cast<int>(desiredHeight * aspect_ratio);
|
||||
if (desiredWidth > maxsize) {
|
||||
desiredWidth = maxsize;
|
||||
desiredHeight = static_cast<int>(maxsize / aspect_ratio);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!sd_is_quiet && sddebugmode==1)
|
||||
{
|
||||
printf("Resize Extraimg: %dx%d to %dx%d\n",nx2,ny2,desiredWidth,desiredHeight);
|
||||
}
|
||||
int resok = stbir_resize_uint8(input_extraimage_buffers[i], nx2, ny2, 0, resized_extraimage_bufs[i].data(), desiredWidth, desiredHeight, 0, desiredchannels);
|
||||
if (!resok) {
|
||||
printf("\nKCPP SD: resize extra image failed!\n");
|
||||
output.data = "";
|
||||
|
|
@ -595,8 +635,8 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
|
|||
return output;
|
||||
}
|
||||
sd_image_t extraimage_reference;
|
||||
extraimage_reference.width = img2imgW;
|
||||
extraimage_reference.height = img2imgH;
|
||||
extraimage_reference.width = desiredWidth;
|
||||
extraimage_reference.height = desiredHeight;
|
||||
extraimage_reference.channel = desiredchannels;
|
||||
extraimage_reference.data = resized_extraimage_bufs[i].data();
|
||||
extraimage_references.push_back(extraimage_reference);
|
||||
|
|
@ -716,6 +756,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
|
|||
}
|
||||
|
||||
// Resize the image
|
||||
if(!sd_is_quiet && sddebugmode==1)
|
||||
{
|
||||
printf("Resize Img2Img: %dx%d to %dx%d\n",nx,ny,img2imgW,img2imgH);
|
||||
}
|
||||
int resok = stbir_resize_uint8(input_image_buffer, nx, ny, 0, resized_image_buf.data(), img2imgW, img2imgH, 0, img2imgC);
|
||||
if (!resok) {
|
||||
printf("\nKCPP SD: resize image failed!\n");
|
||||
|
|
@ -735,6 +779,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
|
|||
image_mask_buffer = kcpp_base64_decode(img2img_mask);
|
||||
input_mask_buffer = stbi_load_from_memory(image_mask_buffer.data(), image_mask_buffer.size(), &nx2, &ny2, &nc2, 1);
|
||||
// Resize the image
|
||||
if(!sd_is_quiet && sddebugmode==1)
|
||||
{
|
||||
printf("Resize Mask: %dx%d to %dx%d\n",nx2,ny2,img2imgW,img2imgH);
|
||||
}
|
||||
int resok = stbir_resize_uint8(input_mask_buffer, nx2, ny2, 0, resized_mask_buf.data(), img2imgW, img2imgH, 0, 1);
|
||||
if (!resok) {
|
||||
printf("\nKCPP SD: resize image failed!\n");
|
||||
|
|
|
|||
|
|
@ -678,7 +678,7 @@ public:
|
|||
|
||||
int64_t t0 = ggml_time_ms();
|
||||
struct ggml_tensor* out = ggml_dup_tensor(work_ctx, x_t);
|
||||
diffusion_model->compute(n_threads, x_t, timesteps, c, concat, NULL, NULL, -1, {}, 0.f, std::vector<struct ggml_tensor*>(), &out);
|
||||
diffusion_model->compute(n_threads, x_t, timesteps, c, concat, NULL, NULL, {}, -1, {}, 0.f, &out);
|
||||
diffusion_model->free_compute_buffer();
|
||||
|
||||
double result = 0.f;
|
||||
|
|
@ -892,12 +892,12 @@ public:
|
|||
const std::vector<float>& sigmas,
|
||||
int start_merge_step,
|
||||
SDCondition id_cond,
|
||||
std::vector<ggml_tensor*> ref_latents = {},
|
||||
std::vector<int> skip_layers = {},
|
||||
float slg_scale = 0,
|
||||
float skip_layer_start = 0.01,
|
||||
float skip_layer_end = 0.2,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
ggml_tensor* noise_mask = NULL) {
|
||||
ggml_tensor* noise_mask = nullptr) {
|
||||
LOG_DEBUG("Sample");
|
||||
struct ggml_init_params params;
|
||||
size_t data_size = ggml_row_size(init_latent->type, init_latent->ne[0]);
|
||||
|
|
@ -980,10 +980,10 @@ public:
|
|||
cond.c_concat,
|
||||
cond.c_vector,
|
||||
guidance_tensor,
|
||||
ref_latents,
|
||||
-1,
|
||||
controls,
|
||||
control_strength,
|
||||
kontext_imgs,
|
||||
&out_cond);
|
||||
} else {
|
||||
diffusion_model->compute(n_threads,
|
||||
|
|
@ -993,10 +993,10 @@ public:
|
|||
cond.c_concat,
|
||||
id_cond.c_vector,
|
||||
guidance_tensor,
|
||||
ref_latents,
|
||||
-1,
|
||||
controls,
|
||||
control_strength,
|
||||
kontext_imgs,
|
||||
&out_cond);
|
||||
}
|
||||
|
||||
|
|
@ -1014,10 +1014,10 @@ public:
|
|||
uncond.c_concat,
|
||||
uncond.c_vector,
|
||||
guidance_tensor,
|
||||
ref_latents,
|
||||
-1,
|
||||
controls,
|
||||
control_strength,
|
||||
kontext_imgs,
|
||||
&out_uncond);
|
||||
negative_data = (float*)out_uncond->data;
|
||||
}
|
||||
|
|
@ -1035,10 +1035,10 @@ public:
|
|||
cond.c_concat,
|
||||
cond.c_vector,
|
||||
guidance_tensor,
|
||||
ref_latents,
|
||||
-1,
|
||||
controls,
|
||||
control_strength,
|
||||
kontext_imgs,
|
||||
&out_skip,
|
||||
NULL,
|
||||
skip_layers);
|
||||
|
|
@ -1416,7 +1416,7 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
|
|||
float style_ratio,
|
||||
bool normalize_input,
|
||||
std::string input_id_images_path,
|
||||
std::vector<struct ggml_tensor*> kontext_imgs = std::vector<struct ggml_tensor*>(),
|
||||
std::vector<ggml_tensor*> ref_latents,
|
||||
std::vector<int> skip_layers = {},
|
||||
float slg_scale = 0,
|
||||
float skip_layer_start = 0.01,
|
||||
|
|
@ -1712,11 +1712,11 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
|
|||
sigmas,
|
||||
start_merge_step,
|
||||
id_cond,
|
||||
ref_latents,
|
||||
skip_layers,
|
||||
slg_scale,
|
||||
skip_layer_start,
|
||||
skip_layer_end,
|
||||
kontext_imgs,
|
||||
noise_mask);
|
||||
|
||||
// struct ggml_tensor* x_0 = load_tensor_from_file(ctx, "samples_ddim.bin");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue