vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-05-09 19:46:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

better quant clip

Browse source
This commit is contained in:
Concedo 2024-08-18 22:15:59 +08:00
parent b3b00750b7
commit 04166d20a4
1 changed files with 13 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

17
examples/llava/quantclip.cpp
View file

@ -14,16 +14,25 @@
int main(int argc, char ** argv) {
ggml_time_init();
if (argc != 3) {
fprintf(stderr, "usage: %s mmproj-f16.gguf output-mmproj-quantized.gguf\n", argv[0]);
if (argc != 3 && argc != 4) {
fprintf(stderr, "usage: %s mmproj-f16.gguf output-mmproj-quantized.gguf TYPE\n", argv[0]);
printf("\nGGML_TYPE_Q4_0 = 2\nGGML_TYPE_Q4_1 = 3\nGGML_TYPE_Q5_0 = 6\nGGML_TYPE_Q5_1 = 7\nGGML_TYPE_Q8_0 = 8\n");
return 1;
}
const std::string fname_inp = argv[1];
const std::string fname_out = argv[2];
printf("quantizing mmproj clip model to q4_1... ");
clip_model_quantize(fname_inp.c_str(), fname_out.c_str(), GGML_TYPE_Q4_1);
int type = GGML_TYPE_Q4_1;
if(argc==4)
{
type = std::stoi(argv[3]);
}
printf("quantizing mmproj clip model to type=%d... ",type);
clip_model_quantize(fname_inp.c_str(), fname_out.c_str(), type);
printf("done\n");
return 0;