mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-10 17:14:36 +00:00
2758fa10da
* examples : add model conversion tool/example This commit adds an "example/tool" that is intended to help in the process of converting models to GGUF. Currently it supports normal causal models and embedding models. The readme contains instructions and command to guide through the process. The motivation for this to have a structured and repeatable process for model conversions and hopefully with time improve upon it to make the process easier and more reliable. We have started to use this for new model conversions internally and will continue doing so and improve it as we go along. Perhaps with time this should be placed in a different directory than the examples directory, but for now it seems like a good place to keep it while we are still developing it. * squash! examples : add model conversion tool/example Remove dependency on scikit-learn in model conversion example. * squash! examples : add model conversion tool/example Update transformer dep to use non-dev version. And also import `AutoModelForCausalLM` instead of `AutoModel` to ensure compatibility with the latest version. * squash! examples : add model conversion tool/example Remove the logits requirements file from the all requirements file.
88 lines
2.8 KiB
Python
Executable file
88 lines
2.8 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
import numpy as np
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
def quick_logits_check(pytorch_file, llamacpp_file):
|
|
"""Lightweight sanity check before NMSE"""
|
|
|
|
try:
|
|
pytorch_logits = np.fromfile(pytorch_file, dtype=np.float32)
|
|
llamacpp_logits = np.fromfile(llamacpp_file, dtype=np.float32)
|
|
except Exception as e:
|
|
print(f"❌ NOK: Failed to load files - {e}")
|
|
return False
|
|
|
|
# Check shapes match
|
|
if pytorch_logits.shape != llamacpp_logits.shape:
|
|
print(f"❌ NOK: Shape mismatch - PyTorch: {pytorch_logits.shape}, llama.cpp: {llamacpp_logits.shape}")
|
|
return False
|
|
|
|
# Calculate key metrics
|
|
diff = pytorch_logits - llamacpp_logits
|
|
abs_diff = np.abs(diff)
|
|
max_diff = np.max(abs_diff)
|
|
|
|
# Get top 10 predictions from both models
|
|
pytorch_top10 = np.argsort(pytorch_logits)[-10:][::-1]
|
|
llamacpp_top10 = np.argsort(llamacpp_logits)[-10:][::-1]
|
|
print(f"Top 10 PyTorch logits: {pytorch_logits[pytorch_top10]}")
|
|
print(f"Top 10 llama.cpp logits: {llamacpp_logits[llamacpp_top10]}")
|
|
print(f"Max absolute difference: {max_diff:.4f}")
|
|
|
|
if max_diff > 1.0:
|
|
print(f"❌ NOK: Large differences detected - max diff: {max_diff:.4f}")
|
|
return False
|
|
|
|
return True
|
|
|
|
def main():
|
|
model_path = os.getenv('MODEL_PATH')
|
|
if not model_path:
|
|
print("Error: MODEL_PATH environment variable not set")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(model_path):
|
|
print(f"Error: Model file not found: {model_path}")
|
|
sys.exit(1)
|
|
|
|
model_name = os.path.splitext(os.path.basename(model_path))[0]
|
|
data_dir = Path("data")
|
|
|
|
pytorch_file = data_dir / f"pytorch-{model_name}.bin"
|
|
llamacpp_file = data_dir / f"llamacpp-{model_name}.bin"
|
|
|
|
if not pytorch_file.exists():
|
|
print(f"Error: PyTorch logits file not found: {pytorch_file}")
|
|
print("Please run scripts/run-org-model.sh first to generate this file.")
|
|
sys.exit(1)
|
|
|
|
if not llamacpp_file.exists():
|
|
print(f"Error: llama.cpp logits file not found: {llamacpp_file}")
|
|
print("Please run scripts/run-converted-model.sh first to generate this file.")
|
|
sys.exit(1)
|
|
|
|
print("Checked all required files were found. Proceeding...\n")
|
|
|
|
|
|
print("🔍 GGML Model Validation for model ", model_name)
|
|
print("=" * 40)
|
|
print(f"PyTorch logits : {pytorch_file}")
|
|
print(f"llama.cpp logits: {llamacpp_file}")
|
|
print()
|
|
|
|
success = quick_logits_check(pytorch_file, llamacpp_file)
|
|
|
|
# Exit with appropriate code
|
|
if success:
|
|
print("✅ OK: Lightweight model check successful!")
|
|
print(" Ok to proceed with NMSE check...")
|
|
sys.exit(0)
|
|
else:
|
|
print(f"❌ NOK: Top 10 predictions don't match - generation will differ")
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|