vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-05-22 03:10:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

unlock custom contextsize

Browse source
This commit is contained in:
Concedo 2024-02-27 18:10:43 +08:00
parent 39ae58ef0d
commit 71898cf728
1 changed files with 13 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

14
koboldcpp.py
View file

@ -2656,6 +2656,18 @@ def start_in_seperate_process(launch_args):
return (output_queue, input_queue, p)
if __name__ == '__main__':
def check_range(value_type, min_value, max_value):
def range_checker(arg: str):
try:
f = value_type(arg)
except ValueError:
raise argparse.ArgumentTypeError(f'must be a valid {value_type}')
if f < min_value or f > max_value:
raise argparse.ArgumentTypeError(f'must be within [{min_value}, {max_value}]')
return f
return range_checker
print("***\nWelcome to KoboldCpp - Version " + KcppVersion) # just update version manually
# print("Python version: " + sys.version)
parser = argparse.ArgumentParser(description='KoboldCpp Server')
@ -2676,7 +2688,7 @@ if __name__ == '__main__':
parser.add_argument("--threads", help="Use a custom number of threads if specified. Otherwise, uses an amount based on CPU cores", type=int, default=default_threads)
parser.add_argument("--blasthreads", help="Use a different number of threads during BLAS if specified. Otherwise, has the same value as --threads",metavar=('[threads]'), type=int, default=0)
parser.add_argument("--highpriority", help="Experimental flag. If set, increases the process CPU priority, potentially speeding up generation. Use caution.", action='store_true')
parser.add_argument("--contextsize", help="Controls the memory allocated for maximum context size, only change if you need more RAM for big contexts. (default 2048)", type=int,choices=[256, 512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536], default=2048)
parser.add_argument("--contextsize", help="Controls the memory allocated for maximum context size, only change if you need more RAM for big contexts. (default 2048). Supported values are [256,512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536]. IF YOU USE ANYTHING ELSE YOU ARE ON YOUR OWN.",metavar=('[256,512,1024,2048,3072,4096,6144,8192,12288,16384,24576,32768,49152,65536]'), type=check_range(int,256,262144), default=2048)
parser.add_argument("--blasbatchsize", help="Sets the batch size used in BLAS processing (default 512). Setting it to -1 disables BLAS mode, but keeps other benefits like GPU offload.", type=int,choices=[-1,32,64,128,256,512,1024,2048], default=512)
parser.add_argument("--ropeconfig", help="If set, uses customized RoPE scaling from configured frequency scale and frequency base (e.g. --ropeconfig 0.25 10000). Otherwise, uses NTK-Aware scaling set automatically based on context size. For linear rope, simply set the freq-scale and ignore the freq-base",metavar=('[rope-freq-scale]', '[rope-freq-base]'), default=[0.0, 10000.0], type=float, nargs='+')
parser.add_argument("--smartcontext", help="Reserving a portion of context to try processing less frequently.", action='store_true')