adjust window size and code formatting

This commit is contained in:
BuildTools 2024-08-04 14:44:39 -07:00
parent 41ebb7d609
commit 3e8d7b1415
1 changed files with 508 additions and 257 deletions

View File

@ -22,15 +22,17 @@
from Logger import Logger from Logger import Logger
from localizations import * from localizations import *
class AutoGGUF(QMainWindow): class AutoGGUF(QMainWindow):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.logger = Logger("AutoGGUF", "logs") self.logger = Logger("AutoGGUF", "logs")
ensure_directory(os.path.abspath("quantized_models"))
self.logger.info(INITIALIZING_AUTOGGUF) self.logger.info(INITIALIZING_AUTOGGUF)
self.setWindowTitle(WINDOW_TITLE) self.setWindowTitle(WINDOW_TITLE)
self.setGeometry(100, 100, 1300, 1100) self.setGeometry(100, 100, 1600, 1200)
ensure_directory(os.path.abspath("quantized_models"))
main_layout = QHBoxLayout() main_layout = QHBoxLayout()
left_layout = QVBoxLayout() left_layout = QVBoxLayout()
@ -89,7 +91,7 @@ def __init__(self):
right_layout.addWidget(download_group) right_layout.addWidget(download_group)
# Initialize releases and backends # Initialize releases and backends
if os.environ.get('AUTOGGUF_CHECK_BACKEND', '').lower() == 'enabled': if os.environ.get("AUTOGGUF_CHECK_BACKEND", "").lower() == "enabled":
self.refresh_releases() self.refresh_releases()
self.refresh_backends() self.refresh_backends()
@ -140,20 +142,64 @@ def __init__(self):
quant_options_layout = QFormLayout() quant_options_layout = QFormLayout()
self.quant_type = QComboBox() self.quant_type = QComboBox()
self.quant_type.addItems([ self.quant_type.addItems(
"Q4_0", "Q4_1", "Q5_0", "Q5_1", "IQ2_XXS", "IQ2_XS", "IQ2_S", "IQ2_M", "IQ1_S", "IQ1_M", [
"Q2_K", "Q2_K_S", "IQ3_XXS", "IQ3_S", "IQ3_M", "Q3_K", "IQ3_XS", "Q3_K_S", "Q3_K_M", "Q3_K_L", "Q4_0",
"IQ4_NL", "IQ4_XS", "Q4_K", "Q4_K_S", "Q4_K_M", "Q5_K", "Q5_K_S", "Q5_K_M", "Q6_K", "Q8_0", "Q4_1",
"Q4_0_4_4", "Q4_0_4_8", "Q4_0_8_8", "F16", "BF16", "F32", "COPY" "Q5_0",
]) "Q5_1",
quant_options_layout.addRow(self.create_label(QUANTIZATION_TYPE, SELECT_QUANTIZATION_TYPE), self.quant_type) "IQ2_XXS",
"IQ2_XS",
"IQ2_S",
"IQ2_M",
"IQ1_S",
"IQ1_M",
"Q2_K",
"Q2_K_S",
"IQ3_XXS",
"IQ3_S",
"IQ3_M",
"Q3_K",
"IQ3_XS",
"Q3_K_S",
"Q3_K_M",
"Q3_K_L",
"IQ4_NL",
"IQ4_XS",
"Q4_K",
"Q4_K_S",
"Q4_K_M",
"Q5_K",
"Q5_K_S",
"Q5_K_M",
"Q6_K",
"Q8_0",
"Q4_0_4_4",
"Q4_0_4_8",
"Q4_0_8_8",
"F16",
"BF16",
"F32",
"COPY",
]
)
quant_options_layout.addRow(
self.create_label(QUANTIZATION_TYPE, SELECT_QUANTIZATION_TYPE),
self.quant_type,
)
self.allow_requantize = QCheckBox(ALLOW_REQUANTIZE) self.allow_requantize = QCheckBox(ALLOW_REQUANTIZE)
self.leave_output_tensor = QCheckBox(LEAVE_OUTPUT_TENSOR) self.leave_output_tensor = QCheckBox(LEAVE_OUTPUT_TENSOR)
self.pure = QCheckBox(PURE) self.pure = QCheckBox(PURE)
quant_options_layout.addRow(self.create_label("", ALLOWS_REQUANTIZING), self.allow_requantize) quant_options_layout.addRow(
quant_options_layout.addRow(self.create_label("", LEAVE_OUTPUT_WEIGHT), self.leave_output_tensor) self.create_label("", ALLOWS_REQUANTIZING), self.allow_requantize
quant_options_layout.addRow(self.create_label("", DISABLE_K_QUANT_MIXTURES), self.pure) )
quant_options_layout.addRow(
self.create_label("", LEAVE_OUTPUT_WEIGHT), self.leave_output_tensor
)
quant_options_layout.addRow(
self.create_label("", DISABLE_K_QUANT_MIXTURES), self.pure
)
self.imatrix = QLineEdit() self.imatrix = QLineEdit()
self.imatrix_button = QPushButton(BROWSE) self.imatrix_button = QPushButton(BROWSE)
@ -161,36 +207,61 @@ def __init__(self):
imatrix_layout = QHBoxLayout() imatrix_layout = QHBoxLayout()
imatrix_layout.addWidget(self.imatrix) imatrix_layout.addWidget(self.imatrix)
imatrix_layout.addWidget(self.imatrix_button) imatrix_layout.addWidget(self.imatrix_button)
quant_options_layout.addRow(self.create_label(IMATRIX, USE_DATA_AS_IMPORTANCE_MATRIX), imatrix_layout) quant_options_layout.addRow(
self.create_label(IMATRIX, USE_DATA_AS_IMPORTANCE_MATRIX), imatrix_layout
)
self.include_weights = QLineEdit() self.include_weights = QLineEdit()
self.exclude_weights = QLineEdit() self.exclude_weights = QLineEdit()
quant_options_layout.addRow(self.create_label(INCLUDE_WEIGHTS, USE_IMPORTANCE_MATRIX_FOR_TENSORS), self.include_weights) quant_options_layout.addRow(
quant_options_layout.addRow(self.create_label(EXCLUDE_WEIGHTS, DONT_USE_IMPORTANCE_MATRIX_FOR_TENSORS), self.exclude_weights) self.create_label(INCLUDE_WEIGHTS, USE_IMPORTANCE_MATRIX_FOR_TENSORS),
self.include_weights,
)
quant_options_layout.addRow(
self.create_label(EXCLUDE_WEIGHTS, DONT_USE_IMPORTANCE_MATRIX_FOR_TENSORS),
self.exclude_weights,
)
self.use_output_tensor_type = QCheckBox(USE_OUTPUT_TENSOR_TYPE) self.use_output_tensor_type = QCheckBox(USE_OUTPUT_TENSOR_TYPE)
self.output_tensor_type = QComboBox() self.output_tensor_type = QComboBox()
self.output_tensor_type.addItems(["F32", "F16", "Q4_0", "Q4_1", "Q5_0", "Q5_1", "Q8_0"]) self.output_tensor_type.addItems(
["F32", "F16", "Q4_0", "Q4_1", "Q5_0", "Q5_1", "Q8_0"]
)
self.output_tensor_type.setEnabled(False) self.output_tensor_type.setEnabled(False)
self.use_output_tensor_type.toggled.connect(lambda checked: self.output_tensor_type.setEnabled(checked)) self.use_output_tensor_type.toggled.connect(
lambda checked: self.output_tensor_type.setEnabled(checked)
)
output_tensor_layout = QHBoxLayout() output_tensor_layout = QHBoxLayout()
output_tensor_layout.addWidget(self.use_output_tensor_type) output_tensor_layout.addWidget(self.use_output_tensor_type)
output_tensor_layout.addWidget(self.output_tensor_type) output_tensor_layout.addWidget(self.output_tensor_type)
quant_options_layout.addRow(self.create_label(OUTPUT_TENSOR_TYPE, USE_THIS_TYPE_FOR_OUTPUT_WEIGHT), output_tensor_layout) quant_options_layout.addRow(
self.create_label(OUTPUT_TENSOR_TYPE, USE_THIS_TYPE_FOR_OUTPUT_WEIGHT),
output_tensor_layout,
)
self.use_token_embedding_type = QCheckBox(USE_TOKEN_EMBEDDING_TYPE) self.use_token_embedding_type = QCheckBox(USE_TOKEN_EMBEDDING_TYPE)
self.token_embedding_type = QComboBox() self.token_embedding_type = QComboBox()
self.token_embedding_type.addItems(["F32", "F16", "Q4_0", "Q4_1", "Q5_0", "Q5_1", "Q8_0"]) self.token_embedding_type.addItems(
["F32", "F16", "Q4_0", "Q4_1", "Q5_0", "Q5_1", "Q8_0"]
)
self.token_embedding_type.setEnabled(False) self.token_embedding_type.setEnabled(False)
self.use_token_embedding_type.toggled.connect(lambda checked: self.token_embedding_type.setEnabled(checked)) self.use_token_embedding_type.toggled.connect(
lambda checked: self.token_embedding_type.setEnabled(checked)
)
token_embedding_layout = QHBoxLayout() token_embedding_layout = QHBoxLayout()
token_embedding_layout.addWidget(self.use_token_embedding_type) token_embedding_layout.addWidget(self.use_token_embedding_type)
token_embedding_layout.addWidget(self.token_embedding_type) token_embedding_layout.addWidget(self.token_embedding_type)
quant_options_layout.addRow(self.create_label(TOKEN_EMBEDDING_TYPE, USE_THIS_TYPE_FOR_TOKEN_EMBEDDINGS), token_embedding_layout) quant_options_layout.addRow(
self.create_label(TOKEN_EMBEDDING_TYPE, USE_THIS_TYPE_FOR_TOKEN_EMBEDDINGS),
token_embedding_layout,
)
self.keep_split = QCheckBox(KEEP_SPLIT) self.keep_split = QCheckBox(KEEP_SPLIT)
self.override_kv = QLineEdit() self.override_kv = QLineEdit()
quant_options_layout.addRow(self.create_label("", WILL_GENERATE_QUANTIZED_MODEL_IN_SAME_SHARDS), self.keep_split) quant_options_layout.addRow(
self.create_label("", WILL_GENERATE_QUANTIZED_MODEL_IN_SAME_SHARDS),
self.keep_split,
)
# KV Override section # KV Override section
self.kv_override_widget = QWidget() self.kv_override_widget = QWidget()
@ -209,7 +280,10 @@ def __init__(self):
kv_override_main_layout.addWidget(kv_override_scroll) kv_override_main_layout.addWidget(kv_override_scroll)
kv_override_main_layout.addWidget(add_override_button) kv_override_main_layout.addWidget(add_override_button)
quant_options_layout.addRow(self.create_label(KV_OVERRIDES, OVERRIDE_MODEL_METADATA), kv_override_main_layout) quant_options_layout.addRow(
self.create_label(KV_OVERRIDES, OVERRIDE_MODEL_METADATA),
kv_override_main_layout,
)
quant_options_widget.setLayout(quant_options_layout) quant_options_widget.setLayout(quant_options_layout)
quant_options_scroll.setWidget(quant_options_widget) quant_options_scroll.setWidget(quant_options_widget)
@ -218,7 +292,10 @@ def __init__(self):
# Add this after the KV override section # Add this after the KV override section
self.extra_arguments = QLineEdit() self.extra_arguments = QLineEdit()
quant_options_layout.addRow(self.create_label(EXTRA_ARGUMENTS, "Additional command-line arguments"), self.extra_arguments) quant_options_layout.addRow(
self.create_label(EXTRA_ARGUMENTS, "Additional command-line arguments"),
self.extra_arguments,
)
# Quantize button layout # Quantize button layout
quantize_layout = QHBoxLayout() quantize_layout = QHBoxLayout()
@ -250,7 +327,10 @@ def __init__(self):
imatrix_datafile_layout = QHBoxLayout() imatrix_datafile_layout = QHBoxLayout()
imatrix_datafile_layout.addWidget(self.imatrix_datafile) imatrix_datafile_layout.addWidget(self.imatrix_datafile)
imatrix_datafile_layout.addWidget(self.imatrix_datafile_button) imatrix_datafile_layout.addWidget(self.imatrix_datafile_button)
imatrix_layout.addRow(self.create_label(DATA_FILE, INPUT_DATA_FILE_FOR_IMATRIX), imatrix_datafile_layout) imatrix_layout.addRow(
self.create_label(DATA_FILE, INPUT_DATA_FILE_FOR_IMATRIX),
imatrix_datafile_layout,
)
self.imatrix_model = QLineEdit() self.imatrix_model = QLineEdit()
self.imatrix_model_button = QPushButton(BROWSE) self.imatrix_model_button = QPushButton(BROWSE)
@ -258,7 +338,9 @@ def __init__(self):
imatrix_model_layout = QHBoxLayout() imatrix_model_layout = QHBoxLayout()
imatrix_model_layout.addWidget(self.imatrix_model) imatrix_model_layout.addWidget(self.imatrix_model)
imatrix_model_layout.addWidget(self.imatrix_model_button) imatrix_model_layout.addWidget(self.imatrix_model_button)
imatrix_layout.addRow(self.create_label(MODEL, MODEL_TO_BE_QUANTIZED), imatrix_model_layout) imatrix_layout.addRow(
self.create_label(MODEL, MODEL_TO_BE_QUANTIZED), imatrix_model_layout
)
self.imatrix_output = QLineEdit() self.imatrix_output = QLineEdit()
self.imatrix_output_button = QPushButton(BROWSE) self.imatrix_output_button = QPushButton(BROWSE)
@ -266,18 +348,27 @@ def __init__(self):
imatrix_output_layout = QHBoxLayout() imatrix_output_layout = QHBoxLayout()
imatrix_output_layout.addWidget(self.imatrix_output) imatrix_output_layout.addWidget(self.imatrix_output)
imatrix_output_layout.addWidget(self.imatrix_output_button) imatrix_output_layout.addWidget(self.imatrix_output_button)
imatrix_layout.addRow(self.create_label(OUTPUT, OUTPUT_PATH_FOR_GENERATED_IMATRIX), imatrix_output_layout) imatrix_layout.addRow(
self.create_label(OUTPUT, OUTPUT_PATH_FOR_GENERATED_IMATRIX),
imatrix_output_layout,
)
self.imatrix_frequency = QSpinBox() self.imatrix_frequency = QSpinBox()
self.imatrix_frequency.setRange(1, 100) # Set the range from 1 to 100 self.imatrix_frequency.setRange(1, 100) # Set the range from 1 to 100
self.imatrix_frequency.setValue(1) # Set a default value self.imatrix_frequency.setValue(1) # Set a default value
imatrix_layout.addRow(self.create_label(OUTPUT_FREQUENCY, HOW_OFTEN_TO_SAVE_IMATRIX), self.imatrix_frequency) imatrix_layout.addRow(
self.create_label(OUTPUT_FREQUENCY, HOW_OFTEN_TO_SAVE_IMATRIX),
self.imatrix_frequency,
)
# Context size input (now a spinbox) # Context size input (now a spinbox)
self.imatrix_ctx_size = QSpinBox() self.imatrix_ctx_size = QSpinBox()
self.imatrix_ctx_size.setRange(1, 1048576) # Up to one million tokens self.imatrix_ctx_size.setRange(1, 1048576) # Up to one million tokens
self.imatrix_ctx_size.setValue(512) # Set a default value self.imatrix_ctx_size.setValue(512) # Set a default value
imatrix_layout.addRow(self.create_label(CONTEXT_SIZE, CONTEXT_SIZE_FOR_IMATRIX), self.imatrix_ctx_size) imatrix_layout.addRow(
self.create_label(CONTEXT_SIZE, CONTEXT_SIZE_FOR_IMATRIX),
self.imatrix_ctx_size,
)
# Threads input with slider and spinbox # Threads input with slider and spinbox
threads_layout = QHBoxLayout() threads_layout = QHBoxLayout()
@ -292,7 +383,9 @@ def __init__(self):
threads_layout.addWidget(self.threads_slider) threads_layout.addWidget(self.threads_slider)
threads_layout.addWidget(self.threads_spinbox) threads_layout.addWidget(self.threads_spinbox)
imatrix_layout.addRow(self.create_label(THREADS, NUMBER_OF_THREADS_FOR_IMATRIX), threads_layout) imatrix_layout.addRow(
self.create_label(THREADS, NUMBER_OF_THREADS_FOR_IMATRIX), threads_layout
)
# GPU Offload for IMatrix (corrected version) # GPU Offload for IMatrix (corrected version)
gpu_offload_layout = QHBoxLayout() gpu_offload_layout = QHBoxLayout()
@ -311,7 +404,9 @@ def __init__(self):
gpu_offload_layout.addWidget(self.gpu_offload_slider) gpu_offload_layout.addWidget(self.gpu_offload_slider)
gpu_offload_layout.addWidget(self.gpu_offload_spinbox) gpu_offload_layout.addWidget(self.gpu_offload_spinbox)
gpu_offload_layout.addWidget(self.gpu_offload_auto) gpu_offload_layout.addWidget(self.gpu_offload_auto)
imatrix_layout.addRow(self.create_label(GPU_OFFLOAD, SET_GPU_OFFLOAD_VALUE), gpu_offload_layout) imatrix_layout.addRow(
self.create_label(GPU_OFFLOAD, SET_GPU_OFFLOAD_VALUE), gpu_offload_layout
)
imatrix_generate_button = QPushButton(GENERATE_IMATRIX) imatrix_generate_button = QPushButton(GENERATE_IMATRIX)
imatrix_generate_button.clicked.connect(self.generate_imatrix) imatrix_generate_button.clicked.connect(self.generate_imatrix)
@ -336,7 +431,10 @@ def __init__(self):
lora_input_layout = QHBoxLayout() lora_input_layout = QHBoxLayout()
lora_input_layout.addWidget(self.lora_input) lora_input_layout.addWidget(self.lora_input)
lora_input_layout.addWidget(lora_input_button) lora_input_layout.addWidget(lora_input_button)
lora_layout.addRow(self.create_label(LORA_INPUT_PATH, SELECT_LORA_INPUT_DIRECTORY), lora_input_layout) lora_layout.addRow(
self.create_label(LORA_INPUT_PATH, SELECT_LORA_INPUT_DIRECTORY),
lora_input_layout,
)
self.lora_output = QLineEdit() self.lora_output = QLineEdit()
lora_output_button = QPushButton(BROWSE) lora_output_button = QPushButton(BROWSE)
@ -344,13 +442,21 @@ def __init__(self):
lora_output_layout = QHBoxLayout() lora_output_layout = QHBoxLayout()
lora_output_layout.addWidget(self.lora_output) lora_output_layout.addWidget(self.lora_output)
lora_output_layout.addWidget(lora_output_button) lora_output_layout.addWidget(lora_output_button)
lora_layout.addRow(self.create_label(LORA_OUTPUT_PATH, SELECT_LORA_OUTPUT_FILE), lora_output_layout) lora_layout.addRow(
self.create_label(LORA_OUTPUT_PATH, SELECT_LORA_OUTPUT_FILE),
lora_output_layout,
)
# Output Type Dropdown # Output Type Dropdown
self.lora_output_type_combo = QComboBox() self.lora_output_type_combo = QComboBox()
self.lora_output_type_combo.addItems(["GGML", "GGUF"]) self.lora_output_type_combo.addItems(["GGML", "GGUF"])
self.lora_output_type_combo.currentIndexChanged.connect(self.update_base_model_visibility) # Connect to update visibility self.lora_output_type_combo.currentIndexChanged.connect(
lora_layout.addRow(self.create_label(OUTPUT_TYPE, SELECT_OUTPUT_TYPE), self.lora_output_type_combo) self.update_base_model_visibility
) # Connect to update visibility
lora_layout.addRow(
self.create_label(OUTPUT_TYPE, SELECT_OUTPUT_TYPE),
self.lora_output_type_combo,
)
# Base Model Path (initially hidden) # Base Model Path (initially hidden)
self.base_model_path = QLineEdit() self.base_model_path = QLineEdit()
@ -362,7 +468,10 @@ def __init__(self):
self.base_model_widget = QWidget() self.base_model_widget = QWidget()
self.base_model_widget.setLayout(base_model_layout) self.base_model_widget.setLayout(base_model_layout)
self.base_model_widget.setVisible(False) # Initially hidden self.base_model_widget.setVisible(False) # Initially hidden
lora_layout.addRow(self.create_label(BASE_MODEL, SELECT_BASE_MODEL_FILE), self.base_model_widget) lora_layout.addRow(
self.create_label(BASE_MODEL, SELECT_BASE_MODEL_FILE),
self.base_model_widget,
)
lora_convert_button = QPushButton(CONVERT_LORA) lora_convert_button = QPushButton(CONVERT_LORA)
lora_convert_button.clicked.connect(self.convert_lora) lora_convert_button.clicked.connect(self.convert_lora)
@ -381,7 +490,9 @@ def __init__(self):
export_lora_model_layout = QHBoxLayout() export_lora_model_layout = QHBoxLayout()
export_lora_model_layout.addWidget(self.export_lora_model) export_lora_model_layout.addWidget(self.export_lora_model)
export_lora_model_layout.addWidget(export_lora_model_button) export_lora_model_layout.addWidget(export_lora_model_button)
export_lora_layout.addRow(self.create_label(MODEL, SELECT_MODEL_FILE), export_lora_model_layout) export_lora_layout.addRow(
self.create_label(MODEL, SELECT_MODEL_FILE), export_lora_model_layout
)
self.export_lora_output = QLineEdit() self.export_lora_output = QLineEdit()
export_lora_output_button = QPushButton(BROWSE) export_lora_output_button = QPushButton(BROWSE)
@ -389,7 +500,9 @@ def __init__(self):
export_lora_output_layout = QHBoxLayout() export_lora_output_layout = QHBoxLayout()
export_lora_output_layout.addWidget(self.export_lora_output) export_lora_output_layout.addWidget(self.export_lora_output)
export_lora_output_layout.addWidget(export_lora_output_button) export_lora_output_layout.addWidget(export_lora_output_button)
export_lora_layout.addRow(self.create_label(OUTPUT, SELECT_OUTPUT_FILE), export_lora_output_layout) export_lora_layout.addRow(
self.create_label(OUTPUT, SELECT_OUTPUT_FILE), export_lora_output_layout
)
# GGML LoRA Adapters # GGML LoRA Adapters
self.export_lora_adapters = QListWidget() self.export_lora_adapters = QListWidget()
@ -400,20 +513,28 @@ def __init__(self):
buttons_layout = QHBoxLayout() buttons_layout = QHBoxLayout()
buttons_layout.addWidget(add_adapter_button) buttons_layout.addWidget(add_adapter_button)
adapters_layout.addLayout(buttons_layout) adapters_layout.addLayout(buttons_layout)
export_lora_layout.addRow(self.create_label(GGML_LORA_ADAPTERS, SELECT_LORA_ADAPTER_FILES), adapters_layout) export_lora_layout.addRow(
self.create_label(GGML_LORA_ADAPTERS, SELECT_LORA_ADAPTER_FILES),
adapters_layout,
)
# Threads # Threads
self.export_lora_threads = QSpinBox() self.export_lora_threads = QSpinBox()
self.export_lora_threads.setRange(1, 64) self.export_lora_threads.setRange(1, 64)
self.export_lora_threads.setValue(8) # Default value self.export_lora_threads.setValue(8) # Default value
export_lora_layout.addRow(self.create_label(THREADS, NUMBER_OF_THREADS_FOR_LORA_EXPORT), self.export_lora_threads) export_lora_layout.addRow(
self.create_label(THREADS, NUMBER_OF_THREADS_FOR_LORA_EXPORT),
self.export_lora_threads,
)
export_lora_button = QPushButton(EXPORT_LORA) export_lora_button = QPushButton(EXPORT_LORA)
export_lora_button.clicked.connect(self.export_lora) export_lora_button.clicked.connect(self.export_lora)
export_lora_layout.addRow(export_lora_button) export_lora_layout.addRow(export_lora_button)
export_lora_group.setLayout(export_lora_layout) export_lora_group.setLayout(export_lora_layout)
right_layout.addWidget(export_lora_group) # Add the Export LoRA group to the right layout right_layout.addWidget(
export_lora_group
) # Add the Export LoRA group to the right layout
# Modify the task list to support right-click menu # Modify the task list to support right-click menu
self.task_list.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu) self.task_list.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu)
@ -445,14 +566,18 @@ def refresh_backends(self):
if valid_backends: if valid_backends:
for name, path in valid_backends: for name, path in valid_backends:
self.backend_combo.addItem(name, userData=path) self.backend_combo.addItem(name, userData=path)
self.backend_combo.setEnabled(True) # Enable the combo box if there are valid backends self.backend_combo.setEnabled(
True
) # Enable the combo box if there are valid backends
else: else:
self.backend_combo.addItem(NO_BACKENDS_AVAILABLE) self.backend_combo.addItem(NO_BACKENDS_AVAILABLE)
self.backend_combo.setEnabled(False) self.backend_combo.setEnabled(False)
self.logger.info(FOUND_VALID_BACKENDS.format(self.backend_combo.count())) self.logger.info(FOUND_VALID_BACKENDS.format(self.backend_combo.count()))
def update_base_model_visibility(self, index): def update_base_model_visibility(self, index):
self.base_model_widget.setVisible(self.lora_output_type_combo.itemText(index) == "GGUF") self.base_model_widget.setVisible(
self.lora_output_type_combo.itemText(index) == "GGUF"
)
def save_preset(self): def save_preset(self):
self.logger.info(SAVING_PRESET) self.logger.info(SAVING_PRESET)
@ -469,15 +594,19 @@ def save_preset(self):
"use_token_embedding_type": self.use_token_embedding_type.isChecked(), "use_token_embedding_type": self.use_token_embedding_type.isChecked(),
"token_embedding_type": self.token_embedding_type.currentText(), "token_embedding_type": self.token_embedding_type.currentText(),
"keep_split": self.keep_split.isChecked(), "keep_split": self.keep_split.isChecked(),
"kv_overrides": [entry.get_override_string() for entry in self.kv_override_entries], "kv_overrides": [
"extra_arguments": self.extra_arguments.text() entry.get_override_string() for entry in self.kv_override_entries
],
"extra_arguments": self.extra_arguments.text(),
} }
file_name, _ = QFileDialog.getSaveFileName(self, SAVE_PRESET, "", JSON_FILES) file_name, _ = QFileDialog.getSaveFileName(self, SAVE_PRESET, "", JSON_FILES)
if file_name: if file_name:
with open(file_name, 'w') as f: with open(file_name, "w") as f:
json.dump(preset, f, indent=4) json.dump(preset, f, indent=4)
QMessageBox.information(self, PRESET_SAVED, PRESET_SAVED_TO.format(file_name)) QMessageBox.information(
self, PRESET_SAVED, PRESET_SAVED_TO.format(file_name)
)
self.logger.info(PRESET_SAVED_TO.format(file_name)) self.logger.info(PRESET_SAVED_TO.format(file_name))
def load_preset(self): def load_preset(self):
@ -485,20 +614,30 @@ def load_preset(self):
file_name, _ = QFileDialog.getOpenFileName(self, LOAD_PRESET, "", JSON_FILES) file_name, _ = QFileDialog.getOpenFileName(self, LOAD_PRESET, "", JSON_FILES)
if file_name: if file_name:
try: try:
with open(file_name, 'r') as f: with open(file_name, "r") as f:
preset = json.load(f) preset = json.load(f)
self.quant_type.setCurrentText(preset.get("quant_type", "")) self.quant_type.setCurrentText(preset.get("quant_type", ""))
self.allow_requantize.setChecked(preset.get("allow_requantize", False)) self.allow_requantize.setChecked(preset.get("allow_requantize", False))
self.leave_output_tensor.setChecked(preset.get("leave_output_tensor", False)) self.leave_output_tensor.setChecked(
preset.get("leave_output_tensor", False)
)
self.pure.setChecked(preset.get("pure", False)) self.pure.setChecked(preset.get("pure", False))
self.imatrix.setText(preset.get("imatrix", "")) self.imatrix.setText(preset.get("imatrix", ""))
self.include_weights.setText(preset.get("include_weights", "")) self.include_weights.setText(preset.get("include_weights", ""))
self.exclude_weights.setText(preset.get("exclude_weights", "")) self.exclude_weights.setText(preset.get("exclude_weights", ""))
self.use_output_tensor_type.setChecked(preset.get("use_output_tensor_type", False)) self.use_output_tensor_type.setChecked(
self.output_tensor_type.setCurrentText(preset.get("output_tensor_type", "")) preset.get("use_output_tensor_type", False)
self.use_token_embedding_type.setChecked(preset.get("use_token_embedding_type", False)) )
self.token_embedding_type.setCurrentText(preset.get("token_embedding_type", "")) self.output_tensor_type.setCurrentText(
preset.get("output_tensor_type", "")
)
self.use_token_embedding_type.setChecked(
preset.get("use_token_embedding_type", False)
)
self.token_embedding_type.setCurrentText(
preset.get("token_embedding_type", "")
)
self.keep_split.setChecked(preset.get("keep_split", False)) self.keep_split.setChecked(preset.get("keep_split", False))
self.extra_arguments.setText(preset.get("extra_arguments", "")) self.extra_arguments.setText(preset.get("extra_arguments", ""))
@ -508,7 +647,9 @@ def load_preset(self):
for override in preset.get("kv_overrides", []): for override in preset.get("kv_overrides", []):
self.add_kv_override(override) self.add_kv_override(override)
QMessageBox.information(self, PRESET_LOADED, PRESET_LOADED_FROM.format(file_name)) QMessageBox.information(
self, PRESET_LOADED, PRESET_LOADED_FROM.format(file_name)
)
except Exception as e: except Exception as e:
QMessageBox.critical(self, ERROR, FAILED_TO_LOAD_PRESET.format(str(e))) QMessageBox.critical(self, ERROR, FAILED_TO_LOAD_PRESET.format(str(e)))
self.logger.info(PRESET_LOADED_FROM.format(file_name)) self.logger.info(PRESET_LOADED_FROM.format(file_name))
@ -520,30 +661,40 @@ def save_task_preset(self, task_item):
preset = { preset = {
"command": thread.command, "command": thread.command,
"backend_path": thread.cwd, "backend_path": thread.cwd,
"log_file": thread.log_file "log_file": thread.log_file,
} }
file_name, _ = QFileDialog.getSaveFileName(self, SAVE_TASK_PRESET, "", JSON_FILES) file_name, _ = QFileDialog.getSaveFileName(
self, SAVE_TASK_PRESET, "", JSON_FILES
)
if file_name: if file_name:
with open(file_name, 'w') as f: with open(file_name, "w") as f:
json.dump(preset, f, indent=4) json.dump(preset, f, indent=4)
QMessageBox.information(self, TASK_PRESET_SAVED, TASK_PRESET_SAVED_TO.format(file_name)) QMessageBox.information(
self, TASK_PRESET_SAVED, TASK_PRESET_SAVED_TO.format(file_name)
)
break break
def browse_export_lora_model(self): def browse_export_lora_model(self):
self.logger.info(BROWSING_FOR_EXPORT_LORA_MODEL_FILE) self.logger.info(BROWSING_FOR_EXPORT_LORA_MODEL_FILE)
model_file, _ = QFileDialog.getOpenFileName(self, SELECT_MODEL_FILE, "", GGUF_FILES) model_file, _ = QFileDialog.getOpenFileName(
self, SELECT_MODEL_FILE, "", GGUF_FILES
)
if model_file: if model_file:
self.export_lora_model.setText(os.path.abspath(model_file)) self.export_lora_model.setText(os.path.abspath(model_file))
def browse_export_lora_output(self): def browse_export_lora_output(self):
self.logger.info(BROWSING_FOR_EXPORT_LORA_OUTPUT_FILE) self.logger.info(BROWSING_FOR_EXPORT_LORA_OUTPUT_FILE)
output_file, _ = QFileDialog.getSaveFileName(self, SELECT_OUTPUT_FILE, "", GGUF_FILES) output_file, _ = QFileDialog.getSaveFileName(
self, SELECT_OUTPUT_FILE, "", GGUF_FILES
)
if output_file: if output_file:
self.export_lora_output.setText(os.path.abspath(output_file)) self.export_lora_output.setText(os.path.abspath(output_file))
def add_lora_adapter(self): def add_lora_adapter(self):
self.logger.info(ADDING_LORA_ADAPTER) self.logger.info(ADDING_LORA_ADAPTER)
adapter_path, _ = QFileDialog.getOpenFileName(self, SELECT_LORA_ADAPTER_FILE, "", LORA_FILES) adapter_path, _ = QFileDialog.getOpenFileName(
self, SELECT_LORA_ADAPTER_FILE, "", LORA_FILES
)
if adapter_path: if adapter_path:
# Create a widget to hold the path and scale input # Create a widget to hold the path and scale input
adapter_widget = QWidget() adapter_widget = QWidget()
@ -557,7 +708,9 @@ def add_lora_adapter(self):
adapter_layout.addWidget(scale_input) adapter_layout.addWidget(scale_input)
delete_button = QPushButton(DELETE_ADAPTER) delete_button = QPushButton(DELETE_ADAPTER)
delete_button.clicked.connect(lambda: self.delete_lora_adapter_item(adapter_widget)) delete_button.clicked.connect(
lambda: self.delete_lora_adapter_item(adapter_widget)
)
adapter_layout.addWidget(delete_button) adapter_layout.addWidget(delete_button)
# Add the widget to the list # Add the widget to the list
@ -568,7 +721,9 @@ def add_lora_adapter(self):
def browse_base_model(self): def browse_base_model(self):
self.logger.info(BROWSING_FOR_BASE_MODEL_FOLDER) # Updated log message self.logger.info(BROWSING_FOR_BASE_MODEL_FOLDER) # Updated log message
base_model_folder = QFileDialog.getExistingDirectory(self, SELECT_BASE_MODEL_FOLDER) base_model_folder = QFileDialog.getExistingDirectory(
self, SELECT_BASE_MODEL_FOLDER
)
if base_model_folder: if base_model_folder:
self.base_model_path.setText(os.path.abspath(base_model_folder)) self.base_model_path.setText(os.path.abspath(base_model_folder))
@ -608,16 +763,22 @@ def export_lora(self):
if not backend_path: if not backend_path:
raise ValueError(NO_BACKEND_SELECTED) raise ValueError(NO_BACKEND_SELECTED)
command = [os.path.join(backend_path, "llama-export-lora"), command = [
"--model", model_path, os.path.join(backend_path, "llama-export-lora"),
"--output", output_path] "--model",
model_path,
"--output",
output_path,
]
for adapter_path, adapter_scale in lora_adapters: for adapter_path, adapter_scale in lora_adapters:
if adapter_path: if adapter_path:
if adapter_scale: if adapter_scale:
try: try:
scale_value = float(adapter_scale) scale_value = float(adapter_scale)
command.extend(["--lora-scaled", adapter_path, str(scale_value)]) command.extend(
["--lora-scaled", adapter_path, str(scale_value)]
)
except ValueError: except ValueError:
raise ValueError(INVALID_LORA_SCALE_VALUE) raise ValueError(INVALID_LORA_SCALE_VALUE)
else: else:
@ -655,11 +816,17 @@ def restart_task(self, task_item):
self.logger.info(RESTARTING_TASK.format(task_item.task_name)) self.logger.info(RESTARTING_TASK.format(task_item.task_name))
for thread in self.quant_threads: for thread in self.quant_threads:
if thread.log_file == task_item.log_file: if thread.log_file == task_item.log_file:
new_thread = QuantizationThread(thread.command, thread.cwd, thread.log_file) new_thread = QuantizationThread(
thread.command, thread.cwd, thread.log_file
)
self.quant_threads.append(new_thread) self.quant_threads.append(new_thread)
new_thread.status_signal.connect(task_item.update_status) new_thread.status_signal.connect(task_item.update_status)
new_thread.finished_signal.connect(lambda: self.task_finished(new_thread)) new_thread.finished_signal.connect(
new_thread.error_signal.connect(lambda err: self.handle_error(err, task_item)) lambda: self.task_finished(new_thread)
)
new_thread.error_signal.connect(
lambda err: self.handle_error(err, task_item)
)
new_thread.model_info_signal.connect(self.update_model_info) new_thread.model_info_signal.connect(self.update_model_info)
new_thread.start() new_thread.start()
task_item.update_status(IN_PROGRESS) task_item.update_status(IN_PROGRESS)
@ -667,14 +834,18 @@ def restart_task(self, task_item):
def browse_lora_input(self): def browse_lora_input(self):
self.logger.info(BROWSING_FOR_LORA_INPUT_DIRECTORY) self.logger.info(BROWSING_FOR_LORA_INPUT_DIRECTORY)
lora_input_path = QFileDialog.getExistingDirectory(self, SELECT_LORA_INPUT_DIRECTORY) lora_input_path = QFileDialog.getExistingDirectory(
self, SELECT_LORA_INPUT_DIRECTORY
)
if lora_input_path: if lora_input_path:
self.lora_input.setText(os.path.abspath(lora_input_path)) self.lora_input.setText(os.path.abspath(lora_input_path))
ensure_directory(lora_input_path) ensure_directory(lora_input_path)
def browse_lora_output(self): def browse_lora_output(self):
self.logger.info(BROWSING_FOR_LORA_OUTPUT_FILE) self.logger.info(BROWSING_FOR_LORA_OUTPUT_FILE)
lora_output_file, _ = QFileDialog.getSaveFileName(self, SELECT_LORA_OUTPUT_FILE, "", GGUF_AND_BIN_FILES) lora_output_file, _ = QFileDialog.getSaveFileName(
self, SELECT_LORA_OUTPUT_FILE, "", GGUF_AND_BIN_FILES
)
if lora_output_file: if lora_output_file:
self.lora_output.setText(os.path.abspath(lora_output_file)) self.lora_output.setText(os.path.abspath(lora_output_file))
@ -691,7 +862,13 @@ def convert_lora(self):
raise ValueError(LORA_OUTPUT_PATH_REQUIRED) raise ValueError(LORA_OUTPUT_PATH_REQUIRED)
if lora_output_type == "GGUF": # Use new file and parameters for GGUF if lora_output_type == "GGUF": # Use new file and parameters for GGUF
command = ["python", "src/convert_lora_to_gguf.py", "--outfile", lora_output_path, lora_input_path] command = [
"python",
"src/convert_lora_to_gguf.py",
"--outfile",
lora_output_path,
lora_input_path,
]
base_model_path = self.base_model_path.text() base_model_path = self.base_model_path.text()
if not base_model_path: if not base_model_path:
raise ValueError(BASE_MODEL_PATH_REQUIRED) raise ValueError(BASE_MODEL_PATH_REQUIRED)
@ -708,7 +885,9 @@ def convert_lora(self):
thread = QuantizationThread(command, os.getcwd(), log_file) thread = QuantizationThread(command, os.getcwd(), log_file)
self.quant_threads.append(thread) self.quant_threads.append(thread)
task_name = LORA_CONVERSION_FROM_TO.format(os.path.basename(lora_input_path), os.path.basename(lora_output_path)) task_name = LORA_CONVERSION_FROM_TO.format(
os.path.basename(lora_input_path), os.path.basename(lora_output_path)
)
task_item = TaskListItem(task_name, log_file) task_item = TaskListItem(task_name, log_file)
list_item = QListWidgetItem(self.task_list) list_item = QListWidgetItem(self.task_list)
list_item.setSizeHint(task_item.sizeHint()) list_item.setSizeHint(task_item.sizeHint())
@ -716,7 +895,11 @@ def convert_lora(self):
self.task_list.setItemWidget(list_item, task_item) self.task_list.setItemWidget(list_item, task_item)
thread.status_signal.connect(task_item.update_status) thread.status_signal.connect(task_item.update_status)
thread.finished_signal.connect(lambda: self.lora_conversion_finished(thread, lora_input_path, lora_output_path)) thread.finished_signal.connect(
lambda: self.lora_conversion_finished(
thread, lora_input_path, lora_output_path
)
)
thread.error_signal.connect(lambda err: self.handle_error(err, task_item)) thread.error_signal.connect(lambda err: self.handle_error(err, task_item))
thread.start() thread.start()
self.logger.info(LORA_CONVERSION_TASK_STARTED) self.logger.info(LORA_CONVERSION_TASK_STARTED)
@ -746,15 +929,28 @@ def download_finished(self, extract_dir):
self.download_button.setEnabled(True) self.download_button.setEnabled(True)
self.download_progress.setValue(100) self.download_progress.setValue(100)
if self.cuda_extract_checkbox.isChecked() and self.cuda_extract_checkbox.isVisible(): if (
self.cuda_extract_checkbox.isChecked()
and self.cuda_extract_checkbox.isVisible()
):
cuda_backend = self.backend_combo_cuda.currentData() cuda_backend = self.backend_combo_cuda.currentData()
if cuda_backend and cuda_backend != NO_SUITABLE_CUDA_BACKENDS: if cuda_backend and cuda_backend != NO_SUITABLE_CUDA_BACKENDS:
self.extract_cuda_files(extract_dir, cuda_backend) self.extract_cuda_files(extract_dir, cuda_backend)
QMessageBox.information(self, DOWNLOAD_COMPLETE, LLAMACPP_DOWNLOADED_AND_EXTRACTED.format(extract_dir, cuda_backend)) QMessageBox.information(
self,
DOWNLOAD_COMPLETE,
LLAMACPP_DOWNLOADED_AND_EXTRACTED.format(extract_dir, cuda_backend),
)
else: else:
QMessageBox.warning(self, CUDA_EXTRACTION_FAILED, NO_SUITABLE_CUDA_BACKEND_FOUND) QMessageBox.warning(
self, CUDA_EXTRACTION_FAILED, NO_SUITABLE_CUDA_BACKEND_FOUND
)
else: else:
QMessageBox.information(self, DOWNLOAD_COMPLETE, LLAMACPP_BINARY_DOWNLOADED_AND_EXTRACTED.format(extract_dir)) QMessageBox.information(
self,
DOWNLOAD_COMPLETE,
LLAMACPP_BINARY_DOWNLOADED_AND_EXTRACTED.format(extract_dir),
)
self.refresh_backends() # Refresh the backends after successful download self.refresh_backends() # Refresh the backends after successful download
self.update_cuda_option() # Update CUDA options in case a CUDA-capable backend was downloaded self.update_cuda_option() # Update CUDA options in case a CUDA-capable backend was downloaded
@ -768,11 +964,13 @@ def download_finished(self, extract_dir):
def refresh_releases(self): def refresh_releases(self):
self.logger.info(REFRESHING_LLAMACPP_RELEASES) self.logger.info(REFRESHING_LLAMACPP_RELEASES)
try: try:
response = requests.get("https://api.github.com/repos/ggerganov/llama.cpp/releases") response = requests.get(
"https://api.github.com/repos/ggerganov/llama.cpp/releases"
)
releases = response.json() releases = response.json()
self.release_combo.clear() self.release_combo.clear()
for release in releases: for release in releases:
self.release_combo.addItem(release['tag_name'], userData=release) self.release_combo.addItem(release["tag_name"], userData=release)
self.release_combo.currentIndexChanged.connect(self.update_assets) self.release_combo.currentIndexChanged.connect(self.update_assets)
self.update_assets() self.update_assets()
except Exception as e: except Exception as e:
@ -783,14 +981,14 @@ def update_assets(self):
self.asset_combo.clear() self.asset_combo.clear()
release = self.release_combo.currentData() release = self.release_combo.currentData()
if release: if release:
for asset in release['assets']: for asset in release["assets"]:
self.asset_combo.addItem(asset['name'], userData=asset) self.asset_combo.addItem(asset["name"], userData=asset)
self.update_cuda_option() self.update_cuda_option()
def update_cuda_option(self): def update_cuda_option(self):
self.logger.debug(UPDATING_CUDA_OPTIONS) self.logger.debug(UPDATING_CUDA_OPTIONS)
asset = self.asset_combo.currentData() asset = self.asset_combo.currentData()
is_cuda = asset and "cudart" in asset['name'].lower() is_cuda = asset and "cudart" in asset["name"].lower()
self.cuda_extract_checkbox.setVisible(is_cuda) self.cuda_extract_checkbox.setVisible(is_cuda)
self.cuda_backend_label.setVisible(is_cuda) self.cuda_backend_label.setVisible(is_cuda)
self.backend_combo_cuda.setVisible(is_cuda) self.backend_combo_cuda.setVisible(is_cuda)
@ -808,9 +1006,9 @@ def download_llama_cpp(self):
if not os.path.exists(llama_bin): if not os.path.exists(llama_bin):
os.makedirs(llama_bin) os.makedirs(llama_bin)
save_path = os.path.join(llama_bin, asset['name']) save_path = os.path.join(llama_bin, asset["name"])
self.download_thread = DownloadThread(asset['browser_download_url'], save_path) self.download_thread = DownloadThread(asset["browser_download_url"], save_path)
self.download_thread.progress_signal.connect(self.update_download_progress) self.download_thread.progress_signal.connect(self.update_download_progress)
self.download_thread.finished_signal.connect(self.download_finished) self.download_thread.finished_signal.connect(self.download_finished)
self.download_thread.error_signal.connect(self.download_error) self.download_thread.error_signal.connect(self.download_error)
@ -843,15 +1041,28 @@ def download_finished(self, extract_dir):
self.download_button.setEnabled(True) self.download_button.setEnabled(True)
self.download_progress.setValue(100) self.download_progress.setValue(100)
if self.cuda_extract_checkbox.isChecked() and self.cuda_extract_checkbox.isVisible(): if (
self.cuda_extract_checkbox.isChecked()
and self.cuda_extract_checkbox.isVisible()
):
cuda_backend = self.backend_combo_cuda.currentData() cuda_backend = self.backend_combo_cuda.currentData()
if cuda_backend: if cuda_backend:
self.extract_cuda_files(extract_dir, cuda_backend) self.extract_cuda_files(extract_dir, cuda_backend)
QMessageBox.information(self, DOWNLOAD_COMPLETE, LLAMACPP_DOWNLOADED_AND_EXTRACTED.format(extract_dir, cuda_backend)) QMessageBox.information(
self,
DOWNLOAD_COMPLETE,
LLAMACPP_DOWNLOADED_AND_EXTRACTED.format(extract_dir, cuda_backend),
)
else: else:
QMessageBox.warning(self, CUDA_EXTRACTION_FAILED, NO_CUDA_BACKEND_SELECTED) QMessageBox.warning(
self, CUDA_EXTRACTION_FAILED, NO_CUDA_BACKEND_SELECTED
)
else: else:
QMessageBox.information(self, DOWNLOAD_COMPLETE, LLAMACPP_BINARY_DOWNLOADED_AND_EXTRACTED.format(extract_dir)) QMessageBox.information(
self,
DOWNLOAD_COMPLETE,
LLAMACPP_BINARY_DOWNLOADED_AND_EXTRACTED.format(extract_dir),
)
self.refresh_backends() self.refresh_backends()
@ -859,12 +1070,11 @@ def extract_cuda_files(self, extract_dir, destination):
self.logger.info(EXTRACTING_CUDA_FILES.format(extract_dir, destination)) self.logger.info(EXTRACTING_CUDA_FILES.format(extract_dir, destination))
for root, dirs, files in os.walk(extract_dir): for root, dirs, files in os.walk(extract_dir):
for file in files: for file in files:
if file.lower().endswith('.dll'): if file.lower().endswith(".dll"):
source_path = os.path.join(root, file) source_path = os.path.join(root, file)
dest_path = os.path.join(destination, file) dest_path = os.path.join(destination, file)
shutil.copy2(source_path, dest_path) shutil.copy2(source_path, dest_path)
def download_error(self, error_message): def download_error(self, error_message):
self.logger.error(DOWNLOAD_ERROR.format(error_message)) self.logger.error(DOWNLOAD_ERROR.format(error_message))
self.download_button.setEnabled(True) self.download_button.setEnabled(True)
@ -874,7 +1084,7 @@ def download_error(self, error_message):
# Clean up any partially downloaded files # Clean up any partially downloaded files
asset = self.asset_combo.currentData() asset = self.asset_combo.currentData()
if asset: if asset:
partial_file = os.path.join(os.path.abspath("llama_bin"), asset['name']) partial_file = os.path.join(os.path.abspath("llama_bin"), asset["name"])
if os.path.exists(partial_file): if os.path.exists(partial_file):
os.remove(partial_file) os.remove(partial_file)
@ -900,7 +1110,9 @@ def show_task_context_menu(self, position):
context_menu.addAction(restart_action) context_menu.addAction(restart_action)
save_preset_action = QAction(SAVE_PRESET, self) save_preset_action = QAction(SAVE_PRESET, self)
save_preset_action.triggered.connect(lambda: self.save_task_preset(task_item)) save_preset_action.triggered.connect(
lambda: self.save_task_preset(task_item)
)
context_menu.addAction(save_preset_action) context_menu.addAction(save_preset_action)
delete_action = QAction(DELETE, self) delete_action = QAction(DELETE, self)
@ -951,10 +1163,13 @@ def retry_task(self, item):
def delete_task(self, item): def delete_task(self, item):
self.logger.info(DELETING_TASK.format(item.text())) self.logger.info(DELETING_TASK.format(item.text()))
reply = QMessageBox.question(self, CONFIRM_DELETION_TITLE, reply = QMessageBox.question(
self,
CONFIRM_DELETION_TITLE,
CONFIRM_DELETION, CONFIRM_DELETION,
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No,
QMessageBox.StandardButton.No) QMessageBox.StandardButton.No,
)
if reply == QMessageBox.StandardButton.Yes: if reply == QMessageBox.StandardButton.Yes:
row = self.task_list.row(item) row = self.task_list.row(item)
self.task_list.takeItem(row) self.task_list.takeItem(row)
@ -981,7 +1196,6 @@ def load_models(self):
self.model_list.addItem(file) self.model_list.addItem(file)
self.logger.info(LOADED_MODELS.format(self.model_list.count())) self.logger.info(LOADED_MODELS.format(self.model_list.count()))
def browse_models(self): def browse_models(self):
self.logger.info(BROWSING_FOR_MODELS_DIRECTORY) self.logger.info(BROWSING_FOR_MODELS_DIRECTORY)
models_path = QFileDialog.getExistingDirectory(self, SELECT_MODELS_DIRECTORY) models_path = QFileDialog.getExistingDirectory(self, SELECT_MODELS_DIRECTORY)
@ -1006,7 +1220,9 @@ def browse_logs(self):
def browse_imatrix(self): def browse_imatrix(self):
self.logger.info(BROWSING_FOR_IMATRIX_FILE) self.logger.info(BROWSING_FOR_IMATRIX_FILE)
imatrix_file, _ = QFileDialog.getOpenFileName(self, SELECT_IMATRIX_FILE, "", DAT_FILES) imatrix_file, _ = QFileDialog.getOpenFileName(
self, SELECT_IMATRIX_FILE, "", DAT_FILES
)
if imatrix_file: if imatrix_file:
self.imatrix.setText(os.path.abspath(imatrix_file)) self.imatrix.setText(os.path.abspath(imatrix_file))
@ -1014,7 +1230,11 @@ def update_system_info(self):
ram = psutil.virtual_memory() ram = psutil.virtual_memory()
cpu = psutil.cpu_percent() cpu = psutil.cpu_percent()
self.ram_bar.setValue(int(ram.percent)) self.ram_bar.setValue(int(ram.percent))
self.ram_bar.setFormat(RAM_USAGE_FORMAT.format(ram.percent, ram.used // 1024 // 1024, ram.total // 1024 // 1024)) self.ram_bar.setFormat(
RAM_USAGE_FORMAT.format(
ram.percent, ram.used // 1024 // 1024, ram.total // 1024 // 1024
)
)
self.cpu_label.setText(CPU_USAGE_FORMAT.format(cpu)) self.cpu_label.setText(CPU_USAGE_FORMAT.format(cpu))
def validate_quantization_inputs(self): def validate_quantization_inputs(self):
@ -1038,8 +1258,8 @@ def add_kv_override(self, override_string=None):
entry = KVOverrideEntry() entry = KVOverrideEntry()
entry.deleted.connect(self.remove_kv_override) entry.deleted.connect(self.remove_kv_override)
if override_string: if override_string:
key, value = override_string.split('=') key, value = override_string.split("=")
type_, val = value.split(':') type_, val = value.split(":")
entry.key_input.setText(key) entry.key_input.setText(key)
entry.type_combo.setCurrentText(type_) entry.type_combo.setCurrentText(type_)
entry.value_input.setText(val) entry.value_input.setText(val)
@ -1070,10 +1290,17 @@ def quantize_model(self):
quant_type = self.quant_type.currentText() quant_type = self.quant_type.currentText()
# Start building the output name # Start building the output name
output_name_parts = [os.path.splitext(model_name)[0], "converted", quant_type] output_name_parts = [
os.path.splitext(model_name)[0],
"converted",
quant_type,
]
# Check for output tensor options # Check for output tensor options
if self.use_output_tensor_type.isChecked() or self.leave_output_tensor.isChecked(): if (
self.use_output_tensor_type.isChecked()
or self.leave_output_tensor.isChecked()
):
output_tensor_part = "o" output_tensor_part = "o"
if self.use_output_tensor_type.isChecked(): if self.use_output_tensor_type.isChecked():
output_tensor_part += "." + self.output_tensor_type.currentText() output_tensor_part += "." + self.output_tensor_type.currentText()
@ -1118,9 +1345,13 @@ def quantize_model(self):
if self.exclude_weights.text(): if self.exclude_weights.text():
command.extend(["--exclude-weights", self.exclude_weights.text()]) command.extend(["--exclude-weights", self.exclude_weights.text()])
if self.use_output_tensor_type.isChecked(): if self.use_output_tensor_type.isChecked():
command.extend(["--output-tensor-type", self.output_tensor_type.currentText()]) command.extend(
["--output-tensor-type", self.output_tensor_type.currentText()]
)
if self.use_token_embedding_type.isChecked(): if self.use_token_embedding_type.isChecked():
command.extend(["--token-embedding-type", self.token_embedding_type.currentText()]) command.extend(
["--token-embedding-type", self.token_embedding_type.currentText()]
)
if self.keep_split.isChecked(): if self.keep_split.isChecked():
command.append("--keep-split") command.append("--keep-split")
if self.override_kv.text(): if self.override_kv.text():
@ -1139,12 +1370,16 @@ def quantize_model(self):
ensure_directory(logs_path) ensure_directory(logs_path)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
log_file = os.path.join(logs_path, f"{model_name}_{timestamp}_{quant_type}.log") log_file = os.path.join(
logs_path, f"{model_name}_{timestamp}_{quant_type}.log"
)
thread = QuantizationThread(command, backend_path, log_file) thread = QuantizationThread(command, backend_path, log_file)
self.quant_threads.append(thread) self.quant_threads.append(thread)
task_item = TaskListItem(QUANTIZING_MODEL_TO.format(model_name, quant_type), log_file) task_item = TaskListItem(
QUANTIZING_MODEL_TO.format(model_name, quant_type), log_file
)
list_item = QListWidgetItem(self.task_list) list_item = QListWidgetItem(self.task_list)
list_item.setSizeHint(task_item.sizeHint()) list_item.setSizeHint(task_item.sizeHint())
self.task_list.addItem(list_item) self.task_list.addItem(list_item)
@ -1188,7 +1423,7 @@ def show_task_details(self, item):
# Load existing content # Load existing content
if os.path.exists(task_item.log_file): if os.path.exists(task_item.log_file):
with open_file_safe(task_item.log_file, 'r') as f: with open_file_safe(task_item.log_file, "r") as f:
log_text.setPlainText(f.read()) log_text.setPlainText(f.read())
# Connect to the thread if it's still running # Connect to the thread if it's still running
@ -1207,13 +1442,17 @@ def browse_imatrix_datafile(self):
def browse_imatrix_model(self): def browse_imatrix_model(self):
self.logger.info(BROWSING_FOR_IMATRIX_MODEL_FILE) self.logger.info(BROWSING_FOR_IMATRIX_MODEL_FILE)
model_file, _ = QFileDialog.getOpenFileName(self, SELECT_MODEL_FILE, "", GGUF_FILES) model_file, _ = QFileDialog.getOpenFileName(
self, SELECT_MODEL_FILE, "", GGUF_FILES
)
if model_file: if model_file:
self.imatrix_model.setText(os.path.abspath(model_file)) self.imatrix_model.setText(os.path.abspath(model_file))
def browse_imatrix_output(self): def browse_imatrix_output(self):
self.logger.info(BROWSING_FOR_IMATRIX_OUTPUT_FILE) self.logger.info(BROWSING_FOR_IMATRIX_OUTPUT_FILE)
output_file, _ = QFileDialog.getSaveFileName(self, SELECT_OUTPUT_FILE, "", DAT_FILES) output_file, _ = QFileDialog.getSaveFileName(
self, SELECT_OUTPUT_FILE, "", DAT_FILES
)
if output_file: if output_file:
self.imatrix_output.setText(os.path.abspath(output_file)) self.imatrix_output.setText(os.path.abspath(output_file))
@ -1241,12 +1480,18 @@ def generate_imatrix(self):
command = [ command = [
os.path.join(backend_path, "llama-imatrix"), os.path.join(backend_path, "llama-imatrix"),
"-f", self.imatrix_datafile.text(), "-f",
"-m", self.imatrix_model.text(), self.imatrix_datafile.text(),
"-o", self.imatrix_output.text(), "-m",
"--output-frequency", str(self.imatrix_frequency.value()), self.imatrix_model.text(),
"--ctx-size", str(self.imatrix_ctx_size.value()), "-o",
"--threads", str(self.threads_spinbox.value()) self.imatrix_output.text(),
"--output-frequency",
str(self.imatrix_frequency.value()),
"--ctx-size",
str(self.imatrix_ctx_size.value()),
"--threads",
str(self.threads_spinbox.value()),
] ]
if self.gpu_offload_auto.isChecked(): if self.gpu_offload_auto.isChecked():
@ -1260,7 +1505,9 @@ def generate_imatrix(self):
thread = QuantizationThread(command, backend_path, log_file) thread = QuantizationThread(command, backend_path, log_file)
self.quant_threads.append(thread) self.quant_threads.append(thread)
task_name = GENERATING_IMATRIX_FOR.format(os.path.basename(self.imatrix_model.text())) task_name = GENERATING_IMATRIX_FOR.format(
os.path.basename(self.imatrix_model.text())
)
task_item = TaskListItem(task_name, log_file) task_item = TaskListItem(task_name, log_file)
list_item = QListWidgetItem(self.task_list) list_item = QListWidgetItem(self.task_list)
list_item.setSizeHint(task_item.sizeHint()) list_item.setSizeHint(task_item.sizeHint())
@ -1287,10 +1534,13 @@ def handle_error(self, error_message, task_item):
def closeEvent(self, event: QCloseEvent): def closeEvent(self, event: QCloseEvent):
self.logger.info(APPLICATION_CLOSING) self.logger.info(APPLICATION_CLOSING)
if self.quant_threads: if self.quant_threads:
reply = QMessageBox.question(self, WARNING, reply = QMessageBox.question(
self,
WARNING,
TASK_RUNNING_WARNING, TASK_RUNNING_WARNING,
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No,
QMessageBox.StandardButton.No) QMessageBox.StandardButton.No,
)
if reply == QMessageBox.StandardButton.Yes: if reply == QMessageBox.StandardButton.Yes:
for thread in self.quant_threads: for thread in self.quant_threads:
@ -1302,6 +1552,7 @@ def closeEvent(self, event: QCloseEvent):
event.accept() event.accept()
self.logger.info(APPLICATION_CLOSED) self.logger.info(APPLICATION_CLOSED)
if __name__ == "__main__": if __name__ == "__main__":
app = QApplication(sys.argv) app = QApplication(sys.argv)
window = AutoGGUF() window = AutoGGUF()