mirror of https://github.com/leafspark/AutoGGUF
feat(core): implement HF upload functionality
- add HF upload GUI definitions - remove old HFTransfer class - update localizations
This commit is contained in:
parent
a80775b0a2
commit
79c74bb54a
121
src/AutoGGUF.py
121
src/AutoGGUF.py
|
@ -298,6 +298,52 @@ def __init__(self, args: List[str]) -> None:
|
||||||
self.split_gguf_layout.addWidget(split_button)
|
self.split_gguf_layout.addWidget(split_button)
|
||||||
self.split_gguf_dialog.setLayout(self.split_gguf_layout)
|
self.split_gguf_dialog.setLayout(self.split_gguf_layout)
|
||||||
|
|
||||||
|
# HF Upload Window
|
||||||
|
self.hf_upload_dialog = QDialog(self)
|
||||||
|
self.hf_upload_dialog.setWindowTitle("HF Upload")
|
||||||
|
self.hf_upload_dialog.setFixedWidth(500)
|
||||||
|
self.hf_upload_layout = QVBoxLayout()
|
||||||
|
|
||||||
|
# Repo input
|
||||||
|
repo_layout = QHBoxLayout()
|
||||||
|
self.hf_repo_input = QLineEdit()
|
||||||
|
repo_layout.addWidget(QLabel("Repository:"))
|
||||||
|
repo_layout.addWidget(self.hf_repo_input)
|
||||||
|
self.hf_upload_layout.addLayout(repo_layout)
|
||||||
|
|
||||||
|
# Remote path input
|
||||||
|
remote_path_layout = QHBoxLayout()
|
||||||
|
self.hf_remote_path_input = QLineEdit()
|
||||||
|
remote_path_layout.addWidget(QLabel("Remote Path:"))
|
||||||
|
remote_path_layout.addWidget(self.hf_remote_path_input)
|
||||||
|
self.hf_upload_layout.addLayout(remote_path_layout)
|
||||||
|
|
||||||
|
# Local file/folder input
|
||||||
|
local_path_layout = QHBoxLayout()
|
||||||
|
self.hf_local_path_input = QLineEdit()
|
||||||
|
local_path_button = QPushButton("Browse")
|
||||||
|
local_path_button.clicked.connect(self.browse_local_path)
|
||||||
|
local_path_layout.addWidget(QLabel("Local Path:"))
|
||||||
|
local_path_layout.addWidget(self.hf_local_path_input)
|
||||||
|
local_path_layout.addWidget(local_path_button)
|
||||||
|
self.hf_upload_layout.addLayout(local_path_layout)
|
||||||
|
|
||||||
|
# Upload type (file or folder)
|
||||||
|
self.upload_type_group = QButtonGroup()
|
||||||
|
self.upload_type_file = QRadioButton("File")
|
||||||
|
self.upload_type_folder = QRadioButton("Folder")
|
||||||
|
self.upload_type_group.addButton(self.upload_type_file)
|
||||||
|
self.upload_type_group.addButton(self.upload_type_folder)
|
||||||
|
self.hf_upload_layout.addWidget(self.upload_type_file)
|
||||||
|
self.hf_upload_layout.addWidget(self.upload_type_folder)
|
||||||
|
|
||||||
|
# Upload button
|
||||||
|
upload_button = QPushButton("Upload")
|
||||||
|
upload_button.clicked.connect(self.transfer_to_hf)
|
||||||
|
self.hf_upload_layout.addWidget(upload_button)
|
||||||
|
|
||||||
|
self.hf_upload_dialog.setLayout(self.hf_upload_layout)
|
||||||
|
|
||||||
# Tools menu
|
# Tools menu
|
||||||
tools_menu = self.menubar.addMenu("&Tools")
|
tools_menu = self.menubar.addMenu("&Tools")
|
||||||
autofp8_action = QAction("&AutoFP8", self)
|
autofp8_action = QAction("&AutoFP8", self)
|
||||||
|
@ -306,6 +352,10 @@ def __init__(self, args: List[str]) -> None:
|
||||||
split_gguf_action = QAction("&Split GGUF", self)
|
split_gguf_action = QAction("&Split GGUF", self)
|
||||||
split_gguf_action.setShortcut(QKeySequence("Shift+G"))
|
split_gguf_action.setShortcut(QKeySequence("Shift+G"))
|
||||||
split_gguf_action.triggered.connect(self.split_gguf_dialog.exec)
|
split_gguf_action.triggered.connect(self.split_gguf_dialog.exec)
|
||||||
|
hf_transfer_action = QAction("&HF Transfer", self)
|
||||||
|
hf_transfer_action.setShortcut(QKeySequence("Shift+H"))
|
||||||
|
hf_transfer_action.triggered.connect(self.hf_upload_dialog.exec)
|
||||||
|
tools_menu.addAction(hf_transfer_action)
|
||||||
tools_menu.addAction(autofp8_action)
|
tools_menu.addAction(autofp8_action)
|
||||||
tools_menu.addAction(split_gguf_action)
|
tools_menu.addAction(split_gguf_action)
|
||||||
|
|
||||||
|
@ -1252,6 +1302,77 @@ def save_task_preset(self, task_item) -> None:
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
def browse_local_path(self) -> None:
|
||||||
|
if self.upload_type_file.isChecked():
|
||||||
|
file_path, _ = QFileDialog.getOpenFileName(self, SELECT_FILE)
|
||||||
|
if file_path:
|
||||||
|
self.hf_local_path_input.setText(file_path)
|
||||||
|
elif self.upload_type_folder.isChecked():
|
||||||
|
folder_path = QFileDialog.getExistingDirectory(self, SELECT_FOLDER)
|
||||||
|
if folder_path:
|
||||||
|
self.hf_local_path_input.setText(folder_path)
|
||||||
|
|
||||||
|
@validate_input("hf_repo_input", "hf_local_path_input", "hf_remote_path_input")
|
||||||
|
def transfer_to_hf(self) -> None:
|
||||||
|
hf_repo = self.hf_repo_input.text()
|
||||||
|
local_path = self.hf_local_path_input.text()
|
||||||
|
remote_path = self.hf_remote_path_input.text()
|
||||||
|
type = "upload"
|
||||||
|
|
||||||
|
if not hf_repo or not local_path or not type:
|
||||||
|
QMessageBox.warning(self, ERROR, ALL_FIELDS_REQUIRED)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
command = ["huggingface-cli", type, hf_repo, local_path]
|
||||||
|
if remote_path:
|
||||||
|
command.append(remote_path)
|
||||||
|
|
||||||
|
logs_path = self.logs_input.text()
|
||||||
|
ensure_directory(logs_path)
|
||||||
|
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
log_file = os.path.join(logs_path, f"hf_transfer_{timestamp}.log")
|
||||||
|
|
||||||
|
# Log command
|
||||||
|
command_str = " ".join(command)
|
||||||
|
self.logger.info(HUGGINGFACE_UPLOAD_COMMAND + command_str)
|
||||||
|
|
||||||
|
thread = QuantizationThread(command, os.getcwd(), log_file)
|
||||||
|
self.quant_threads.append(thread)
|
||||||
|
|
||||||
|
task_name_temp = (
|
||||||
|
UPLOADING if self.upload_type_file.isChecked() else UPLOADING_FOLDER
|
||||||
|
)
|
||||||
|
|
||||||
|
task_name = HF_TRANSFER_TASK_NAME.format(
|
||||||
|
task_name_temp, local_path, hf_repo, local_path
|
||||||
|
)
|
||||||
|
task_item = TaskListItem(
|
||||||
|
task_name,
|
||||||
|
log_file,
|
||||||
|
show_progress_bar=False,
|
||||||
|
logger=self.logger,
|
||||||
|
quant_threads=self.quant_threads,
|
||||||
|
)
|
||||||
|
list_item = QListWidgetItem(self.task_list)
|
||||||
|
list_item.setSizeHint(task_item.sizeHint())
|
||||||
|
self.task_list.addItem(list_item)
|
||||||
|
self.task_list.setItemWidget(list_item, task_item)
|
||||||
|
|
||||||
|
thread.status_signal.connect(task_item.update_status)
|
||||||
|
thread.finished_signal.connect(
|
||||||
|
lambda: self.task_finished(thread, task_item)
|
||||||
|
)
|
||||||
|
thread.error_signal.connect(
|
||||||
|
lambda err: handle_error(self.logger, err, task_item)
|
||||||
|
)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
show_error(self.logger, ERROR_STARTING_HF_TRANSFER.format(str(e)))
|
||||||
|
self.logger.info(STARTED_HUGGINGFACE_TRANSFER.format(type))
|
||||||
|
|
||||||
def quantize_to_fp8_dynamic(self, model_dir: str, output_dir: str) -> None:
|
def quantize_to_fp8_dynamic(self, model_dir: str, output_dir: str) -> None:
|
||||||
if model_dir or output_dir == "":
|
if model_dir or output_dir == "":
|
||||||
show_error(
|
show_error(
|
||||||
|
|
|
@ -1,30 +0,0 @@
|
||||||
from huggingface_hub import HfApi, snapshot_download, hf_hub_url
|
|
||||||
|
|
||||||
|
|
||||||
class HFTransfer:
|
|
||||||
def __init__(self, token) -> None:
|
|
||||||
self.api = HfApi(token=token)
|
|
||||||
|
|
||||||
def upload_file(self, path, remote_path, repo, repo_type) -> str:
|
|
||||||
status = self.api.upload_file(
|
|
||||||
path_or_fileobj=path,
|
|
||||||
path_in_repo=remote_path,
|
|
||||||
repo_id=repo,
|
|
||||||
repo_type=repo_type,
|
|
||||||
)
|
|
||||||
return status
|
|
||||||
|
|
||||||
def upload_folder(self, local_folder, remote_path, repo, repo_type) -> str:
|
|
||||||
status = self.api.upload_folder(
|
|
||||||
folder_path=local_folder,
|
|
||||||
path_in_repo=remote_path,
|
|
||||||
repo_id=repo,
|
|
||||||
repo_type=repo_type,
|
|
||||||
)
|
|
||||||
return status
|
|
||||||
|
|
||||||
def download_repo(self, repo) -> str:
|
|
||||||
return snapshot_download(repo_id=repo)
|
|
||||||
|
|
||||||
def get_download_link(self, repo, file) -> str:
|
|
||||||
return hf_hub_url(repo_id=repo, filename=file)
|
|
|
@ -416,6 +416,17 @@ def __init__(self):
|
||||||
self.MODEL_RENAMED_SUCCESSFULLY = "Model renamed successfully."
|
self.MODEL_RENAMED_SUCCESSFULLY = "Model renamed successfully."
|
||||||
self.MODEL_DELETED_SUCCESSFULLY = "Model deleted successfully."
|
self.MODEL_DELETED_SUCCESSFULLY = "Model deleted successfully."
|
||||||
|
|
||||||
|
# HuggingFace Transfer
|
||||||
|
self.ALL_FIELDS_REQUIRED = "All fields are required."
|
||||||
|
self.HUGGINGFACE_UPLOAD_COMMAND = "HuggingFace Upload Command: "
|
||||||
|
self.UPLOADING = "Uploading"
|
||||||
|
self.UPLOADING_FOLDER = "Uploading folder"
|
||||||
|
self.HF_TRANSFER_TASK_NAME = "{} {} to {} from {}"
|
||||||
|
self.ERROR_STARTING_HF_TRANSFER = "Error starting HF transfer: {}"
|
||||||
|
self.STARTED_HUGGINGFACE_TRANSFER = "Started HuggingFace {} operation."
|
||||||
|
self.SELECT_FOLDER = "Select Folder"
|
||||||
|
self.SELECT_FILE = "Select File"
|
||||||
|
|
||||||
|
|
||||||
class _French(_Localization):
|
class _French(_Localization):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|
Loading…
Reference in New Issue