pengjunjie/ragflow
1
0
Fork 1
mirror of https://github.com/infiniflow/ragflow.git synced 2025-12-08 12:32:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: add huggingface model download functionality (#11101)

Browse Source 
### What problem does this PR solve?

reverse #11048

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Liu An
2025-11-07 15:12:47 +08:00
committed by GitHub
parent c58d95ed69
commit fa98cc2bb9
1 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
download_deps.py
View File

@ -14,6 +14,7 @@ import urllib.request
from typing import Union
import nltk
from huggingface_hub import snapshot_download
def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
@ -39,6 +40,19 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
]
repos = [
"InfiniFlow/text_concat_xgb_v1.0",
"InfiniFlow/deepdoc",
"InfiniFlow/huqie",
]
def download_model(repo_id):
local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
os.makedirs(local_dir, exist_ok=True)
snapshot_download(repo_id=repo_id, local_dir=local_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support")
parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads")
@ -57,3 +71,7 @@ if __name__ == "__main__":
for data in ["wordnet", "punkt", "punkt_tab"]:
print(f"Downloading nltk {data}...")
nltk.download(data, download_dir=local_dir)
for repo_id in repos:
print(f"Downloading huggingface repo {repo_id}...")
download_model(repo_id)