Updated Dockefile to use cache (#2703)

### What problem does this PR solve?

Updated Dockefile to use cache

### Type of change

- [ ] Bug Fix (non-breaking change which fixes an issue)
- [ ] New Feature (non-breaking change which adds functionality)
- [ ] Documentation Update
- [ ] Refactoring
- [ ] Performance Improvement
- [x] Other (please describe): CI
This commit is contained in:
Zhichang Yu
2024-10-01 17:41:38 +08:00
committed by GitHub
parent 62202b7eff
commit 5f4d2dc4fe
9 changed files with 60 additions and 41 deletions

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
from huggingface_hub import snapshot_download
import nltk
import os
repos = [
@ -12,13 +13,17 @@ repos = [
"maidalun1020/bce-reranker-base_v1",
]
def download_model(repo_id):
local_dir = os.path.join("huggingface.co", repo_id)
local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
os.makedirs(local_dir, exist_ok=True)
snapshot_download(repo_id=repo_id, local_dir=local_dir)
if __name__ == "__main__":
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'wordnet']:
print(f"Downloading nltk {data}...")
nltk.download(data, download_dir=local_dir)
for repo_id in repos:
print(f"Downloading huggingface repo {repo_id}...")
download_model(repo_id)