Maharshi Gor
commited on
Commit
·
bd9e200
1
Parent(s):
98bfd66
Updated requirements
Browse files- requirements.txt +22 -8
- src/hf_datasets_utils.py +3 -2
requirements.txt
CHANGED
|
@@ -1,20 +1,25 @@
|
|
| 1 |
APScheduler
|
| 2 |
black
|
| 3 |
-
|
|
|
|
| 4 |
gradio
|
| 5 |
-
modelscope_studio
|
| 6 |
gradio[oauth]
|
| 7 |
gradio_leaderboard
|
| 8 |
gradio_client
|
|
|
|
|
|
|
|
|
|
| 9 |
huggingface-hub>=0.18.0
|
| 10 |
-
matplotlib
|
| 11 |
-
numpy<2.0.0
|
| 12 |
-
pandas>=2.0.0
|
| 13 |
-
python-dateutil
|
| 14 |
-
tqdm
|
| 15 |
transformers
|
| 16 |
tokenizers>=0.15.0
|
| 17 |
sentencepiece
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
litellm
|
| 19 |
diskcache
|
| 20 |
openai
|
|
@@ -28,6 +33,15 @@ langchain-openai
|
|
| 28 |
langchain-cohere
|
| 29 |
langchain-deepseek
|
| 30 |
json_repair
|
|
|
|
|
|
|
|
|
|
| 31 |
inflect
|
| 32 |
unidecode
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
APScheduler
|
| 2 |
black
|
| 3 |
+
|
| 4 |
+
# UI
|
| 5 |
gradio
|
|
|
|
| 6 |
gradio[oauth]
|
| 7 |
gradio_leaderboard
|
| 8 |
gradio_client
|
| 9 |
+
|
| 10 |
+
# Hugging Face
|
| 11 |
+
datasets
|
| 12 |
huggingface-hub>=0.18.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
transformers
|
| 14 |
tokenizers>=0.15.0
|
| 15 |
sentencepiece
|
| 16 |
+
|
| 17 |
+
# Data Science
|
| 18 |
+
matplotlib
|
| 19 |
+
numpy<2.0.0
|
| 20 |
+
pandas>=2.0.0
|
| 21 |
+
|
| 22 |
+
# LLMs / APIs
|
| 23 |
litellm
|
| 24 |
diskcache
|
| 25 |
openai
|
|
|
|
| 33 |
langchain-cohere
|
| 34 |
langchain-deepseek
|
| 35 |
json_repair
|
| 36 |
+
|
| 37 |
+
# NLP
|
| 38 |
+
nltk
|
| 39 |
inflect
|
| 40 |
unidecode
|
| 41 |
+
|
| 42 |
+
# Misc
|
| 43 |
+
python-dateutil
|
| 44 |
+
tqdm
|
| 45 |
+
rich
|
| 46 |
+
loguru
|
| 47 |
+
tabulate
|
src/hf_datasets_utils.py
CHANGED
|
@@ -7,10 +7,11 @@ api = HfApi()
|
|
| 7 |
def check_and_create_dataset_repo(repo_id: str):
|
| 8 |
try:
|
| 9 |
api.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 10 |
-
print(f"{repo_id} exists")
|
| 11 |
except Exception:
|
| 12 |
-
print(f"
|
| 13 |
api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
def download_dataset_snapshot(repo_id, local_dir):
|
|
|
|
| 7 |
def check_and_create_dataset_repo(repo_id: str):
|
| 8 |
try:
|
| 9 |
api.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 10 |
+
print(f"💁🏻♂️ Repo {repo_id} exists ✅")
|
| 11 |
except Exception:
|
| 12 |
+
print(f"⚠️ Repo {repo_id} doesn't exist, Creating...")
|
| 13 |
api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
|
| 14 |
+
print("💁🏻♂️ Repo created ✅")
|
| 15 |
|
| 16 |
|
| 17 |
def download_dataset_snapshot(repo_id, local_dir):
|