Jimin Huang
commited on
Commit
·
47f193c
1
Parent(s):
25729c8
fix: alter dataset names
Browse files
backend/app/config/hf_config.py
CHANGED
@@ -24,11 +24,11 @@ API = HfApi(token=HF_TOKEN)
|
|
24 |
|
25 |
# Repository configuration
|
26 |
QUEUE_REPO = f"{HF_ORGANIZATION}/requests"
|
27 |
-
AGGREGATED_REPO = f"{HF_ORGANIZATION}/contents"
|
28 |
VOTES_REPO = f"{HF_ORGANIZATION}/votes"
|
29 |
MAINTAINERS_HIGHLIGHT_REPO = f"{HF_ORGANIZATION}/maintainers-highlight"
|
30 |
|
31 |
# File paths from cache config
|
32 |
VOTES_PATH = cache_config.votes_file
|
33 |
EVAL_REQUESTS_PATH = cache_config.eval_requests_file
|
34 |
-
MODEL_CACHE_DIR = cache_config.models_cache
|
|
|
24 |
|
25 |
# Repository configuration
|
26 |
QUEUE_REPO = f"{HF_ORGANIZATION}/requests"
|
27 |
+
AGGREGATED_REPO = f"{HF_ORGANIZATION}/greek-contents"
|
28 |
VOTES_REPO = f"{HF_ORGANIZATION}/votes"
|
29 |
MAINTAINERS_HIGHLIGHT_REPO = f"{HF_ORGANIZATION}/maintainers-highlight"
|
30 |
|
31 |
# File paths from cache config
|
32 |
VOTES_PATH = cache_config.votes_file
|
33 |
EVAL_REQUESTS_PATH = cache_config.eval_requests_file
|
34 |
+
MODEL_CACHE_DIR = cache_config.models_cache
|
backend/app/services/leaderboard.py
CHANGED
@@ -17,10 +17,10 @@ class LeaderboardService:
|
|
17 |
"""Fetch raw leaderboard data from HuggingFace dataset"""
|
18 |
try:
|
19 |
logger.info(LogFormatter.section("FETCHING LEADERBOARD DATA"))
|
20 |
-
logger.info(LogFormatter.info(f"Loading dataset from {HF_ORGANIZATION}/contents"))
|
21 |
|
22 |
dataset = datasets.load_dataset(
|
23 |
-
f"{HF_ORGANIZATION}/contents",
|
24 |
cache_dir=cache_config.get_cache_path("datasets")
|
25 |
)["train"]
|
26 |
|
|
|
17 |
"""Fetch raw leaderboard data from HuggingFace dataset"""
|
18 |
try:
|
19 |
logger.info(LogFormatter.section("FETCHING LEADERBOARD DATA"))
|
20 |
+
logger.info(LogFormatter.info(f"Loading dataset from {HF_ORGANIZATION}/greek-contents"))
|
21 |
|
22 |
dataset = datasets.load_dataset(
|
23 |
+
f"{HF_ORGANIZATION}/greek-contents",
|
24 |
cache_dir=cache_config.get_cache_path("datasets")
|
25 |
)["train"]
|
26 |
|
backend/utils/analyze_prod_models.py
CHANGED
@@ -31,13 +31,13 @@ def count_evaluated_models():
|
|
31 |
"""Count the number of evaluated models"""
|
32 |
try:
|
33 |
# Get dataset info
|
34 |
-
dataset_info = api.dataset_info(repo_id=f"{HF_ORGANIZATION}/contents", repo_type="dataset")
|
35 |
|
36 |
# Get file list
|
37 |
-
files = api.list_repo_files(f"{HF_ORGANIZATION}/contents", repo_type="dataset")
|
38 |
|
39 |
# Get last commit info
|
40 |
-
commits = api.list_repo_commits(f"{HF_ORGANIZATION}/contents", repo_type="dataset")
|
41 |
last_commit = next(commits, None)
|
42 |
|
43 |
# Count lines in jsonl files
|
@@ -47,7 +47,7 @@ def count_evaluated_models():
|
|
47 |
try:
|
48 |
# Download file content
|
49 |
content = api.hf_hub_download(
|
50 |
-
repo_id=f"{HF_ORGANIZATION}/contents",
|
51 |
filename=file,
|
52 |
repo_type="dataset"
|
53 |
)
|
@@ -103,4 +103,4 @@ def main():
|
|
103 |
return {"error": str(e)}
|
104 |
|
105 |
if __name__ == "__main__":
|
106 |
-
main()
|
|
|
31 |
"""Count the number of evaluated models"""
|
32 |
try:
|
33 |
# Get dataset info
|
34 |
+
dataset_info = api.dataset_info(repo_id=f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
|
35 |
|
36 |
# Get file list
|
37 |
+
files = api.list_repo_files(f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
|
38 |
|
39 |
# Get last commit info
|
40 |
+
commits = api.list_repo_commits(f"{HF_ORGANIZATION}/greek-contents", repo_type="dataset")
|
41 |
last_commit = next(commits, None)
|
42 |
|
43 |
# Count lines in jsonl files
|
|
|
47 |
try:
|
48 |
# Download file content
|
49 |
content = api.hf_hub_download(
|
50 |
+
repo_id=f"{HF_ORGANIZATION}/greek-contents",
|
51 |
filename=file,
|
52 |
repo_type="dataset"
|
53 |
)
|
|
|
103 |
return {"error": str(e)}
|
104 |
|
105 |
if __name__ == "__main__":
|
106 |
+
main()
|
backend/utils/sync_datasets_locally.py
CHANGED
@@ -29,7 +29,7 @@ DATASET_NAMES = [
|
|
29 |
"votes",
|
30 |
"results",
|
31 |
"requests",
|
32 |
-
"contents",
|
33 |
"maintainers-highlight",
|
34 |
]
|
35 |
|
@@ -127,4 +127,4 @@ def copy_datasets():
|
|
127 |
logger.error(f"❌ Global error: {str(e)}")
|
128 |
|
129 |
if __name__ == "__main__":
|
130 |
-
copy_datasets()
|
|
|
29 |
"votes",
|
30 |
"results",
|
31 |
"requests",
|
32 |
+
"greek-contents",
|
33 |
"maintainers-highlight",
|
34 |
]
|
35 |
|
|
|
127 |
logger.error(f"❌ Global error: {str(e)}")
|
128 |
|
129 |
if __name__ == "__main__":
|
130 |
+
copy_datasets()
|