Spaces:
Running
Running
Upload 6 files
Browse files- Dockerfile +0 -6
- add_model_explanations.py +57 -93
- app.py +9 -98
- build_index.py +3 -8
- huggingface_model_descriptions.py +6 -10
- requirements.txt +1 -3
Dockerfile
CHANGED
@@ -19,12 +19,6 @@ WORKDIR /app
|
|
19 |
# This ensures the directory exists and is writable by the user running the process
|
20 |
RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
|
21 |
|
22 |
-
# --- Create the persistent storage mount point directory ---
|
23 |
-
# Create /data within the image and set permissions.
|
24 |
-
# This only helps if HF Spaces actually mounts a writable volume here.
|
25 |
-
RUN mkdir -p /data && chmod -R 777 /data
|
26 |
-
# ---
|
27 |
-
|
28 |
# Copy the requirements file into the container at /app
|
29 |
COPY requirements.txt .
|
30 |
|
|
|
19 |
# This ensures the directory exists and is writable by the user running the process
|
20 |
RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
|
21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
22 |
# Copy the requirements file into the container at /app
|
23 |
COPY requirements.txt .
|
24 |
|
add_model_explanations.py
CHANGED
@@ -3,49 +3,36 @@ import json
|
|
3 |
from typing import Dict, Any, Optional
|
4 |
import logging
|
5 |
import time
|
6 |
-
|
7 |
-
from openai import OpenAI, APIError # Add back OpenAI imports
|
8 |
|
9 |
# Configure logging
|
10 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
11 |
|
12 |
-
|
13 |
-
PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
|
14 |
-
|
15 |
-
# Point to the JSON data within persistent storage
|
16 |
-
MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
|
17 |
EXPLANATION_KEY = "model_explanation_gemini"
|
18 |
DESCRIPTION_KEY = "description"
|
19 |
MAX_RETRIES = 3 # Retries for API calls
|
20 |
RETRY_DELAY_SECONDS = 5 # Delay between retries
|
21 |
|
22 |
-
# --- DeepSeek API Configuration
|
23 |
-
DEEPSEEK_API_KEY_ENV_VAR = "DEEPSEEK_API_KEY"
|
24 |
DEEPSEEK_BASE_URL = "https://api.deepseek.com"
|
25 |
DEEPSEEK_MODEL_NAME = "deepseek-chat"
|
26 |
-
# ---
|
27 |
-
|
28 |
-
# Remove Gemini configuration
|
29 |
-
# GEMINI_API_KEY_ENV_VAR = "GEMINI_API_KEY"
|
30 |
-
# GEMINI_MODEL_NAME = "gemini-1.5-flash-latest"
|
31 |
|
32 |
-
# Global client variable
|
33 |
-
client: Optional[OpenAI] = None
|
34 |
-
# gemini_model: Optional[genai.GenerativeModel] = None # Remove Gemini model variable
|
35 |
|
36 |
def configure_llm_client():
|
37 |
"""Configures the OpenAI client for DeepSeek API using the API key from environment variables."""
|
38 |
global client
|
39 |
-
|
40 |
-
api_key = os.getenv(DEEPSEEK_API_KEY_ENV_VAR) # Use DeepSeek env var
|
41 |
if not api_key:
|
42 |
logging.error(f"Error: {DEEPSEEK_API_KEY_ENV_VAR} environment variable not set.")
|
43 |
-
logging.error("Please set the environment variable
|
44 |
return False
|
45 |
try:
|
46 |
-
# Configure OpenAI client for DeepSeek
|
47 |
client = OpenAI(api_key=api_key, base_url=DEEPSEEK_BASE_URL)
|
48 |
-
logging.info(
|
49 |
return True
|
50 |
except Exception as e:
|
51 |
logging.error(f"Failed to configure DeepSeek API client: {e}")
|
@@ -66,8 +53,7 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
|
|
66 |
Returns:
|
67 |
A short English explanation string from DeepSeek, or None if generation fails.
|
68 |
"""
|
69 |
-
global client
|
70 |
-
# global gemini_model # Remove
|
71 |
if not client:
|
72 |
logging.error(f"[{model_id}] DeepSeek client not configured. Cannot generate explanation.")
|
73 |
return None
|
@@ -76,13 +62,13 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
|
|
76 |
logging.warning(f"[{model_id}] Description is empty or not a string. Skipping explanation generation.")
|
77 |
return None
|
78 |
|
79 |
-
# Truncate very long descriptions
|
80 |
max_desc_length = 4000
|
81 |
if len(description) > max_desc_length:
|
82 |
logging.warning(f"[{model_id}] Description truncated to {max_desc_length} chars for API call.")
|
83 |
description = description[:max_desc_length] + "... [truncated]"
|
84 |
|
85 |
-
# Construct the messages for DeepSeek API
|
86 |
messages = [
|
87 |
{"role": "system", "content": "You are an AI assistant tasked with summarizing Hugging Face model descriptions concisely."},
|
88 |
{"role": "user", "content": (
|
@@ -94,14 +80,10 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
|
|
94 |
)}
|
95 |
]
|
96 |
|
97 |
-
# Remove Gemini prompt construction
|
98 |
-
# prompt = (...)
|
99 |
-
|
100 |
retries = 0
|
101 |
while retries < MAX_RETRIES:
|
102 |
try:
|
103 |
logging.info(f"[{model_id}] Calling DeepSeek API (Attempt {retries + 1}/{MAX_RETRIES})...")
|
104 |
-
# Use OpenAI client call format
|
105 |
response = client.chat.completions.create(
|
106 |
model=DEEPSEEK_MODEL_NAME,
|
107 |
messages=messages,
|
@@ -110,20 +92,13 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
|
|
110 |
temperature=0.2 # Lower temperature for more focused summary
|
111 |
)
|
112 |
|
113 |
-
|
114 |
-
# if not response.candidates: ...
|
115 |
-
|
116 |
-
explanation = response.choices[0].message.content.strip() # Get explanation from OpenAI response structure
|
117 |
logging.info(f"[{model_id}] Explanation received from DeepSeek: '{explanation}'")
|
118 |
-
|
119 |
# Basic post-processing: remove potential quotes
|
120 |
if explanation.startswith('"') and explanation.endswith('"'):
|
121 |
explanation = explanation[1:-1]
|
122 |
-
# Remove Gemini specific post-processing
|
123 |
-
# explanation = explanation.replace('**', '')
|
124 |
return explanation
|
125 |
|
126 |
-
# Restore specific APIError catch for OpenAI client
|
127 |
except APIError as e:
|
128 |
retries += 1
|
129 |
logging.error(f"[{model_id}] DeepSeek API Error (Attempt {retries}/{MAX_RETRIES}): {e}")
|
@@ -133,21 +108,14 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
|
|
133 |
else:
|
134 |
logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation via DeepSeek.")
|
135 |
return None
|
136 |
-
#
|
137 |
-
|
138 |
-
|
139 |
-
logging.error(f"[{model_id}] Unexpected Error during API call (Attempt {retries}/{MAX_RETRIES}): {e}")
|
140 |
-
if retries < MAX_RETRIES:
|
141 |
-
logging.info(f"Retrying in {RETRY_DELAY_SECONDS} seconds...")
|
142 |
-
time.sleep(RETRY_DELAY_SECONDS)
|
143 |
-
else:
|
144 |
-
logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation due to unexpected errors.")
|
145 |
-
return None
|
146 |
|
147 |
-
return None
|
148 |
|
149 |
def process_json_file(filepath: str):
|
150 |
-
"""Reads, updates
|
151 |
model_id = os.path.basename(filepath).replace('.json', '')
|
152 |
logging.info(f"Processing {filepath}...")
|
153 |
|
@@ -156,58 +124,58 @@ def process_json_file(filepath: str):
|
|
156 |
data = json.load(f)
|
157 |
except json.JSONDecodeError:
|
158 |
logging.error(f"[{model_id}] Invalid JSON format in {filepath}. Skipping.")
|
159 |
-
return
|
160 |
except FileNotFoundError:
|
161 |
logging.error(f"[{model_id}] File not found: {filepath}. Skipping.")
|
162 |
-
return
|
163 |
except Exception as e:
|
164 |
logging.error(f"[{model_id}] Error reading {filepath}: {e}. Skipping.")
|
165 |
-
return
|
166 |
|
167 |
if not isinstance(data, dict):
|
168 |
logging.error(f"[{model_id}] Expected JSON object (dict) but got {type(data)} in {filepath}. Skipping.")
|
169 |
-
return
|
170 |
|
171 |
-
|
172 |
-
|
173 |
-
logging.debug(f"[{model_id}] Checking for existing explanation. Key: '{EXPLANATION_KEY}'. Found value: '{existing_explanation}' (Type: {type(existing_explanation)})")
|
174 |
-
if existing_explanation: # Simplified check: Checks for non-empty string, non-None
|
175 |
-
logging.info(f"[{model_id}] Explanation already exists. Skipping generation.")
|
176 |
-
return False # Indicate no update was needed
|
177 |
|
178 |
-
# --- Deletion Logic
|
179 |
-
|
|
|
|
|
|
|
180 |
|
181 |
# --- Generation Logic ---
|
182 |
-
logging.info(f"[{model_id}] Existing explanation is missing or empty. Proceeding with generation.")
|
183 |
-
description = data.get(DESCRIPTION_KEY)
|
184 |
if not description:
|
185 |
logging.warning(f"[{model_id}] Description field is missing or empty. Cannot generate explanation.")
|
186 |
-
return
|
187 |
|
188 |
explanation = generate_explanation(model_id, description) # Try to generate a new one
|
189 |
|
190 |
-
# --- Update and Write Logic ---
|
191 |
if explanation: # Only update if generation was successful
|
192 |
data[EXPLANATION_KEY] = explanation
|
193 |
try:
|
194 |
with open(filepath, 'w', encoding='utf-8') as f:
|
195 |
json.dump(data, f, ensure_ascii=False, indent=4)
|
196 |
-
|
197 |
-
|
|
|
|
|
198 |
except IOError as e:
|
199 |
logging.error(f"[{model_id}] Error writing updated data to {filepath}: {e}")
|
200 |
-
return False
|
201 |
except Exception as e:
|
202 |
logging.error(f"[{model_id}] Unexpected error writing {filepath}: {e}")
|
203 |
-
return False
|
204 |
else: # Explanation generation failed
|
205 |
-
|
206 |
-
|
|
|
|
|
207 |
|
208 |
|
209 |
def main():
|
210 |
"""Main function to iterate through the directory and process files."""
|
|
|
211 |
if not configure_llm_client():
|
212 |
return # Stop if API key is not configured
|
213 |
|
@@ -217,9 +185,8 @@ def main():
|
|
217 |
|
218 |
logging.info(f"Starting processing directory: {MODEL_DATA_DIR}")
|
219 |
processed_files = 0
|
220 |
-
updated_files = 0
|
221 |
-
|
222 |
-
skipped_error = 0 # Count files skipped due to read/write/API errors or no description
|
223 |
|
224 |
all_files = [f for f in os.listdir(MODEL_DATA_DIR) if f.lower().endswith(".json")]
|
225 |
total_files = len(all_files)
|
@@ -229,32 +196,29 @@ def main():
|
|
229 |
filepath = os.path.join(MODEL_DATA_DIR, filename)
|
230 |
logging.info(f"--- Processing file {i+1}/{total_files}: {filename} ---")
|
231 |
try:
|
232 |
-
#
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
pass # Logging within the function indicates reason (skipped existing, API fail, etc.)
|
242 |
|
243 |
except Exception as e:
|
244 |
-
logging.error(f"Unexpected error processing file
|
245 |
-
|
246 |
# Add a small delay between files to potentially avoid hitting rate limits
|
247 |
-
# Adjust delay
|
248 |
-
time.sleep(0.2)
|
249 |
|
250 |
|
251 |
logging.info(f"--- Processing complete ---")
|
|
|
252 |
logging.info(f"Total JSON files found: {total_files}")
|
253 |
logging.info(f"Files processed (attempted): {processed_files}")
|
254 |
-
|
255 |
-
|
256 |
-
# logging.info(f"Files skipped (existing explanation, errors, or no description): {total_files - updated_files}")
|
257 |
-
|
258 |
|
259 |
if __name__ == "__main__":
|
260 |
main()
|
|
|
3 |
from typing import Dict, Any, Optional
|
4 |
import logging
|
5 |
import time
|
6 |
+
from openai import OpenAI, APIError
|
|
|
7 |
|
8 |
# Configure logging
|
9 |
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
10 |
|
11 |
+
MODEL_DATA_DIR = "model_data_json"
|
|
|
|
|
|
|
|
|
12 |
EXPLANATION_KEY = "model_explanation_gemini"
|
13 |
DESCRIPTION_KEY = "description"
|
14 |
MAX_RETRIES = 3 # Retries for API calls
|
15 |
RETRY_DELAY_SECONDS = 5 # Delay between retries
|
16 |
|
17 |
+
# --- DeepSeek API Configuration ---
|
18 |
+
DEEPSEEK_API_KEY_ENV_VAR = "DEEPSEEK_API_KEY"
|
19 |
DEEPSEEK_BASE_URL = "https://api.deepseek.com"
|
20 |
DEEPSEEK_MODEL_NAME = "deepseek-chat"
|
|
|
|
|
|
|
|
|
|
|
21 |
|
22 |
+
# Global client variable
|
23 |
+
client: Optional[OpenAI] = None
|
|
|
24 |
|
25 |
def configure_llm_client():
|
26 |
"""Configures the OpenAI client for DeepSeek API using the API key from environment variables."""
|
27 |
global client
|
28 |
+
api_key = os.getenv(DEEPSEEK_API_KEY_ENV_VAR)
|
|
|
29 |
if not api_key:
|
30 |
logging.error(f"Error: {DEEPSEEK_API_KEY_ENV_VAR} environment variable not set.")
|
31 |
+
logging.error("Please set the environment variable before running the script.")
|
32 |
return False
|
33 |
try:
|
|
|
34 |
client = OpenAI(api_key=api_key, base_url=DEEPSEEK_BASE_URL)
|
35 |
+
logging.info("DeepSeek API client configured successfully.")
|
36 |
return True
|
37 |
except Exception as e:
|
38 |
logging.error(f"Failed to configure DeepSeek API client: {e}")
|
|
|
53 |
Returns:
|
54 |
A short English explanation string from DeepSeek, or None if generation fails.
|
55 |
"""
|
56 |
+
global client
|
|
|
57 |
if not client:
|
58 |
logging.error(f"[{model_id}] DeepSeek client not configured. Cannot generate explanation.")
|
59 |
return None
|
|
|
62 |
logging.warning(f"[{model_id}] Description is empty or not a string. Skipping explanation generation.")
|
63 |
return None
|
64 |
|
65 |
+
# Truncate very long descriptions
|
66 |
max_desc_length = 4000
|
67 |
if len(description) > max_desc_length:
|
68 |
logging.warning(f"[{model_id}] Description truncated to {max_desc_length} chars for API call.")
|
69 |
description = description[:max_desc_length] + "... [truncated]"
|
70 |
|
71 |
+
# Construct the messages for DeepSeek API
|
72 |
messages = [
|
73 |
{"role": "system", "content": "You are an AI assistant tasked with summarizing Hugging Face model descriptions concisely."},
|
74 |
{"role": "user", "content": (
|
|
|
80 |
)}
|
81 |
]
|
82 |
|
|
|
|
|
|
|
83 |
retries = 0
|
84 |
while retries < MAX_RETRIES:
|
85 |
try:
|
86 |
logging.info(f"[{model_id}] Calling DeepSeek API (Attempt {retries + 1}/{MAX_RETRIES})...")
|
|
|
87 |
response = client.chat.completions.create(
|
88 |
model=DEEPSEEK_MODEL_NAME,
|
89 |
messages=messages,
|
|
|
92 |
temperature=0.2 # Lower temperature for more focused summary
|
93 |
)
|
94 |
|
95 |
+
explanation = response.choices[0].message.content.strip()
|
|
|
|
|
|
|
96 |
logging.info(f"[{model_id}] Explanation received from DeepSeek: '{explanation}'")
|
|
|
97 |
# Basic post-processing: remove potential quotes
|
98 |
if explanation.startswith('"') and explanation.endswith('"'):
|
99 |
explanation = explanation[1:-1]
|
|
|
|
|
100 |
return explanation
|
101 |
|
|
|
102 |
except APIError as e:
|
103 |
retries += 1
|
104 |
logging.error(f"[{model_id}] DeepSeek API Error (Attempt {retries}/{MAX_RETRIES}): {e}")
|
|
|
108 |
else:
|
109 |
logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation via DeepSeek.")
|
110 |
return None
|
111 |
+
except Exception as e: # Catch other potential errors
|
112 |
+
logging.error(f"[{model_id}] Unexpected error during DeepSeek API call: {e}")
|
113 |
+
return None # Don't retry for unexpected errors
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
114 |
|
115 |
+
return None
|
116 |
|
117 |
def process_json_file(filepath: str):
|
118 |
+
"""Reads, updates, and writes a single JSON file."""
|
119 |
model_id = os.path.basename(filepath).replace('.json', '')
|
120 |
logging.info(f"Processing {filepath}...")
|
121 |
|
|
|
124 |
data = json.load(f)
|
125 |
except json.JSONDecodeError:
|
126 |
logging.error(f"[{model_id}] Invalid JSON format in {filepath}. Skipping.")
|
127 |
+
return
|
128 |
except FileNotFoundError:
|
129 |
logging.error(f"[{model_id}] File not found: {filepath}. Skipping.")
|
130 |
+
return
|
131 |
except Exception as e:
|
132 |
logging.error(f"[{model_id}] Error reading {filepath}: {e}. Skipping.")
|
133 |
+
return
|
134 |
|
135 |
if not isinstance(data, dict):
|
136 |
logging.error(f"[{model_id}] Expected JSON object (dict) but got {type(data)} in {filepath}. Skipping.")
|
137 |
+
return
|
138 |
|
139 |
+
description = data.get(DESCRIPTION_KEY)
|
140 |
+
explanation_overwritten = False
|
|
|
|
|
|
|
|
|
141 |
|
142 |
+
# --- Deletion Logic: Always remove existing explanation before trying to regenerate ---
|
143 |
+
if EXPLANATION_KEY in data:
|
144 |
+
logging.info(f"[{model_id}] Existing explanation found. Deleting before regenerating.")
|
145 |
+
del data[EXPLANATION_KEY]
|
146 |
+
explanation_overwritten = True # Mark that we intend to replace it
|
147 |
|
148 |
# --- Generation Logic ---
|
|
|
|
|
149 |
if not description:
|
150 |
logging.warning(f"[{model_id}] Description field is missing or empty. Cannot generate explanation.")
|
151 |
+
return
|
152 |
|
153 |
explanation = generate_explanation(model_id, description) # Try to generate a new one
|
154 |
|
155 |
+
# --- Update and Write Logic ---
|
156 |
if explanation: # Only update if generation was successful
|
157 |
data[EXPLANATION_KEY] = explanation
|
158 |
try:
|
159 |
with open(filepath, 'w', encoding='utf-8') as f:
|
160 |
json.dump(data, f, ensure_ascii=False, indent=4)
|
161 |
+
if explanation_overwritten:
|
162 |
+
logging.info(f"[{model_id}] Successfully overwrote and updated {filepath} with new explanation.")
|
163 |
+
else:
|
164 |
+
logging.info(f"[{model_id}] Successfully generated and updated {filepath} with new explanation.")
|
165 |
except IOError as e:
|
166 |
logging.error(f"[{model_id}] Error writing updated data to {filepath}: {e}")
|
|
|
167 |
except Exception as e:
|
168 |
logging.error(f"[{model_id}] Unexpected error writing {filepath}: {e}")
|
|
|
169 |
else: # Explanation generation failed
|
170 |
+
log_message = f"[{model_id}] Failed to generate new explanation for {filepath} via API."
|
171 |
+
if explanation_overwritten:
|
172 |
+
log_message += " Existing explanation was removed but not replaced due to API failure."
|
173 |
+
logging.warning(log_message)
|
174 |
|
175 |
|
176 |
def main():
|
177 |
"""Main function to iterate through the directory and process files."""
|
178 |
+
# Configure LLM client at the start
|
179 |
if not configure_llm_client():
|
180 |
return # Stop if API key is not configured
|
181 |
|
|
|
185 |
|
186 |
logging.info(f"Starting processing directory: {MODEL_DATA_DIR}")
|
187 |
processed_files = 0
|
188 |
+
updated_files = 0
|
189 |
+
skipped_files = 0
|
|
|
190 |
|
191 |
all_files = [f for f in os.listdir(MODEL_DATA_DIR) if f.lower().endswith(".json")]
|
192 |
total_files = len(all_files)
|
|
|
196 |
filepath = os.path.join(MODEL_DATA_DIR, filename)
|
197 |
logging.info(f"--- Processing file {i+1}/{total_files}: {filename} ---")
|
198 |
try:
|
199 |
+
# Check if explanation exists before calling process_json_file
|
200 |
+
# to potentially save API calls if already done.
|
201 |
+
# However, process_json_file already has this check.
|
202 |
+
process_json_file(filepath)
|
203 |
+
processed_files +=1 # Count as processed even if skipped due to existing explanation
|
204 |
+
|
205 |
+
# Check if file was actually updated (optional metric)
|
206 |
+
# Re-read might be inefficient, could return status from process_json_file
|
207 |
+
# For simplicity, we just log success/failure in process_json_file
|
|
|
208 |
|
209 |
except Exception as e:
|
210 |
+
logging.error(f"Unexpected error processing file {filename}: {e}")
|
211 |
+
skipped_files += 1
|
212 |
# Add a small delay between files to potentially avoid hitting rate limits
|
213 |
+
time.sleep(0.5) # Adjust delay as needed
|
|
|
214 |
|
215 |
|
216 |
logging.info(f"--- Processing complete ---")
|
217 |
+
# Refine reporting slightly
|
218 |
logging.info(f"Total JSON files found: {total_files}")
|
219 |
logging.info(f"Files processed (attempted): {processed_files}")
|
220 |
+
# A more accurate count of updated files would require modifying process_json_file to return status
|
221 |
+
logging.info(f"Files skipped due to unexpected errors: {skipped_files}")
|
|
|
|
|
222 |
|
223 |
if __name__ == "__main__":
|
224 |
main()
|
app.py
CHANGED
@@ -5,39 +5,17 @@ from flask_cors import CORS
|
|
5 |
import numpy as np
|
6 |
import json
|
7 |
import traceback
|
8 |
-
import logging # Added for background task logging
|
9 |
-
import threading # Added for background task
|
10 |
-
import time # Added for background task
|
11 |
-
import schedule # Added for background task
|
12 |
-
|
13 |
-
# --- Import the daily update function ---
|
14 |
-
try:
|
15 |
-
from daily_update import main as run_daily_update
|
16 |
-
# Set up logging for the daily_update module if it uses logging
|
17 |
-
# logging.getLogger('daily_update').setLevel(logging.INFO) # Example
|
18 |
-
except ImportError:
|
19 |
-
logging.error("Failed to import daily_update.py. The daily update task will not run.")
|
20 |
-
run_daily_update = None # Define as None if import fails
|
21 |
-
# ---
|
22 |
|
23 |
app = Flask(__name__) # Create app object FIRST
|
24 |
-
|
25 |
-
# Define the base persistent storage path (must match other scripts)
|
26 |
-
PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
|
27 |
-
|
28 |
-
# Configure Flask app logging (optional but recommended)
|
29 |
-
# app.logger.setLevel(logging.INFO)
|
30 |
-
|
31 |
# Allow requests from the Vercel frontend and localhost for development
|
32 |
CORS(app, origins=["http://127.0.0.1:3000", "http://localhost:3000", "https://rag-huggingface.vercel.app"], supports_credentials=True)
|
33 |
|
34 |
# --- Configuration ---
|
35 |
-
|
36 |
-
|
37 |
-
MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
|
38 |
EMBEDDING_MODEL = 'all-mpnet-base-v2'
|
39 |
-
#
|
40 |
-
MODEL_DATA_DIR = os.path.join(
|
41 |
# ---
|
42 |
|
43 |
# --- Global variables for resources ---
|
@@ -76,8 +54,7 @@ def load_resources():
|
|
76 |
print("Sentence transformer model loaded successfully.")
|
77 |
|
78 |
# Load FAISS Index
|
79 |
-
|
80 |
-
index_path = INDEX_FILE # Use configured path
|
81 |
print(f"Loading FAISS index from: {index_path}")
|
82 |
if not os.path.exists(index_path):
|
83 |
raise FileNotFoundError(f"FAISS index file not found at {index_path}")
|
@@ -86,8 +63,7 @@ def load_resources():
|
|
86 |
print("FAISS index loaded successfully.")
|
87 |
|
88 |
# Load Index-to-Metadata Map
|
89 |
-
|
90 |
-
map_path = MAP_FILE # Use configured path
|
91 |
print(f"Loading index-to-Metadata map from: {map_path}")
|
92 |
if not os.path.exists(map_path):
|
93 |
raise FileNotFoundError(f"Metadata map file not found at {map_path}")
|
@@ -101,8 +77,8 @@ def load_resources():
|
|
101 |
|
102 |
except FileNotFoundError as fnf_error:
|
103 |
print(f"Error: {fnf_error}")
|
104 |
-
print(f"Please ensure {
|
105 |
-
print("You might need to run
|
106 |
RESOURCES_LOADED = False # Keep as False
|
107 |
except ImportError as import_error:
|
108 |
print(f"Import Error loading resources: {import_error}")
|
@@ -118,71 +94,6 @@ def load_resources():
|
|
118 |
load_resources()
|
119 |
# ---
|
120 |
|
121 |
-
# --- Background Update Task ---
|
122 |
-
|
123 |
-
UPDATE_INTERVAL_HOURS = 24 # Check every 24 hours
|
124 |
-
UPDATE_TIME = "02:00" # Time to run the update (24-hour format)
|
125 |
-
|
126 |
-
def run_update_task():
|
127 |
-
"""Wrapper function to run the daily update and handle errors."""
|
128 |
-
if run_daily_update is None:
|
129 |
-
logging.warning("run_daily_update function not available (import failed). Skipping task.")
|
130 |
-
return
|
131 |
-
|
132 |
-
logging.info(f"Background task: Starting daily update check (scheduled for {UPDATE_TIME})...")
|
133 |
-
try:
|
134 |
-
# Make sure the DEEPSEEK_API_KEY is set before running
|
135 |
-
if not os.getenv("DEEPSEEK_API_KEY"):
|
136 |
-
logging.error("Background task: DEEPSEEK_API_KEY not set. Daily update cannot run.")
|
137 |
-
return # Don't run if key is missing
|
138 |
-
|
139 |
-
run_daily_update() # Call the main function from daily_update.py
|
140 |
-
logging.info("Background task: Daily update process finished.")
|
141 |
-
except Exception as e:
|
142 |
-
logging.error(f"Background task: Error during daily update execution: {e}")
|
143 |
-
logging.error(traceback.format_exc())
|
144 |
-
|
145 |
-
def background_scheduler():
|
146 |
-
"""Runs the scheduler loop in a background thread."""
|
147 |
-
logging.info(f"Background scheduler started. Will run update task daily around {UPDATE_TIME}.")
|
148 |
-
|
149 |
-
if run_daily_update is None:
|
150 |
-
logging.error("Background scheduler: daily_update.py could not be imported. Scheduler will not run tasks.")
|
151 |
-
return # Stop the thread if the core function isn't available
|
152 |
-
|
153 |
-
# Schedule the job
|
154 |
-
# schedule.every(UPDATE_INTERVAL_HOURS).hours.do(run_update_task) # Alternative: run every X hours
|
155 |
-
schedule.every().day.at(UPDATE_TIME).do(run_update_task)
|
156 |
-
logging.info(f"Scheduled daily update task for {UPDATE_TIME}.")
|
157 |
-
|
158 |
-
# --- Run once immediately on startup ---
|
159 |
-
logging.info("Background task: Running initial update check on startup...")
|
160 |
-
run_update_task() # Call the task function directly
|
161 |
-
logging.info("Background task: Initial update check finished.")
|
162 |
-
# ---
|
163 |
-
|
164 |
-
while True:
|
165 |
-
schedule.run_pending()
|
166 |
-
time.sleep(60) # Check every 60 seconds if a task is due
|
167 |
-
|
168 |
-
# Start the background scheduler thread only if this is the main process
|
169 |
-
# This check helps prevent duplicate schedulers when using workers (like Gunicorn)
|
170 |
-
# Note: This might not be perfectly reliable with all WSGI servers/configs.
|
171 |
-
# Consider using a more robust method for ensuring single execution if needed (e.g., file lock, external process manager)
|
172 |
-
if os.environ.get("WERKZEUG_RUN_MAIN") == "true" or os.environ.get("FLASK_ENV") != "development":
|
173 |
-
# Start only in main Werkzeug process OR if not in Flask development mode (like production with Gunicorn)
|
174 |
-
# Check if the function is available before starting thread
|
175 |
-
if run_daily_update is not None:
|
176 |
-
scheduler_thread = threading.Thread(target=background_scheduler, daemon=True)
|
177 |
-
scheduler_thread.start()
|
178 |
-
logging.info("Background scheduler thread started.")
|
179 |
-
else:
|
180 |
-
logging.warning("Background scheduler thread NOT started because daily_update.py failed to import.")
|
181 |
-
else:
|
182 |
-
logging.info("Skipping background scheduler start in Werkzeug reloader process.")
|
183 |
-
|
184 |
-
# --- End Background Update Task ---
|
185 |
-
|
186 |
@app.route('/search', methods=['POST'])
|
187 |
def search():
|
188 |
"""Handles search requests, embedding the query and searching the FAISS index."""
|
@@ -241,7 +152,7 @@ def search():
|
|
241 |
# --- Add description from model_data_json ---
|
242 |
model_id = metadata.get('model_id')
|
243 |
description = None
|
244 |
-
# Use the globally defined
|
245 |
if model_id and MODEL_DATA_DIR:
|
246 |
filename = model_id.replace('/', '_') + '.json'
|
247 |
filepath = os.path.join(MODEL_DATA_DIR, filename)
|
|
|
5 |
import numpy as np
|
6 |
import json
|
7 |
import traceback
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
app = Flask(__name__) # Create app object FIRST
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
# Allow requests from the Vercel frontend and localhost for development
|
11 |
CORS(app, origins=["http://127.0.0.1:3000", "http://localhost:3000", "https://rag-huggingface.vercel.app"], supports_credentials=True)
|
12 |
|
13 |
# --- Configuration ---
|
14 |
+
INDEX_FILE = "index.faiss"
|
15 |
+
MAP_FILE = "index_to_metadata.pkl"
|
|
|
16 |
EMBEDDING_MODEL = 'all-mpnet-base-v2'
|
17 |
+
# Corrected path joining for model_data_json - relative to app.py location
|
18 |
+
MODEL_DATA_DIR = os.path.join(os.path.dirname(__file__), 'model_data_json')
|
19 |
# ---
|
20 |
|
21 |
# --- Global variables for resources ---
|
|
|
54 |
print("Sentence transformer model loaded successfully.")
|
55 |
|
56 |
# Load FAISS Index
|
57 |
+
index_path = os.path.join(os.path.dirname(__file__), INDEX_FILE)
|
|
|
58 |
print(f"Loading FAISS index from: {index_path}")
|
59 |
if not os.path.exists(index_path):
|
60 |
raise FileNotFoundError(f"FAISS index file not found at {index_path}")
|
|
|
63 |
print("FAISS index loaded successfully.")
|
64 |
|
65 |
# Load Index-to-Metadata Map
|
66 |
+
map_path = os.path.join(os.path.dirname(__file__), MAP_FILE)
|
|
|
67 |
print(f"Loading index-to-Metadata map from: {map_path}")
|
68 |
if not os.path.exists(map_path):
|
69 |
raise FileNotFoundError(f"Metadata map file not found at {map_path}")
|
|
|
77 |
|
78 |
except FileNotFoundError as fnf_error:
|
79 |
print(f"Error: {fnf_error}")
|
80 |
+
print(f"Please ensure {INDEX_FILE} and {MAP_FILE} exist in the 'backend' directory relative to app.py.")
|
81 |
+
print("You might need to run 'python build_index.py' first.")
|
82 |
RESOURCES_LOADED = False # Keep as False
|
83 |
except ImportError as import_error:
|
84 |
print(f"Import Error loading resources: {import_error}")
|
|
|
94 |
load_resources()
|
95 |
# ---
|
96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
97 |
@app.route('/search', methods=['POST'])
|
98 |
def search():
|
99 |
"""Handles search requests, embedding the query and searching the FAISS index."""
|
|
|
152 |
# --- Add description from model_data_json ---
|
153 |
model_id = metadata.get('model_id')
|
154 |
description = None
|
155 |
+
# Use the globally defined and corrected MODEL_DATA_DIR
|
156 |
if model_id and MODEL_DATA_DIR:
|
157 |
filename = model_id.replace('/', '_') + '.json'
|
158 |
filepath = os.path.join(MODEL_DATA_DIR, filename)
|
build_index.py
CHANGED
@@ -7,15 +7,10 @@ import pickle
|
|
7 |
import json # Import json module
|
8 |
from tqdm import tqdm
|
9 |
|
10 |
-
# Define the base persistent storage path (must match other scripts)
|
11 |
-
PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
|
12 |
-
|
13 |
# --- Configuration ---
|
14 |
-
#
|
15 |
-
|
16 |
-
|
17 |
-
INDEX_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index.faiss")
|
18 |
-
MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
|
19 |
EMBEDDING_MODEL = 'all-mpnet-base-v2' # Efficient and good quality model
|
20 |
ENCODE_BATCH_SIZE = 32 # Process descriptions in smaller batches
|
21 |
# Tags to exclude from indexing text
|
|
|
7 |
import json # Import json module
|
8 |
from tqdm import tqdm
|
9 |
|
|
|
|
|
|
|
10 |
# --- Configuration ---
|
11 |
+
MODEL_DATA_DIR = "model_data_json" # Path to downloaded JSON data
|
12 |
+
INDEX_FILE = "index.faiss"
|
13 |
+
MAP_FILE = "index_to_metadata.pkl" # Changed filename to reflect content
|
|
|
|
|
14 |
EMBEDDING_MODEL = 'all-mpnet-base-v2' # Efficient and good quality model
|
15 |
ENCODE_BATCH_SIZE = 32 # Process descriptions in smaller batches
|
16 |
# Tags to exclude from indexing text
|
huggingface_model_descriptions.py
CHANGED
@@ -10,11 +10,8 @@ from requests.exceptions import RequestException
|
|
10 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
11 |
import pickle # Add pickle for caching
|
12 |
|
13 |
-
#
|
14 |
-
|
15 |
-
|
16 |
-
# Create a directory to store JSON data within persistent storage
|
17 |
-
OUTPUT_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
|
18 |
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
19 |
|
20 |
# Number of worker threads for parallel processing - REDUCED
|
@@ -44,8 +41,7 @@ def clean_readme_content(text):
|
|
44 |
return text
|
45 |
# ---
|
46 |
|
47 |
-
|
48 |
-
MODELS_CACHE_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "models_list_cache.pkl") # File to cache the raw model list
|
49 |
|
50 |
def get_all_models_with_downloads(min_downloads=10000):
|
51 |
"""Fetch all models from Hugging Face with at least min_downloads, using a local cache for the list."""
|
@@ -70,7 +66,7 @@ def get_all_models_with_downloads(min_downloads=10000):
|
|
70 |
api = HfApi()
|
71 |
print("HfApi initialized. Calling list_models...")
|
72 |
# Fetch the iterator
|
73 |
-
models_iterator = api.list_models(sort="downloads", direction=-1, fetch_config=False, cardData=
|
74 |
print("list_models call returned. Converting iterator to list...")
|
75 |
# Convert the iterator to a list TO ALLOW CACHING
|
76 |
models_list = list(models_iterator)
|
@@ -158,9 +154,9 @@ def get_model_readme(model_id):
|
|
158 |
return None
|
159 |
|
160 |
def get_filename_for_model(model_id):
|
161 |
-
"""Generate JSON filename for a model
|
162 |
safe_id = model_id.replace("/", "_")
|
163 |
-
return os.path.join(OUTPUT_DIR, f"{safe_id}.json") #
|
164 |
|
165 |
def save_model_data(model_id, data):
|
166 |
"""Save model data (description, tags, downloads) to a JSON file."""
|
|
|
10 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
11 |
import pickle # Add pickle for caching
|
12 |
|
13 |
+
# Create a directory to store JSON data
|
14 |
+
OUTPUT_DIR = "model_data_json"
|
|
|
|
|
|
|
15 |
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
16 |
|
17 |
# Number of worker threads for parallel processing - REDUCED
|
|
|
41 |
return text
|
42 |
# ---
|
43 |
|
44 |
+
MODELS_CACHE_FILE = "models_list_cache.pkl" # File to cache the raw model list
|
|
|
45 |
|
46 |
def get_all_models_with_downloads(min_downloads=10000):
|
47 |
"""Fetch all models from Hugging Face with at least min_downloads, using a local cache for the list."""
|
|
|
66 |
api = HfApi()
|
67 |
print("HfApi initialized. Calling list_models...")
|
68 |
# Fetch the iterator
|
69 |
+
models_iterator = api.list_models(sort="downloads", direction=-1, fetch_config=False, cardData=True)
|
70 |
print("list_models call returned. Converting iterator to list...")
|
71 |
# Convert the iterator to a list TO ALLOW CACHING
|
72 |
models_list = list(models_iterator)
|
|
|
154 |
return None
|
155 |
|
156 |
def get_filename_for_model(model_id):
|
157 |
+
"""Generate JSON filename for a model"""
|
158 |
safe_id = model_id.replace("/", "_")
|
159 |
+
return os.path.join(OUTPUT_DIR, f"{safe_id}.json") # Change extension to .json
|
160 |
|
161 |
def save_model_data(model_id, data):
|
162 |
"""Save model data (description, tags, downloads) to a JSON file."""
|
requirements.txt
CHANGED
@@ -4,6 +4,4 @@ sentence-transformers>=2.3.0
|
|
4 |
numpy>=1.20.0
|
5 |
faiss-cpu>=1.7.0 # Use faiss-gpu if you need GPU support on HF Spaces
|
6 |
huggingface-hub>=0.15.1 # Version compatible with sentence-transformers >= 2.3.0
|
7 |
-
gunicorn # Added for deployment on Hugging Face Spaces
|
8 |
-
openai>=1.0.0 # Added back for DeepSeek API via OpenAI client
|
9 |
-
schedule>=1.0.0 # Added for in-app scheduling
|
|
|
4 |
numpy>=1.20.0
|
5 |
faiss-cpu>=1.7.0 # Use faiss-gpu if you need GPU support on HF Spaces
|
6 |
huggingface-hub>=0.15.1 # Version compatible with sentence-transformers >= 2.3.0
|
7 |
+
gunicorn # Added for deployment on Hugging Face Spaces
|
|
|
|