shayan5422 commited on
Commit
8181a7b
·
verified ·
1 Parent(s): e4b732c

Upload 6 files

Browse files
Dockerfile CHANGED
@@ -19,12 +19,6 @@ WORKDIR /app
19
  # This ensures the directory exists and is writable by the user running the process
20
  RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
21
 
22
- # --- Create the persistent storage mount point directory ---
23
- # Create /data within the image and set permissions.
24
- # This only helps if HF Spaces actually mounts a writable volume here.
25
- RUN mkdir -p /data && chmod -R 777 /data
26
- # ---
27
-
28
  # Copy the requirements file into the container at /app
29
  COPY requirements.txt .
30
 
 
19
  # This ensures the directory exists and is writable by the user running the process
20
  RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
21
 
 
 
 
 
 
 
22
  # Copy the requirements file into the container at /app
23
  COPY requirements.txt .
24
 
add_model_explanations.py CHANGED
@@ -3,49 +3,36 @@ import json
3
  from typing import Dict, Any, Optional
4
  import logging
5
  import time
6
- # import google.generativeai as genai # Remove Gemini import
7
- from openai import OpenAI, APIError # Add back OpenAI imports
8
 
9
  # Configure logging
10
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
11
 
12
- # Define the base persistent storage path (must match other scripts)
13
- PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
14
-
15
- # Point to the JSON data within persistent storage
16
- MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
17
  EXPLANATION_KEY = "model_explanation_gemini"
18
  DESCRIPTION_KEY = "description"
19
  MAX_RETRIES = 3 # Retries for API calls
20
  RETRY_DELAY_SECONDS = 5 # Delay between retries
21
 
22
- # --- DeepSeek API Configuration (Restored) ---
23
- DEEPSEEK_API_KEY_ENV_VAR = "DEEPSEEK_API_KEY" # Environment variable for the key
24
  DEEPSEEK_BASE_URL = "https://api.deepseek.com"
25
  DEEPSEEK_MODEL_NAME = "deepseek-chat"
26
- # ---
27
-
28
- # Remove Gemini configuration
29
- # GEMINI_API_KEY_ENV_VAR = "GEMINI_API_KEY"
30
- # GEMINI_MODEL_NAME = "gemini-1.5-flash-latest"
31
 
32
- # Global client variable for DeepSeek/OpenAI client
33
- client: Optional[OpenAI] = None # Use OpenAI client type
34
- # gemini_model: Optional[genai.GenerativeModel] = None # Remove Gemini model variable
35
 
36
  def configure_llm_client():
37
  """Configures the OpenAI client for DeepSeek API using the API key from environment variables."""
38
  global client
39
- # global gemini_model # Remove
40
- api_key = os.getenv(DEEPSEEK_API_KEY_ENV_VAR) # Use DeepSeek env var
41
  if not api_key:
42
  logging.error(f"Error: {DEEPSEEK_API_KEY_ENV_VAR} environment variable not set.")
43
- logging.error("Please set the environment variable with your DeepSeek API key before running the script.")
44
  return False
45
  try:
46
- # Configure OpenAI client for DeepSeek
47
  client = OpenAI(api_key=api_key, base_url=DEEPSEEK_BASE_URL)
48
- logging.info(f"DeepSeek API client configured successfully for model: {DEEPSEEK_MODEL_NAME}.")
49
  return True
50
  except Exception as e:
51
  logging.error(f"Failed to configure DeepSeek API client: {e}")
@@ -66,8 +53,7 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
66
  Returns:
67
  A short English explanation string from DeepSeek, or None if generation fails.
68
  """
69
- global client # Use OpenAI client
70
- # global gemini_model # Remove
71
  if not client:
72
  logging.error(f"[{model_id}] DeepSeek client not configured. Cannot generate explanation.")
73
  return None
@@ -76,13 +62,13 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
76
  logging.warning(f"[{model_id}] Description is empty or not a string. Skipping explanation generation.")
77
  return None
78
 
79
- # Truncate very long descriptions (adjust limit back if needed for DeepSeek)
80
  max_desc_length = 4000
81
  if len(description) > max_desc_length:
82
  logging.warning(f"[{model_id}] Description truncated to {max_desc_length} chars for API call.")
83
  description = description[:max_desc_length] + "... [truncated]"
84
 
85
- # Construct the messages for DeepSeek API (Restore original format)
86
  messages = [
87
  {"role": "system", "content": "You are an AI assistant tasked with summarizing Hugging Face model descriptions concisely."},
88
  {"role": "user", "content": (
@@ -94,14 +80,10 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
94
  )}
95
  ]
96
 
97
- # Remove Gemini prompt construction
98
- # prompt = (...)
99
-
100
  retries = 0
101
  while retries < MAX_RETRIES:
102
  try:
103
  logging.info(f"[{model_id}] Calling DeepSeek API (Attempt {retries + 1}/{MAX_RETRIES})...")
104
- # Use OpenAI client call format
105
  response = client.chat.completions.create(
106
  model=DEEPSEEK_MODEL_NAME,
107
  messages=messages,
@@ -110,20 +92,13 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
110
  temperature=0.2 # Lower temperature for more focused summary
111
  )
112
 
113
- # Remove Gemini response handling
114
- # if not response.candidates: ...
115
-
116
- explanation = response.choices[0].message.content.strip() # Get explanation from OpenAI response structure
117
  logging.info(f"[{model_id}] Explanation received from DeepSeek: '{explanation}'")
118
-
119
  # Basic post-processing: remove potential quotes
120
  if explanation.startswith('"') and explanation.endswith('"'):
121
  explanation = explanation[1:-1]
122
- # Remove Gemini specific post-processing
123
- # explanation = explanation.replace('**', '')
124
  return explanation
125
 
126
- # Restore specific APIError catch for OpenAI client
127
  except APIError as e:
128
  retries += 1
129
  logging.error(f"[{model_id}] DeepSeek API Error (Attempt {retries}/{MAX_RETRIES}): {e}")
@@ -133,21 +108,14 @@ def generate_explanation(model_id: str, description: str) -> Optional[str]:
133
  else:
134
  logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation via DeepSeek.")
135
  return None
136
- # Keep general Exception catch
137
- except Exception as e:
138
- retries += 1 # Consider retrying general errors too or handle differently
139
- logging.error(f"[{model_id}] Unexpected Error during API call (Attempt {retries}/{MAX_RETRIES}): {e}")
140
- if retries < MAX_RETRIES:
141
- logging.info(f"Retrying in {RETRY_DELAY_SECONDS} seconds...")
142
- time.sleep(RETRY_DELAY_SECONDS)
143
- else:
144
- logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation due to unexpected errors.")
145
- return None
146
 
147
- return None # Should not be reached if loop finishes without returning
148
 
149
  def process_json_file(filepath: str):
150
- """Reads, updates (only if explanation missing), and writes a single JSON file."""
151
  model_id = os.path.basename(filepath).replace('.json', '')
152
  logging.info(f"Processing {filepath}...")
153
 
@@ -156,58 +124,58 @@ def process_json_file(filepath: str):
156
  data = json.load(f)
157
  except json.JSONDecodeError:
158
  logging.error(f"[{model_id}] Invalid JSON format in {filepath}. Skipping.")
159
- return False # Indicate failure/skip
160
  except FileNotFoundError:
161
  logging.error(f"[{model_id}] File not found: {filepath}. Skipping.")
162
- return False
163
  except Exception as e:
164
  logging.error(f"[{model_id}] Error reading {filepath}: {e}. Skipping.")
165
- return False
166
 
167
  if not isinstance(data, dict):
168
  logging.error(f"[{model_id}] Expected JSON object (dict) but got {type(data)} in {filepath}. Skipping.")
169
- return False
170
 
171
- # --- Check if explanation already exists ---
172
- existing_explanation = data.get(EXPLANATION_KEY)
173
- logging.debug(f"[{model_id}] Checking for existing explanation. Key: '{EXPLANATION_KEY}'. Found value: '{existing_explanation}' (Type: {type(existing_explanation)})")
174
- if existing_explanation: # Simplified check: Checks for non-empty string, non-None
175
- logging.info(f"[{model_id}] Explanation already exists. Skipping generation.")
176
- return False # Indicate no update was needed
177
 
178
- # --- Deletion Logic REMOVED ---
179
- # if EXPLANATION_KEY in data: ...
 
 
 
180
 
181
  # --- Generation Logic ---
182
- logging.info(f"[{model_id}] Existing explanation is missing or empty. Proceeding with generation.")
183
- description = data.get(DESCRIPTION_KEY)
184
  if not description:
185
  logging.warning(f"[{model_id}] Description field is missing or empty. Cannot generate explanation.")
186
- return False # Cannot generate, so no update possible
187
 
188
  explanation = generate_explanation(model_id, description) # Try to generate a new one
189
 
190
- # --- Update and Write Logic ---
191
  if explanation: # Only update if generation was successful
192
  data[EXPLANATION_KEY] = explanation
193
  try:
194
  with open(filepath, 'w', encoding='utf-8') as f:
195
  json.dump(data, f, ensure_ascii=False, indent=4)
196
- logging.info(f"[{model_id}] Successfully generated and updated {filepath} with new explanation.")
197
- return True # Indicate success/update
 
 
198
  except IOError as e:
199
  logging.error(f"[{model_id}] Error writing updated data to {filepath}: {e}")
200
- return False
201
  except Exception as e:
202
  logging.error(f"[{model_id}] Unexpected error writing {filepath}: {e}")
203
- return False
204
  else: # Explanation generation failed
205
- logging.warning(f"[{model_id}] Failed to generate new explanation for {filepath} via API. File not updated.")
206
- return False # Indicate failure/no update
 
 
207
 
208
 
209
  def main():
210
  """Main function to iterate through the directory and process files."""
 
211
  if not configure_llm_client():
212
  return # Stop if API key is not configured
213
 
@@ -217,9 +185,8 @@ def main():
217
 
218
  logging.info(f"Starting processing directory: {MODEL_DATA_DIR}")
219
  processed_files = 0
220
- updated_files = 0 # Count files actually updated
221
- skipped_existing = 0 # Count files skipped because explanation existed
222
- skipped_error = 0 # Count files skipped due to read/write/API errors or no description
223
 
224
  all_files = [f for f in os.listdir(MODEL_DATA_DIR) if f.lower().endswith(".json")]
225
  total_files = len(all_files)
@@ -229,32 +196,29 @@ def main():
229
  filepath = os.path.join(MODEL_DATA_DIR, filename)
230
  logging.info(f"--- Processing file {i+1}/{total_files}: {filename} ---")
231
  try:
232
- # process_json_file now returns True if updated, False otherwise
233
- updated = process_json_file(filepath)
234
- processed_files += 1
235
- if updated:
236
- updated_files += 1
237
- else:
238
- # Need to differentiate why it wasn't updated. Re-read is inefficient.
239
- # Let's rely on logs from process_json_file for now.
240
- # A better way would be for process_json_file to return status codes.
241
- pass # Logging within the function indicates reason (skipped existing, API fail, etc.)
242
 
243
  except Exception as e:
244
- logging.error(f"Unexpected error processing file loop for {filename}: {e}")
245
- skipped_error += 1 # Count generic loop errors
246
  # Add a small delay between files to potentially avoid hitting rate limits
247
- # Adjust delay based on Gemini quota/limits (might need less than 0.5s)
248
- time.sleep(0.2)
249
 
250
 
251
  logging.info(f"--- Processing complete ---")
 
252
  logging.info(f"Total JSON files found: {total_files}")
253
  logging.info(f"Files processed (attempted): {processed_files}")
254
- logging.info(f"Files successfully updated with new explanation: {updated_files}")
255
- # Cannot precisely count skipped_existing vs skipped_error without better return values
256
- # logging.info(f"Files skipped (existing explanation, errors, or no description): {total_files - updated_files}")
257
-
258
 
259
  if __name__ == "__main__":
260
  main()
 
3
  from typing import Dict, Any, Optional
4
  import logging
5
  import time
6
+ from openai import OpenAI, APIError
 
7
 
8
  # Configure logging
9
  logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
10
 
11
+ MODEL_DATA_DIR = "model_data_json"
 
 
 
 
12
  EXPLANATION_KEY = "model_explanation_gemini"
13
  DESCRIPTION_KEY = "description"
14
  MAX_RETRIES = 3 # Retries for API calls
15
  RETRY_DELAY_SECONDS = 5 # Delay between retries
16
 
17
+ # --- DeepSeek API Configuration ---
18
+ DEEPSEEK_API_KEY_ENV_VAR = "DEEPSEEK_API_KEY"
19
  DEEPSEEK_BASE_URL = "https://api.deepseek.com"
20
  DEEPSEEK_MODEL_NAME = "deepseek-chat"
 
 
 
 
 
21
 
22
+ # Global client variable
23
+ client: Optional[OpenAI] = None
 
24
 
25
  def configure_llm_client():
26
  """Configures the OpenAI client for DeepSeek API using the API key from environment variables."""
27
  global client
28
+ api_key = os.getenv(DEEPSEEK_API_KEY_ENV_VAR)
 
29
  if not api_key:
30
  logging.error(f"Error: {DEEPSEEK_API_KEY_ENV_VAR} environment variable not set.")
31
+ logging.error("Please set the environment variable before running the script.")
32
  return False
33
  try:
 
34
  client = OpenAI(api_key=api_key, base_url=DEEPSEEK_BASE_URL)
35
+ logging.info("DeepSeek API client configured successfully.")
36
  return True
37
  except Exception as e:
38
  logging.error(f"Failed to configure DeepSeek API client: {e}")
 
53
  Returns:
54
  A short English explanation string from DeepSeek, or None if generation fails.
55
  """
56
+ global client
 
57
  if not client:
58
  logging.error(f"[{model_id}] DeepSeek client not configured. Cannot generate explanation.")
59
  return None
 
62
  logging.warning(f"[{model_id}] Description is empty or not a string. Skipping explanation generation.")
63
  return None
64
 
65
+ # Truncate very long descriptions
66
  max_desc_length = 4000
67
  if len(description) > max_desc_length:
68
  logging.warning(f"[{model_id}] Description truncated to {max_desc_length} chars for API call.")
69
  description = description[:max_desc_length] + "... [truncated]"
70
 
71
+ # Construct the messages for DeepSeek API
72
  messages = [
73
  {"role": "system", "content": "You are an AI assistant tasked with summarizing Hugging Face model descriptions concisely."},
74
  {"role": "user", "content": (
 
80
  )}
81
  ]
82
 
 
 
 
83
  retries = 0
84
  while retries < MAX_RETRIES:
85
  try:
86
  logging.info(f"[{model_id}] Calling DeepSeek API (Attempt {retries + 1}/{MAX_RETRIES})...")
 
87
  response = client.chat.completions.create(
88
  model=DEEPSEEK_MODEL_NAME,
89
  messages=messages,
 
92
  temperature=0.2 # Lower temperature for more focused summary
93
  )
94
 
95
+ explanation = response.choices[0].message.content.strip()
 
 
 
96
  logging.info(f"[{model_id}] Explanation received from DeepSeek: '{explanation}'")
 
97
  # Basic post-processing: remove potential quotes
98
  if explanation.startswith('"') and explanation.endswith('"'):
99
  explanation = explanation[1:-1]
 
 
100
  return explanation
101
 
 
102
  except APIError as e:
103
  retries += 1
104
  logging.error(f"[{model_id}] DeepSeek API Error (Attempt {retries}/{MAX_RETRIES}): {e}")
 
108
  else:
109
  logging.error(f"[{model_id}] Max retries reached. Failed to generate explanation via DeepSeek.")
110
  return None
111
+ except Exception as e: # Catch other potential errors
112
+ logging.error(f"[{model_id}] Unexpected error during DeepSeek API call: {e}")
113
+ return None # Don't retry for unexpected errors
 
 
 
 
 
 
 
114
 
115
+ return None
116
 
117
  def process_json_file(filepath: str):
118
+ """Reads, updates, and writes a single JSON file."""
119
  model_id = os.path.basename(filepath).replace('.json', '')
120
  logging.info(f"Processing {filepath}...")
121
 
 
124
  data = json.load(f)
125
  except json.JSONDecodeError:
126
  logging.error(f"[{model_id}] Invalid JSON format in {filepath}. Skipping.")
127
+ return
128
  except FileNotFoundError:
129
  logging.error(f"[{model_id}] File not found: {filepath}. Skipping.")
130
+ return
131
  except Exception as e:
132
  logging.error(f"[{model_id}] Error reading {filepath}: {e}. Skipping.")
133
+ return
134
 
135
  if not isinstance(data, dict):
136
  logging.error(f"[{model_id}] Expected JSON object (dict) but got {type(data)} in {filepath}. Skipping.")
137
+ return
138
 
139
+ description = data.get(DESCRIPTION_KEY)
140
+ explanation_overwritten = False
 
 
 
 
141
 
142
+ # --- Deletion Logic: Always remove existing explanation before trying to regenerate ---
143
+ if EXPLANATION_KEY in data:
144
+ logging.info(f"[{model_id}] Existing explanation found. Deleting before regenerating.")
145
+ del data[EXPLANATION_KEY]
146
+ explanation_overwritten = True # Mark that we intend to replace it
147
 
148
  # --- Generation Logic ---
 
 
149
  if not description:
150
  logging.warning(f"[{model_id}] Description field is missing or empty. Cannot generate explanation.")
151
+ return
152
 
153
  explanation = generate_explanation(model_id, description) # Try to generate a new one
154
 
155
+ # --- Update and Write Logic ---
156
  if explanation: # Only update if generation was successful
157
  data[EXPLANATION_KEY] = explanation
158
  try:
159
  with open(filepath, 'w', encoding='utf-8') as f:
160
  json.dump(data, f, ensure_ascii=False, indent=4)
161
+ if explanation_overwritten:
162
+ logging.info(f"[{model_id}] Successfully overwrote and updated {filepath} with new explanation.")
163
+ else:
164
+ logging.info(f"[{model_id}] Successfully generated and updated {filepath} with new explanation.")
165
  except IOError as e:
166
  logging.error(f"[{model_id}] Error writing updated data to {filepath}: {e}")
 
167
  except Exception as e:
168
  logging.error(f"[{model_id}] Unexpected error writing {filepath}: {e}")
 
169
  else: # Explanation generation failed
170
+ log_message = f"[{model_id}] Failed to generate new explanation for {filepath} via API."
171
+ if explanation_overwritten:
172
+ log_message += " Existing explanation was removed but not replaced due to API failure."
173
+ logging.warning(log_message)
174
 
175
 
176
  def main():
177
  """Main function to iterate through the directory and process files."""
178
+ # Configure LLM client at the start
179
  if not configure_llm_client():
180
  return # Stop if API key is not configured
181
 
 
185
 
186
  logging.info(f"Starting processing directory: {MODEL_DATA_DIR}")
187
  processed_files = 0
188
+ updated_files = 0
189
+ skipped_files = 0
 
190
 
191
  all_files = [f for f in os.listdir(MODEL_DATA_DIR) if f.lower().endswith(".json")]
192
  total_files = len(all_files)
 
196
  filepath = os.path.join(MODEL_DATA_DIR, filename)
197
  logging.info(f"--- Processing file {i+1}/{total_files}: {filename} ---")
198
  try:
199
+ # Check if explanation exists before calling process_json_file
200
+ # to potentially save API calls if already done.
201
+ # However, process_json_file already has this check.
202
+ process_json_file(filepath)
203
+ processed_files +=1 # Count as processed even if skipped due to existing explanation
204
+
205
+ # Check if file was actually updated (optional metric)
206
+ # Re-read might be inefficient, could return status from process_json_file
207
+ # For simplicity, we just log success/failure in process_json_file
 
208
 
209
  except Exception as e:
210
+ logging.error(f"Unexpected error processing file {filename}: {e}")
211
+ skipped_files += 1
212
  # Add a small delay between files to potentially avoid hitting rate limits
213
+ time.sleep(0.5) # Adjust delay as needed
 
214
 
215
 
216
  logging.info(f"--- Processing complete ---")
217
+ # Refine reporting slightly
218
  logging.info(f"Total JSON files found: {total_files}")
219
  logging.info(f"Files processed (attempted): {processed_files}")
220
+ # A more accurate count of updated files would require modifying process_json_file to return status
221
+ logging.info(f"Files skipped due to unexpected errors: {skipped_files}")
 
 
222
 
223
  if __name__ == "__main__":
224
  main()
app.py CHANGED
@@ -5,39 +5,17 @@ from flask_cors import CORS
5
  import numpy as np
6
  import json
7
  import traceback
8
- import logging # Added for background task logging
9
- import threading # Added for background task
10
- import time # Added for background task
11
- import schedule # Added for background task
12
-
13
- # --- Import the daily update function ---
14
- try:
15
- from daily_update import main as run_daily_update
16
- # Set up logging for the daily_update module if it uses logging
17
- # logging.getLogger('daily_update').setLevel(logging.INFO) # Example
18
- except ImportError:
19
- logging.error("Failed to import daily_update.py. The daily update task will not run.")
20
- run_daily_update = None # Define as None if import fails
21
- # ---
22
 
23
  app = Flask(__name__) # Create app object FIRST
24
-
25
- # Define the base persistent storage path (must match other scripts)
26
- PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
27
-
28
- # Configure Flask app logging (optional but recommended)
29
- # app.logger.setLevel(logging.INFO)
30
-
31
  # Allow requests from the Vercel frontend and localhost for development
32
  CORS(app, origins=["http://127.0.0.1:3000", "http://localhost:3000", "https://rag-huggingface.vercel.app"], supports_credentials=True)
33
 
34
  # --- Configuration ---
35
- # Point to index/map files in persistent storage
36
- INDEX_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index.faiss")
37
- MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
38
  EMBEDDING_MODEL = 'all-mpnet-base-v2'
39
- # Point to model data JSON in persistent storage
40
- MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
41
  # ---
42
 
43
  # --- Global variables for resources ---
@@ -76,8 +54,7 @@ def load_resources():
76
  print("Sentence transformer model loaded successfully.")
77
 
78
  # Load FAISS Index
79
- # index_path = os.path.join(os.path.dirname(__file__), INDEX_FILE) # Old path
80
- index_path = INDEX_FILE # Use configured path
81
  print(f"Loading FAISS index from: {index_path}")
82
  if not os.path.exists(index_path):
83
  raise FileNotFoundError(f"FAISS index file not found at {index_path}")
@@ -86,8 +63,7 @@ def load_resources():
86
  print("FAISS index loaded successfully.")
87
 
88
  # Load Index-to-Metadata Map
89
- # map_path = os.path.join(os.path.dirname(__file__), MAP_FILE) # Old path
90
- map_path = MAP_FILE # Use configured path
91
  print(f"Loading index-to-Metadata map from: {map_path}")
92
  if not os.path.exists(map_path):
93
  raise FileNotFoundError(f"Metadata map file not found at {map_path}")
@@ -101,8 +77,8 @@ def load_resources():
101
 
102
  except FileNotFoundError as fnf_error:
103
  print(f"Error: {fnf_error}")
104
- print(f"Please ensure {os.path.basename(INDEX_FILE)} and {os.path.basename(MAP_FILE)} exist in the persistent storage directory ({PERSISTENT_STORAGE_PATH}).")
105
- print("You might need to run the update process first or manually place initial files there.")
106
  RESOURCES_LOADED = False # Keep as False
107
  except ImportError as import_error:
108
  print(f"Import Error loading resources: {import_error}")
@@ -118,71 +94,6 @@ def load_resources():
118
  load_resources()
119
  # ---
120
 
121
- # --- Background Update Task ---
122
-
123
- UPDATE_INTERVAL_HOURS = 24 # Check every 24 hours
124
- UPDATE_TIME = "02:00" # Time to run the update (24-hour format)
125
-
126
- def run_update_task():
127
- """Wrapper function to run the daily update and handle errors."""
128
- if run_daily_update is None:
129
- logging.warning("run_daily_update function not available (import failed). Skipping task.")
130
- return
131
-
132
- logging.info(f"Background task: Starting daily update check (scheduled for {UPDATE_TIME})...")
133
- try:
134
- # Make sure the DEEPSEEK_API_KEY is set before running
135
- if not os.getenv("DEEPSEEK_API_KEY"):
136
- logging.error("Background task: DEEPSEEK_API_KEY not set. Daily update cannot run.")
137
- return # Don't run if key is missing
138
-
139
- run_daily_update() # Call the main function from daily_update.py
140
- logging.info("Background task: Daily update process finished.")
141
- except Exception as e:
142
- logging.error(f"Background task: Error during daily update execution: {e}")
143
- logging.error(traceback.format_exc())
144
-
145
- def background_scheduler():
146
- """Runs the scheduler loop in a background thread."""
147
- logging.info(f"Background scheduler started. Will run update task daily around {UPDATE_TIME}.")
148
-
149
- if run_daily_update is None:
150
- logging.error("Background scheduler: daily_update.py could not be imported. Scheduler will not run tasks.")
151
- return # Stop the thread if the core function isn't available
152
-
153
- # Schedule the job
154
- # schedule.every(UPDATE_INTERVAL_HOURS).hours.do(run_update_task) # Alternative: run every X hours
155
- schedule.every().day.at(UPDATE_TIME).do(run_update_task)
156
- logging.info(f"Scheduled daily update task for {UPDATE_TIME}.")
157
-
158
- # --- Run once immediately on startup ---
159
- logging.info("Background task: Running initial update check on startup...")
160
- run_update_task() # Call the task function directly
161
- logging.info("Background task: Initial update check finished.")
162
- # ---
163
-
164
- while True:
165
- schedule.run_pending()
166
- time.sleep(60) # Check every 60 seconds if a task is due
167
-
168
- # Start the background scheduler thread only if this is the main process
169
- # This check helps prevent duplicate schedulers when using workers (like Gunicorn)
170
- # Note: This might not be perfectly reliable with all WSGI servers/configs.
171
- # Consider using a more robust method for ensuring single execution if needed (e.g., file lock, external process manager)
172
- if os.environ.get("WERKZEUG_RUN_MAIN") == "true" or os.environ.get("FLASK_ENV") != "development":
173
- # Start only in main Werkzeug process OR if not in Flask development mode (like production with Gunicorn)
174
- # Check if the function is available before starting thread
175
- if run_daily_update is not None:
176
- scheduler_thread = threading.Thread(target=background_scheduler, daemon=True)
177
- scheduler_thread.start()
178
- logging.info("Background scheduler thread started.")
179
- else:
180
- logging.warning("Background scheduler thread NOT started because daily_update.py failed to import.")
181
- else:
182
- logging.info("Skipping background scheduler start in Werkzeug reloader process.")
183
-
184
- # --- End Background Update Task ---
185
-
186
  @app.route('/search', methods=['POST'])
187
  def search():
188
  """Handles search requests, embedding the query and searching the FAISS index."""
@@ -241,7 +152,7 @@ def search():
241
  # --- Add description from model_data_json ---
242
  model_id = metadata.get('model_id')
243
  description = None
244
- # Use the globally defined MODEL_DATA_DIR pointing to persistent storage
245
  if model_id and MODEL_DATA_DIR:
246
  filename = model_id.replace('/', '_') + '.json'
247
  filepath = os.path.join(MODEL_DATA_DIR, filename)
 
5
  import numpy as np
6
  import json
7
  import traceback
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  app = Flask(__name__) # Create app object FIRST
 
 
 
 
 
 
 
10
  # Allow requests from the Vercel frontend and localhost for development
11
  CORS(app, origins=["http://127.0.0.1:3000", "http://localhost:3000", "https://rag-huggingface.vercel.app"], supports_credentials=True)
12
 
13
  # --- Configuration ---
14
+ INDEX_FILE = "index.faiss"
15
+ MAP_FILE = "index_to_metadata.pkl"
 
16
  EMBEDDING_MODEL = 'all-mpnet-base-v2'
17
+ # Corrected path joining for model_data_json - relative to app.py location
18
+ MODEL_DATA_DIR = os.path.join(os.path.dirname(__file__), 'model_data_json')
19
  # ---
20
 
21
  # --- Global variables for resources ---
 
54
  print("Sentence transformer model loaded successfully.")
55
 
56
  # Load FAISS Index
57
+ index_path = os.path.join(os.path.dirname(__file__), INDEX_FILE)
 
58
  print(f"Loading FAISS index from: {index_path}")
59
  if not os.path.exists(index_path):
60
  raise FileNotFoundError(f"FAISS index file not found at {index_path}")
 
63
  print("FAISS index loaded successfully.")
64
 
65
  # Load Index-to-Metadata Map
66
+ map_path = os.path.join(os.path.dirname(__file__), MAP_FILE)
 
67
  print(f"Loading index-to-Metadata map from: {map_path}")
68
  if not os.path.exists(map_path):
69
  raise FileNotFoundError(f"Metadata map file not found at {map_path}")
 
77
 
78
  except FileNotFoundError as fnf_error:
79
  print(f"Error: {fnf_error}")
80
+ print(f"Please ensure {INDEX_FILE} and {MAP_FILE} exist in the 'backend' directory relative to app.py.")
81
+ print("You might need to run 'python build_index.py' first.")
82
  RESOURCES_LOADED = False # Keep as False
83
  except ImportError as import_error:
84
  print(f"Import Error loading resources: {import_error}")
 
94
  load_resources()
95
  # ---
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  @app.route('/search', methods=['POST'])
98
  def search():
99
  """Handles search requests, embedding the query and searching the FAISS index."""
 
152
  # --- Add description from model_data_json ---
153
  model_id = metadata.get('model_id')
154
  description = None
155
+ # Use the globally defined and corrected MODEL_DATA_DIR
156
  if model_id and MODEL_DATA_DIR:
157
  filename = model_id.replace('/', '_') + '.json'
158
  filepath = os.path.join(MODEL_DATA_DIR, filename)
build_index.py CHANGED
@@ -7,15 +7,10 @@ import pickle
7
  import json # Import json module
8
  from tqdm import tqdm
9
 
10
- # Define the base persistent storage path (must match other scripts)
11
- PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
12
-
13
  # --- Configuration ---
14
- # Point to the JSON data within persistent storage
15
- MODEL_DATA_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
16
- # Save index and map to persistent storage
17
- INDEX_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index.faiss")
18
- MAP_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "index_to_metadata.pkl")
19
  EMBEDDING_MODEL = 'all-mpnet-base-v2' # Efficient and good quality model
20
  ENCODE_BATCH_SIZE = 32 # Process descriptions in smaller batches
21
  # Tags to exclude from indexing text
 
7
  import json # Import json module
8
  from tqdm import tqdm
9
 
 
 
 
10
  # --- Configuration ---
11
+ MODEL_DATA_DIR = "model_data_json" # Path to downloaded JSON data
12
+ INDEX_FILE = "index.faiss"
13
+ MAP_FILE = "index_to_metadata.pkl" # Changed filename to reflect content
 
 
14
  EMBEDDING_MODEL = 'all-mpnet-base-v2' # Efficient and good quality model
15
  ENCODE_BATCH_SIZE = 32 # Process descriptions in smaller batches
16
  # Tags to exclude from indexing text
huggingface_model_descriptions.py CHANGED
@@ -10,11 +10,8 @@ from requests.exceptions import RequestException
10
  from concurrent.futures import ThreadPoolExecutor, as_completed
11
  import pickle # Add pickle for caching
12
 
13
- # Define the base persistent storage path
14
- PERSISTENT_STORAGE_PATH = "/data" # <-- ADJUST IF YOUR PATH IS DIFFERENT
15
-
16
- # Create a directory to store JSON data within persistent storage
17
- OUTPUT_DIR = os.path.join(PERSISTENT_STORAGE_PATH, "model_data_json")
18
  os.makedirs(OUTPUT_DIR, exist_ok=True)
19
 
20
  # Number of worker threads for parallel processing - REDUCED
@@ -44,8 +41,7 @@ def clean_readme_content(text):
44
  return text
45
  # ---
46
 
47
- # Use persistent storage for the cache file
48
- MODELS_CACHE_FILE = os.path.join(PERSISTENT_STORAGE_PATH, "models_list_cache.pkl") # File to cache the raw model list
49
 
50
  def get_all_models_with_downloads(min_downloads=10000):
51
  """Fetch all models from Hugging Face with at least min_downloads, using a local cache for the list."""
@@ -70,7 +66,7 @@ def get_all_models_with_downloads(min_downloads=10000):
70
  api = HfApi()
71
  print("HfApi initialized. Calling list_models...")
72
  # Fetch the iterator
73
- models_iterator = api.list_models(sort="downloads", direction=-1, fetch_config=False, cardData=False)
74
  print("list_models call returned. Converting iterator to list...")
75
  # Convert the iterator to a list TO ALLOW CACHING
76
  models_list = list(models_iterator)
@@ -158,9 +154,9 @@ def get_model_readme(model_id):
158
  return None
159
 
160
  def get_filename_for_model(model_id):
161
- """Generate JSON filename for a model (uses global OUTPUT_DIR)"""
162
  safe_id = model_id.replace("/", "_")
163
- return os.path.join(OUTPUT_DIR, f"{safe_id}.json") # OUTPUT_DIR is already correct path
164
 
165
  def save_model_data(model_id, data):
166
  """Save model data (description, tags, downloads) to a JSON file."""
 
10
  from concurrent.futures import ThreadPoolExecutor, as_completed
11
  import pickle # Add pickle for caching
12
 
13
+ # Create a directory to store JSON data
14
+ OUTPUT_DIR = "model_data_json"
 
 
 
15
  os.makedirs(OUTPUT_DIR, exist_ok=True)
16
 
17
  # Number of worker threads for parallel processing - REDUCED
 
41
  return text
42
  # ---
43
 
44
+ MODELS_CACHE_FILE = "models_list_cache.pkl" # File to cache the raw model list
 
45
 
46
  def get_all_models_with_downloads(min_downloads=10000):
47
  """Fetch all models from Hugging Face with at least min_downloads, using a local cache for the list."""
 
66
  api = HfApi()
67
  print("HfApi initialized. Calling list_models...")
68
  # Fetch the iterator
69
+ models_iterator = api.list_models(sort="downloads", direction=-1, fetch_config=False, cardData=True)
70
  print("list_models call returned. Converting iterator to list...")
71
  # Convert the iterator to a list TO ALLOW CACHING
72
  models_list = list(models_iterator)
 
154
  return None
155
 
156
  def get_filename_for_model(model_id):
157
+ """Generate JSON filename for a model"""
158
  safe_id = model_id.replace("/", "_")
159
+ return os.path.join(OUTPUT_DIR, f"{safe_id}.json") # Change extension to .json
160
 
161
  def save_model_data(model_id, data):
162
  """Save model data (description, tags, downloads) to a JSON file."""
requirements.txt CHANGED
@@ -4,6 +4,4 @@ sentence-transformers>=2.3.0
4
  numpy>=1.20.0
5
  faiss-cpu>=1.7.0 # Use faiss-gpu if you need GPU support on HF Spaces
6
  huggingface-hub>=0.15.1 # Version compatible with sentence-transformers >= 2.3.0
7
- gunicorn # Added for deployment on Hugging Face Spaces
8
- openai>=1.0.0 # Added back for DeepSeek API via OpenAI client
9
- schedule>=1.0.0 # Added for in-app scheduling
 
4
  numpy>=1.20.0
5
  faiss-cpu>=1.7.0 # Use faiss-gpu if you need GPU support on HF Spaces
6
  huggingface-hub>=0.15.1 # Version compatible with sentence-transformers >= 2.3.0
7
+ gunicorn # Added for deployment on Hugging Face Spaces