Reduce QUEUE_THRESHOLD to 10 for more frequent batch processing and update logging for queue status in sync loop
Browse files
main.py
CHANGED
@@ -327,7 +327,7 @@ async def search(
|
|
327 |
# new constants
|
328 |
QUEUE_FILE = "./save_queue.jsonl"
|
329 |
PUSH_INTERVAL_S = 300 # seconds
|
330 |
-
QUEUE_THRESHOLD =
|
331 |
MAX_PUSH_INTERVAL_S = 47 * 3600 # 44 hours
|
332 |
|
333 |
# background task to batch-push queued records
|
@@ -353,7 +353,7 @@ async def _hf_sync_loop():
|
|
353 |
now = datetime.now(timezone.utc).timestamp()
|
354 |
time_since_last_push = now - last_push_time
|
355 |
|
356 |
-
|
357 |
# Only push if threshold met or max interval
|
358 |
if queue_len >= QUEUE_THRESHOLD or time_since_last_push >= MAX_PUSH_INTERVAL_S:
|
359 |
if not lines:
|
|
|
327 |
# new constants
|
328 |
QUEUE_FILE = "./save_queue.jsonl"
|
329 |
PUSH_INTERVAL_S = 300 # seconds
|
330 |
+
QUEUE_THRESHOLD = 10
|
331 |
MAX_PUSH_INTERVAL_S = 47 * 3600 # 44 hours
|
332 |
|
333 |
# background task to batch-push queued records
|
|
|
353 |
now = datetime.now(timezone.utc).timestamp()
|
354 |
time_since_last_push = now - last_push_time
|
355 |
|
356 |
+
print(f"Queue length: {queue_len}, Time since last push: {time_since_last_push}")
|
357 |
# Only push if threshold met or max interval
|
358 |
if queue_len >= QUEUE_THRESHOLD or time_since_last_push >= MAX_PUSH_INTERVAL_S:
|
359 |
if not lines:
|