ai: Implement parallel request to the server.

jarvis.py

@@ -17,6 +17,7 @@ import pptx
 import fitz
 import io
 import uuid
+import concurrent.futures
 
 from openai import OpenAI
 
@@ -116,6 +117,13 @@ def process_ai_response(ai_text):
     except Exception:
         return ai_text
 
+def fetch_response(host, provider_key, selected_model, messages, model_config, session_id):
+    client = OpenAI(base_url=host, api_key=provider_key)
+    data = {"model": selected_model, "messages": messages, **model_config}
+    response = client.chat.completions.create(extra_body={"optillm_approach": "rto|re2|cot_reflection|self_consistency|plansearch|leap|z3|bon|moa|mcts|mcp|router|privacy|executecode|json", "session_id": session_id}, **data)
+    ai_text = response.choices[0].message.content if response.choices and response.choices[0].message and response.choices[0].message.content else RESPONSES["RESPONSE_2"]
+    return process_ai_response(ai_text)
+
 def chat_with_model(history, user_input, selected_model_display, sess):
     if not LINUX_SERVER_PROVIDER_KEYS or not LINUX_SERVER_HOSTS:
         return RESPONSES["RESPONSE_3"]
@@ -126,14 +134,15 @@ def chat_with_model(history, user_input, selected_model_display, sess):
     messages = [{"role": "user", "content": user} for user, _ in history]
     messages += [{"role": "assistant", "content": assistant} for _, assistant in history if assistant]
     messages.append({"role": "user", "content": user_input})
-    random.shuffle(LINUX_SERVER_PROVIDER_KEYS)
-    random.shuffle(LINUX_SERVER_HOSTS)
-    client = OpenAI(base_url=LINUX_SERVER_HOSTS[0], api_key=LINUX_SERVER_PROVIDER_KEYS[0])
-    data = {"model": selected_model, "messages": messages, **model_config}
-    response = client.chat.completions.create(extra_body={"optillm_approach": "rto|re2|cot_reflection|self_consistency|plansearch|leap|z3|bon|moa|mcts|mcp|router|privacy|executecode|json", "session_id": sess.session_id}, **data)
-    ai_text = response.choices[0].message.content if response.choices and response.choices[0].message and response.choices[0].message.content else RESPONSES["RESPONSE_2"]
-    processed_text = process_ai_response(ai_text)
-    return processed_text
+    futures = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(LINUX_SERVER_HOSTS)) as executor:
+        for host, key in zip(LINUX_SERVER_HOSTS, LINUX_SERVER_PROVIDER_KEYS):
+            futures.append(executor.submit(fetch_response, host, key, selected_model, messages, model_config, sess.session_id))
+        done, not_done = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
+        for future in not_done:
+            future.cancel()
+        result = list(done)[0].result() if done else RESPONSES["RESPONSE_2"]
+    return result
 
 def respond(multi_input, history, selected_model_display, sess):
     message = {"text": multi_input.get("text", "").strip(), "files": multi_input.get("files", [])}