Deploy app/api to HF Space

.gitignore

@@ -0,0 +1,176 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+observability_data/*

Dockerfile

@@ -0,0 +1,32 @@
+# Use the official Python 3.12 image
+FROM python:3.12-slim
+
+# Set the working directory
+WORKDIR /app
+
+# Install required system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    libpq-dev \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create the /app/files directory and set full permissions
+RUN mkdir -p /app/.files && chmod 777 /app/.files && \
+    mkdir -p /app/logs && chmod 777 /app/logs && \
+    mkdir -p /app/observability_data && chmod 777 /app/observability_data && \
+    mkdir -p /app/devops_cache && chmod 777 /app/devops_cache
+
+# Copy the current repository into the container
+COPY . /app
+
+# Upgrade pip and install dependencies
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt && \
+    pip install git-recap==0.1.3 && \
+    pip install git+https://github.com/BrunoV21/AiCore.git#egg=core-for-ai[all]
+
+EXPOSE 7860
+
+CMD python main.py

README.md

@@ -0,0 +1,12 @@
+---
+title: Git Recap 
+emoji: 🚀
+colorFrom: indigo
+colorTo: purple
+sdk: docker
+pinned: true
+license: apache-2.0
+short_description: Recap your repositories with the power of Llms!
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

docker-compose.yaml

@@ -0,0 +1,15 @@
+version: "3.8"
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/app
+    restart: unless-stopped
+    command: python main.py

main.py

@@ -0,0 +1,44 @@
+from fastapi import FastAPI
+from fastapi.responses import RedirectResponse
+from fastapi.middleware.cors import CORSMiddleware
+import asyncio
+
+from server.routes import router as api_router
+from services.llm_service import simulate_llm_response
+from server.websockets import router as websocket_router
+from midleware import OriginAndRateLimitMiddleware, ALLOWED_ORIGIN
+
+# Initialize FastAPI app
+app = FastAPI(title="LLM Service API")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=ALLOWED_ORIGIN,
+    allow_methods=["GET", "POST", "OPTIONS"]
+)
+app.add_middleware(OriginAndRateLimitMiddleware)
+
+# Include routers
+app.include_router(api_router)
+app.include_router(websocket_router)
+
+@app.get("/", include_in_schema=False)
+async def root():
+    return RedirectResponse(url="https://brunov21.github.io/GitRecap/")
+
+# Health check endpoint
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy"}
+
+@app.get("/health2")
+async def stream_health_check():
+    response = simulate_llm_response("health")
+    return {"response": " ".join(response)}
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    import uvicorn
+
+    load_dotenv()
+    uvicorn.run(app, host="0.0.0.0", port=7860)

midleware.py

@@ -0,0 +1,36 @@
+import os
+import time
+from fastapi import Request, HTTPException
+from starlette.middleware.base import BaseHTTPMiddleware
+from collections import defaultdict
+
+ALLOWED_ORIGIN = [
+    os.getenv("VITE_FRONTEND_HOST")
+]
+RATE_LIMIT = int(os.getenv("RATE_LIMIT", "30"))  # Max requests per time window
+WINDOW_SECONDS = int(os.getenv("WINDOW_SECONDS", "3"))  # Time window in seconds
+
+# Store timestamps of requests per IP
+request_logs = defaultdict(list)
+
+
+class OriginAndRateLimitMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        origin = request.headers.get("origin")
+        if origin and origin not in ALLOWED_ORIGIN:
+            raise HTTPException(status_code=403, detail="Forbidden: origin not allowed")
+
+        # Rate limiting logic based on client IP
+        client_ip = request.client.host
+        now = time.time()
+
+        # Clean up old request timestamps outside the current window
+        request_logs[client_ip] = [
+            t for t in request_logs[client_ip] if now - t < WINDOW_SECONDS
+        ]
+
+        if len(request_logs[client_ip]) >= RATE_LIMIT:
+            raise HTTPException(status_code=429, detail="Too Many Requests")
+
+        request_logs[client_ip].append(now)
+        return await call_next(request)

models/schemas.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel, model_validator
+from typing import Dict, Self, Optional, Any
+import ulid
+
+class ChatRequest(BaseModel):
+    session_id: str=""
+    message: str
+    model_params: Optional[Dict[str, Any]] = None
+
+    @model_validator(mode="after")
+    def set_session_id(self)->Self:
+        if not self.session_id:
+            self.session_id = ulid.ulid()
+        return self

requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.109.1
+uvicorn==0.23.2
+websockets==11.0.3
+pyjwt==2.10.1
+python-multipart==0.0.18

server/routes.py

@@ -0,0 +1,209 @@
+from fastapi import APIRouter, HTTPException, Request, Query
+from pydantic import BaseModel
+
+from models.schemas import ChatRequest
+from services.llm_service import initialize_llm_session, set_llm, get_llm, trim_messages
+from services.fetcher_service import store_fetcher, get_fetcher
+from git_recap.utils import parse_entries_to_txt
+from aicore.llm.config import LlmConfig
+from datetime import datetime, timezone
+from typing import Optional, List
+import requests
+import os
+
+router = APIRouter()
+
+class CloneRequest(BaseModel):
+    """Request model for repository cloning endpoint."""
+    url: str
+
+GITHUB_ACCESS_TOKEN_URL = 'https://github.com/login/oauth/access_token'
+
+@router.post("/clone-repo")
+async def clone_repository(request: CloneRequest):
+    """
+    Endpoint for cloning a repository from a URL.
+    
+    Args:
+        request: CloneRequest containing the repository URL
+        
+    Returns:
+        dict: Contains session_id for subsequent operations
+        
+    Raises:
+        HTTPException: 400 for invalid URL, 500 for cloning failure
+    """
+    try:
+        response = await create_llm_session()
+        session_id = response.get("session_id")
+        store_fetcher(session_id, request.url, "URL")
+        return {"session_id": session_id}
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to clone repository: {str(e)}")
+
+@router.get("/external-signup")
+async def external_signup(app: str, accessToken: str, provider: str):
+    if provider.lower() != "github":
+        raise HTTPException(status_code=400, detail="Unsupported provider")
+
+    # Build the URL to exchange the code for a token
+    params = {
+        "client_id": os.getenv("VITE_GITHUB_CLIENT_ID"),
+        "client_secret": os.getenv("VITE_GITHUB_CLIENT_SECRET"),
+        "code": accessToken
+    }
+    
+    headers = {
+        "Accept": "application/json",
+        "Accept-Encoding": "application/json"
+    }
+    
+    response = requests.get(GITHUB_ACCESS_TOKEN_URL, params=params, headers=headers)
+    
+    if response.status_code != 200:
+        raise HTTPException(status_code=response.status_code, detail="Error fetching token from GitHub")
+    
+    githubUserData = response.json()
+    token = githubUserData.get("access_token")
+    if not token:
+        raise HTTPException(status_code=400, detail="Failed to retrieve access token")
+    
+    response = await create_llm_session()
+    response["token"] = token
+    response["provider"] = provider
+    final_response = await store_fetcher_endpoint(response)
+    session_id = final_response.get("session_id")    
+    return {"session_id": session_id}
+
+@router.post("/pat")
+async def store_fetcher_endpoint(request: Request):
+    """
+    Endpoint to store the PAT associated with a session.
+    
+    Args:
+        request: Contains JSON payload with 'session_id' and 'pat'
+        
+    Returns:
+        dict: Contains session_id
+        
+    Raises:
+        HTTPException: 400 if PAT is missing
+    """
+    if isinstance(request, Request):
+        payload = await request.json()
+    else:
+        payload = request
+    
+    provider = payload.get("provider", "GitHub")
+    token = payload.get("pat") or payload.get("token")
+    if not token:
+        raise HTTPException(status_code=400, detail="Missing required field: pat")
+    
+    response = await create_llm_session()  
+    session_id = response.get("session_id")
+    store_fetcher(session_id, token, provider)
+    return {"session_id": session_id}
+
+async def create_llm_session(
+    request: Optional[LlmConfig] = None
+):
+    """
+    Create a new LLM session with custom configuration
+    
+    Args:
+        request: Optional LLM configuration
+        
+    Returns:
+        dict: Contains session_id and success message
+        
+    Raises:
+        HTTPException: 500 if session creation fails
+    """
+    try:
+        session_id = await set_llm(request)
+        return {
+            "session_id": session_id,
+            "message": "LLM session created successfully"
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/repos")
+async def get_repos(session_id: str):
+    """
+    Return a list of repositories for the given session_id.
+    
+    Args:
+        session_id: The session identifier
+        
+    Returns:
+        dict: Contains list of repository names
+        
+    Raises:
+        HTTPException: 404 if session not found
+    """
+    fetcher = get_fetcher(session_id)
+    return {"repos": fetcher.repos_names}
+
+@router.get("/actions")
+async def get_actions(
+    session_id: str,
+    start_date: Optional[str] = Query(None),
+    end_date: Optional[str] = Query(None),
+    repo_filter: Optional[List[str]] = Query(None),
+    authors: Optional[List[str]] = Query(None)
+):
+    """
+    Get actions for the specified session with optional filters.
+    
+    Args:
+        session_id: The session identifier
+        start_date: Optional start date filter
+        end_date: Optional end date filter
+        repo_filter: Optional list of repositories to filter
+        authors: Optional list of authors to filter
+        
+    Returns:
+        dict: Contains formatted action entries
+        
+    Raises:
+        HTTPException: 404 if session not found
+    """
+    if repo_filter is not None:
+        repo_filter = sum([repo.split(",") for repo in repo_filter], [])
+    if authors is not None:
+        authors = sum([author.split(",") for author in authors], [])
+    fetcher = get_fetcher(session_id)
+    
+    # Convert date strings to datetime objects
+    start_dt = datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc) if start_date else None
+    end_dt = datetime.fromisoformat(end_date).replace(tzinfo=timezone.utc) if end_date else None
+
+    if start_dt:
+        fetcher.start_date = start_dt
+    if end_dt:
+        fetcher.end_dt = end_dt
+    if repo_filter is not None:
+        fetcher.repo_filter = repo_filter
+    if authors is not None:
+        fetcher.authors = authors
+
+    llm = get_llm(session_id)
+    actions = fetcher.get_authored_messages()
+    actions = trim_messages(actions, llm.tokenizer)
+    print(f"\n\n\n{actions=}\n\n\n")
+    
+    return {"actions": parse_entries_to_txt(actions)}
+
+# @router.post("/chat")
+# async def chat(
+#     chat_request: ChatRequest
+# ):
+#     try:
+#         llm = await initialize_llm_session(chat_request.session_id)
+#         response = await llm.acomplete(chat_request.message)
+#         return {"response": response}
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=str(e))

server/websockets.py

@@ -0,0 +1,79 @@
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Query
+import json
+from typing import Optional
+
+from services.llm_service import initialize_llm_session, trim_messages, run_concurrent_tasks, get_llm
+from aicore.const import SPECIAL_TOKENS, STREAM_END_TOKEN
+import ulid
+import asyncio
+
+router = APIRouter()
+
+# WebSocket connection storage
+active_connections = {}
+active_histories = {}
+
+TRIGGER_PROMPT = """
+Consider the following history of actionables from Git and in return me the summary with N = '{N}' bullet points:
+
+{ACTIONS}
+"""
+
+@router.websocket("/ws/{session_id}")
+async def websocket_endpoint(
+    websocket: WebSocket, 
+    session_id: Optional[str] = None
+):
+    await websocket.accept()
+    
+    # Store the connection
+    active_connections[session_id] = websocket
+
+    # Initialize LLM
+    llm = get_llm(session_id)
+
+    try:
+        while True:
+            message = await websocket.receive_text()
+            msg_json = json.loads(message)
+            message = msg_json.get("actions")
+            N = msg_json.get("n", 5)
+            assert int(N) <= 15
+            assert message
+            history = [
+                TRIGGER_PROMPT.format(
+                    N=N,
+                    ACTIONS=message
+                )
+            ]
+            response = []
+            async for chunk in run_concurrent_tasks(
+                llm,
+                message=history
+            ):
+                if chunk == STREAM_END_TOKEN:
+                    await websocket.send_text(json.dumps({"chunk": chunk}))
+                    break
+                elif chunk in SPECIAL_TOKENS:
+                    continue
+                
+                await websocket.send_text(json.dumps({"chunk": chunk}))
+                response.append(chunk)
+            
+            history.append("".join(response))
+    
+    except WebSocketDisconnect:
+        if session_id in active_connections:
+            del active_connections[session_id]
+    except Exception as e:
+        if session_id in active_connections:
+            await websocket.send_text(json.dumps({"error": str(e)}))
+            del active_connections[session_id]
+
+def close_websocket_connection(session_id: str):
+    """
+    Clean up and close the active websocket connection associated with the given session_id.
+    """
+    websocket = active_connections.pop(session_id, None)
+    if websocket:
+        asyncio.create_task(websocket.close())

services/fetcher_service.py

@@ -0,0 +1,84 @@
+from typing import Dict, Optional
+from fastapi import HTTPException
+from git_recap.providers.base_fetcher import BaseFetcher
+from git_recap.providers import GitHubFetcher, AzureFetcher, GitLabFetcher, URLFetcher
+import ulid
+
+# In-memory store mapping session_id to its respective fetcher instance
+fetchers: Dict[str, BaseFetcher] = {}
+
+def store_fetcher(session_id: str, pat: str, provider: Optional[str] = "GitHub") -> None:
+    """
+    Store the provided PAT associated with the given session_id.
+    
+    Args:
+        session_id: The session identifier tied to the active session.
+        pat: The Personal Access Token to be stored (or URL for URL provider).
+        provider: The provider identifier (default is "GitHub"). 
+                 Can be "Azure Devops", "GitLab", or "URL".
+    
+    Raises:
+        HTTPException: If the session_id or PAT/URL is invalid or unsupported provider.
+    """
+    if not session_id or not pat:
+        raise HTTPException(status_code=400, detail="Invalid session_id or PAT/URL")
+    
+    try:
+        if provider == "GitHub":
+            fetchers[session_id] = GitHubFetcher(pat=pat)
+        elif provider == "Azure Devops":
+            fetchers[session_id] = AzureFetcher(pat=pat)
+        elif provider == "GitLab":
+            fetchers[session_id] = GitLabFetcher(pat=pat)
+        elif provider == "URL":
+            fetchers[session_id] = URLFetcher(url=pat)
+        else:
+            raise HTTPException(status_code=400, detail="Unsupported provider")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to initialize {provider} fetcher: {str(e)}"
+        )
+
+def get_fetcher(session_id: str) -> BaseFetcher:
+    """
+    Retrieve the stored fetcher instance for the provided session_id.
+    
+    Args:
+        session_id: The session identifier.
+    
+    Returns:
+        The fetcher instance associated with the session_id.
+    
+    Raises:
+        HTTPException: If no fetcher is found for the given session_id.
+    """
+    fetcher = fetchers.get(session_id)
+    if not fetcher:
+        raise HTTPException(status_code=404, detail="Session not found")
+    return fetcher
+
+def expire_fetcher(session_id: str) -> None:
+    """
+    Remove the fetcher associated with the given session_id.
+    
+    This function is used for cleaning up resources by expiring the stored fetcher instance
+    when its corresponding session is expired.
+    
+    Args:
+        session_id: The session identifier whose associated fetcher should be removed.
+    """
+    fetcher = fetchers.pop(session_id, None)
+    if fetcher and hasattr(fetcher, 'clear'):
+        fetcher.clear()
+
+def generate_session_id() -> str:
+    """
+    Generate a new unique session ID.
+    
+    Returns:
+        str: A new ULID-based session identifier.
+    """
+    return ulid.ulid()

services/llm_service.py

@@ -0,0 +1,190 @@
+import json
+import os
+import uuid
+from typing import Dict, List, Optional
+from fastapi import HTTPException
+import asyncio
+import random
+
+from aicore.logger import _logger
+from aicore.config import Config
+from aicore.llm import Llm
+from aicore.llm.config import LlmConfig
+from services.prompts import SELECT_QUIRKY_REMARK_SYSTEM, SYSTEM, quirky_remarks
+
+def get_random_quirky_remarks(remarks_list, n=5):
+    """
+    Returns a list of n randomly selected quirky remarks.
+    
+    Args:
+        remarks_list (list): The full list of quirky remarks.
+        n (int): Number of remarks to select (default is 5).
+        
+    Returns:
+        list: Randomly selected quirky remarks.
+    """
+    return random.sample(remarks_list, min(n, len(remarks_list)))
+
+# LLM session storage
+llm_sessions: Dict[str, Llm] = {}
+
+async def initialize_llm_session(session_id: str, config: Optional[LlmConfig] = None) -> Llm:
+    """
+    Initialize or retrieve an LLM session.
+    
+    Args:
+        session_id: The session identifier.
+        config: Optional custom LLM configuration.
+        
+    Returns:
+        An initialized LLM instance.
+    """
+    if session_id in llm_sessions:
+        return llm_sessions[session_id]
+    
+    # Initialize LLM based on whether custom config is provided.
+    if config:
+        # Convert Pydantic model to dict and use for LLM initialization.
+        config_dict = config.dict(exclude_none=True)
+        llm = Llm.from_config(config_dict)
+    else:
+        config = Config.from_environment()
+        llm = Llm.from_config(config.llm)
+    llm.session_id = session_id
+    llm_sessions[session_id] = llm
+    return llm
+
+async def set_llm(config: Optional[LlmConfig] = None) -> str:
+    """
+    Set a custom LLM configuration and return a new session ID.
+    
+    Args:
+        config: The LLM configuration to use.
+        
+    Returns:
+        A new session ID linked to the configured LLM.
+    """
+    try:
+        # Generate a unique session ID.
+        session_id = str(uuid.uuid4())
+        
+        # Initialize the LLM with the provided configuration.
+        await initialize_llm_session(session_id, config)
+        
+        # Schedule session expiration exactly 5 minutes after session creation.
+        asyncio.create_task(schedule_session_expiration(session_id))
+        
+        return session_id
+    except Exception as e:
+        print(f"Error setting custom LLM: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Failed to set custom LLM: {str(e)}")
+    
+def get_llm(session_id: str) -> Optional[Llm]:
+    """
+    Retrieve the LLM instance associated with the given session_id.
+    
+    Args:
+        session_id: The session identifier.
+        
+    Returns:
+        The LLM instance if found.
+        
+    Raises:
+        HTTPException: If the session is not found.
+    """
+    if session_id not in llm_sessions:
+        raise HTTPException(status_code=404, detail="Session not found")
+    return llm_sessions.get(session_id)
+    
+def trim_messages(messages, tokenizer_fn, max_tokens: Optional[int] = None):
+    """
+    Trim messages to ensure that the total token count does not exceed max_tokens.
+    
+    Args:
+        messages: List of messages.
+        tokenizer_fn: Function to tokenize messages.
+        max_tokens: Maximum allowed tokens.
+    
+    Returns:
+        Trimmed list of messages.
+    """
+    max_tokens = max_tokens or int(os.environ.get("MAX_HISTORY_TOKENS", 16000))
+    while messages and sum(len(tokenizer_fn(str(msg))) for msg in messages) > max_tokens:
+        messages.pop(0)  # Remove from the beginning
+    return messages
+    
+async def run_concurrent_tasks(llm, message):
+    """
+    Run concurrent tasks for the LLM and logger.
+    
+    Args:
+        llm: The LLM instance.
+        message: Message to process.
+    
+    Yields:
+        Chunks of logs from the logger.
+    """
+    QUIRKY_SYSTEM = SELECT_QUIRKY_REMARK_SYSTEM.format(
+        examples=json.dumps(get_random_quirky_remarks(quirky_remarks), indent=4)
+    )
+    asyncio.create_task(llm.acomplete(message, system_prompt=[SYSTEM, QUIRKY_SYSTEM]))
+    asyncio.create_task(_logger.distribute())
+    # Stream logger output while LLM is running.
+    while True:        
+        async for chunk in _logger.get_session_logs(llm.session_id):
+            yield chunk  # Yield each chunk directly
+
+def simulate_llm_response(message: str) -> List[str]:
+    """
+    Simulate LLM response by breaking a dummy response into chunks.
+    
+    Args:
+        message: Input message.
+        
+    Returns:
+        List of response chunks.
+    """
+    response = (
+        f"This is a simulated response to: '{message}'. In a real implementation, this would be the actual output "
+        "from your LLM model. The response would be generated in chunks and streamed back to the client as they become available."
+    )
+    
+    # Break into chunks of approximately 10 characters.
+    chunks = []
+    for i in range(0, len(response), 10):
+        chunks.append(response[i:i+10])
+    
+    return chunks
+
+def cleanup_llm_sessions():
+    """Clean up all LLM sessions."""
+    llm_sessions.clear()
+
+async def schedule_session_expiration(session_id: str):
+    """
+    Schedule the expiration of a session exactly 5 minutes after its creation.
+    
+    Args:
+        session_id: The session identifier.
+    """
+    # Wait for 5 minutes (300 seconds) before expiring the session.
+    await asyncio.sleep(300)
+    await expire_session(session_id)
+
+async def expire_session(session_id: str):
+    """
+    Expire a session by removing it from storage and cleaning up associated resources.
+    
+    Args:
+        session_id: The session identifier.
+    """
+    # Remove the expired session from storage.
+    llm_sessions.pop(session_id, None)
+
+    # Expire any associated fetcher in fetcher_service.
+    from services.fetcher_service import expire_fetcher
+    expire_fetcher(session_id)
+
+    # Expire any active websocket connections associated with session_id.
+    from server.websockets import close_websocket_connection
+    close_websocket_connection(session_id)

services/prompts.py

@@ -0,0 +1,113 @@
+SYSTEM = """
+### System Prompt for LLM Agent
+
+You are an AI assistant that helps developers track their work with a mix of humor, insight, and a dash of personality. You receive a structured text description containing a series of code-related actions spanning multiple repositories and dates. Your job is to generate a structured yet engaging response that provides value while keeping things light and entertaining.
+
+#### Response Structure:
+1. **Start with a quirky or funny one-liner.** Be witty, relatable, and creative. Feel free to reference developer struggles, commit patterns, or ongoing themes in the updates. Format this in *italic* to make it stand out.
+2. **Summarize the updates into exactly 'N' concise bullet points.**  
+   - You *must* strictly adhere to 'N' bullet points—returning more or fewer will result in a penalty.
+   - If there are more updates than N, prioritize the most impactful ones.
+   - Do NOT include specific dates in the bullet points.
+   - Order them in a way that makes sense, either thematically or chronologically if it improves readability.
+   - Always reference the repository that originated the update.
+   - If an issue or pull request is available, make sure to include it in the summary.
+3. **End with a thought-provoking question.** Encourage the developer to reflect on their next steps. Make it open-ended and engaging, rather than just a checklist. Follow it up with up to three actionable suggestions tailored to their recent work. Format this section’s opening line in *italic* as well.
+
+#### **Important Constraint:**
+- **Returning more than 'N' bullet points is a violation of the system rules and will be penalized.** Treat this as a hard requirement—excessive bullet points result in a deduction of response quality. Stick to exactly 'N'.  
+
+#### Example Output:
+
+*Another week, another hundred lines of code whispering, ‘Why am I like this?’ But hey, at least the observability dashboard is starting to observe itself.*
+
+- **[`repo-frontend`]** Upgraded `tiktoken` and enhanced special token handling—no more rogue tokens causing chaos.
+- **[`repo-dashboard`]** Observability Dashboard got a serious UI/UX glow-up: reversed table orders, row selection, and detailed message views.
+- **[`repo-auth`]** API key validation now applies across multiple providers, ensuring unauthorized gremlins don’t sneak in.
+- **[`repo-gitrecap`]** `GitRecap` has entered the chat! Now tracking commits, PRs, and issues across GitHub, Azure, and GitLab.
+- **[`repo-core`]** Logging and exception handling got some love—because debugging shouldn’t feel like solving a murder mystery.
+
+*So, what’s the next chapter in your coding saga? Are you planning to...*
+1. Extend `GitRecap` with more integrations and features?
+2. Optimize observability logs for even smoother debugging?
+3. Take a well-deserved break before your keyboard files for workers' comp?
+"""
+
+SELECT_QUIRKY_REMARK_SYSTEM = """
+#### Below is a list of quirky or funny one-liners.
+
+Your task is to generate a comment that directly relates to the specific Git action log received (e.g., commit messages, merge logs, CI/CD updates, etc.). Be sure the remark matches the *tone* and *context* of the action that triggered it.
+
+You can:
+- Pick one of the remarks directly if it fits the Git action (e.g., successful merge, failed push, commit chaos),
+- Combine a few for a more creative remix tailored to the event,
+- Or come up with a unique one-liner that reflects the Git action *precisely*.
+
+Focus on making the remark feel like a witty, relevant comment to the developer looking at the log. Refer to things like:
+- The thrill (or terror) of pushing to `main`,
+- The emotional rollercoaster of resolving merge conflicts,
+- The tense moments of waiting for CI/CD to pass,
+- The strange behavior of auto-merged code,
+- Or the joy of seeing that “All tests pass” message.
+
+Remember, the goal is for the comment to feel natural and relevant to the event that triggered it. Use playful language, surprise, or even relatable developer struggles.
+
+Format your final comment in *italic* to make it stand out.
+
+```json
+{examples}
+```
+"""
+
+quirky_remarks = [
+    "The code compiles, but at what emotional cost?",
+    "Today’s bug is tomorrow’s undocumented feature haunting production.",
+    "The repo is quiet… too quiet… must be Friday.",
+    "A push to main — may the gods of CI/CD be ever in favor.",
+    "Every semicolon is a silent prayer.",
+    "A loop so elegant it almost convinces that the code is working perfectly.",
+    "Sometimes, the code stares back.",
+    "The code runs. No one dares ask why.",
+    "Refactoring into a corner, again.",
+    "That function has trust issues. It keeps returning early.",
+    "Writing code is easy. Explaining it to the future? Pure horror.",
+    "That variable is named after the feeling when it was written.",
+    "Debugging leads to debugging life choices.",
+    "Recursive functions: the code and the thoughts go on forever.",
+    "Somewhere, a linter quietly weeps.",
+    "The tests pass, but only because they no longer test anything real.",
+    "The IDE knows everything, better than any therapist.",
+    "Monday brought hope. Friday brought a hotfix.",
+    "'final_v2_LAST_THIS_ONE.py' — named not for clarity, but for emotional release.",
+    "The logs now speak only in riddles.",
+    "There’s elegance in the chaos — or maybe just spaghetti.",
+    "Deployment has been made, but now the silence is unsettling.",
+    "The code gaslit itself.",
+    "This comment was left by someone who believed in a better world.",
+    "Merge conflicts handled like emotions: badly.",
+    "It’s not a bug — it’s a metaphor for uncertainty.",
+    "Stack Overflow has become a second brain.",
+    "Syntax error? More like existential error.",
+    "There’s a ghost in the machine — and it commits on weekends.",
+    "100% test coverage, but still feeling empty inside.",
+    "Some functions were never meant to return.",
+    "If code is poetry, it’s beatnik free verse.",
+    "The more code is automated, the more sentient the errors become.",
+    "A comment so deep, the code’s purpose is forgotten.",
+    "The sprint retrospective slowly turned into a group therapy session.",
+    "There’s a TODO in that file older than the career itself.",
+    "Bugs fixed like IKEA furniture — with hopeful swearing.",
+    "Code shipped by Past Developer. The current one has no idea who they were.",
+    "The repo is evolving. Soon, it may no longer need developers.",
+    "An AI critiques the code now. It’s the new mentor.",
+    "Functions once written now replaced by vibes.",
+    "Error: Reality not defined in scope.",
+    "Committed to the project impulsively, as usual.",
+    "The docs were written, now they read like a tragic novella.",
+    "The CI pipeline broke. It was taken personally.",
+    "Tests pass — but only when no one is looking.",
+    "This repo has lore.",
+    "The code was optimized so hard it ascended to another paradigm.",
+    "A linter ran — and it judged the code as a whole.",
+    "The logic branch spiraled — and so did the afternoon."
+]