Init files

.gitignore

@@ -0,0 +1,4 @@
+.idea/ C:\Users\james/.ssh/id_ed25518
+cat C:\Users\james\.ssh\id_ed25518.pub
+
+ssh -i C:\Users\james\.ssh\id_ed25518 [email protected]

Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.11
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN pip install -U gptqmodel --no-build-isolation -v
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+	SYSTEM=spaces
+ 
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

README.md

@@ -1,14 +1,22 @@
----
-title: Tiny GPTQ V2 LLM On CPU
-emoji: 🌖
-colorFrom: green
-colorTo: gray
-sdk: gradio
-sdk_version: 5.27.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: A Simple app to test out the potentials of small GPTQ LLM mo
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Tiny GPTQ V2 LLM On CPU
+emoji: 🌖
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 5.27.1
+app_file: app.py
+pinned: false
+license: mit
+short_description: A Simple app to test out the potentials of small GPTQ LLM mo
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+### Build and Run on Docker
+
+```text
+docker build -t my_gradio_app .  
+docker run -d -p 7860:7860 my_gradio_app  
+```

app.py

@@ -0,0 +1,127 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM, GPTQConfig
+import inspect
+from typing import get_type_hints, Callable, Any
+import gradio as gr
+
+model_name = "wolfofbackstreet/SmolLM2-135M-int4-qptq-v2"
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+# Define GPTQ configuration
+gptq_config = GPTQConfig(bits=4, use_exllama=False, use_cuda_fp16=False)
+
+# Load pre-quantized model on CPU
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    quantization_config=gptq_config,
+    device_map="cpu"  # Explicitly enforce CPU execution
+)
+
+
+def parse_docstring(func):
+    doc = inspect.getdoc(func)
+    if not doc:
+        return {"title": "Untitled", "description": ""}
+
+    lines = doc.splitlines()
+    title = next((line.replace("Title:", "").strip() for line in lines if line.startswith("Title:")), "Untitled")
+    description = "\n".join(line.strip() for line in lines if line.startswith("Description:"))
+    description = description.replace("Description:", "").strip()
+
+    return {"title": title, "description": description}
+
+def gradio_app_with_docs(func: Callable) -> Callable:
+    sig = inspect.signature(func)
+    type_hints = get_type_hints(func)
+    metadata = parse_docstring(func)
+
+    """
+    A decorator that automatically builds and launches a Gradio interface
+    based on function type hints.
+
+    Args:
+        func: A callable with type-hinted parameters and return type.
+
+    Returns:
+        The wrapped function with a `.launch()` method to start the app.
+    """
+    # Infer Gradio components from type hints
+    def _map_type(t: type) -> gr.Component:
+        if t == str:
+            return gr.Textbox(label="Input")
+        elif t == int:
+            return gr.Number(precision=0)
+        elif t == float:
+            return gr.Number()
+        elif t == bool:
+            return gr.Checkbox()
+        elif hasattr(t, "__origin__") and t.__origin__ == list:  # Handle List[type]
+            elem_type = t.__args__[0]
+            if elem_type == str:
+                return gr.Dropdown(choices=["Option1", "Option2"])
+            else:
+                raise ValueError(f"Unsupported list element type: {elem_type}")
+        else:
+            raise ValueError(f"Unsupported type: {t}")
+
+    # Extract function signature and type hints
+    sig = inspect.signature(func)
+    type_hints = get_type_hints(func)
+
+    # Map parameters to Gradio inputs
+    inputs = []
+    for name, param in sig.parameters.items():
+        if name == "self":
+            continue  # Skip self in class methods
+        param_type = type_hints.get(name, Any)
+        component = _map_type(param_type)
+        component.label = name.replace("_", " ").title()
+        inputs.append(component)
+
+    # Map return type to Gradio output
+    return_type = type_hints.get("return", Any)
+    outputs = _map_type(return_type)
+
+    # Wrap function with Gradio interface
+    interface = gr.Interface(fn=func, inputs=inputs, outputs=outputs)
+
+    with gr.Blocks() as demo:
+        gr.Markdown(f"## {metadata['title']}\n{metadata['description']}")
+        interface = gr.Interface(fn=func, inputs=inputs, outputs=outputs)
+
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    wrapper.launch = lambda: demo.launch()
+    return wrapper
+
+
+@gradio_app_with_docs
+def generate_response(prompt: str) -> str:
+    """
+    Title: Super Tiny GPTQ V2 Model on CPU
+    Description: A Simple app to test out the potentials of small GPTQ LLM model.
+
+    Args:
+        prompt (str): A simple prompt.
+
+    Returns:
+        str: Simplified response.
+    """
+    inputs = tokenizer(prompt, return_tensors="pt").to("cpu")  # Move inputs to CPU
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=50,
+        temperature=0.7,
+        top_p=0.9
+    )
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+# # Example usage
+# prompt = "Explain quantum computing in simple terms."
+# response = generate_response(prompt)
+# print(response)
+
+
+if __name__ == "__main__":
+    generate_response.launch()

requirements.txt

@@ -0,0 +1,4 @@
+optimum
+gradio
+torch
+transformers