new changes

.gitignore

@@ -1 +1,3 @@
-hf_env/
+hf_env/
+Y
+Y.pub

0.26.0

@@ -0,0 +1,28 @@
+Collecting accelerate
+  Using cached accelerate-1.5.2-py3-none-any.whl.metadata (19 kB)
+Requirement already satisfied: numpy<3.0.0,>=1.17 in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (2.2.4)
+Requirement already satisfied: packaging>=20.0 in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (24.2)
+Requirement already satisfied: psutil in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (7.0.0)
+Requirement already satisfied: pyyaml in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (6.0.2)
+Requirement already satisfied: torch>=2.0.0 in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (2.6.0+cu118)
+Requirement already satisfied: huggingface-hub>=0.21.0 in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (0.29.3)
+Requirement already satisfied: safetensors>=0.4.3 in f:\gemma testing\gemma_env\lib\site-packages (from accelerate) (0.5.3)
+Requirement already satisfied: filelock in f:\gemma testing\gemma_env\lib\site-packages (from huggingface-hub>=0.21.0->accelerate) (3.18.0)
+Requirement already satisfied: fsspec>=2023.5.0 in f:\gemma testing\gemma_env\lib\site-packages (from huggingface-hub>=0.21.0->accelerate) (2025.3.0)
+Requirement already satisfied: requests in f:\gemma testing\gemma_env\lib\site-packages (from huggingface-hub>=0.21.0->accelerate) (2.32.3)
+Requirement already satisfied: tqdm>=4.42.1 in f:\gemma testing\gemma_env\lib\site-packages (from huggingface-hub>=0.21.0->accelerate) (4.67.1)
+Requirement already satisfied: typing-extensions>=3.7.4.3 in f:\gemma testing\gemma_env\lib\site-packages (from huggingface-hub>=0.21.0->accelerate) (4.12.2)
+Requirement already satisfied: networkx in f:\gemma testing\gemma_env\lib\site-packages (from torch>=2.0.0->accelerate) (3.3)
+Requirement already satisfied: jinja2 in f:\gemma testing\gemma_env\lib\site-packages (from torch>=2.0.0->accelerate) (3.1.6)
+Requirement already satisfied: setuptools in f:\gemma testing\gemma_env\lib\site-packages (from torch>=2.0.0->accelerate) (77.0.3)
+Requirement already satisfied: sympy==1.13.1 in f:\gemma testing\gemma_env\lib\site-packages (from torch>=2.0.0->accelerate) (1.13.1)
+Requirement already satisfied: mpmath<1.4,>=1.1.0 in f:\gemma testing\gemma_env\lib\site-packages (from sympy==1.13.1->torch>=2.0.0->accelerate) (1.3.0)
+Requirement already satisfied: colorama in f:\gemma testing\gemma_env\lib\site-packages (from tqdm>=4.42.1->huggingface-hub>=0.21.0->accelerate) (0.4.6)
+Requirement already satisfied: MarkupSafe>=2.0 in f:\gemma testing\gemma_env\lib\site-packages (from jinja2->torch>=2.0.0->accelerate) (3.0.2)
+Requirement already satisfied: charset-normalizer<4,>=2 in f:\gemma testing\gemma_env\lib\site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (3.4.1)
+Requirement already satisfied: idna<4,>=2.5 in f:\gemma testing\gemma_env\lib\site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (3.10)
+Requirement already satisfied: urllib3<3,>=1.21.1 in f:\gemma testing\gemma_env\lib\site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (2.3.0)
+Requirement already satisfied: certifi>=2017.4.17 in f:\gemma testing\gemma_env\lib\site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (2025.1.31)
+Using cached accelerate-1.5.2-py3-none-any.whl (345 kB)
+Installing collected packages: accelerate
+Successfully installed accelerate-1.5.2

app.ipynb

@@ -0,0 +1,526 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoProcessor, Gemma3ForConditionalGeneration, TextIteratorStreamer\n",
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_id =  \"google/gemma-3-12b-it\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(model_id, padding_side=\"left\")\n",
+    "model = Gemma3ForConditionalGeneration.from_pretrained(\n",
+    "    model_id, device_map=\"auto\", torch_dtype=torch.bfloat16, attn_implementation=\"eager\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "import torch\n",
+    "\n",
+    "pipe = pipeline(\n",
+    "    \"image-text-to-text\",\n",
+    "    model=\"google/gemma-3-4b-it\",\n",
+    "    device=\"cuda\",\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    cache_dir=\"F:\\\\huggingface_cache\"\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pip install accelerate\n",
+    "print(\"Hi\")\n",
+    "from transformers import AutoProcessor, Gemma3ForConditionalGeneration\n",
+    "import requests\n",
+    "import torch\n",
+    "from PIL import Image\n",
+    "\n",
+    "print(\"Done\")\n",
+    "model_id = \"google/gemma-3-4b-it\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "pip install bitsandbytes\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Using device: cuda\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "70d5eaba90854abfb20ffc43970c226c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some parameters are on the meta device because they were offloaded to the cpu.\n",
+      "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.50, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import Gemma3ForConditionalGeneration, AutoProcessor\n",
+    "from transformers import BitsAndBytesConfig\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Using device: {device}\")\n",
+    "\n",
+    "model_id = \"google/gemma-3-4b-it\"\n",
+    "\n",
+    "\n",
+    "quantization_config = BitsAndBytesConfig(load_in_8bit=True,  # Load in 8-bit mode\n",
+    "    llm_int8_enable_fp32_cpu_offload=True  # Allow CPU offloading for layers that don't fit in VRAM\n",
+    ")\n",
+    "# Load the model and move it to the correct device\n",
+    "model = Gemma3ForConditionalGeneration.from_pretrained(\n",
+    "    model_id,\n",
+    "    cache_dir=\"F:\\\\huggingface_cache\",\n",
+    "    device_map=\"auto\",  # Automatically assigns layers to available devices\n",
+    "    quantization_config=quantization_config\n",
+    ").eval()\n",
+    "\n",
+    "# Load the processor\n",
+    "processor = AutoProcessor.from_pretrained(model_id, cache_dir=\"F:\\\\huggingface_cache\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "messages = [\n",
+    "    {\n",
+    "        \"role\": \"system\",\n",
+    "        \"content\": [{\"type\": \"text\", \"text\": \"You are a helpful assistant.\"}]\n",
+    "    },\n",
+    "    {\n",
+    "        \"role\": \"user\",\n",
+    "        \"content\": [\n",
+    "            \n",
+    "            {\"type\": \"text\", \"text\": \"Whats the color of sky?.\"}\n",
+    "        ]\n",
+    "    }\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs = processor.apply_chat_template(\n",
+    "    messages, add_generation_prompt=True, tokenize=True,\n",
+    "    return_dict=True, return_tensors=\"pt\"\n",
+    ").to(model.device, dtype=torch.bfloat16)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_len = inputs[\"input_ids\"].shape[-1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "F:\\HF\\gemma-examples\\hf_env\\Lib\\site-packages\\transformers\\generation\\configuration_utils.py:633: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.95` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
+      "  warnings.warn(\n",
+      "F:\\HF\\gemma-examples\\hf_env\\Lib\\site-packages\\transformers\\generation\\configuration_utils.py:650: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `64` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.\n",
+      "  warnings.warn(\n",
+      "\n",
+      "KeyboardInterrupt\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "with torch.inference_mode():\n",
+    "    generation = model.generate(**inputs, max_new_tokens=100, do_sample=False)\n",
+    "    generation = generation[0][input_len:]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "with torch.inference_mode():\n",
+    "    generation = model.generate(**inputs, max_new_tokens=100, do_sample=False)\n",
+    "    generation = generation[0][input_len:]\n",
+    "\n",
+    "decoded = processor.decode(generation, skip_special_tokens=True)\n",
+    "print(decoded)\n",
+    "\n",
+    "# **Overall Impression:** The image is a close-up shot of a vibrant garden scene, \n",
+    "# focusing on a cluster of pink cosmos flowers and a busy bumblebee. \n",
+    "# It has a slightly soft, natural feel, likely captured in daylight.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ssh-keygen -t ed25519 -C \"[email protected]\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "access_token='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'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SHA256:5m1PnI3v38H/z5cecFQQKtSAveRgolJirSjhjixvkVs [email protected]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use a pipeline as a high-level helper\n",
+    "from transformers import pipeline\n",
+    "\n",
+    "messages = [\n",
+    "    {\"role\": \"user\", \"content\": \"Who are you?\"},\n",
+    "]\n",
+    "pipe = pipeline(\"image-text-to-text\", model=\"google/gemma-3-4b-it\")\n",
+    "pipe(messages)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Model ID\n",
+    "model_id = \"gemma3:latest\"\n",
+    "\n",
+    "from ollama import chat\n",
+    "from ollama import ChatResponse\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "That's a fantastic question, and one that's fascinated people for centuries! The short answer is: **it's due to a phenomenon called Rayleigh scattering.** Here's a more detailed explanation:\n",
+      "\n",
+      "**1. Sunlight and Colors:**\n",
+      "\n",
+      "* Sunlight appears white, but it's actually made up of *all* the colors of the rainbow – red, orange, yellow, green, blue, indigo, and violet.  Think of a prism splitting light.\n",
+      "\n",
+      "**2. The Atmosphere and Molecules:**\n",
+      "\n",
+      "* The Earth's atmosphere is filled with tiny particles – mostly nitrogen and oxygen molecules.\n",
+      "* These molecules are much smaller than the wavelengths of visible light.\n",
+      "\n",
+      "**3. Rayleigh Scattering:**\n",
+      "\n",
+      "* When sunlight enters the atmosphere, it collides with these tiny air molecules.\n",
+      "* **Rayleigh scattering** is the scattering of electromagnetic radiation (like light) by particles of a much smaller wavelength.\n",
+      "* **Crucially, shorter wavelengths of light (blue and violet) are scattered *much* more strongly than longer wavelengths (red and orange).**  This is because the amount of scattering is inversely proportional to the fourth power of the wavelength.  (That's a fancy way of saying that blue light gets scattered *way* more intensely than red light.)\n",
+      "\n",
+      "**4. Why Blue Specifically?**\n",
+      "\n",
+      "* Violet light is scattered even *more* than blue light. However, there are a couple of reasons why we see a blue sky instead of a violet one:\n",
+      "    * **Sunlight emits less violet light than blue light.**\n",
+      "    * **Our eyes are more sensitive to blue light than violet light.**\n",
+      "\n",
+      "\n",
+      "**In simpler terms:**\n",
+      "\n",
+      "Imagine throwing a small ball (blue light) and a large ball (red light) at a bumpy surface (the atmosphere). The small ball is more likely to bounce off in all directions, while the large ball is more likely to continue straight on.  Blue light is \"bounced around\" more in the atmosphere, spreading it across the sky.\n",
+      "\n",
+      "**Why sunsets are red/orange:**\n",
+      "\n",
+      "At sunset (and sunrise), the sunlight has to travel through *much* more of the atmosphere to reach our eyes.  During this longer journey, most of the blue light has already been scattered away.  The longer wavelengths – red and orange – are less affected and can make it through, giving us those beautiful sunset colors.\n",
+      "\n",
+      "\n",
+      "\n",
+      "**Resources for further learning:**\n",
+      "\n",
+      "* **NASA - Why is the sky blue?:** [https://science.nasa.gov/sky-facts/why-is-the-sky-blue/](https://science.nasa.gov/sky-facts/why-is-the-sky-blue/)\n",
+      "* **Wikipedia - Rayleigh scattering:** [https://en.wikipedia.org/wiki/Rayleigh_scattering](https://en.wikipedia.org/wiki/Rayleigh_scattering)\n",
+      "\n",
+      "\n",
+      "Do you want me to delve into any specific aspect of this explanation, such as:\n",
+      "\n",
+      "*   The math behind Rayleigh scattering?\n",
+      "*   How this affects other celestial phenomena?\n",
+      "That's a fantastic question, and one that's fascinated people for centuries! The short answer is: **it's due to a phenomenon called Rayleigh scattering.** Here's a more detailed explanation:\n",
+      "\n",
+      "**1. Sunlight and Colors:**\n",
+      "\n",
+      "* Sunlight appears white, but it's actually made up of *all* the colors of the rainbow – red, orange, yellow, green, blue, indigo, and violet.  Think of a prism splitting light.\n",
+      "\n",
+      "**2. The Atmosphere and Molecules:**\n",
+      "\n",
+      "* The Earth's atmosphere is filled with tiny particles – mostly nitrogen and oxygen molecules.\n",
+      "* These molecules are much smaller than the wavelengths of visible light.\n",
+      "\n",
+      "**3. Rayleigh Scattering:**\n",
+      "\n",
+      "* When sunlight enters the atmosphere, it collides with these tiny air molecules.\n",
+      "* **Rayleigh scattering** is the scattering of electromagnetic radiation (like light) by particles of a much smaller wavelength.\n",
+      "* **Crucially, shorter wavelengths of light (blue and violet) are scattered *much* more strongly than longer wavelengths (red and orange).**  This is because the amount of scattering is inversely proportional to the fourth power of the wavelength.  (That's a fancy way of saying that blue light gets scattered *way* more intensely than red light.)\n",
+      "\n",
+      "**4. Why Blue Specifically?**\n",
+      "\n",
+      "* Violet light is scattered even *more* than blue light. However, there are a couple of reasons why we see a blue sky instead of a violet one:\n",
+      "    * **Sunlight emits less violet light than blue light.**\n",
+      "    * **Our eyes are more sensitive to blue light than violet light.**\n",
+      "\n",
+      "\n",
+      "**In simpler terms:**\n",
+      "\n",
+      "Imagine throwing a small ball (blue light) and a large ball (red light) at a bumpy surface (the atmosphere). The small ball is more likely to bounce off in all directions, while the large ball is more likely to continue straight on.  Blue light is \"bounced around\" more in the atmosphere, spreading it across the sky.\n",
+      "\n",
+      "**Why sunsets are red/orange:**\n",
+      "\n",
+      "At sunset (and sunrise), the sunlight has to travel through *much* more of the atmosphere to reach our eyes.  During this longer journey, most of the blue light has already been scattered away.  The longer wavelengths – red and orange – are less affected and can make it through, giving us those beautiful sunset colors.\n",
+      "\n",
+      "\n",
+      "\n",
+      "**Resources for further learning:**\n",
+      "\n",
+      "* **NASA - Why is the sky blue?:** [https://science.nasa.gov/sky-facts/why-is-the-sky-blue/](https://science.nasa.gov/sky-facts/why-is-the-sky-blue/)\n",
+      "* **Wikipedia - Rayleigh scattering:** [https://en.wikipedia.org/wiki/Rayleigh_scattering](https://en.wikipedia.org/wiki/Rayleigh_scattering)\n",
+      "\n",
+      "\n",
+      "Do you want me to delve into any specific aspect of this explanation, such as:\n",
+      "\n",
+      "*   The math behind Rayleigh scattering?\n",
+      "*   How this affects other celestial phenomena?\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "response: ChatResponse = chat(model=model_id, messages=[\n",
+    "  {\n",
+    "    'role': 'user',\n",
+    "    'content': 'Why is the sky blue?',\n",
+    "  },\n",
+    "])\n",
+    "print(response['message']['content'])\n",
+    "# or access fields directly from the response object\n",
+    "print(response.message.content)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

app2.py

@@ -0,0 +1,27 @@
+from PIL import Image
+import requests
+import torch
+import os
+# Set the cache directory
+cache_dir = "F:\\huggingface_cache"
+
+# Set environment variables for good measure
+# os.environ["TRANSFORMERS_CACHE"] = cache_dir
+# os.environ["HF_HOME"] = cache_dir
+# os.environ["HUGGINGFACE_HUB_CACHE"] = cache_dir
+
+# Model ID
+model_id = "google/gemma-3-4b-it"
+
+from ollama import chat
+from ollama import ChatResponse
+
+response: ChatResponse = chat(model='llama3.2', messages=[
+  {
+    'role': 'user',
+    'content': 'Why is the sky blue?',
+  },
+])
+print(response['message']['content'])
+# or access fields directly from the response object
+print(response.message.content)