{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "db4208b9-5da4-46df-b77a-0f1836c9e4ec", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/raid/sourab/transformers/src/transformers/utils/hub.py:122: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n", " warnings.warn(\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] } ], "source": [ "import os\n", "\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n", "from peft import PeftConfig, PeftModel\n", "from peft import PeftModel, PeftConfig\n", "from transformers import AutoModelForCausalLM, AutoTokenizer\n", "from datasets import load_dataset\n", "import torch\n", "import random\n", "\n", "peft_model_id = \"smangrul/tinyllama_lora_norobots\"\n", "device = \"cuda\"\n", "config = PeftConfig.from_pretrained(peft_model_id)\n", "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_4bit=True, device_map=\"auto\")\n", "tokenizer = AutoTokenizer.from_pretrained(peft_model_id)\n", "model.resize_token_embeddings(len(tokenizer))\n", "model = PeftModel.from_pretrained(model, peft_model_id, adapter_name=\"norobots\")\n", "_ = model.load_adapter(\"smangrul/tinyllama_lora_sql\", adapter_name=\"sql\")\n", "_ = model.load_adapter(\"smangrul/tinyllama_lora_adcopy\", adapter_name=\"adcopy\")" ] }, { "cell_type": "code", "execution_count": 684, "id": "541dab43-9675-42a2-8d90-7437df9f0fa0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 23.3 s, sys: 535 ms, total: 23.8 s\n", "Wall time: 796 ms\n" ] } ], "source": [ "%%time\n", "# [0.8, 0.1, 0.1] linear #[1.0, 0.2] 0.7 density dare_linear #[1.5, 0.3] 0.5 density ties #[0.8, 0.5] cat\n", "adapters = [\"norobots\", \"adcopy\", \"sql\"]\n", "weights = [2.0, 0.3, 0.7]\n", "adapter_name = \"merge\"\n", "density = 0.2\n", "combination_type = \"ties\"\n", "if adapter_name in model.peft_config:\n", " model.delete_adapter(adapter_name)\n", "model.add_weighted_adapter(adapters, weights, adapter_name, combination_type=combination_type, density=density)" ] }, { "cell_type": "code", "execution_count": 685, "id": "76596671-3677-47f0-9d66-81f40bc4d726", "metadata": {}, "outputs": [], "source": [ "model.eval()\n", "model.set_adapter(\"merge\")" ] }, { "cell_type": "code", "execution_count": 691, "id": "9d59f9f3-6313-43d8-be36-4ca2bbb105b2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "<|im_start|>user \n", "Write an essay about Generative AI.<|im_end|> \n", "<|im_start|>assistant \n", "Generative Artificial Intelligence (GAI) is a type of artificial intelligence that uses machine learning to create art, music and other creations. It's like having a human artist who creates something new without the need for inspiration or motivation.<|im_end|>\n" ] } ], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Write an essay about Generative AI.\"},\n", "]\n", "text = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(\"cuda\") for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs,\n", " max_new_tokens=256,\n", " do_sample=True,\n", " top_p=0.95,\n", " temperature=0.2,\n", " repetition_penalty=1.2,\n", " eos_token_id=tokenizer.eos_token_id,\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] }, { "cell_type": "code", "execution_count": 689, "id": "e5c1daeb-59c8-41d7-bebb-7abd052ab917", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "<|im_start|>system \n", "Create a text ad given the following product and description.<|im_end|> \n", "<|im_start|>user \n", "Product: Sony PS5 PlayStation Console\n", "Description: The PS5™ console unleashes new gaming possibilities that you never anticipated.<|im_end|> \n", "<|im_start|>assistant \n", "Ad Text: Experience the next-gen power of the all-new Sony PS5 with its stunning visuals, innovative gameplay features, and more! Get ready to play in style as you experience the future of gaming on your own terms.<|im_end|>\n" ] } ], "source": [ "messages = [\n", " {\"role\": \"system\", \"content\": \"Create a text ad given the following product and description.\"},\n", " {\n", " \"role\": \"user\",\n", " \"content\": \"Product: Sony PS5 PlayStation Console\\nDescription: The PS5™ console unleashes new gaming possibilities that you never anticipated.\",\n", " },\n", "]\n", "text = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(\"cuda\") for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs,\n", " max_new_tokens=128,\n", " do_sample=True,\n", " top_p=0.95,\n", " temperature=0.2,\n", " repetition_penalty=1.2,\n", " eos_token_id=tokenizer.eos_token_id,\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] }, { "cell_type": "code", "execution_count": 690, "id": "5bb08b46-90ae-48a8-8783-ca74b3e26e42", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Table: 2-11365528-2\n", "Columns: ['Team', 'Head Coach', 'President', 'Home Ground', 'Location']\n", "Natural Query: Who is the Head Coach of the team whose President is Mario Volarevic?\n", "SQL Query: SELECT Head Coach FROM 2-11365528-2 WHERE President = Mario Volarevic\n" ] } ], "source": [ "text = \"\"\"Table: 2-11365528-2\n", "Columns: ['Team', 'Head Coach', 'President', 'Home Ground', 'Location']\n", "Natural Query: Who is the Head Coach of the team whose President is Mario Volarevic?\n", "SQL Query:\"\"\"\n", "\n", "inputs = tokenizer(text, return_tensors=\"pt\") # , add_special_tokens=False)\n", "inputs = {k: v.to(\"cuda\") for k, v in inputs.items()}\n", "outputs = model.generate(\n", " **inputs, max_new_tokens=64, repetition_penalty=1.1, eos_token_id=tokenizer(\"\").input_ids[-1]\n", ")\n", "print(tokenizer.decode(outputs[0]))" ] }, { "cell_type": "code", "execution_count": null, "id": "cc927536-bb58-4270-876f-10ff1a94802d", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 5 }