Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1 @@
+*config*

generate_data.py

@@ -0,0 +1,41 @@
+import openai
+import json
+import threading
+
+config: dict = json.loads(open("config.json").read())
+
+dataset = []
+
+def gen():
+    try:
+        response = openai.OpenAI(
+            api_key=config["key"],
+            base_url=config["url"],
+            default_headers={
+                "User-Agent": "OpenAI SDK",
+                "Cookie": config.get("cookie", "")
+            }
+        ).chat.completions.create(
+            model=config["model"],
+            extra_body={"model_id": config["model"]},
+            messages=[
+                {"role": "user", "content": "Generate some texts for training LLM. Do not add any other elements to your response."}
+            ],
+            stream=False
+        )
+        dataset.append(response.choices[0].message.content)
+    except Exception as e:
+        print(e)
+
+thpool = []
+
+for i in range(100):
+    th = threading.Thread(target=gen)
+    th.start()
+    thpool.append(th)
+
+for th in thpool:
+    th.join()
+
+with open("train_data.txt", "a", encoding="utf-8") as f:
+    f.write("\n".join(dataset))

tiny_llm.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e667fcebb47365f15110285118cf5941aeca57b08cea65da785850c1e3f282da
+size 24274

train.py

@@ -0,0 +1,108 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import os
+import json
+from tqdm import tqdm, trange
+import time
+
+# Generate simple training data
+training_text = open("train_data.txt", encoding="utf-8").read()
+chars = sorted(list(set(training_text)))  # Unique characters
+char_to_idx = {ch: i for i, ch in enumerate(chars)}
+idx_to_char = {i: ch for i, ch in enumerate(chars)}
+
+# Model parameters
+input_size = len(chars)
+hidden_size = 32
+output_size = len(chars)
+sequence_length = 5
+epochs = 1000
+learning_rate = 0.0001
+model_path = "tiny_llm.pth"
+
+# Create training data (input-output pairs)
+train_data = []
+for i in range(len(training_text) - sequence_length):
+    input_seq = training_text[i : i + sequence_length]
+    target_char = training_text[i + sequence_length]
+    train_data.append((torch.tensor([char_to_idx[ch] for ch in input_seq]), char_to_idx[target_char]))
+
+# Define the simple RNN model
+class SimpleRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, output_size):
+        super(SimpleRNN, self).__init__()
+        self.hidden_size = hidden_size
+        self.rnn = nn.RNN(input_size, hidden_size, batch_first=True)
+        self.fc = nn.Linear(hidden_size, output_size)
+
+    def forward(self, x, hidden):
+        x = torch.nn.functional.one_hot(x, num_classes=input_size).float()
+        out, hidden = self.rnn(x.unsqueeze(0), hidden)
+        out = self.fc(out[:, -1, :])  # Take last time step's output
+        return out, hidden
+
+# Load model if available
+if os.path.exists(model_path):
+    model = torch.load(model_path, weights_only=False)
+    with open("vocab.json", "r") as f:
+        chars = json.loads(f.read())
+    char_to_idx = {ch: i for i, ch in enumerate(chars)}
+    idx_to_char = {i: ch for i, ch in enumerate(chars)}
+    print("Loaded pre-trained model.")
+else:
+    print("Training new model...")
+    # Initialize the model
+    model = SimpleRNN(input_size, hidden_size, output_size)
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=learning_rate)
+    for epoch in range(epochs):
+        try:
+            total_loss = 0
+            hidden = torch.zeros(1, 1, hidden_size)
+            
+            pbar = tqdm(train_data, desc=f"Epoch={epoch}, Loss=N/A")
+            count = 0
+            for input_seq, target in pbar:
+                count += 1
+                optimizer.zero_grad()
+                output, hidden = model(input_seq, hidden.detach())
+                loss = criterion(output, torch.tensor([target]))
+                loss.backward()
+                optimizer.step()
+                total_loss += loss.item()
+                pbar.desc = f"Epoch={epoch}, Loss={total_loss / count:.12f}"
+            
+            pbar.close()
+            time.sleep(1)
+        except KeyboardInterrupt:
+            break
+    
+    hidden = torch.zeros(1, 1, hidden_size)
+    output, hidden = model(input_seq, hidden.detach())
+
+    # Save the trained model
+    torch.save(model, model_path)
+    with open("vocab.json", "w") as f:
+        f.write(json.dumps(chars))
+    print("Model saved.")
+
+# Text generation function
+def generate_text(start_text, length=10000):
+    model.eval()
+    hidden = torch.zeros(1, 1, hidden_size)
+    input_seq = torch.tensor([char_to_idx[ch] for ch in start_text])
+
+    generated_text = start_text
+    for _ in trange(length):
+        output, hidden = model(input_seq, hidden)
+        predicted_idx = output.argmax().item()
+        generated_text += idx_to_char[predicted_idx]
+        input_seq = torch.cat((input_seq[1:], torch.tensor([predicted_idx])))
+
+    return generated_text
+
+# Generate some text
+
+while True:
+    print("LLM Output: ", generate_text(input("Ask LLM: ")))

train_data.txt

@@ -0,0 +1,193 @@
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet.
+
+Artificial intelligence is a wonderful field that combines computer science and large datasets to create systems capable of performing complex tasks.
+
+In the heart of the city, a small coffee shop thrives, attracting customers with its aromatic brews and cozy atmosphere.
+
+Quantum mechanics is a fundamental theory in physics that describes nature at the smallest scales, such as atoms and subatomic particles.
+
+Exploring space has always been a dream of humanity, with missions to the Moon, Mars, and beyond capturing our imagination.
+
+Learning new languages opens up opportunities for communication and understanding across different cultures and communities.
+
+The history of ancient civilizations reveals how humans have evolved socially, politically, and technologically over thousands of years.
+
+Cooking is both an art and a science, requiring creativity, precision, and a deep understanding of flavors and ingredients.
+
+Climate change poses significant challenges to our planet, influencing weather patterns, ecosystems, and biodiversity worldwide.
+
+Books are portals to other worlds, offering knowledge, entertainment, and inspiration through stories and ideas crafted by authors.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet.
+
+Artificial intelligence is a wonderful field that combines computer science and data to create intelligent systems.
+
+In the heart of the city, a small café serves the best coffee, attracting visitors from all around the neighborhood.
+
+Exploring space has always been a dream of humanity, with countless scientists working tirelessly to make it a reality.
+
+Books open doors to new worlds, allowing readers to experience lives and cultures different from their own.
+
+Climate change poses significant challenges, but innovative solutions are being developed to combat its effects worldwide.
+
+Learning new skills can be both challenging and rewarding, often leading to personal and professional growth.
+
+History is filled with fascinating stories of individuals who changed the world with their ideas and determination.
+
+Music has the power to evoke deep emotions, bringing people together across different cultures and languages.
+
+Technology continues to evolve at a rapid pace, transforming the way we live, work, and communicate with each other.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet.
+
+In the heart of the city, a small café serves the best coffee, attracting visitors from all around the neighborhood.
+
+Autumn leaves fall gently to the ground, painting the earth in shades of red, orange, and yellow.
+
+A journey of a thousand miles begins with a single step, as the ancient philosopher once said.
+
+The scientist conducted numerous experiments to test her hypothesis about the behavior of light in different mediums.
+
+History books are filled with stories of great leaders who shaped the world through their vision and determination.
+
+Under the starlit sky, a group of friends gathered around a campfire, sharing stories and laughter late into the night.
+
+Deep in the forest, a hidden cabin provided shelter for travelers seeking refuge from the storm.
+
+Every morning, the fisherman sets out to sea, hoping for a bountiful catch to bring back to his family.
+
+Artificial intelligence is transforming industries, offering new possibilities for innovation and efficiency in everyday tasks.
+The quick brown fox jumps over the lazy dog.  
+A journey of a thousand miles begins with a single step.  
+The sun sets beautifully over the horizon, painting the sky in hues of orange and pink.  
+Learning is a lifelong process that enriches our minds and souls.  
+In the heart of the forest, a gentle stream winds its way through ancient trees.  
+Books are windows to countless worlds, waiting to be explored.  
+Every challenge presents an opportunity for growth and self-discovery.  
+The aroma of freshly brewed coffee filled the cozy kitchen on a rainy morning.  
+Friendship is built on trust, understanding, and shared experiences.  
+The stars twinkled like diamonds scattered across the velvet night sky.  
+
+Technology continues to evolve at an unprecedented pace, shaping the future of humanity.  
+A small act of kindness can have a ripple effect, touching lives in unexpected ways.  
+Mountains stood tall against the clear blue sky, their peaks dusted with snow.  
+Music has the power to evoke emotions words alone cannot express.  
+Curiosity drives us to ask questions, seek answers, and expand our horizons.  
+The ocean waves crashed against the shore, a symphony of nature's rhythm.  
+History teaches us valuable lessons if we take the time to listen and reflect.  
+Children laughed as they played under the warm glow of the afternoon sun.  
+Dreams inspire us to reach beyond our limits and create something extraordinary.  
+Life is a tapestry woven from moments of joy, struggle, love, and resilience.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet, making it a pangram often used for typing practice and font displays.
+
+Artificial intelligence is a wonderful field that combines computer science and robust datasets to enable problem-solving. It encompasses various subfields such as machine learning, neural networks, and natural language processing.
+
+In the heart of the city, there is a small café known for its aromatic coffee and freshly baked pastries. Every morning, people from all walks of life gather here to enjoy a warm cup of coffee and share stories.
+
+The sun sets behind the mountains, painting the sky with hues of orange and pink. As night falls, the stars begin to twinkle, creating a mesmerizing view that captivates everyone who gazes upon it.
+
+Books are a great source of knowledge and entertainment. They transport readers to different worlds, introduce them to diverse characters, and offer insights into various aspects of life, history, and culture.
+
+Walking through the forest, one can hear the rustling of leaves and the chirping of birds. The air is fresh, filled with the scent of pine and earth, providing a peaceful escape from the hustle and bustle of daily life.
+
+Technology continues to evolve at a rapid pace, transforming industries and reshaping the way we live and work. From smartphones to autonomous vehicles, innovations are constantly emerging to enhance convenience and efficiency.
+
+Music has the power to evoke emotions and bring people together. Whether it’s a classical symphony, a lively jazz tune, or a modern pop song, melodies have a unique way of resonating with listeners across the globe. 
+
+Exploring new places can be an exhilarating experience. Each destination offers its own charm, from historical landmarks and vibrant cultures to stunning landscapes and hidden gems waiting to be discovered.
+
+Science fiction novels often explore futuristic concepts and imaginative worlds. These stories challenge readers to think beyond the present and consider possibilities about space exploration, time travel, and advanced technologies.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet.
+
+Artificial intelligence is a wonderful field that combines computer science and data to create intelligent systems.
+
+The sun sets in the west, painting the sky with hues of orange and pink, marking the end of another day.
+
+Reading books is an excellent way to expand your knowledge and improve your vocabulary.
+
+In the heart of the city, a small café serves the best coffee, attracting visitors from all around.
+
+Learning new skills can be challenging, but it is also incredibly rewarding and fulfilling.
+
+The mountain trail was steep and rocky, but the view from the top was absolutely breathtaking.
+
+Music has the power to evoke deep emotions and bring back memories from the past.
+
+Scientists are constantly exploring new ways to harness renewable energy sources like wind and solar power.
+
+A journey of a thousand miles begins with a single step, highlighting the importance of starting small.
+The quick brown fox jumps over the lazy dog.  
+A journey of a thousand miles begins with a single step.  
+The sun rises in the east and sets in the west.  
+Water is essential for all forms of life on Earth.  
+Reading books can open doors to new worlds and ideas.  
+The cat slept on the couch while the rain poured outside.  
+Technology continues to evolve at a rapid pace.  
+Healthy eating habits contribute to a longer life.  
+Artificial intelligence is transforming various industries.  
+The mountain peak was covered in snow all year round.  
+Friendship is built on trust, respect, and shared experiences.  
+Learning a new language can be both challenging and rewarding.  
+The river flows gently through the lush green valley.  
+Music has the power to evoke deep emotions in people.  
+Innovation often arises from solving everyday problems.  
+Children played happily in the park under the warm sun.  
+History teaches us valuable lessons about the past.  
+The scientist conducted numerous experiments to test the theory.  
+Walking in nature helps reduce stress and improve mental health.  
+Good communication skills are vital for personal and professional success.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet.
+
+In the heart of the city, skyscrapers tower above the bustling streets, where people from all walks of life go about their daily routines.
+
+A gentle breeze rustled the leaves of the old oak tree, casting dappled shadows on the ground below. Birds chirped melodiously, adding to the serene ambiance of the park.
+
+History is filled with tales of great explorers who ventured into the unknown, driven by curiosity and the desire to uncover new lands and cultures.
+
+Quantum physics challenges our understanding of reality, introducing concepts that seem to defy common sense, such as entanglement and superposition.
+
+Artificial intelligence has rapidly evolved over the past decade, transforming industries and reshaping the way we interact with technology in our everyday lives.
+
+The aroma of freshly brewed coffee wafted through the air, enticing customers to step into the cozy café tucked away on a quiet street corner.
+
+Music has the power to evoke deep emotions, transporting listeners to different times and places with just a single chord or melody.
+
+Ancient civilizations built remarkable structures that continue to astonish us today, from the pyramids of Egypt to the temples of Angkor Wat.
+
+Climate change poses one of the greatest challenges humanity has ever faced, requiring urgent action to mitigate its impacts and protect future generations.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet, making it a pangram often used for typing practice and font displays.
+
+Artificial intelligence is transforming industries by enabling machines to learn from data and make decisions with minimal human intervention. From healthcare to finance, AI applications are expanding rapidly.
+
+The history of the Roman Empire spans centuries, beginning with the founding of Rome in 753 BC and ending with the fall of Constantinople in 1453 AD. It was one of the most powerful empires in world history.
+
+Photosynthesis is the process by which green plants and some other organisms use sunlight to synthesize foods with the help of chlorophyll. This natural phenomenon is essential for life on Earth.
+
+Quantum mechanics describes the behavior of matter and energy at very small scales, such as atoms and subatomic particles. It has led to groundbreaking technologies like semiconductors and lasers.
+
+Mount Everest, located in the Himalayan mountain range, is the highest peak on Earth above sea level. Climbers from around the world attempt to summit it each year, despite its challenging conditions.
+
+In literature, metaphors are figures of speech that compare two unlike things by stating that one thing is another. They add depth and creativity to writing, allowing readers to see connections in new ways.
+
+Renewable energy sources, such as solar, wind, and hydroelectric power, are becoming increasingly important as the world seeks sustainable alternatives to fossil fuels. These technologies aim to reduce carbon emissions.
+
+The periodic table organizes chemical elements based on their atomic number and properties. Created by Dmitri Mendeleev in 1869, it remains a cornerstone of chemistry education and research.
+
+Cultural diversity enriches societies by bringing together people with different traditions, languages, and perspectives. Embracing this diversity fosters mutual understanding and innovation across communities.
+The quick brown fox jumps over the lazy dog. This sentence contains every letter of the English alphabet, making it a popular choice for typing practice and font displays.
+
+Artificial intelligence is transforming industries by automating repetitive tasks, analyzing vast datasets, and enabling smarter decision-making. Its applications range from healthcare to finance, and its potential continues to grow as technology advances.
+
+In a small village nestled at the foot of towering mountains, life moved at a slower pace. Farmers tended to their fields, children played by the river, and elders shared stories under the ancient oak tree that had stood there for centuries.
+
+Scientific discoveries often begin with curiosity and persistence. Marie Curie’s groundbreaking work on radioactivity not only earned her two Nobel Prizes but also paved the way for advancements in medicine and energy production.
+
+Rain poured down in sheets, drenching everything in its path. Despite the storm, she walked briskly through the empty streets, her mind racing with thoughts of what lay ahead. Each step brought her closer to answers—and more questions.
+
+Books have the power to transport readers to different worlds, offering insights into lives and cultures far removed from their own. Whether fiction or nonfiction, they inspire imagination and deepen understanding.
+
+Technology has revolutionized communication, allowing people across the globe to connect instantly. From handwritten letters to emails and video calls, the evolution of how we share ideas reflects humanity's relentless pursuit of progress.
+
+A chef carefully plated the final dish, garnishing it with delicate herbs and edible flowers. The aroma filled the kitchen, promising a feast that would delight both the palate and the eyes. Culinary artistry turns simple ingredients into extraordinary experiences.
+
+History teaches us valuable lessons about resilience, innovation, and human nature. By studying past civilizations, we gain perspective on our present challenges and inspiration for shaping a better future.
+
+Music transcends language barriers, evoking emotions that words alone cannot express. From classical symphonies to modern pop songs, melodies resonate deeply within us, creating connections that span generations and geographies.

vocab.json

@@ -0,0 +1 @@
+["\n", " ", "'", ",", "-", ".", "1", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "G", "H", "I", "L", "M", "N", "P", "Q", "R", "S", "T", "U", "W", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "\u00e9", "\u2014", "\u2019"]