First Commit

README.md

@@ -1,3 +1,47 @@
----
-license: mit
----
+---
+language:
+  - ar
+  - en
+
+dataset:
+  - fka/awesome-chatgpt-prompts
+  - open-r1/codeforces
+
+
+license: mit
+---
+
+## Miscovery Tokenizer
+
+A SentencePiece unigram tokenizer trained on a mix of Arabic and English text, with a vocabulary size of 70,000 tokens.
+
+## Training Data
+This tokenizer was trained on:
+- Arabic Quran.
+- awesome-chatgpt-prompts
+- open-r1/codeforces
+
+## Usage
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("miscovery/arabic-english-tokenizer")
+
+# Example usage
+text = "بسم الله الرحمن الرحيم Hello World"
+encoded = tokenizer(text)
+print(encoded)
+```
+
+## Features
+
+- Vocabulary size: 70,000
+- Model type: Unigram
+- Model Max Length: 512
+- Handles both Arabic and English text
+- Supports Arabic normalization
+
+
+
+

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "right",
+  "unk_token": "<unk>"
+}