Spaces:
Running
Running
Commit
·
be641c0
1
Parent(s):
1d21e47
Init for HF space
Browse files- .gitattributes +35 -0
- README.md +13 -0
- app.py +112 -0
- original_blt_requirements.txt +24 -0
- requirements.txt +364 -24
.gitattributes
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
@@ -1,3 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
# Byte Latent Transformer
|
2 |
|
3 |
This repository contains code for our paper: "Byte Latent Transformer: Patches Scale Better Than Tokens"
|
|
|
1 |
+
---
|
2 |
+
title: Blt Entropy Patcher
|
3 |
+
emoji: ⚡
|
4 |
+
colorFrom: green
|
5 |
+
colorTo: red
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 5.27.0
|
8 |
+
python_version: 3.11.11
|
9 |
+
app_file: app.py
|
10 |
+
pinned: false
|
11 |
+
license: cc-by-nc-4.0
|
12 |
+
---
|
13 |
+
|
14 |
# Byte Latent Transformer
|
15 |
|
16 |
This repository contains code for our paper: "Byte Latent Transformer: Patches Scale Better Than Tokens"
|
app.py
ADDED
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import gradio as gr
|
3 |
+
import torch
|
4 |
+
# Removed matplotlib and plot_entropies imports
|
5 |
+
|
6 |
+
# Assuming bytelatent library and its dependencies are installed
|
7 |
+
from bytelatent.data.file_util import get_fs
|
8 |
+
# from bytelatent.distributed import DistributedArgs, setup_torch_distributed # Not needed
|
9 |
+
from bytelatent.generate_patcher import patcher_nocache
|
10 |
+
from bytelatent.tokenizers.blt_tokenizer import BltTokenizer
|
11 |
+
# Removed: from bytelatent.plotting.entropy_figure_via_matplot_lib import plot_entropies
|
12 |
+
from bytelatent.args import TrainArgs
|
13 |
+
from download_blt_weights import main as ensure_present
|
14 |
+
|
15 |
+
# --- Global Setup (Consider loading models outside if necessary) ---
|
16 |
+
# Kept inside the function for simplicity as before.
|
17 |
+
|
18 |
+
def process_text(prompt: str, model_name: str = "blt-1b"):
|
19 |
+
"""
|
20 |
+
Processes the input prompt using the ByteLatent model and returns decoded characters.
|
21 |
+
|
22 |
+
Args:
|
23 |
+
prompt: The input text string from the Gradio interface.
|
24 |
+
model_name: The name of the model to use.
|
25 |
+
|
26 |
+
Returns:
|
27 |
+
A string containing the decoded characters after processing, or an error message.
|
28 |
+
"""
|
29 |
+
try:
|
30 |
+
# --- Model and Tokenizer Loading ---
|
31 |
+
consolidated_path = os.path.join("hf-weights", model_name)
|
32 |
+
train_args_path = os.path.join(consolidated_path, "params.json")
|
33 |
+
|
34 |
+
if not os.path.exists(train_args_path):
|
35 |
+
raise FileNotFoundError(f"Training args not found at {train_args_path}. "
|
36 |
+
f"Ensure model '{model_name}' is downloaded/available.")
|
37 |
+
|
38 |
+
fs = get_fs(train_args_path)
|
39 |
+
train_args = TrainArgs.model_validate_json(fs.read_text(train_args_path))
|
40 |
+
|
41 |
+
tokenizer = train_args.data.tokenizer_args.build()
|
42 |
+
assert isinstance(tokenizer, BltTokenizer)
|
43 |
+
|
44 |
+
patcher_args = train_args.data.patcher_args.model_copy(deep=True)
|
45 |
+
patcher_args.realtime_patching = True
|
46 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
47 |
+
print(f"Using device: {device}")
|
48 |
+
patcher_args.patching_device = device
|
49 |
+
patcher_args.device = device
|
50 |
+
|
51 |
+
print("Loading entropy model and patcher...")
|
52 |
+
entropy_model_dir = os.path.join(consolidated_path, "entropy_model")
|
53 |
+
if not os.path.exists(entropy_model_dir):
|
54 |
+
raise FileNotFoundError(f"Entropy model directory not found at {entropy_model_dir}.")
|
55 |
+
|
56 |
+
patcher_args.entropy_model_checkpoint_dir = entropy_model_dir
|
57 |
+
patcher = patcher_args.build()
|
58 |
+
# --- End Loading ---
|
59 |
+
|
60 |
+
# --- Processing ---
|
61 |
+
prompts = [prompt]
|
62 |
+
print(f"Processing prompt: '{prompt}'")
|
63 |
+
results = patcher_nocache(
|
64 |
+
prompts, tokenizer=tokenizer, patcher=patcher
|
65 |
+
)
|
66 |
+
|
67 |
+
if not results:
|
68 |
+
print("Processing returned no results.")
|
69 |
+
return "Processing completed, but no results were generated." # Return info message
|
70 |
+
|
71 |
+
batch_patch_lengths, batch_scores, batch_tokens = results
|
72 |
+
# Decode the first (and only) result in the batch
|
73 |
+
decoded_chars_list = [tokenizer.decode(row_tokens.tolist()) for row_tokens in batch_tokens]
|
74 |
+
decoded_output = decoded_chars_list[0] if decoded_chars_list else "No characters decoded."
|
75 |
+
|
76 |
+
print("Processing and decoding complete.")
|
77 |
+
# --- End Processing ---
|
78 |
+
|
79 |
+
|
80 |
+
# Return the decoded text string
|
81 |
+
return decoded_output
|
82 |
+
|
83 |
+
except FileNotFoundError as e:
|
84 |
+
print(f"Error: {e}")
|
85 |
+
# raise gr.Error(str(e)) # Display specific error in Gradio UI
|
86 |
+
return f"Error: {str(e)}" # Return error as text output
|
87 |
+
except Exception as e:
|
88 |
+
print(f"An unexpected error occurred: {e}")
|
89 |
+
import traceback
|
90 |
+
traceback.print_exc()
|
91 |
+
# raise gr.Error(f"An error occurred during processing: {e}")
|
92 |
+
return f"An unexpected error occurred: {e}" # Return error as text output
|
93 |
+
|
94 |
+
|
95 |
+
# --- Gradio Interface Definition ---
|
96 |
+
iface = gr.Interface(
|
97 |
+
fn=process_text,
|
98 |
+
inputs=gr.Textbox(
|
99 |
+
label="Input Prompt",
|
100 |
+
placeholder="Enter your text here..."
|
101 |
+
),
|
102 |
+
# Changed output to display the decoded text
|
103 |
+
outputs=gr.Text(label="Decoded Output"),
|
104 |
+
title="ByteLatent Text Processor",
|
105 |
+
description="Enter text to process it with the ByteLatent model ('blt-1b' by default). The decoded output will be shown.",
|
106 |
+
allow_flagging="never",
|
107 |
+
)
|
108 |
+
|
109 |
+
# --- Launch the Gradio App ---
|
110 |
+
if __name__ == "__main__":
|
111 |
+
ensure_present(["blt-1b"])
|
112 |
+
iface.launch()
|
original_blt_requirements.txt
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
numpy
|
2 |
+
omegaconf
|
3 |
+
msgspec
|
4 |
+
rouge-score
|
5 |
+
sacrebleu
|
6 |
+
sentencepiece
|
7 |
+
tiktoken
|
8 |
+
fsspec
|
9 |
+
blobfile
|
10 |
+
wandb
|
11 |
+
viztracer
|
12 |
+
lm-eval
|
13 |
+
scipy
|
14 |
+
pynvml
|
15 |
+
datatrove
|
16 |
+
orjson
|
17 |
+
luigi
|
18 |
+
pydantic
|
19 |
+
altair
|
20 |
+
submitit
|
21 |
+
typer
|
22 |
+
rich
|
23 |
+
fsspec[full]
|
24 |
+
orjson
|
requirements.txt
CHANGED
@@ -1,24 +1,364 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# This file may be used to create an environment using:
|
2 |
+
# $ conda create --name <env> --file <this file>
|
3 |
+
# platform: linux-64
|
4 |
+
_libgcc_mutex=0.1=main
|
5 |
+
_openmp_mutex=5.1=1_gnu
|
6 |
+
absl-py=2.2.2=pypi_0
|
7 |
+
accelerate=1.6.0=pypi_0
|
8 |
+
adlfs=2024.12.0=pypi_0
|
9 |
+
aiobotocore=2.21.1=pypi_0
|
10 |
+
aiohappyeyeballs=2.6.1=pypi_0
|
11 |
+
aiohttp=3.11.16=pypi_0
|
12 |
+
aioitertools=0.12.0=pypi_0
|
13 |
+
aiosignal=1.3.2=pypi_0
|
14 |
+
alabaster=1.0.0=pypi_0
|
15 |
+
altair=5.5.0=pypi_0
|
16 |
+
annotated-types=0.7.0=pypi_0
|
17 |
+
antlr4-python3-runtime=4.9.3=pypi_0
|
18 |
+
astunparse=1.6.3=pypi_0
|
19 |
+
attrs=25.3.0=pypi_0
|
20 |
+
aws-sam-translator=1.97.0=pypi_0
|
21 |
+
aws-xray-sdk=2.14.0=pypi_0
|
22 |
+
azure-core=1.33.0=pypi_0
|
23 |
+
azure-datalake-store=0.0.53=pypi_0
|
24 |
+
azure-identity=1.21.0=pypi_0
|
25 |
+
azure-storage-blob=12.25.1=pypi_0
|
26 |
+
babel=2.17.0=pypi_0
|
27 |
+
bcrypt=4.3.0=pypi_0
|
28 |
+
bleach=6.2.0=pypi_0
|
29 |
+
blinker=1.9.0=pypi_0
|
30 |
+
blis=1.3.0=pypi_0
|
31 |
+
blobfile=3.0.0=pypi_0
|
32 |
+
bokeh=3.7.2=pypi_0
|
33 |
+
boto3=1.37.1=pypi_0
|
34 |
+
botocore=1.37.1=pypi_0
|
35 |
+
botok=0.9.0=pypi_0
|
36 |
+
bzip2=1.0.8=h5eee18b_6
|
37 |
+
ca-certificates=2025.2.25=h06a4308_0
|
38 |
+
cachetools=5.5.2=pypi_0
|
39 |
+
catalogue=2.0.10=pypi_0
|
40 |
+
certifi=2025.1.31=pypi_0
|
41 |
+
cffi=1.17.1=pypi_0
|
42 |
+
cfn-lint=1.33.2=pypi_0
|
43 |
+
chardet=5.2.0=pypi_0
|
44 |
+
charset-normalizer=3.4.1=pypi_0
|
45 |
+
circuitbreaker=2.1.3=pypi_0
|
46 |
+
click=8.1.8=pypi_0
|
47 |
+
cloudpathlib=0.21.0=pypi_0
|
48 |
+
cloudpickle=3.1.1=pypi_0
|
49 |
+
colorama=0.4.6=pypi_0
|
50 |
+
confection=0.1.5=pypi_0
|
51 |
+
contourpy=1.3.1=pypi_0
|
52 |
+
courlan=1.3.2=pypi_0
|
53 |
+
cryptography=44.0.2=pypi_0
|
54 |
+
cycler=0.12.1=pypi_0
|
55 |
+
cymem=2.0.11=pypi_0
|
56 |
+
dask=2025.3.0=pypi_0
|
57 |
+
dataproperty=1.1.0=pypi_0
|
58 |
+
datasets=3.5.0=pypi_0
|
59 |
+
datatrove=0.4.0=pypi_0
|
60 |
+
dateparser=1.2.1=pypi_0
|
61 |
+
decorator=5.2.1=pypi_0
|
62 |
+
dill=0.3.8=pypi_0
|
63 |
+
distributed=2025.3.0=pypi_0
|
64 |
+
docker=7.1.0=pypi_0
|
65 |
+
docker-pycreds=0.4.0=pypi_0
|
66 |
+
docutils=0.21.2=pypi_0
|
67 |
+
dropbox=12.0.2=pypi_0
|
68 |
+
dropboxdrivefs=1.4.1=pypi_0
|
69 |
+
emoji=2.14.1=pypi_0
|
70 |
+
evaluate=0.4.3=pypi_0
|
71 |
+
execnet=2.1.1=pypi_0
|
72 |
+
fasteners=0.19=pypi_0
|
73 |
+
fasttext-numpy2-wheel=0.9.2=pypi_0
|
74 |
+
faust-cchardet=2.1.19=pypi_0
|
75 |
+
filelock=3.13.1=pypi_0
|
76 |
+
flask=3.1.0=pypi_0
|
77 |
+
flask-cors=5.0.1=pypi_0
|
78 |
+
flatbuffers=25.2.10=pypi_0
|
79 |
+
fonttools=4.57.0=pypi_0
|
80 |
+
frozenlist=1.5.0=pypi_0
|
81 |
+
fsspec=2024.6.1=pypi_0
|
82 |
+
ftfy=6.3.1=pypi_0
|
83 |
+
fusepy=3.0.1=pypi_0
|
84 |
+
gast=0.6.0=pypi_0
|
85 |
+
gcsfs=2024.6.1=pypi_0
|
86 |
+
gitdb=4.0.12=pypi_0
|
87 |
+
gitpython=3.1.44=pypi_0
|
88 |
+
google-api-core=2.24.2=pypi_0
|
89 |
+
google-auth=2.38.0=pypi_0
|
90 |
+
google-auth-oauthlib=1.2.1=pypi_0
|
91 |
+
google-cloud-core=2.4.3=pypi_0
|
92 |
+
google-cloud-storage=3.1.0=pypi_0
|
93 |
+
google-crc32c=1.7.1=pypi_0
|
94 |
+
google-pasta=0.2.0=pypi_0
|
95 |
+
google-resumable-media=2.7.2=pypi_0
|
96 |
+
googleapis-common-protos=1.69.2=pypi_0
|
97 |
+
graphql-core=3.2.6=pypi_0
|
98 |
+
grpcio=1.71.0=pypi_0
|
99 |
+
h5py=3.13.0=pypi_0
|
100 |
+
htmldate=1.9.3=pypi_0
|
101 |
+
huggingface-hub=0.30.2=pypi_0
|
102 |
+
humanize=4.12.2=pypi_0
|
103 |
+
idna=3.10=pypi_0
|
104 |
+
imagesize=1.4.1=pypi_0
|
105 |
+
importlib-metadata=8.6.1=pypi_0
|
106 |
+
indic-nlp-library=0.92=pypi_0
|
107 |
+
iniconfig=2.1.0=pypi_0
|
108 |
+
inscriptis=2.6.0=pypi_0
|
109 |
+
isodate=0.7.2=pypi_0
|
110 |
+
itsdangerous=2.2.0=pypi_0
|
111 |
+
jieba=0.42.1=pypi_0
|
112 |
+
jinja2=3.1.4=pypi_0
|
113 |
+
jmespath=1.0.1=pypi_0
|
114 |
+
joblib=1.4.2=pypi_0
|
115 |
+
joserfc=1.0.4=pypi_0
|
116 |
+
jsonlines=4.0.0=pypi_0
|
117 |
+
jsonpatch=1.33=pypi_0
|
118 |
+
jsonpath-ng=1.7.0=pypi_0
|
119 |
+
jsonpointer=3.0.0=pypi_0
|
120 |
+
jsonschema=4.23.0=pypi_0
|
121 |
+
jsonschema-path=0.3.4=pypi_0
|
122 |
+
jsonschema-specifications=2024.10.1=pypi_0
|
123 |
+
justext=3.0.2=pypi_0
|
124 |
+
kenlm=0.3.0=pypi_0
|
125 |
+
keras=3.9.2=pypi_0
|
126 |
+
khmer-nltk=1.6=pypi_0
|
127 |
+
kiwipiepy=0.20.4=pypi_0
|
128 |
+
kiwipiepy-model=0.20.0=pypi_0
|
129 |
+
kiwisolver=1.4.8=pypi_0
|
130 |
+
langcodes=3.5.0=pypi_0
|
131 |
+
language-data=1.3.0=pypi_0
|
132 |
+
laonlp=1.2.0=pypi_0
|
133 |
+
lazy-object-proxy=1.10.0=pypi_0
|
134 |
+
ld_impl_linux-64=2.40=h12ee557_0
|
135 |
+
libarchive-c=5.2=pypi_0
|
136 |
+
libclang=18.1.1=pypi_0
|
137 |
+
libffi=3.4.4=h6a678d5_1
|
138 |
+
libgcc-ng=11.2.0=h1234567_1
|
139 |
+
libgomp=11.2.0=h1234567_1
|
140 |
+
libstdcxx-ng=11.2.0=h1234567_1
|
141 |
+
libuuid=1.41.5=h5eee18b_0
|
142 |
+
linkify-it-py=2.0.3=pypi_0
|
143 |
+
lm-eval=0.4.8=pypi_0
|
144 |
+
locket=1.0.0=pypi_0
|
145 |
+
lockfile=0.12.2=pypi_0
|
146 |
+
loguru=0.7.3=pypi_0
|
147 |
+
luigi=3.6.0=pypi_0
|
148 |
+
lxml=5.3.2=pypi_0
|
149 |
+
lxml-html-clean=0.4.2=pypi_0
|
150 |
+
marisa-trie=1.2.1=pypi_0
|
151 |
+
markdown=3.8=pypi_0
|
152 |
+
markdown-it-py=3.0.0=pypi_0
|
153 |
+
markupsafe=2.1.5=pypi_0
|
154 |
+
matplotlib=3.10.1=pypi_0
|
155 |
+
mbstrdecoder=1.1.4=pypi_0
|
156 |
+
mdit-py-plugins=0.4.2=pypi_0
|
157 |
+
mdurl=0.1.2=pypi_0
|
158 |
+
ml-dtypes=0.5.1=pypi_0
|
159 |
+
more-itertools=10.6.0=pypi_0
|
160 |
+
morfessor=2.0.6=pypi_0
|
161 |
+
moto=5.1.3=pypi_0
|
162 |
+
mpmath=1.3.0=pypi_0
|
163 |
+
msal=1.32.0=pypi_0
|
164 |
+
msal-extensions=1.3.1=pypi_0
|
165 |
+
msgpack=1.1.0=pypi_0
|
166 |
+
msgspec=0.19.0=pypi_0
|
167 |
+
multidict=6.4.3=pypi_0
|
168 |
+
multiprocess=0.70.16=pypi_0
|
169 |
+
murmurhash=1.0.12=pypi_0
|
170 |
+
namex=0.0.8=pypi_0
|
171 |
+
narwhals=1.34.1=pypi_0
|
172 |
+
ncurses=6.4=h6a678d5_0
|
173 |
+
networkx=3.3=pypi_0
|
174 |
+
ninja=1.11.1.4=pypi_0
|
175 |
+
nltk=3.9.1=pypi_0
|
176 |
+
numexpr=2.10.2=pypi_0
|
177 |
+
numpy=2.1.2=pypi_0
|
178 |
+
nvidia-cublas-cu12=12.1.3.1=pypi_0
|
179 |
+
nvidia-cuda-cupti-cu12=12.1.105=pypi_0
|
180 |
+
nvidia-cuda-nvrtc-cu12=12.1.105=pypi_0
|
181 |
+
nvidia-cuda-runtime-cu12=12.1.105=pypi_0
|
182 |
+
nvidia-cudnn-cu12=9.1.0.70=pypi_0
|
183 |
+
nvidia-cufft-cu12=11.0.2.54=pypi_0
|
184 |
+
nvidia-curand-cu12=10.3.2.106=pypi_0
|
185 |
+
nvidia-cusolver-cu12=11.4.5.107=pypi_0
|
186 |
+
nvidia-cusparse-cu12=12.1.0.106=pypi_0
|
187 |
+
nvidia-ml-py=12.570.86=pypi_0
|
188 |
+
nvidia-nccl-cu12=2.21.5=pypi_0
|
189 |
+
nvidia-nvjitlink-cu12=12.1.105=pypi_0
|
190 |
+
nvidia-nvtx-cu12=12.1.105=pypi_0
|
191 |
+
oauthlib=3.2.2=pypi_0
|
192 |
+
objprint=0.3.0=pypi_0
|
193 |
+
oci=2.150.0=pypi_0
|
194 |
+
ocifs=1.3.2=pypi_0
|
195 |
+
omegaconf=2.3.0=pypi_0
|
196 |
+
openapi-schema-validator=0.6.3=pypi_0
|
197 |
+
openapi-spec-validator=0.7.1=pypi_0
|
198 |
+
openssl=3.0.16=h5eee18b_0
|
199 |
+
opt-einsum=3.4.0=pypi_0
|
200 |
+
optree=0.15.0=pypi_0
|
201 |
+
orjson=3.10.16=pypi_0
|
202 |
+
packaging=24.2=pypi_0
|
203 |
+
pandas=2.2.3=pypi_0
|
204 |
+
panel=1.6.2=pypi_0
|
205 |
+
param=2.2.0=pypi_0
|
206 |
+
paramiko=3.5.1=pypi_0
|
207 |
+
partd=1.4.2=pypi_0
|
208 |
+
pathable=0.4.4=pypi_0
|
209 |
+
pathvalidate=3.2.3=pypi_0
|
210 |
+
peft=0.15.1=pypi_0
|
211 |
+
pillow=11.1.0=pypi_0
|
212 |
+
pip=25.0=py311h06a4308_0
|
213 |
+
platformdirs=4.3.7=pypi_0
|
214 |
+
pluggy=1.5.0=pypi_0
|
215 |
+
ply=3.11=pypi_0
|
216 |
+
portalocker=3.1.1=pypi_0
|
217 |
+
preshed=3.0.9=pypi_0
|
218 |
+
propcache=0.3.1=pypi_0
|
219 |
+
proto-plus=1.26.1=pypi_0
|
220 |
+
protobuf=5.29.4=pypi_0
|
221 |
+
psutil=7.0.0=pypi_0
|
222 |
+
py-partiql-parser=0.6.1=pypi_0
|
223 |
+
pyahocorasick=2.1.0=pypi_0
|
224 |
+
pyarrow=19.0.1=pypi_0
|
225 |
+
pyasn1=0.6.1=pypi_0
|
226 |
+
pyasn1-modules=0.4.2=pypi_0
|
227 |
+
pybind11=2.13.6=pypi_0
|
228 |
+
pycparser=2.22=pypi_0
|
229 |
+
pycryptodomex=3.22.0=pypi_0
|
230 |
+
pydantic=2.11.3=pypi_0
|
231 |
+
pydantic-core=2.33.1=pypi_0
|
232 |
+
pygit2=1.17.0=pypi_0
|
233 |
+
pygments=2.19.1=pypi_0
|
234 |
+
pyidaungsu-numpy2=0.1.4=pypi_0
|
235 |
+
pyjwt=2.10.1=pypi_0
|
236 |
+
pynacl=1.5.0=pypi_0
|
237 |
+
pynvml=12.0.0=pypi_0
|
238 |
+
pyopenssl=24.3.0=pypi_0
|
239 |
+
pyparsing=3.2.3=pypi_0
|
240 |
+
pyspnego=0.11.2=pypi_0
|
241 |
+
pytablewriter=1.2.1=pypi_0
|
242 |
+
pytest=8.3.5=pypi_0
|
243 |
+
pytest-timeout=2.3.1=pypi_0
|
244 |
+
pytest-xdist=3.6.1=pypi_0
|
245 |
+
pythainlp=5.1.1=pypi_0
|
246 |
+
python=3.11.11=he870216_0
|
247 |
+
python-crfsuite=0.9.11=pypi_0
|
248 |
+
python-daemon=3.1.2=pypi_0
|
249 |
+
python-dateutil=2.9.0.post0=pypi_0
|
250 |
+
python-magic=0.4.27=pypi_0
|
251 |
+
pytz=2025.2=pypi_0
|
252 |
+
pyvi=0.1.1=pypi_0
|
253 |
+
pyviz-comms=3.0.4=pypi_0
|
254 |
+
pyyaml=6.0.2=pypi_0
|
255 |
+
readline=8.2=h5eee18b_0
|
256 |
+
referencing=0.36.2=pypi_0
|
257 |
+
regex=2024.11.6=pypi_0
|
258 |
+
requests=2.32.3=pypi_0
|
259 |
+
requests-file=2.1.0=pypi_0
|
260 |
+
requests-oauthlib=2.0.0=pypi_0
|
261 |
+
responses=0.25.7=pypi_0
|
262 |
+
rfc3339-validator=0.1.4=pypi_0
|
263 |
+
rich=14.0.0=pypi_0
|
264 |
+
roman-numerals-py=3.1.0=pypi_0
|
265 |
+
rouge-score=0.1.2=pypi_0
|
266 |
+
rpds-py=0.24.0=pypi_0
|
267 |
+
rsa=4.9=pypi_0
|
268 |
+
ruff=0.11.5=pypi_0
|
269 |
+
s3fs=2024.6.1=pypi_0
|
270 |
+
s3transfer=0.11.3=pypi_0
|
271 |
+
sacrebleu=2.5.1=pypi_0
|
272 |
+
safetensors=0.5.3=pypi_0
|
273 |
+
scikit-learn=1.6.1=pypi_0
|
274 |
+
scipy=1.15.2=pypi_0
|
275 |
+
sentencepiece=0.2.0=pypi_0
|
276 |
+
sentry-sdk=2.25.1=pypi_0
|
277 |
+
setproctitle=1.3.5=pypi_0
|
278 |
+
setuptools=75.8.0=py311h06a4308_0
|
279 |
+
shellingham=1.5.4=pypi_0
|
280 |
+
six=1.17.0=pypi_0
|
281 |
+
sklearn-crfsuite=0.5.0=pypi_0
|
282 |
+
smart-open=7.1.0=pypi_0
|
283 |
+
smbprotocol=1.15.0=pypi_0
|
284 |
+
smmap=5.0.2=pypi_0
|
285 |
+
snowballstemmer=2.2.0=pypi_0
|
286 |
+
sortedcontainers=2.4.0=pypi_0
|
287 |
+
spacy=3.8.5=pypi_0
|
288 |
+
spacy-legacy=3.0.12=pypi_0
|
289 |
+
spacy-loggers=1.0.5=pypi_0
|
290 |
+
sphinx=8.2.3=pypi_0
|
291 |
+
sphinx-argparse=0.5.2=pypi_0
|
292 |
+
sphinx-rtd-theme=3.0.2=pypi_0
|
293 |
+
sphinxcontrib-applehelp=2.0.0=pypi_0
|
294 |
+
sphinxcontrib-devhelp=2.0.0=pypi_0
|
295 |
+
sphinxcontrib-htmlhelp=2.1.0=pypi_0
|
296 |
+
sphinxcontrib-jquery=4.1=pypi_0
|
297 |
+
sphinxcontrib-jsmath=1.0.1=pypi_0
|
298 |
+
sphinxcontrib-qthelp=2.0.0=pypi_0
|
299 |
+
sphinxcontrib-serializinghtml=2.0.0=pypi_0
|
300 |
+
sqlite=3.45.3=h5eee18b_0
|
301 |
+
sqlitedict=2.1.0=pypi_0
|
302 |
+
srsly=2.5.1=pypi_0
|
303 |
+
stanza=1.10.1=pypi_0
|
304 |
+
stone=3.3.1=pypi_0
|
305 |
+
submitit=1.5.2=pypi_0
|
306 |
+
sudachidict-core=20250129=pypi_0
|
307 |
+
sudachipy=0.6.10=pypi_0
|
308 |
+
sympy=1.13.1=pypi_0
|
309 |
+
tabledata=1.3.4=pypi_0
|
310 |
+
tabulate=0.9.0=pypi_0
|
311 |
+
tblib=3.1.0=pypi_0
|
312 |
+
tcolorpy=0.1.7=pypi_0
|
313 |
+
tenacity=8.5.0=pypi_0
|
314 |
+
tensorboard=2.19.0=pypi_0
|
315 |
+
tensorboard-data-server=0.7.2=pypi_0
|
316 |
+
tensorflow=2.19.0=pypi_0
|
317 |
+
tensorflow-io-gcs-filesystem=0.37.1=pypi_0
|
318 |
+
termcolor=3.0.1=pypi_0
|
319 |
+
thinc=8.3.6=pypi_0
|
320 |
+
threadpoolctl=3.6.0=pypi_0
|
321 |
+
tiktoken=0.9.0=pypi_0
|
322 |
+
tk=8.6.14=h39e8969_0
|
323 |
+
tld=0.13=pypi_0
|
324 |
+
tldextract=5.2.0=pypi_0
|
325 |
+
tokenizers=0.21.1=pypi_0
|
326 |
+
toolz=1.0.0=pypi_0
|
327 |
+
torch=2.5.0+cu121=pypi_0
|
328 |
+
tornado=6.4.2=pypi_0
|
329 |
+
tqdm=4.67.1=pypi_0
|
330 |
+
tqdm-multiprocess=0.0.11=pypi_0
|
331 |
+
trafilatura=1.11.0=pypi_0
|
332 |
+
transformers=4.51.2=pypi_0
|
333 |
+
triton=3.1.0=pypi_0
|
334 |
+
typepy=1.3.4=pypi_0
|
335 |
+
typer=0.15.2=pypi_0
|
336 |
+
typing-extensions=4.12.2=pypi_0
|
337 |
+
typing-inspection=0.4.0=pypi_0
|
338 |
+
tzdata=2025.2=pypi_0
|
339 |
+
tzlocal=5.3.1=pypi_0
|
340 |
+
uc-micro-py=1.0.3=pypi_0
|
341 |
+
urduhack=0.1.4=pypi_0
|
342 |
+
urllib3=2.4.0=pypi_0
|
343 |
+
viztracer=1.0.3=pypi_0
|
344 |
+
vl-convert-python=1.7.0=pypi_0
|
345 |
+
wandb=0.19.9=pypi_0
|
346 |
+
warcio=1.7.5=pypi_0
|
347 |
+
wasabi=1.1.3=pypi_0
|
348 |
+
wcwidth=0.2.13=pypi_0
|
349 |
+
weasel=0.4.1=pypi_0
|
350 |
+
webencodings=0.5.1=pypi_0
|
351 |
+
werkzeug=3.1.3=pypi_0
|
352 |
+
wheel=0.45.1=py311h06a4308_0
|
353 |
+
word2number=1.1=pypi_0
|
354 |
+
wrapt=1.17.2=pypi_0
|
355 |
+
xformers=0.0.28.post2=pypi_0
|
356 |
+
xmltodict=0.14.2=pypi_0
|
357 |
+
xxhash=3.5.0=pypi_0
|
358 |
+
xyzservices=2025.1.0=pypi_0
|
359 |
+
xz=5.6.4=h5eee18b_1
|
360 |
+
yarl=1.19.0=pypi_0
|
361 |
+
zict=3.0.0=pypi_0
|
362 |
+
zipp=3.21.0=pypi_0
|
363 |
+
zlib=1.2.13=h5eee18b_1
|
364 |
+
zstandard=0.23.0=pypi_0
|