File size: 1,085 Bytes
b51c975 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
import openai
import json
import threading
config: dict = json.loads(open("config.json").read())
dataset = []
def gen():
try:
response = openai.OpenAI(
api_key=config["key"],
base_url=config["url"],
default_headers={
"User-Agent": "OpenAI SDK",
"Cookie": config.get("cookie", "")
}
).chat.completions.create(
model=config["model"],
extra_body={"model_id": config["model"]},
messages=[
{"role": "user", "content": "Generate some texts for training LLM. Do not add any other elements to your response."}
],
stream=False
)
dataset.append(response.choices[0].message.content)
except Exception as e:
print(e)
thpool = []
for i in range(100):
th = threading.Thread(target=gen)
th.start()
thpool.append(th)
for th in thpool:
th.join()
with open("train_data.txt", "a", encoding="utf-8") as f:
f.write("\n".join(dataset))
|