File size: 436 Bytes
c8497a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from fastapi import FastAPI
import spacy

app = FastAPI()
nlp = spacy.blank("en")

@app.post("/tokenize")
async def tokenize_text(text: str):
    doc = nlp(text)
    tokens = []
    for token in doc:
        tokens.append({
            "text": token.text,
            "start_char": token.idx,
            "end_char": token.idx + len(token.text),
            "start": token.i,
            "end": token.i + 1
        })
    return tokens