Spaces:
Sleeping
Sleeping
File size: 2,129 Bytes
05d3571 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
import os
import tempfile
import pytest
from laser_encoders.download_models import LaserModelDownloader
from laser_encoders.language_list import LASER2_LANGUAGE, LASER3_LANGUAGE
from laser_encoders.laser_tokenizer import initialize_tokenizer
from laser_encoders.models import initialize_encoder
def test_validate_achnese_models_and_tokenize_laser3(lang="acehnese"):
with tempfile.TemporaryDirectory() as tmp_dir:
print(f"Created temporary directory for {lang}", tmp_dir)
downloader = LaserModelDownloader(model_dir=tmp_dir)
downloader.download_laser3(lang)
encoder = initialize_encoder(lang, model_dir=tmp_dir)
tokenizer = initialize_tokenizer(lang, model_dir=tmp_dir)
# Test tokenization with a sample sentence
tokenized = tokenizer.tokenize("This is a sample sentence.")
print(f"{lang} model validated successfully")
def test_validate_english_models_and_tokenize_laser2(lang="english"):
with tempfile.TemporaryDirectory() as tmp_dir:
print(f"Created temporary directory for {lang}", tmp_dir)
downloader = LaserModelDownloader(model_dir=tmp_dir)
downloader.download_laser2()
encoder = initialize_encoder(lang, model_dir=tmp_dir)
tokenizer = initialize_tokenizer(lang, model_dir=tmp_dir)
# Test tokenization with a sample sentence
tokenized = tokenizer.tokenize("This is a sample sentence.")
print(f"{lang} model validated successfully")
def test_validate_kashmiri_models_and_tokenize_laser3(lang="kas"):
with tempfile.TemporaryDirectory() as tmp_dir:
print(f"Created temporary directory for {lang}", tmp_dir)
downloader = LaserModelDownloader(model_dir=tmp_dir)
with pytest.raises(ValueError):
downloader.download_laser3(lang)
encoder = initialize_encoder(lang, model_dir=tmp_dir)
tokenizer = initialize_tokenizer(lang, model_dir=tmp_dir)
# Test tokenization with a sample sentence
tokenized = tokenizer.tokenize("This is a sample sentence.")
print(f"{lang} model validated successfully")
|