Spaces:
Sleeping
Sleeping
File size: 4,270 Bytes
05d3571 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
#!/usr/bin/python3
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
#
# LASER Language-Agnostic SEntence Representations
# is a toolkit to calculate multilingual sentence embeddings
# and to use them for document classification, bitext filtering
# and mining
#
# --------------------------------------------------------
#
# Quora Q&A paraphrase detection
import os
import sys
import argparse
import faiss
import numpy as np
# get environment
assert os.environ.get('LASER'), 'Please set the enviornment variable LASER'
LASER = os.environ['LASER']
sys.path.append(LASER + '/source')
sys.path.append(LASER + '/source/lib')
from embed import SentenceEncoder, EncodeLoad, EncodeFile
from text_processing import Token, BPEfastApply
from indexing import IndexCreate, IndexSearchMultiple, IndexPrintConfusionMatrix
###############################################################################
parser = argparse.ArgumentParser('LASER: similarity search')
parser.add_argument('--base-dir', type=str, default='.',
help='Base directory for all data files')
parser.add_argument('--data', type=str, required=True,
help='Direcory and basename of input data (language name will be added)')
parser.add_argument('--output', type=str, required=True,
help='Directory and basename of created data (language name will be added)')
parser.add_argument('--textual', action='store_true',
help='Use textual comparison instead of indicies')
parser.add_argument(
'--lang', '-l', nargs='+', required=True,
help="List of languages to test on")
# preprocessing
parser.add_argument('--bpe-codes', type=str, required=True,
help='Fast BPPE codes and vocabulary')
parser.add_argument('--verbose', action='store_true',
help='Detailed output')
# options for encoder
parser.add_argument('--encoder', type=str, required=True,
help='encoder to be used')
parser.add_argument('--buffer-size', type=int, default=100,
help='Buffer size (sentences)')
parser.add_argument('--max-tokens', type=int, default=12000,
help='Maximum number of tokens to process in a batch')
parser.add_argument('--max-sentences', type=int, default=None,
help='Maximum number of sentences to process in a batch')
parser.add_argument('--cpu', action='store_true',
help='Use CPU instead of GPU')
args = parser.parse_args()
print('LASER: similarity search')
print('\nProcessing:')
all_texts = []
if args.textual:
print(' - using textual comparision')
for l in args.lang:
with open(os.path.join(args.base_dir, args.data + '.' + l),
encoding='utf-8', errors='surrogateescape') as f:
texts = f.readlines()
print(' - {:s}: {:d} lines'.format(args.data + '.' + l, len(texts)))
all_texts.append(texts)
enc = EncodeLoad(args)
out_dir = os.path.dirname(args.output)
if not os.path.exists(out_dir):
print(' - creating directory {}'.format(out_dir))
os.mkdir(out_dir)
all_data = []
all_index = []
for l in args.lang:
Token(os.path.join(args.base_dir, args.data + '.' + l),
os.path.join(args.base_dir, args.output + '.tok.' + l),
lang=l,
romanize=True if l == 'el' else False,
lower_case=True,
verbose=args.verbose, over_write=False)
BPEfastApply(os.path.join(args.base_dir, args.output + '.tok.' + l),
os.path.join(args.base_dir, args.output + '.bpe.' + l),
args.bpe_codes,
verbose=args.verbose, over_write=False)
EncodeFile(enc,
os.path.join(args.base_dir, args.output + '.bpe.' + l),
os.path.join(args.base_dir, args.output + '.enc.' + l),
verbose=args.verbose, over_write=False)
d, idx = IndexCreate(os.path.join(args.base_dir, args.output + '.enc.' + l),
'FlatL2',
verbose=args.verbose, save_index=False)
all_data.append(d)
all_index.append(idx)
err = IndexSearchMultiple(all_data, all_index, args.lang, texts=all_texts,
verbose=False, print_errors=False)
IndexPrintConfusionMatrix(err, args.lang)
|