File size: 5,141 Bytes
6551065 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
#!/usr/bin/env python
# Python wrapper for METEOR implementation, by Xinlei Chen
# Acknowledge Michael Denkowski for the generous discussion and help
from __future__ import division
import atexit
import logging
import os
import re
import subprocess
import sys
import threading
import psutil
# Assumes meteor-1.5.jar is in the same directory as meteor.py. Change as needed.
METEOR_JAR = 'meteor-1.5.jar'
def enc(s):
return s.encode('utf-8')
def dec(s):
return s.decode('utf-8')
class Meteor:
def __init__(self):
# Used to guarantee thread safety
self.lock = threading.Lock()
mem = '1G'
mem_available_G = psutil.virtual_memory().available / 1E9
if mem_available_G < 2:
logging.warning("There is less than 2GB of available memory.\n"
"Will try with limiting Meteor to 1GB of memory but this might cause issues.\n"
"If you have problems using Meteor, "
"then you can try to lower the `mem` variable in meteor.py")
mem = '1G'
meteor_cmd = ['java', '-jar', '-Xmx{}'.format(mem), METEOR_JAR,
'-', '-', '-stdio', '-l', 'en', '-norm']
env = os.environ.copy()
env['LC_ALL'] = "C"
self.meteor_p = subprocess.Popen(meteor_cmd,
cwd=os.path.dirname(os.path.abspath(__file__)),
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
atexit.register(self.close)
def close(self):
with self.lock:
if self.meteor_p:
self.meteor_p.kill()
self.meteor_p.wait()
self.meteor_p = None
# if the user calls close() manually, remove the
# reference from atexit so the object can be garbage-collected.
if atexit is not None and atexit.unregister is not None:
atexit.unregister(self.close)
def compute_score(self, gts, res):
assert (gts.keys() == res.keys())
imgIds = gts.keys()
scores = []
eval_line = 'EVAL'
with self.lock:
for i in imgIds:
assert (len(res[i]) == 1)
stat = self._stat(res[i][0], gts[i])
eval_line += ' ||| {}'.format(stat)
self.meteor_p.stdin.write(enc('{}\n'.format(eval_line)))
self.meteor_p.stdin.flush()
for i in range(0, len(imgIds)):
v = self.meteor_p.stdout.readline()
try:
scores.append(float(dec(v.strip())))
except:
sys.stderr.write("Error handling value: {}\n".format(v))
sys.stderr.write("Decoded value: {}\n".format(dec(v.strip())))
sys.stderr.write("eval_line: {}\n".format(eval_line))
# You can try uncommenting the next code line to show stderr from the Meteor JAR.
# If the Meteor JAR is not writing to stderr, then the line will just hang.
# sys.stderr.write("Error from Meteor:\n{}".format(self.meteor_p.stderr.read()))
raise
score = float(dec(self.meteor_p.stdout.readline()).strip())
self.close()
return score, scores
def method(self):
return "METEOR"
def _stat(self, hypothesis_str, reference_list):
# SCORE ||| reference 1 words ||| reference n words ||| hypothesis words
hypothesis_str = hypothesis_str.replace('|||', '')
score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str))
score_line = re.sub(r'\s+', ' ', score_line)
self.meteor_p.stdin.write(enc(score_line))
self.meteor_p.stdin.write(enc('\n'))
self.meteor_p.stdin.flush()
return dec(self.meteor_p.stdout.readline()).strip()
def _score(self, hypothesis_str, reference_list):
with self.lock:
# SCORE ||| reference 1 words ||| reference n words ||| hypothesis words
hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ')
score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str))
self.meteor_p.stdin.write(enc('{}\n'.format(score_line)))
self.meteor_p.stdin.flush()
stats = dec(self.meteor_p.stdout.readline()).strip()
eval_line = 'EVAL ||| {}'.format(stats)
# EVAL ||| stats
self.meteor_p.stdin.write(enc('{}\n'.format(eval_line)))
self.meteor_p.stdin.flush()
score = float(dec(self.meteor_p.stdout.readline()).strip())
# bug fix: there are two values returned by the jar file, one average, and one all, so do it twice
# thanks for Andrej for pointing this out
score = float(dec(self.meteor_p.stdout.readline()).strip())
return score
def __del__(self):
self.close()
|