Spaces:
Running
Running
| import numpy as np | |
| from sklearn.metrics import confusion_matrix | |
| from spiga.eval.benchmark.metrics.metrics import Metrics | |
| class MetricsHeadpose(Metrics): | |
| def __init__(self, name='headpose'): | |
| super().__init__(name) | |
| # Angles | |
| self.angles = ['yaw', 'pitch', 'roll'] | |
| # Confusion matrix intervals | |
| self.pose_labels = [-90, -75, -60, -45, -30, -15, 0, 15, 30, 45, 60, 75, 90] | |
| # Percentile reference angles | |
| self.error_labels = [2.5, 5, 10, 15, 30] | |
| # Cumulative plot axis length | |
| self.bins = 1000 | |
| def compute_error(self, data_anns, data_pred, database, select_ids=None): | |
| # Initialize global logs and variables of Computer Error function | |
| self.init_ce(data_anns, data_pred, database) | |
| # Generate annotations if needed | |
| if data_anns[0]['headpose'] is None: | |
| print('Database anns generated by posit...') | |
| data_anns = self._posit_anns() | |
| print('Posit generation done...') | |
| # Dictionary variables | |
| self.error['data_pred'] = [] | |
| self.error['data_anns'] = [] | |
| self.error['data_pred_trl'] = [] | |
| self.error['data_anns_trl'] = [] | |
| self.error['mae_ypr'] = [] | |
| self.error['mae_mean'] = [] | |
| # Order data | |
| for img_id, img_anns in enumerate(data_anns): | |
| pose_anns = img_anns['headpose'][0:3] | |
| self.error['data_anns'].append(pose_anns) | |
| pose_pred = data_pred[img_id]['headpose'][0:3] | |
| self.error['data_pred'].append(pose_pred) | |
| # Compute MAE error | |
| anns_array = np.array(self.error['data_anns']) | |
| pred_array = np.array(self.error['data_pred']) | |
| mae_ypr = np.abs((anns_array-pred_array)) | |
| self.error['mae_ypr'] = mae_ypr.tolist() | |
| self.error['mae_mean'] = np.mean(mae_ypr, axis=-1).tolist() | |
| # Quantize labeled data | |
| label_anns = self._nearest_label(anns_array) | |
| label_pred = self._nearest_label(pred_array) | |
| self.error['label_anns'] = label_anns | |
| self.error['label_pred'] = label_pred | |
| for angle_id, angle in enumerate(self.angles): | |
| # Confusion matrix | |
| self.error['cm_%s' % angle] = confusion_matrix(label_anns[:, angle_id], label_pred[:, angle_id]) | |
| # Cumulative error | |
| self.error['cumulative_%s' % angle] = self._cumulative_error(mae_ypr[:, angle_id], bins=self.bins) | |
| return self.error | |
| def metrics(self): | |
| # Initialize global logs and variables of Metrics function | |
| self.init_metrics() | |
| # Mean Absolute Error | |
| mae_ypr = np.array(self.error['mae_ypr']) | |
| mae_ypr_mean = np.mean(mae_ypr, axis=0) | |
| self.metrics_log['mae_ypr'] = mae_ypr_mean.tolist() | |
| self.metrics_log['mae_mean'] = np.mean(mae_ypr_mean) | |
| print('MAE [yaw, pitch, roll]: [%.3f, %.3f, %.3f]' % (mae_ypr_mean[0], mae_ypr_mean[1], mae_ypr_mean[2])) | |
| print('MAE mean: %.3f' % self.metrics_log['mae_mean']) | |
| # Per angle measurements | |
| self.metrics_log['acc_label'] = [] | |
| self.metrics_log['acc_adj_label'] = [] | |
| for angle_id, angle in enumerate(self.angles): | |
| # Accuracy per label | |
| cm = self.error['cm_%s' % angle] | |
| diagonal = np.diagonal(cm, offset=0).sum() | |
| acc_main = diagonal / cm.sum().astype('float') | |
| self.metrics_log['acc_label'].append(acc_main) | |
| # Permissive accuracy | |
| diagonal_adj = diagonal.sum() + np.diagonal(cm, offset=-1).sum() + np.diagonal(cm, offset=1).sum() | |
| acc_adj = diagonal_adj / cm.sum().astype('float') | |
| self.metrics_log['acc_adj_label'].append(acc_adj) | |
| # Percentile of relevant angles | |
| self.metrics_log['sr_%s' % angle] = {} | |
| for angle_num in self.error_labels: | |
| if max(mae_ypr[:, angle_id]) > angle_num: | |
| [cumulative, base] = self.error['cumulative_%s' % angle] | |
| perc = [cumulative[x[0] - 1] for x in enumerate(base) if x[1] > angle_num][0] | |
| else: | |
| perc = 1. | |
| self.metrics_log['sr_%s' % angle][angle_num] = perc | |
| print('Accuracy [yaw, pitch, roll]: ', self.metrics_log['acc_label']) | |
| print('Accuracy [yaw, pitch, roll] (adjacency as TP): ', self.metrics_log['acc_adj_label']) | |
| for angle in self.angles: | |
| print('Success Rate %s: ' % angle, self.metrics_log['sr_%s' % angle]) | |
| return self.metrics_log | |
| def get_pimg_err(self, data_dict, img_select=None): | |
| mae_mean = self.error['mae_mean'] | |
| mae_ypr = self.error['mae_ypr'] | |
| if img_select is not None: | |
| mae_mean = [mae_mean[img_id] for img_id in img_select] | |
| mae_ypr = [mae_ypr[img_id] for img_id in img_select] | |
| name_dict = self.name + '/%s' | |
| data_dict[name_dict % 'mae'] = mae_mean | |
| mae_ypr = np.array(mae_ypr) | |
| data_dict[name_dict % 'mae_yaw'] = mae_ypr[:, 0].tolist() | |
| data_dict[name_dict % 'mae_pitch'] = mae_ypr[:, 1].tolist() | |
| data_dict[name_dict % 'mae_roll'] = mae_ypr[:, 2].tolist() | |
| return data_dict | |
| def _posit_anns(self): | |
| import spiga.data.loaders.dl_config as dl_config | |
| import spiga.data.loaders.dataloader as dl | |
| # Load configuration | |
| data_config = dl_config.AlignConfig(self.database, self.data_type) | |
| data_config.image_size = (256, 256) | |
| data_config.generate_pose = True | |
| data_config.aug_names = [] | |
| data_config.shuffle = False | |
| dataloader, _ = dl.get_dataloader(1, data_config, debug=True) | |
| data_anns = [] | |
| for num_batch, batch_dict in enumerate(dataloader): | |
| pose = batch_dict['pose'].numpy() | |
| data_anns.append({'headpose': pose[0].tolist()}) | |
| return data_anns | |
| def _nearest_label(self, data): | |
| data_tile = data[:, :, np.newaxis] | |
| data_tile = np.tile(data_tile, len(self.pose_labels)) | |
| diff_tile = np.abs(data_tile - self.pose_labels) | |
| label_idx = diff_tile.argmin(axis=-1) | |
| return label_idx | |
| def _cumulative_error(self, error, bins=1000): | |
| num_imgs, base = np.histogram(error, bins=bins) | |
| cumulative = [x / float(len(error)) for x in np.cumsum(num_imgs)] | |
| return [cumulative[:bins], base[:bins]] | |