File size: 4,577 Bytes
2913579 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 |
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"\n",
"# Paths (adjust if needed)\n",
"eval_dir = \"/home/jianingy/research/fast3r/notebooks/RealEstate10K_eval\"\n",
"test_file = \"/home/jianingy/research/fast3r/notebooks/re10k_test_1800.txt\"\n",
"\n",
"# Read the list of scene IDs into a set for fast lookup\n",
"with open(test_file, \"r\") as f:\n",
" test_scenes = set(line.strip() for line in f if line.strip())\n",
"\n",
"# Metrics we want to average\n",
"metrics = [\"RRA_at_5\", \"RRA_at_15\", \"RRA_at_30\",\n",
" \"RTA_at_5\", \"RTA_at_15\", \"RTA_at_30\",\n",
" \"mAA_30\"]\n",
"\n",
"# Initialize accumulators\n",
"acc = {m: 0.0 for m in metrics}\n",
"count = 0\n",
"\n",
"# Go through each file in the eval_dir\n",
"for filename in os.listdir(eval_dir):\n",
" if filename.endswith(\".txt\"):\n",
" filepath = os.path.join(eval_dir, filename)\n",
"\n",
" # Read JSON text. If it's single-quoted JSON, handle with `json.loads` carefully\n",
" # or use `ast.literal_eval`. If your data uses proper double-quoted JSON, regular json.loads will do.\n",
" with open(filepath, \"r\") as ff:\n",
" # If your data is strictly valid JSON, you can do:\n",
" # data = json.load(ff)\n",
" # If it has single quotes or other slight deviations, you can do:\n",
" text = ff.read()\n",
" # Convert single quotes to double quotes if your data is actually single-quoted\n",
" text = text.replace(\"'\", \"\\\"\")\n",
" data = json.loads(text)\n",
"\n",
" video_name = data.get(\"video_name\")\n",
"\n",
" # Check if this video is in our test set\n",
" if video_name in test_scenes:\n",
" # Accumulate each metric\n",
" for m in metrics:\n",
" acc[m] += data.get(m, 0.0)\n",
" count += 1\n",
"\n",
"# Compute averages if we found matches\n",
"if count > 0:\n",
" for m in metrics:\n",
" acc[m] /= count\n",
"\n",
"print(f\"Number of matched scenes: {count}\")\n",
"print(\"Averaged metrics across matched scenes:\")\n",
"for m in metrics:\n",
" print(f\"{m}: {acc[m]:.6f}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# find the worst 10 scenes on mAA_30\n",
"worst_10 = []\n",
"for filename in os.listdir(eval_dir):\n",
" if filename.endswith(\".txt\"):\n",
" filepath = os.path.join(eval_dir, filename)\n",
"\n",
" # Read JSON text. If it's single-quoted JSON, handle with `json.loads` carefully\n",
" # or use `ast.literal_eval`. If your data uses proper double-quoted JSON, regular json.loads will do.\n",
" with open(filepath, \"r\") as ff:\n",
" # If your data is strictly valid JSON, you can do:\n",
" # data = json.load(ff)\n",
" # If it has single quotes or other slight deviations, you can do:\n",
" text = ff.read()\n",
" # Convert single quotes to double quotes if your data is actually single-quoted\n",
" text = text.replace(\"'\", \"\\\"\")\n",
" data = json.loads(text)\n",
"\n",
" video_name = data.get(\"video_name\")\n",
"\n",
" # Check if this video is in our test set\n",
" if video_name in test_scenes:\n",
" worst_10.append((video_name, data.get(\"mAA_30\", 0.0)))\n",
"\n",
"print(\"Worst 10 scenes on mAA_30:\")\n",
"worst_10.sort(key=lambda x: x[1])\n",
"for video_name, score in worst_10[:10]:\n",
" print(f\"{video_name}: {score:.6f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "fast3r",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|