From 6bcf6b8306ce4903734fb31824799a50281cea69 Mon Sep 17 00:00:00 2001 From: fredeee Date: Sat, 23 Mar 2024 13:27:00 +0100 Subject: add bouncingball experiment and ablation studies --- .../lambda/evaluation_loci_looped_abl1.ipynb | 479 +++++++++++++++++++++ .../lambda/evaluation_loci_looped_abl2.ipynb | 469 ++++++++++++++++++++ .../lambda/evaluation_loci_looped_abl3.ipynb | 469 ++++++++++++++++++++ .../recon/evaluation_loci_looped_norecon.ipynb | 464 ++++++++++++++++++++ .../recon/evaluation_loci_looped_recon.ipynb | 466 ++++++++++++++++++++ 5 files changed, 2347 insertions(+) create mode 100644 evaluation/adept_ablation/lambda/evaluation_loci_looped_abl1.ipynb create mode 100644 evaluation/adept_ablation/lambda/evaluation_loci_looped_abl2.ipynb create mode 100644 evaluation/adept_ablation/lambda/evaluation_loci_looped_abl3.ipynb create mode 100644 evaluation/adept_ablation/recon/evaluation_loci_looped_norecon.ipynb create mode 100644 evaluation/adept_ablation/recon/evaluation_loci_looped_recon.ipynb (limited to 'evaluation/adept_ablation') diff --git a/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl1.ipynb b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl1.ipynb new file mode 100644 index 0000000..8fde028 --- /dev/null +++ b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl1.ipynb @@ -0,0 +1,479 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import warnings\n", + "import scipy.stats as stats\n", + "import os\n", + "\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)\n", + "pd.options.mode.chained_assignment = None \n", + "plt.style.use('ggplot')\n", + "sns.color_palette(\"Paired\");\n", + "sns.set_theme();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '../../../out/pretrained/adept_ablations/lambda/results/adept_level1_ablation_lambda.run311/trialframe.csv'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/Users/fredericbecker/Documents/Master/Masterarbeit/code/local/loci_public/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl1.ipynb Cell 3\u001b[0m line \u001b[0;36m1\n\u001b[1;32m 13\u001b[0m \u001b[39mfor\u001b[39;00m net \u001b[39min\u001b[39;00m nets:\n\u001b[1;32m 14\u001b[0m path \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(root_path, \u001b[39m'\u001b[39m\u001b[39mresults\u001b[39m\u001b[39m'\u001b[39m, net)\n\u001b[0;32m---> 15\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39;49m(os\u001b[39m.\u001b[39;49mpath\u001b[39m.\u001b[39;49mjoin(path, \u001b[39m'\u001b[39;49m\u001b[39mtrialframe.csv\u001b[39;49m\u001b[39m'\u001b[39;49m), \u001b[39m'\u001b[39;49m\u001b[39mrb\u001b[39;49m\u001b[39m'\u001b[39;49m) \u001b[39mas\u001b[39;00m f:\n\u001b[1;32m 16\u001b[0m tf_temp \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mread_csv(f, index_col\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[1;32m 17\u001b[0m tf_temp[\u001b[39m'\u001b[39m\u001b[39mnet\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m net\n", + "File \u001b[0;32m~/miniconda3/envs/loci23/lib/python3.9/site-packages/IPython/core/interactiveshell.py:282\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[39mif\u001b[39;00m file \u001b[39min\u001b[39;00m {\u001b[39m0\u001b[39m, \u001b[39m1\u001b[39m, \u001b[39m2\u001b[39m}:\n\u001b[1;32m 276\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m 277\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mIPython won\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt let you open fd=\u001b[39m\u001b[39m{\u001b[39;00mfile\u001b[39m}\u001b[39;00m\u001b[39m by default \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 278\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 279\u001b[0m \u001b[39m\"\u001b[39m\u001b[39myou can use builtins\u001b[39m\u001b[39m'\u001b[39m\u001b[39m open.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 280\u001b[0m )\n\u001b[0;32m--> 282\u001b[0m \u001b[39mreturn\u001b[39;00m io_open(file, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../../../out/pretrained/adept_ablations/lambda/results/adept_level1_ablation_lambda.run311/trialframe.csv'" + ] + } + ], + "source": [ + "# setting path to results folder\n", + "root_path = '../../../out/pretrained/adept_ablations/lambda'\n", + "\n", + "# list all folders in root path that don't stat with a dot\n", + "nets = ['adept_level1_ablation_lambda.run311']\n", + "\n", + "# read pickle file\n", + "tf = pd.DataFrame()\n", + "sf = pd.DataFrame()\n", + "af = pd.DataFrame()\n", + "\n", + "# load statistics files from nets\n", + "for net in nets:\n", + " path = os.path.join(root_path, net, 'results')\n", + " with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n", + " tf_temp = pd.read_csv(f, index_col=0)\n", + " tf_temp['net'] = net\n", + " tf = pd.concat([tf,tf_temp])\n", + "\n", + " with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n", + " sf_temp = pd.read_csv(f, index_col=0)\n", + " sf_temp['net'] = net\n", + " sf = pd.concat([sf,sf_temp])\n", + "\n", + " with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n", + " af_temp = pd.read_csv(f, index_col=0)\n", + " af_temp['net'] = net\n", + " af = pd.concat([af,af_temp])\n", + "\n", + "# cast variables\n", + "sf['visible'] = sf['visible'].astype(bool)\n", + "sf['bound'] = sf['bound'].astype(bool)\n", + "sf['occluder'] = sf['occluder'].astype(bool)\n", + "sf['inimage'] = sf['inimage'].astype(bool)\n", + "sf['vanishing'] = sf['vanishing'].astype(bool)\n", + "sf['alpha_pos'] = 1-sf['alpha_pos']\n", + "sf['alpha_ges'] = 1-sf['alpha_ges']\n", + "\n", + "# scale to percentage\n", + "sf['TE'] = sf['TE'] * 100\n", + "\n", + "# add surprise as dummy code\n", + "tf['control'] = [('control' in set) for set in tf['set']]\n", + "sf['control'] = [('control' in set) for set in sf['set']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Tracking Error (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tracking Error when visible: M: 11.6 , STD: 13.5, Count: 2281\n", + "Tracking Error when occluded: M: 7.1 , STD: 4.02, Count: 564\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "\n", + "def get_stats(col):\n", + " return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n", + "\n", + "# When Visible\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n", + "\n", + "# When Occluded\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Tracking Error when occluded:' + get_stats(temp['TE']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Succesfull Trackings (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
setevalmodetracked_postracked_negtracked_pos_protracked_neg_pro
0controlopen0540.01.0
\n", + "
" + ], + "text/plain": [ + " set evalmode tracked_pos tracked_neg tracked_pos_pro \\\n", + "0 control open 0 54 0.0 \n", + "\n", + " tracked_neg_pro \n", + "0 1.0 " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# succesfull trackings: In the last visible moment of the target, the slot was less than 10% away from the target\n", + "# determine last visible frame numeric\n", + "grouping_factors = ['net','set','evalmode','scene','slot']\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).max()\n", + "ff.rename(columns = {'frame':'last_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['last_visible']], on=grouping_factors, how='left')\n", + "\n", + "# same for first bound frame\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).min()\n", + "ff.rename(columns = {'frame':'first_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['first_visible']], on=grouping_factors, how='left')\n", + "\n", + "# add dummy variable to sf\n", + "sf['last_visible'] = (sf['last_visible'] == sf['frame'])\n", + "\n", + "# extract the trials where the target was last visible and threshold the TE\n", + "ff = sf[sf['last_visible']] \n", + "ff['tracked_pos'] = (ff['TE'] < 10)\n", + "ff['tracked_neg'] = (ff['TE'] >= 10)\n", + "\n", + "# fill NaN with 0\n", + "sf = sf.merge(ff[grouping_factors + ['tracked_pos', 'tracked_neg']], on=grouping_factors, how='left')\n", + "sf['tracked_pos'].fillna(False, inplace=True)\n", + "sf['tracked_neg'].fillna(False, inplace=True)\n", + "\n", + "# Aggreagte over all scenes\n", + "temp = sf[(sf['frame']== 1) & ~sf.occluder & sf.control & (sf.first_visible < 20)]\n", + "temp = temp.groupby(['set', 'evalmode']).sum()\n", + "temp = temp[['tracked_pos', 'tracked_neg']]\n", + "temp = temp.reset_index()\n", + "\n", + "temp['tracked_pos_pro'] = temp['tracked_pos'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "temp['tracked_neg_pro'] = temp['tracked_neg'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "\n", + "temp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mostly Tracked stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEmCAYAAAB20LuSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAir0lEQVR4nO3dd3hUVf7H8fek0UsIoSoiCkdABUGKBVFkXeuqix10V0SKCy7KqvijN8FdUVAUFURBARfBLljoFqSJICoHBBsQIYagJBAgmfn9MQk7xEAmMOFmTj6v5+Ex994zd74mkw+Hc8891xcIBBAREXfEeF2AiIhEloJdRMQxCnYREcco2EVEHKNgFxFxTJzH718GaAWkADke1yIiEi1igdrASmB//oNeB3sr4GOPaxARiVbtgE/y7/Q62FMA0tMz8fs1nz4SkpIqkpaW4XUZIn+gz2bkxMT4SEysALkZmp/XwZ4D4PcHFOwRpO+llFT6bEZcgUPYungqIuIYBbuIiGO8HooRkeOUk5NNenoq2dkHvC7lqHbujMHv93tdRlSJi0sgMTGZ2NiiRbWCXSTKpaenUrZseSpUqIXP5/O6nCOKi4shO1vBHq5AIEBm5u+kp6dSvXrtIr02rGA3xlQGPgOuttb+kO9Yc2AyUBlYCvS01mYXqQoROWbZ2QdKfKhL0fl8PipUqExGxu4iv7bQMXZjTBuC8yQbHaHJK0Bva20jwAfcXeQqROS4KNTddKw/13Aunt4N/APYnv+AMeYUoJy19vPcXS8BNx5TJSIiEhGFDsVYa7sBGGMKOlyHwyfIpwAnFbWIpKSKRX3JCXcg5yAJsfFelxGW5ORKXpdQqGj6fpZ0O3b6iIs7vI9WrnwZypaJ/CW0rP3Z7Nv7hzvYw5a/znB8/fV6Fi1aQO/e/+Tdd9/miy9WM3jwsCKdIyNjDyNGDOXRR8cW2jYQCBy1p9yr191069aDli3PLVINAJMmPQvA3Xf3DPs1MTExRf6dPt6ffAwQeseBDyjy1ZG0tIwSf+NCcnIlbvpvL6/LcMasmyeSmrrH6zKc4MPH5l0/Hbbv3MTGXNPvrYi/1ztjr2X91k0RP+/RLF+/kh9/+YnNu35iZ2Yaew5k/OH/tzCpO3bw9bfrw3rdadXqHfUibyAQICfHf0wXgvNyriiv9fv9f/hdiYnxHbVDfLzBvpXgQjR5alHAkI2IlB7frPuKt16dTVx8HDt/2UHLtq0pU7Ysq5ctJ0CAB4cNZvPGTbz28nQC/gA1atXkrj73UCWxKtMnv8j6NWvx+Xy0PK8Nl197NXNemUnWvizefPU1EpOqAbD+y7XMfnkmQ8eOAWDJRwvYbDfRtXfBPeFpz04mfVc6T4wYTZfuXXl00HAqVa5MQkIC/xz4EJPGTWDXr2mkp+3i/Lbn8+CDAwGYOPEpli5dTFxcLH/5y1+56aZbD50zPX0X997bk+7d76Fdu4t5+eWXWLToI3Jy/LRp05Zeve7F5/MxY8Y03n77DapUqUqlSpVo3LhpMf8EjvMGJWvtj0CWMeaC3F23A/OOuyoRiWrf2Y107d2TkePH8uE7c6lcpQojnxxLvfr1mT/3faZMmMj9gx5mzDPjadSkMS9NfJ7UHTtZu+oLRj89jiGPjWHbTz8Tn5BApy630qJtK6675X+X75o2O5vd6ensSAmOBH+yYDEXdexwxHru6NmNxGqJ3DfoYQBStm7jngf68vAjw/hyxSpOaXAqwx5/lMcnP8OaNauxdgOLFi3gq6/WMm3aqzz//FTmzn2HtLRfAcjMzOCBB/rStWt32rW7mM8//wxrv2XSpGm8+OJ0UlNT+fDDeWzY8A3vvfc2U6ZMZ9y4Z0hN3VmM3/X/OaYeuzFmLjDYWrsK6AxMyp0S+QXwZATrE5EodHL9eiQlJwNQqXIlmjY/G4DqNZJZs3wlpzVqSHLNmgB0uOIy3p41h2rVk0hISGBov/6c0/pcbu16BwkJCQWe3+fz0e7SS/hk4RLa/+lSftu9m9PPONLEvT+qXLXKofc//+KL2Gw3Mu/Nt9n+81Z+++039u3by5dfrqZDhz+RkJBAQkICL70049Dr//Of0VSrlkT79sG/TFatWsE336znrrtuB2D//ixq1qxFWloabdteQPny5QG45JKO5OQU/wrlYQe7tbZ+yNdXhny9Fmgd2bJEJJrFxR0eLbGx/xscCAQOv54WHLPOITY2luFP/Jtvv/qaL1etZuj9/Rn46MgjvsdFHTvw78HDSUhIoN2llxSpvtC/MD54+11WfLKMDldcxpnNm5G6dQeBQIC4uDhCr6GmpGynatVEADp3voNlyz7ljTdm06nTTfj9Odx0063ccksXAPbs2UNsbCxvvfU6oZchY2NjT0iwa60YETmhTjON+G7DRlJ37ABg4bwPadLsLH7YvIURDw3kjLOa0rnbndStdxIp27YRGxuLP+ePFxuTa9agWlIS89+bx4UdLj7qe8bExpJTwDkA1q9ZS4crLuOCS9pz8MBBNm2y+P1+mjVrweLFC8nOziYrK4t+/focGkpp2NDQr19/XnxxEqmpO2nRohUffDCXvXv3kp2dzcMP92Px4gWce24rPv30YzIyMti/fz9Lly46vm9emLSkgIicUFWqVuGue3vxxIgxZGdnU71GMnf37U1itWo0bGx4qNe9lClThkZNzqDZuS3ZmfILr09/lVdfnEbtk+oedq7z2l/Iik+XHbqoeuT3rEr1GtUZ2X8gPe7rc9ixy6+9hheffpa3Z82hfIUKnHVWM1JStnPNNdexYcM3dO3aGb8/wI033kq9eqccet3JJ9fjr3+9kccf/zejRz/Gd99tpHv3v+P359CmzflcccXV+Hw+brzxVrp1u4NKlSpRs2bRlgY4Vr78/yw6weoD32u6Y+mj6Y6Rs3Pnz+wre/i868Z1G1ChbJmIv1dm1n6+3bYl4uc9Fjk5OUx8bBxtLjyfVhecF7HzFjbd8UT75ZcfqVXrlMP2hUx3PBX4If9r1GMXcVBJCd/iEggE6N2lK2ee04yW57UBYMP6r5n67KQC2z84bHChvXqXKNhFJOr4fD4mzpx62L4zzmzK6AnjvCmohNHFUxERxyjYRUQco2AXEXGMgl1ExDEKdhERx2hWjIiDzq57Cglly0f8vAey9rJu248RP2+evZmZPPf4k9w36GHS03YxafwEHhw++Ijt57wyE4BOXW6l85XXMX3um0V+z9mvzOTM5mdzxpnHt+ri3LnvsGbNagYMGFrk16akbKdPnx7Mnv3OcdWQR8Eu4qCEsuXZMqpTxM/bYMCciJ8zVGZGBj9sDs7BT0yqdtRQj5QNX62nydlnFvv7nEgKdhGJqG/WfcWbM2cRGxdH6i87aNCoIXf37c0bM/7L+i/Xkbkng8SkRPr0f4AqiVXpecsdnNrwdHan76JqYrXD1k0f+dBAxr80iZ9/+JGpz05i/74sftv9G3+5qRMdr7r8D+/t9/u5r2sP+o8cSu2T6pKVlcUD3f/B2MkTC1wp8uMFi9iyaTOTxz9N34H9mfbsJGpUq8GWLZsZPnw069Z9yfvvzyUrax/x8fEMHTqKevXqs3LlciZMGEcg4KdWrdoMGXL4YmXjx49l1640Bg8ewcaNG3jyycfZvz+LKlWq8sAD/0edOnXZuHEDY8aMAOD008NfmTIcGmMXkYjb+K3l9h7d+M/zT3Pw4EHef/Mdtv+8laFjxzB28jMkJSfzyaLFAOz5/XeuufF6Rk8YR9fePQ5bNz3P4g8+4rpbbmTE+McYOGYEM6e8VOD7xsTE0K5jBz5dtASAlZ8u45zWrY64/G+7Sy+hQcPT6PbPf1Dv1PoAnH56Q2bOfJ06deqydOkSJkx4jpdfnsX557djzpxZHDhwgOHDBzFw4FCmTfsvDRqczrx57x465wsvPEdq6k4GDRqO3+9nzJiRDBkyiilTpnPLLV149NFRAIwcOYRevfowZcp06tSpW1B5x0w9dhGJuDPObEKd3AW7LuxwMQvnfcjf77mbxR98xPat29i0wVKjdq1D7U83R++xdu52J2tXr+Gt/87m5x9+JGtf1hHbtv9TB0b/3xBuuP02Pp6/kJv+dnuRam/a9CwAKlSoyNChI5k//0N+/vknli//jIYNDVu2fEdycjINGwafA92zZ28gOMb++eefsXt3OpMmTSMuLo4tW75j+/at9O9//6HzZ2Zmsnv3bn799VdatWoLwBVXXM2770buUYYKdhGJuNjY2ENfB/x+fD4YM2AYV1z/F9pceD4xMTEQsgBhQpmjL1j25Oj/UKFiRc5p04rz2rdj2ZKPj9g2uWZNqtdIZuWny/ht929FegAHQJncWnbs+IU+fXrQqdNNtG17PtWqJbFpkyU2No7g452DMjIy2Ls3E4BatWrTo8c9PP74ozz77BRycvzUqVP30EM6cnJySE/fhc93+Lr0wXNGjoZiRCTi7NffsuvXNPx+Px8vXIxp2oTGZzel41WXU6tuHdasWIXf/8cVFI+0bvpXa9Zyw+23ce55bVi7ajUA/qM8sKL9ZR2Z+uzkQtdpz3vPgtZ737DhG0466WRuvrkzjRs3YenSRfj9OdSrdwq7d6fz/ffBi7zTp0/lzTeDF5Xr1z+Vq6++jnLlyvH667M45ZT6/P7776xduwaA9957m6FDB1ClSlVq1arFZ599AsBHH71faJ1FoR67iIMOZO0tlhksB7L2htUuMakaz44dx660XZx1TjMuuKQ940aO4aFe9wLQoOHppO744/M/j7RueqfOtzDsgYeJj4/nlAanklyzBjsLeH2eVue3ZfKTT4cV7M1atmDKhIn07Nf38HO0assbb8ymS5cbCQQCNG/egi1bNlOmTBkGDRrOyJFDyM4+SJ06JzFo0HAWL15w6LX9+vXnnnvu4qKLLmHEiDGMH/8YBw4coHz5CgwcOAyAQYNGMHr0MCZNeoamTc8utM6i0HrsYdJ67JGl9dgjp6D12L30zbqveH36qwzMvUh4ogUCAdauWs2CuR/Qb8iAIr9e67GLiJQwrzz/Al8sX3nYHPiR/QeSmZHxh7aXXnF5gdMmo52CXUQiqsnZZ9Hk7LM8e//be3Tj9h7dDts3cMyRH4rtIl08FRFxjIJdRMQxCnYREcco2EVEHKOLpyIOanxSfSqUKRfx82bu38e3W3+I+HmPZrPdyIpPl3Fr17+x5KMFfPvVenre/88ineNYp2BmZGQwatRQRo9+rEiv85qCXcRBFcqUK5b7LmbdPDHi5yzM1p9+5rfdu0/4+wLs2fM7mzZZT977eCjYRSSivln3FW+9Opu4+Dh2/rKDlm1bU6ZsWVYvW06AAA8OG8zmjZt47eXpBPwBatSqyV197qFKYlWmT36R9WvW4vP5aHleGy6/9mrmvDKTrH1ZvPnqayQmVQNg/Zdrmf3yTIaOHQPAko8WsNluomvvnoXWl7J1Gy889QwZezIoU7Ysd/TsxmmNGvLpoiW8O+cNysWXpXbtOgwaNIJx4/7Dr7+m8vDD/4qqXrvG2EUk4r6zG+nauycjx4/lw3fmUrlKFUY+OZZ69eszf+77TJkwkfsHPcyYZ8bTqEljXpr4PKk7drJ21ReMfnocQx4bw7affiY+IYFOXW6lRdtWXHfLjYfO37TZ2exOT2dHSgoAnyxYzEUdO4RV2zOPjePPf7maMc+Mp0v3rowf9W8OHjzIay/PoP/IoUydOoPatevy008/0LfvA1SvnhxVoQ4KdhEpBifXr0dScjJlypahUuVKNG0eXAuleo1k1ixfyWmNGpJcsyYAHa64jK+/XEe16kkkJCQwtF9/3n/rHW7tescR11H3+Xy0u/QSPlm4hF93pvLb7t1hreKYtW8fO7an0OqC8wBoeIahYqWKpGzdRovWrRj2r4d56qlxtG/f4dCyvNFIwS4iERcXd/gob2zs/6Im//pUgUCAnJwcYmNjGf7Ev7nx9tvI2LOHoff3J2XrtiO+x0UdO/D50k9YtuRj2l16SVh1+QtYGytA8P3v6NmNvgMeonLlyowYMYgPPpgb1jlLIgW7iJxQp5lGfLdhI6k7dgCwcN6HNGl2Fj9s3sKIhwZyxllN6dztTurWO4mUbduIPcKyusk1a1AtKYn5780LaxVHgPLly1OjVk1WfroMgE0bLL+l7+bkU+pxf7deVKpcmb/9rSuXX34VGzdaYmNjyTnK8sAlVVgXT40xtwEDgXhgnLX26XzHWwDPAQnAz0AXa+3uyJYqIi6oUrUKd93biydGjCE7O5vqNZK5u29vEqtVo2Fjw0O97qVMmTI0anIGzc5tyc6UX3h9+qu8+uI0ap90+CPkzmt/ISs+XXboomo47nngPqZMmMjsV2YSHx9P3wH9iYuP54YutzJ6wBAqla9I1aqJDBgwlEqVKlOzZi369OnBU089F+lvRbEpdNleY0xd4BOgJbAf+Ay41Vr7TUibj4FHrLXzjDFjgX3W2oFhvH99tGxvqaRleyOnoGV7XZrHfiQ5OTlMfGwcbS48/9CYeSSUlmV7OwILrbW7AIwxs4EbgOEhbWKByrlflwd2FbF2EYmgkhK+xSUQCNC7S1fOPKcZLc9rA8CG9V8z9dlJBbZ/cNjgIvXqo104wV4HSAnZTgFa52tzP/ChMWYckAm0iUh1IiIF8Pl8TJw59bB9Z5zZlNETxnlTUAkTTrDHAKHjJD7g0L9TjDHlgBeAjtbaFcaY+4FpwFXhFpH7TwopZZKTK3ldghN2HvkJcXKM4uJKzrySmJiYIv+uhBPsW4F2Idu1gO0h22cSHFNfkbv9HDCiKEVEyxi7RJbG2CMoEABfyXk8XrQrKWPsgUAAv9//h9+VkDH2AoXz19J84FJjTLIxpjzQCQh9pPZ3wMnGmLzZ/NcCK4tSvIgcu3LlyuI/cCAY7uKMQCBAZubvxMUVfJPW0RTaY7fWbjPGDAAWEZzOODl3yGUuMNhau8oY83dgljHGB+wE7ixyJSJyTE4++WRWfT6X5DKJ+FCv/Xhtzwr2kkuCuLgEEhOTi/66cBpZa2cAM/LtuzLk63nAvCK/u4gct/j4eGZvf7/whhIWF6bilpwrBCIiEhEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQcE+d1AdHiQPYBZt080esynHEg+4DXJYg4S8EepoS4BLaM6uR1Gc5oMGAOsN/rMkScpKEYERHHKNhFRByjYBcRcYyCXUTEMQp2ERHHKNhFRByjYBcRcUxY89iNMbcBA4F4YJy19ul8xw3wHJAI/ALcYq1Nj3CtIiIShkJ77MaYusAo4EKgOdDdGNMk5LgPeBsYY61tBqwB+hdLtSIiUqhwhmI6AguttbustZnAbOCGkOMtgExr7fu5248ATyMiIp4IZyimDpASsp0CtA7ZPh34xRjzAnAO8C3QJ2IVishRaR2jyHJhHaNwgj0GCIRs+wB/vnNcDFxkrV1ljBkBPA78PdwikpIqhttUHJKcXMnrEpyhdYwip8GAOSQnJ3hdxnEJJ9i3Au1CtmsB20O2fwE2WWtX5W7PJDhcE7a0tAz8/kDhDT2kEIq81NQ9XpfgBH02I6+kfzZjYnxH7RCHM8Y+H7jUGJNsjCkPdALeDzn+GZBsjGmWu30NsPoY6xURkeNUaLBba7cBA4BFwJfADGvtCmPMXGPMudbafcD1wCRjzNdAB6BfMdYsIiJHEdY8dmvtDGBGvn1Xhny9nMMvqIqIiEd056mIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGMU7CIijlGwi4g4RsEuIuIYBbuIiGPCCnZjzG3GmG+MMZuMMf84SrurjDHfR648EREpqkKD3RhTFxgFXAg0B7obY5oU0K4m8Bjgi3CNIiJSBHFhtOkILLTW7gIwxswGbgCG52s3GRgGjIlohSWE/+ABGgyY43UZzvAfPOB1CSLOCifY6wApIdspQOvQBsaYe4EvgM+PpYikpIrH8rIT7pp+b3ldgjPeGXstyckJXpchUqDk5Epel3Bcwgn2GCAQsu0D/HkbxpgzgU7ApcBJx1JEWloGfn+g8IYeivYfdEmUmrrH6xKcoM9m5JX0z2ZMjO+oHeJwLp5uBWqHbNcCtods35h7fBUwF6hjjPm46KWKiEgkhNNjnw8MNcYkA5kEe+fd8w5aa4cAQwCMMfWBxdbadpEvVUREwlFoj91auw0YACwCvgRmWGtXGGPmGmPOLeb6RESkiMLpsWOtnQHMyLfvygLa/QDUj0RhIiJybHTnqYiIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4piwblASkZJLS0pHlgtLSivYRaJcTHyClpSOoHfGXgvs97qM46KhGBERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQcExdOI2PMbcBAIB4YZ619Ot/xa4FhgA/4HrjTWpse4VpFRCQMhfbYjTF1gVHAhUBzoLsxpknI8crAROAqa20zYB0wtDiKFRGRwoUzFNMRWGit3WWtzQRmAzeEHI8H/mGt3Za7vQ6oF9kyRUQkXOEMxdQBUkK2U4DWeRvW2jTgDQBjTDmgP/BUBGsUEZEiCCfYY4BAyLYP8OdvZIypQjDg11prpxaliKSkikVpLo5ITq7kdQkiBYr2z2Y4wb4VaBeyXQvYHtrAGFMb+ABYCNxX1CLS0jLw+wOFN/RQtP+gS6LU1D1el+AEfTYjr6R/NmNifEftEIcT7POBocaYZCAT6AR0zztojIkF3gFmWWtHHl+5IiJyvAoNdmvtNmPMAGARkABMttauMMbMBQYDJwMtgDhjTN5F1VXW2m7FVbSIiBxZWPPYrbUzgBn59l2Z++UqdKOTiEiJoUAWEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXFMXDiNjDG3AQOBeGCctfbpfMebA5OBysBSoKe1NjuypYqISDgK7bEbY+oCo4ALgeZAd2NMk3zNXgF6W2sbAT7g7gjXKSIiYQqnx94RWGit3QVgjJkN3AAMz90+BShnrf08t/1LwDBgYhjnjgWIifEVrWqP1Egs53UJTomWn3s00Gczskr6ZzOkvtiCjocT7HWAlJDtFKB1IcdPCrO+2gCJiRXCbO6tFwZe5nUJTklKquh1Cc7QZzOyouizWRvYnH9nOMEeAwRCtn2AvwjHj2Yl0I7gXwY5Yb5GRKS0iyUY6isLOhhOsG8lGL55agHb8x2vfZTjR7Mf+CTMtiIi8j9/6KnnCWe643zgUmNMsjGmPNAJeD/voLX2RyDLGHNB7q7bgXnHUayIiByHQoPdWrsNGAAsAr4EZlhrVxhj5hpjzs1t1hl4whizAagIPFlM9YqISCF8gUCg8FYiIhI1dOepiIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo4Ja9leKXmMMYOPdtxaO/xE1SJyJLlLfjcluELsDdbaaR6XVCqoxx69fIX8EfGUMWYMcCXwV4KdyDuNMWO9rap00A1KDjDGJANtCP7yLLPW7vC4JBGMMWuAFsAX1tpzjDFxwDprbf7nOUiEqcce5Ywxfya41MOdwN+AdcaYqz0tSiQob5XXvN5jGcJf+VWOg8bYo98o4EJr7fcAxpgGwOvAu55WJQKzgP8C1YwxfYE7gBmeVlRKqMce/eLzQh3AWrsF/VylBLDWPgq8ALwG1AMGWWsf8baq0kEBEP1+Msb0NcZUyv1zH/Cj10WJGGMSgBRr7QPAauBiY0x1j8sqFRTs0e8u4DxgC/B97tfdPa1IJOgVoLMxpjUwBPid4DORpZhpjD363WutvdnrIkQKcKq19iZjzKPAC9baR40xBT7KTSJLPfbod40xRvPWpSSKyx16uR54zxhTCyjncU2lgnrs0S8N2GCM+QLYl7fTWtvVu5JEAPgPsBx421q73hizERjkcU2lgoI9+k0tYJ/uOhPPWWtncPj0xsbW2hyv6ilNFOzRr461dnToDmOMppSJZ4wx71prrzbGfE++ToYxBmttA49KKzUU7FEqdx2OGsBfjDENQw7FAW2B//OkMBG4O/e/NwE7vSyktFKwR685QBPgUmBJyP5sYIQnFYkA1tqU3C+nWWsbe1pMKaVFwKKcMaaKtfY3r+sQyc8Y8yrwHrCCwy/s/+RZUaWEeuzR77rcpVATc7d9QMBaG+thTSIQXHG0Tb59AUBj7MVMPfYoZ4zZDFxrrV3vdS0iUjKoxx79tivUpSQyxtQDngQ6ELz2Mxe4z1qb6mlhpYCCPfqtNsbMBj4EsvJ26hFkUgJMJ7hs7+1ALMFnBkwl+FQlKUYK9uhXBdhDcPGvPAFAwS5eq2ytnRCy/YQx5u9eFVOaKNijnLX2TmNMPGAI/jzXW2uzPS5LBOAzY0wXa+0rAMaYq4A1HtdUKujiaZQzxrQkOKc9jeCibjWB6621yz0tTEo9Y8wOIJngVEc/UCHksGZuFSP12KPfk8DNeUFujGkLPAW09rQqEbjMWrvW6yJKIy3bG/0qhvbOrbWfA2U9rEckz6teF1Baqcce/XYZY6611r4FYIy5juCwjIjXvjHGDCa4dG/onadLvSupdFCwR7/uwLvGmBfIvesUON/bkkQAqAZckvsnT4DgvHYpRgr26HcFsBdoAZxGcN7wxcBGD2sSwVp7SeGtpDgo2KNfd6C1tXYvsC53lsxy4Hlvy5LSzhiziAIe+mKtVY+9mCnYo188cCBk+wB6gpKUDENDvo4HrgXSvSmldFGwR783gYXGmFkEA70T8JanFYkA1tol+XbNN8YsBwZ7UU9pohuUHGCMuQFoDxwEllpr3/S2IpFDi4Dl8QFnAuOttad7VFKpoR67A6y1s4HZXtchks8Sgv+K9BG88/RXoLenFZUSukFJRIrLLcDTwBnAZoKztuQEULCLSHEZD3wF/JXglNxz0PN4TwgFu4gUlxhr7YfA1cAca+3PaPj3hFCwi0hx2WuM6UfwTtN3jTH3Enx2gBQzBbuIFJfOBJfq7WStTQfqArd5W1LpoOmOIiKOUY9dRMQxCnYREcco2EWKwBhzqjFmjtd1iByNgl2kaE4h+OBwkRJLF0+lVDPGVAReBBoSvO19NdADuAoYCCQQvLnmX8AKwBKc3bHUWvtnL2oWKYx67FLaXQ9UstY2B1rl7jsNeAS40lp7DsE1718n+CzZbsBmhbqUZLoLTEq7T4BHjDGLgY+AccCfgNrAAmMOjbr4Aa1KKFFBPXYp1ay13xMM7NFAZWA+EAsssNY2z/sDtAXWe1aoSBEo2KVUM8b0IjjG/qG19iHgA4IPYb7MGHNGbpsrgXVAOSCb4NOAREosXTyVUs0YUwGYApxN8CLpT0BXoCMwgOBa4tlAX2vtx8aYasAiYD/QxlqrXyApcRTsIiKO0VCMiIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLimP8HPUx1x9OfirsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "temp = af[af.index == 'OVERALL']\n", + "temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n", + "temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n", + "temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n", + "g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MOTA " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idf1idpidrrecallprecisionnum_unique_objectsmostly_trackedpartially_trackedmostly_lostnum_false_positivesnum_missesnum_switchesnum_fragmentationsmotamotpnum_transfernum_ascendnum_migrate
set
control0.7220420.8794720.6124160.6581580.94516186.035.022.029.0187.01674.026.038.00.6146620.0339575.021.00.0
surprise0.7381220.9230040.6149460.6466120.97053233.011.09.013.031.0558.05.06.00.6238130.0310670.05.00.0
\n", + "
" + ], + "text/plain": [ + " idf1 idp idr recall precision \\\n", + "set \n", + "control 0.722042 0.879472 0.612416 0.658158 0.945161 \n", + "surprise 0.738122 0.923004 0.614946 0.646612 0.970532 \n", + "\n", + " num_unique_objects mostly_tracked partially_tracked mostly_lost \\\n", + "set \n", + "control 86.0 35.0 22.0 29.0 \n", + "surprise 33.0 11.0 9.0 13.0 \n", + "\n", + " num_false_positives num_misses num_switches num_fragmentations \\\n", + "set \n", + "control 187.0 1674.0 26.0 38.0 \n", + "surprise 31.0 558.0 5.0 6.0 \n", + "\n", + " mota motp num_transfer num_ascend num_migrate \n", + "set \n", + "control 0.614662 0.033957 5.0 21.0 0.0 \n", + "surprise 0.623813 0.031067 0.0 5.0 0.0 " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "af[af.index == 'OVERALL'].groupby(['set']).mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gate Openings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percept gate openings when visible: M: 1.02 , STD: 0.681, Count: 2281\n", + "Percept gate openings when occluded: M: 0.216 , STD: 0.48, Count: 564\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Percept gate openings when visible:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Percept gate openings when occluded:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl2.ipynb b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl2.ipynb new file mode 100644 index 0000000..5b8e275 --- /dev/null +++ b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl2.ipynb @@ -0,0 +1,469 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import warnings\n", + "import scipy.stats as stats\n", + "import os\n", + "\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)\n", + "pd.options.mode.chained_assignment = None \n", + "plt.style.use('ggplot')\n", + "sns.color_palette(\"Paired\");\n", + "sns.set_theme();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# setting path to results folder\n", + "root_path = '../../../out/pretrained/adept_ablations/lambda'\n", + "\n", + "# list all folders in root path that don't stat with a dot\n", + "nets = ['adept_level1_ablation_lambda.run411']\n", + "\n", + "# read pickle file\n", + "tf = pd.DataFrame()\n", + "sf = pd.DataFrame()\n", + "af = pd.DataFrame()\n", + "\n", + "# load statistics files from nets\n", + "for net in nets:\n", + " path = os.path.join(root_path, net, 'results')\n", + " with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n", + " tf_temp = pd.read_csv(f, index_col=0)\n", + " tf_temp['net'] = net\n", + " tf = pd.concat([tf,tf_temp])\n", + "\n", + " with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n", + " sf_temp = pd.read_csv(f, index_col=0)\n", + " sf_temp['net'] = net\n", + " sf = pd.concat([sf,sf_temp])\n", + "\n", + " with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n", + " af_temp = pd.read_csv(f, index_col=0)\n", + " af_temp['net'] = net\n", + " af = pd.concat([af,af_temp])\n", + "\n", + "# cast variables\n", + "sf['visible'] = sf['visible'].astype(bool)\n", + "sf['bound'] = sf['bound'].astype(bool)\n", + "sf['occluder'] = sf['occluder'].astype(bool)\n", + "sf['inimage'] = sf['inimage'].astype(bool)\n", + "sf['vanishing'] = sf['vanishing'].astype(bool)\n", + "sf['alpha_pos'] = 1-sf['alpha_pos']\n", + "sf['alpha_ges'] = 1-sf['alpha_ges']\n", + "\n", + "# scale to percentage\n", + "sf['TE'] = sf['TE'] * 100\n", + "\n", + "# add surprise as dummy code\n", + "tf['control'] = [('control' in set) for set in tf['set']]\n", + "sf['control'] = [('control' in set) for set in sf['set']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Tracking Error (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tracking Error when visible: M: 5.25 , STD: 7.52, Count: 1246\n", + "Tracking Error when occluded: M: 5.16 , STD: 4.93, Count: 342\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "\n", + "def get_stats(col):\n", + " return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n", + "\n", + "# When Visible\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n", + "\n", + "# When Occluded\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Tracking Error when occluded:' + get_stats(temp['TE']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Succesfull Trackings (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
setevalmodetracked_postracked_negtracked_pos_protracked_neg_pro
0controlopen18150.5454550.454545
\n", + "
" + ], + "text/plain": [ + " set evalmode tracked_pos tracked_neg tracked_pos_pro \\\n", + "0 control open 18 15 0.545455 \n", + "\n", + " tracked_neg_pro \n", + "0 0.454545 " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# succesfull trackings: In the last visible moment of the target, the slot was less than 10% away from the target\n", + "# determine last visible frame numeric\n", + "grouping_factors = ['net','set','evalmode','scene','slot']\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).max()\n", + "ff.rename(columns = {'frame':'last_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['last_visible']], on=grouping_factors, how='left')\n", + "\n", + "# same for first bound frame\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).min()\n", + "ff.rename(columns = {'frame':'first_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['first_visible']], on=grouping_factors, how='left')\n", + "\n", + "# add dummy variable to sf\n", + "sf['last_visible'] = (sf['last_visible'] == sf['frame'])\n", + "\n", + "# extract the trials where the target was last visible and threshold the TE\n", + "ff = sf[sf['last_visible']] \n", + "ff['tracked_pos'] = (ff['TE'] < 10)\n", + "ff['tracked_neg'] = (ff['TE'] >= 10)\n", + "\n", + "# fill NaN with 0\n", + "sf = sf.merge(ff[grouping_factors + ['tracked_pos', 'tracked_neg']], on=grouping_factors, how='left')\n", + "sf['tracked_pos'].fillna(False, inplace=True)\n", + "sf['tracked_neg'].fillna(False, inplace=True)\n", + "\n", + "# Aggreagte over all scenes\n", + "temp = sf[(sf['frame']== 1) & ~sf.occluder & sf.control & (sf.first_visible < 20)]\n", + "temp = temp.groupby(['set', 'evalmode']).sum()\n", + "temp = temp[['tracked_pos', 'tracked_neg']]\n", + "temp = temp.reset_index()\n", + "\n", + "temp['tracked_pos_pro'] = temp['tracked_pos'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "temp['tracked_neg_pro'] = temp['tracked_neg'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "\n", + "temp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mostly Tracked stats" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEmCAYAAAB20LuSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmP0lEQVR4nO3deUCUdeLH8fcAgmIqiDPgbeVNoB0mkWm6rtfPKzXzpFYjj1WSWlfNM4+FSvPIsqTMLK3MvNg1JM9KTdNNzfK28oZR1ERBYGZ+f1izy3oAOjjMw+f1F888zzzzAcaPX57nme9jcjgcDkRExDC83B1ARERcS8UuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYH3cHADh37hJ2uy6nd4WgoLs4ezbd3TFErqH3put4eZkIDCx9w/VFotjtdoeK3YX0s5SiSu/NO0OHYkREDEbFLiJiMEXiUIyI3DqbLYdz56zk5GS5O8pNpaZ6Ybfb3R3Do/j4+BIYaMbbu2BVrWIX8XDnzlkpWdKf0qVDMJlM7o5zQz4+XuTkqNjzy+FwcOnSb5w7Z6VChYoFem6+DsWkp6fTvn17jh8/fs26vXv30qVLF1q3bs3o0aPJyckpUAARuT05OVmULl22SJe6FJzJZKJ06bK39JdYnsW+a9cuevbsyS+//HLd9cOHD2fcuHGsXr0ah8PB4sWLCxxCRG6PSt2YbvX3mmexL168mPHjx2OxWK5Zd+LECTIzM2nYsCEAXbp0ISkp6ZaCiIiIa+R5jH3KlCk3XJeamorZbHYum81mUlJSChwiKOiuAj/nTsuyZePrXcLdMfLFbC7j7gh58qSfZ1GXkmrCxyf3GK2Uvx8l/Vx/Ci3zSg4Zl6/c8vP/N2d+/PjjHtavX8uQIc/zz3+u5N//3sG4cS8XaB/p6ReZNGkCr7wyLc9tHQ7HTUfKgwZF8+yzA3jwwYcKlAEgIeFtAKKjB+b7OV5eXgX+N31bv3m73Z7rB5DXD+RGzp5NL/IfXDCby9D900HujmEYi5+ag9V60d0xDMGEicNpR3M99lBgPTq8uMLlr5U4rRN7jh90+X5vZuue7/j19FEOpx0l9dJZLmalX/P95sWaksKPe/fk63n3lq9205O8DocDm81+SyeC/+i5gjzXbrdf82/Fy8t00wHxbRV7SEgIVqvVuXzmzJnrHrIRkeLjp90/sOKTJfiU8CH1dAoPRjyMX8mS7NiyFQcO/v7yOA4fOMhnHy7EYXdgCQmm/9DBlAsMYOG777Pn+12YTCYefKQxbTq15/OPPiYzI5Pln3xGYFB5APbs3MWSDz9mwrR4ADZ+uZbD+w/Sb8j1R8IL3n6Xc2nnmD4pjj7P9eOVsRMpU7Ysvr6+PD9mBAkzZpN25iznzqYRGRHJ3/8+BoA5c97gq6824OPjTceOXejevadzn+fOpRETM5DnnhvMY489zocfzmf9+i+x2ew0bhzBoEExmEwmFi1awMqVyyhXLoAyZcpQr15oIf8GbvMDSpUrV8bPz48dO3YAsGLFCpo2beqSYCLiuQ7tP0C/IQOZPHMayYmrKFuuHJNnTaNajRqsWZXEvNlzeGHsKOLfmknt+vWYP2cu1pRUdm3/N3FvzmD81HhOHD1GCV9fuvbpyQMRjejc40nn/kMbhHP+3DlSTp0C4Ju1G2jassUN80QNfJbA8oHEjh0FwKnjJxg8fBij/vEyO7dtp/o9d/Py66/w+rtv8f33O9i/fx/r16/lhx92sWDBJ8yd+wGrViVy9uwZAC5dSmf48GH06/ccjz32ON9+u5n9+/eSkLCA999fiNVqJTn5C/bt+4l//Wsl8+YtZMaMt7BaUwvxp/4ftzRij46OJiYmhrCwMKZOncqYMWNIT08nNDSUqKgoV2cUEQ9TtUY1gn4//1ambBlCG4YDUMFi5vut33Fv7VqYg4MBaNG2FSsXf075CkH4+voy4cWR3P/wQ/TsF4Wvr+91928ymXjsT835Zt1Gmv35T1w4f56adWvnO1/ZgHLO1498vCmH9x/gi+UrOXnsOBcuXCAj4zI7d+6gRYs/4+vri6+vL/PnL3I+/7XX4ihfPohmza7+Z7J9+zZ++mkP/fv3BeDKlUyCg0M4e/YsERGP4u/vD0Dz5i2x2WwF+VHeknwX+7p165xfJyQkOL+uW7cuS5YscW0qEfFoPj65q8Xb+z8HBxyO3OfTrh6ztuHt7c3E6a+y94cf2bl9BxNeGMmYVybf8DWatmzBq+Mm4uvry2N/al6gfP/9H8bqlf9k2zdbaNG2Ffc1bID1eAoOhwMfHx/++5ThqVMnCQgIBKB37yi2bNnEsmVL6Nq1O3a7je7de9KjRx8ALl68iLe3NytWLAX+8/16e3vfkWLXXDEickfdW6c2h/YdwPr7FXTrvkimfoMwfjl8hEkjxlA3LJTez/6FytWqcOrECby9vbHbrj3ZaA62UD4oiDX/+oImLR6/6Wt6eXtju84+APZ8v4sWbVvxaPNmZGdlc/Dgfux2Ow0aPMCGDevIyckhMzOTF18c6jyUUqtWHV58cSTvv5+A1ZrKAw80YvXqVVy+fJmcnBxGjXqRDRvW8tBDjdi06WvS09O5cuUKX321/vZ+ePmkKQVE5I4qF1CO/jGDmD4pnpycHCpYzEQPG0Jg+fLUqleHEYNi8PPzo3b9ujR46EFST51m6cJP+OT9BVSsUjnXvh5p1oRtm7Y4T6re+DUDqGCpwOSRYxgQOzTXujadOvD+m2+zcvHn+JcuTVhYA06dOkmHDp3Zt+8n+vXrjd3u4Mkne1KtWnXn86pWrUaXLk/y+uuvEhc3lUOHDvDcc89gt9to3DiStm3bYzKZePLJnjz7bBRlypQhOLhgUwPcKpPjf/8ucgNd7lj86HJH10lNPUZGydyXGderfA+lS/q5/LUuZV5h74kjLt/vrbDZbMyZOoPGTSJp9OgjLttvXpc73mmnT/9KSEj1XI8V6uWOIlI0FZXyLSwOh4Mhffpx3/0NePCRxgDs2/MjH7ydcN3t//7yuDxH9UaiYhcRj2MymZjz8Qe5Hqt7Xyhxs2e4J1ARo5OnIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMLoqRsSAwitXx7ekv8v3m5V5md0nfnX5fv9w+dIl3nl9FrFjR3HubBoJM2fz94njbrj95x99DEDXPj3p3a4zC1ctL/BrLvnoY+5rGE7d+25v1sVVqxL5/vsdjB49ocDPPXXqJEOHDmDJksTbyvAHFbuIAfmW9OfIlK4u3+89oz93+T7/26X0dH45fPUa/MCg8jctdVfZ98Me6offV+ivcyep2EXEpX7a/QPLP16Mt48P1tMp3FO7FtHDhrBs0afs2bmbSxfTCQwKZOjI4ZQLDGBgjyjurlWT8+fSCAgsn2ve9MkjxjBzfgLHfvmVD95O4EpGJhfOX6Bj9660/L8217y23W4ntt8ARk6eQMUqlcnMzGT4c39l2rtzrjtT5Ndr13Pk4GHenfkmw8aMZMHbCVjKWzhy5DATJ8axe/dOkpJWkZmZQYkSJZgwYQrVqtXgu++2Mnv2DBwOOyEhFRk/PvdkZTNnTiMt7Szjxk3iwIF9zJr1OleuZFKuXADDh79EpUqVOXBgH/HxkwCoWTP/M1Pmh46xi4jLHdi7n74DnuW1uW+SnZ1N0vJETh47zoRp8Ux79y2CzGa+Wb8BgIu//UaHJ58gbvYM+g0ZkGve9D9sWP0lnXs8yaSZUxkTP4mP582/7ut6eXnxWMsWbFq/EYDvNm3h/ocb3XD638f+1Jx7at3Ls8//lWp31wCgZs1afPzxUipVqsxXX21k9ux3+PDDxURGPsbnny8mKyuLiRPHMmbMBBYs+JR77qnJF1/807nP9957B6s1lbFjJ2K324mPn8z48VOYN28hPXr04ZVXrt5udPLk8QwaNJR58xZSqVLl68W7ZRqxi4jL1b2vPpV+n7CrSYvHWfdFMs8MjmbD6i85efwEB/ftx1IxxLl9zTo3H7H2fvYv7NrxPSs+XcKxX34lMyPzhts2+3ML4l4aT7e+vfh6zTq6P923QNlDQ8MAKF36LiZMmMyaNckcO3aUrVs3U6tWHY4cOYTZbKZWrToADBw4BLh6jP3bbzdz/vw5EhIW4OPjw5Ejhzh58jgjR77g3P+lS5c4f/48Z86coVGjCADatm3PP//pulsZqthFxOW8vb2dXzvsdkwmiB/9Mm2f6EjjJpF4eXnBf80/6Ot38wnLZsW9Rum77uL+xo14pNljbNn49Q23NQcHU8Fi5rtNW7hw/kKBbsAB4Pd7lpSU0wwdOoCuXbsTERFJ+fJBHDy4H29vH+A/k66lp6dz+fIlAEJCKjJgwGBef/0V3n57HjabnUqVKjtv0mGz2Th3Lg2TKfe89Ff36To6FCMiLrf/x72knTmL3W7n63UbqBNan3rhobT8vzaEVK7E99u2Y7dfO4PijeZN/+H7XXTr24uHHmnMru1Xb8Vpv8kNK5q1askHb7+b5zztf7zm9eZ737fvJ6pUqcpTT/WmXr36fPXVeux2G9WqVef8+XP8/PPVk7wLF37A8uVXTyrXqHE37dt3plSpUixdupjq1Wvw22+/sWvX9wD8618rmTBhNOXKBRASEsLmzd8A8OWXSXnmLAiN2EUMKCvzcqFcwZKVeTlf2wUGleftaTNIO5tG2P0NeLR5M2ZMjmfEoBgA7qlVE2vKtff/vNG86V179+Dl4aMoUaIE1e+5G3OwhdTrPP8PjSIjeHfWm/kq9gYPPsC82XMY+OKw3PtoFMGyZUvo0+dJHA4HDRs+wJEjh/Hz82Ps2IlMnjyenJxsKlWqwtixE9mwYa3zuS++OJLBg/vTtGlzJk2KZ+bMqWRlZeHvX5oxY14GYOzYScTFvUxCwluEhobnmbMgNB97Pmk+dtfSfOyuc7352N3pp90/sHThJ4z5/SThneZwONi1fQdrV63mxfGjC/x8zccuIlLEfDT3Pf699btc18BPHjmGS+np12z7p7ZtrnvZpKfTiD2fNGJ3LY3YXaeojdg9nRFG7Dp5KiJiMCp2ERGDUbGLiBiMil1ExGB0VYyIAdWrUoPSfqVcvt9LVzLYe/wXl+/3Zg7vP8C2TVvo2e9pNn65lr0/7GHgC88XaB+3eglmeno6U6ZMIC5uaoGe524qdhEDKu1XqlCu4lr81ByX7zMvx48e48L583f8dQEuXvyNgwf3u+W1b4eKXURc6qfdP7DikyX4lPAh9XQKD0Y8jF/JkuzYshUHDv7+8jgOHzjIZx8uxGF3YAkJpv/QwZQLDGDhu++z5/tdmEwmHnykMW06tefzjz4mMyOT5Z98RmBQeQD27NzFkg8/ZsK0eAA2frmWw/sP0m/IwDzznTp+gvfeeIv0i+n4lSxJ1MBnubd2LTat38g/P19GqRIlqVixEmPHTmLGjNc4c8bKqFF/86hRu46xi4jLHdp/gH5DBjJ55jSSE1dRtlw5Js+aRrUaNVizKol5s+fwwthRxL81k9r16zF/zlysKans2v5v4t6cwfip8Zw4eowSvr507dOTByIa0bnHk879hzYI5/y5c6ScOgXAN2s30LRli3xle2vqDFp3bE/8WzPp81w/Zk55lezsbD77cBEjJ0/ggw8WUbFiZY4e/YVhw4ZToYLZo0odVOwiUgiq1qhGkNmMX0k/ypQtQ2jDq3OhVLCY+X7rd9xbuxbm4GAAWrRtxY87d1O+QhC+vr5MeHEkSSsS6dkv6obzqJtMJh77U3O+WbeRM6lWLpw/n69ZHDMzMkg5eYpGjz4CQK26dbirzF2cOn6CBx5uxMt/G8Ubb8ygWbMWzml5PZGKXURczscn91Feb+//VM3/ftjd4XBgs9nw9vZm4vRXebJvL9IvXmTCCyM5dfzEDV+jacsWfPvVN2zZ+DWP/al5vnLZr/NBewdXXz9q4LMMGz2CsmXLMmnSWFavXpWvfRZFKnYRuaPurVObQ/sOYE1JAWDdF8nUbxDGL4ePMGnEGOqGhdL72b9QuVoVTp04gfcNptU1B1soHxTEmn99ka9ZHAH8/f2xhATz3aYtABzct58L585TtXo1Xnh2EGXKluXpp/vRps3/ceDAfry9vbHdZHrgoipfJ08TExOZM2cOOTk5PP300/Tu3TvX+h9//JFx48aRnZ1NxYoVee211yhbtmyhBBYRz1YuoBz9YwYxfVI8OTk5VLCYiR42hMDy5alVrw4jBsXg5+dH7fp1afDQg6SeOs3ShZ/wyfsLqFgl9y3kHmnWhG2btjhPqubH4OGxzJs9hyUffUyJEiUYNnokPiVK0K1PT+JGj6eM/10EBAQyevQEypQpS3BwCEOHDuCNN95x9Y+i0OQ5CVhKSgo9e/Zk6dKl+Pr60qNHD15//XVq1qzp3KZXr14MGDCAZs2aER8fj5+fH7GxsfkOoUnAih9NAuY615sEzEjXsd+IzWZjztQZNG4S6Txm7gpGmAQszxH75s2biYiIICAgAIDWrVuTlJTEkCFDnNvY7XYuXbp6a6iMjAzKlSt3K/lFxEWKSvkWFofDwZA+/bjv/gY8+EhjAPbt+ZEP3k647vZ/f3lcgUb1ni7PYk9NTcVsNjuXLRYLu3fvzrXNyJEj6devH//4xz8oVaoUixcvdn1SEZHfmUwm5nz8Qa7H6t4XStzsGe4JVMTkWex2ux2T6T9/5jkcjlzLmZmZjB49mvnz5xMeHs7777/PiBEjmDt3br5D3OxPCjEus7mMuyMYQuqN7xAnt8jHp+hcV+Ll5VXgfyt5FntISAjbt293LlutViwWi3P5wIED+Pn5ER5+9TrVp556ipkzZxYohKccYxfX0jF2F3I4wKSbbbhKUTnG7nA4sNvt1/xbue0bbURGRrJlyxbS0tLIyMggOTmZpk2bOtdXr16d06dPc+TI1Tt2r127lrCwsFv9PkSkgEqVKok9K+tquYthOBwOLl36DR+f639I62byHLEHBwcTGxtLVFQU2dnZdOvWjfDwcKKjo4mJiSEsLIy4uDiGDRuGw+EgKCiIf/zjH7f0jYhIwVWtWpXt367C7BeICY3ab9fJzKuj5KLAx8eXwEBz3hv+D93zNJ90uaNr6XJH19F707U84b2pe56KiBQzKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDCZfxZ6YmEi7du1o1aoVCxcuvGb9kSNH6Nu3Lx07dqR///5cuHDB5UFFRCR/8iz2lJQUpk+fzqJFi1i+fDmffvophw4dcq53OBwMGjSI6OhoVq5cSb169Zg7d26hhhYRkRvLs9g3b95MREQEAQEB+Pv707p1a5KSkpzrf/zxR/z9/WnatCkAAwcOpHfv3oWXWEREbirPYk9NTcVsNjuXLRYLKSkpzuWjR49SoUIFXnrpJZ544gnGjx+Pv79/4aQVEZE8+eS1gd1ux2QyOZcdDkeu5ZycHLZt28ZHH31EWFgYM2bMID4+nvj4+HyHCAq6q4CxxQjM5jLujiByXZ7+3syz2ENCQti+fbtz2Wq1YrFYnMtms5nq1asTFhYGQPv27YmJiSlQiLNn07HbHQV6zp3m6b/ooshqvejuCIag96brFfX3ppeX6aYD4jwPxURGRrJlyxbS0tLIyMggOTnZeTwd4P777yctLY19+/YBsG7dOkJDQ10QXUREbkWeI/bg4GBiY2OJiooiOzubbt26ER4eTnR0NDExMYSFhfHmm28yZswYMjIyCAkJ4dVXX70T2UVE5DryLHaADh060KFDh1yPJSQkOL9u0KABS5YscW0yERG5JfkqdhEpurJyslj81Bx3xzCMrJwsd0e4bSp2EQ/n6+PLkSld3R3DMO4Z/Tlwxd0xbovmihERMRgVu4iIwehQTD7pOKZrGeE4pkhRpWLPJx3HdC0jHMcUKap0KEZExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajKQXyyZ6d9fvH4MUV7NmaK0aksKjY88mrhC8dXlzh7hiGkTitE5orRqRw6FCMiIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGk69iT0xMpF27drRq1YqFCxfecLsNGzbQokULl4UTEZGCy/NGGykpKUyfPp2lS5fi6+tLjx49aNy4MTVr1sy13ZkzZ3jllVcKLaiIiORPniP2zZs3ExERQUBAAP7+/rRu3ZqkpKRrthszZgxDhgwplJAiIpJ/eY7YU1NTMZvNzmWLxcLu3btzbbNgwQLq169PgwYNbilEUNBdt/Q88Wxmcxl3RxC5Lk9/b+ZZ7Ha7HZPJ5Fx2OBy5lg8cOEBycjLz58/n9OnTtxTi7Nl07HbHLT33TvH0X3RRZLVedHcEQ9B70/WK+nvTy8t00wFxnsUeEhLC9u3bnctWqxWLxeJcTkpKwmq10rVrV7Kzs0lNTaVXr14sWrToNqOLSH7Ys7O4Z/Tn7o5hGPbsLHdHuG15FntkZCRvvPEGaWlplCpViuTkZCZNmuRcHxMTQ0xMDADHjx8nKipKpS5yB3mV8KXDiyvcHcMwEqd1Aq64O8ZtyfPkaXBwMLGxsURFRdG5c2fat29PeHg40dHR/PDDD3cio4iIFECeI3aADh060KFDh1yPJSQkXLNdlSpVWLdunWuSiYjILdEnT0VEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjB5KvYExMTadeuHa1atWLhwoXXrF+zZg2dOnWiY8eODB48mAsXLrg8qIiI5E+exZ6SksL06dNZtGgRy5cv59NPP+XQoUPO9enp6UyYMIG5c+eycuVK6tSpwxtvvFGooUVE5MbyLPbNmzcTERFBQEAA/v7+tG7dmqSkJOf67Oxsxo8fT3BwMAB16tTh1KlThZdYRERuKs9iT01NxWw2O5ctFgspKSnO5cDAQP785z8DkJmZydy5c2nZsmUhRBURkfzwyWsDu92OyWRyLjscjlzLf7h48SJ//etfqVu3Lk888USBQgQF3VWg7cUYzOYy7o4gcl2e/t7Ms9hDQkLYvn27c9lqtWKxWHJtk5qaSv/+/YmIiOCll14qcIizZ9Ox2x0Fft6d5Om/6KLIar3o7giGoPem6xX196aXl+mmA+I8D8VERkayZcsW0tLSyMjIIDk5maZNmzrX22w2Bg4cSNu2bRk9evR1R/MiInLn5DliDw4OJjY2lqioKLKzs+nWrRvh4eFER0cTExPD6dOn+emnn7DZbKxevRqA++67jylTphR6eBERuVaexQ7QoUMHOnTokOuxhIQEAMLCwti3b5/rk4mIyC3RJ09FRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMJl/FnpiYSLt27WjVqhULFy68Zv3evXvp0qULrVu3ZvTo0eTk5Lg8qIiI5E+exZ6SksL06dNZtGgRy5cv59NPP+XQoUO5thk+fDjjxo1j9erVOBwOFi9eXGiBRUTk5nzy2mDz5s1EREQQEBAAQOvWrUlKSmLIkCEAnDhxgszMTBo2bAhAly5dmDVrFr169cp3CC8vU8GTu4ElsJS7IxiKp/zePYHem65V1N+beeXLs9hTU1Mxm83OZYvFwu7du2+43mw2k5KSUqCQgYGlC7S9u7w3ppW7IxhKUNBd7o5gGHpvupanvzfzPBRjt9sxmf7zv4PD4ci1nNd6ERG5s/Is9pCQEKxWq3PZarVisVhuuP7MmTO51ouIyJ2VZ7FHRkayZcsW0tLSyMjIIDk5maZNmzrXV65cGT8/P3bs2AHAihUrcq0XEZE7y+RwOBx5bZSYmMg777xDdnY23bp1Izo6mujoaGJiYggLC2Pfvn2MGTOG9PR0QkNDiYuLw9fX907kFxGR/5GvYhcREc+hT56KiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAwmz7lipGiaPXv2Tdf/MUmbiDslJiZy6NAhBg4cyOrVq+ncubO7IxULGrGLSKGYOnUqGzduJDk5GZvNxueff058fLy7YxUL+oCSAaSlpbFr1y5sNhsNGzakQoUK7o4kQufOnVm2bBlPPPEEy5cvJycnh44dO7Jq1Sp3RzM8jdg93Ndff02nTp1YunQpy5Yto2PHjqxfv97dsUTw8rpaL3/M9pqVleV8TAqXjrF7uD/ublW1alUAjh07xpAhQ2jevLmbk0lx16ZNG4YNG8aFCxeYP38+K1asoH379u6OVSyo2D1cTk6Os9QBqlatit1ud2Mikauee+45vv76aypVqsSpU6d4/vnnefzxx90dq1jQ30UerlKlSsyfP5/09HTS09OZP38+lStXdncsEbKysjCbzYwYMYLQ0FC2bt1KWlqau2MVCzp56uHOnj3LpEmT+Pbbb3E4HERERDB69Gjd7ETc7vnnn6dKlSq0bt2a4cOH07FjR3bv3s0777zj7miGp0MxHm7BggXMmDHD3TFErnH8+HFmzpzJa6+9RteuXXnuuefo2rWru2MVCzoU4+HWr1+P/uiSoshms5GWlsaaNWt4/PHHsVqtXLlyxd2xigWN2D1cQEAAbdq0ITQ0FD8/P+fjcXFxbkwlAv3796d79+60aNGC2rVr07p1a55//nl3xyoWdIzdwy1btuyax0wmkz66LUWOzWbD29vb3TGKBY3YPVxqaioDBgzI9djrr7/upjQiMGDAAN555x1atGjh/HDSf1u7dq0bUhUvGrF7qKlTp3L27FnWrVtHixYtnI/bbDZ27drF6tWr3ZhOirPU1FQsFgu7d+8mKCjomvW6HLfwqdg91O7duzl8+DCzZs0iJibG+bi3tzfh4eHUqFHDfeFEgLZt2/LFF1+4O0axpGL3cBcvXqRMmTLujiFyjdjYWJo1a0Z4eDglS5Z0Pl6pUiU3pioeVOwebtmyZcTHx/Pbb78B4HA4MJlM7N27183JpLj770OEfzCZTDrGfgeo2D1cy5Yteeutt6hdu7a7o4hIEaEPKHk4i8WiUpci6eTJkwwePJgHHniAhx9+mL/97W+aK+YO0Yjdw02ZMoWUlBQeffTRXB9Q0nXs4m69evWiXbt2dO7cGbvdztKlS9m0aRMJCQnujmZ4uo7dw6Wnp1O6dGl27tyZ63EVu7hbeno6ffr0cS4/88wzLF261I2Jig8Vu4eLi4sjOzubn3/+GZvNRq1atfDx0a9V3O/+++9nxYoVdOrUCYANGzZQv359N6cqHnQoxsPt2bOHmJgYAgICsNvtnDlzhjfffJMGDRq4O5oUc5GRkaSlpVGyZElMJhMZGRnOdbpyq3Cp2D1cjx49GDVqlLPId+7cyeTJk1myZImbk0lxt2/fPurWrevuGMWSrorxcJcvX841Om/YsKGmRpUiITY21t0Rii0djPVw5cqVY82aNbRs2RKANWvWEBAQ4N5QIkDNmjWZPXs2DRo0yPXJ00aNGrkxVfGgQzEe7pdffmHAgAGcP3/e+dgnn3zC3Xff7b5QIkDfvn2vecxkMrFgwQI3pCleNGL3cF999RWlSpVi2bJlHD16lNjYWLZt26ZiF7f78MMP3R2h2NKI3cO1b9+ezz77jFKlSgGQkZFB9+7dSUxMdHMyKe769u173fnYNWIvfBqxe7js7GxKlCjhXP7vr0XcaejQoc6vc3JyWLt2LWXLlnVjouJDI3YP99prr7Fz507atm2LyWRi9erVPPDAAwwbNszd0USu8eSTT/LZZ5+5O4bhacTu4YYPH05SUhLfffcdPj4+REVFOa+QEXGnkydPOr92OBwcPHgw10l+KTwasYtIofjjnqcOhwMvLy8CAwMZOnQoTZs2dXc0w9MHlESkUEyfPp3evXuTlJREtWrVOHr0qLsjFRsqdhEpFFOmTKF27dokJydTsmRJli9fzsyZM90dq1hQsYtIobDb7TRp0oT169fTqlUrKlasiM1mc3esYkHFLiKFolSpUsybN4+tW7fSvHlzFixYQOnSpd0dq1hQsYtIoZg6dSqXL19m1qxZlCtXjpSUFKZNm+buWMWCrooRETEYjdhFRAxGxS4iYjAqdpECOHbsWK45UESKIhW7SAGcPHmSn3/+2d0xRG5KJ0+lWLt06RKjRo3i119/xcvLi9DQUCZOnMiGDRuYM2cO2dnZlCxZkhEjRhAeHk6bNm1ISUmhUaNGvPfee+6OL3JdGrFLsfbll19y6dIlVqxY4bwB+NGjR5k+fTpz585l+fLlTJo0iaFDh3LlyhUmT55MtWrVVOpSpGl2RynWHnzwQaZPn07fvn2JjIzk6aefZtOmTaSmpvLMM884tzOZTJrrRDyGil2KtapVq/Lll1+ydetWvv32W/7yl78wYMAAHnnkEWbMmOHc7tSpU1gsFrZv3+6+sCL5pEMxUqwtWrSIUaNG0aRJE4YPH06TJk24cOECmzZt4vDhwwBs3LiRjh07kpmZibe3N9nZ2W5OLXJzGrFLsda5c2e2bdtGu3btKFWqFBUrVqRv377ce++9vPDCCzgcDnx8fJgzZw6lS5emZs2a+Pn50a1bNz777LPr3tNTxN10VYyIiMHoUIyIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExmP8HXuChHBTEVwUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "temp = af[af.index == 'OVERALL']\n", + "temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n", + "temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n", + "temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n", + "g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MOTA " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idf1idpidrrecallprecisionnum_unique_objectsmostly_trackedpartially_trackedmostly_lostnum_false_positivesnum_missesnum_switchesnum_fragmentationsmotamotpnum_transfernum_ascendnum_migrate
set
control0.7539590.9341990.6320200.6399840.94597086.043.05.038.0179.01763.07.022.00.6020010.0598560.07.00.0
surprise0.7519820.9308410.6307790.6440790.95046733.013.06.014.053.0562.03.011.00.6086130.0581770.03.00.0
\n", + "
" + ], + "text/plain": [ + " idf1 idp idr recall precision \\\n", + "set \n", + "control 0.753959 0.934199 0.632020 0.639984 0.945970 \n", + "surprise 0.751982 0.930841 0.630779 0.644079 0.950467 \n", + "\n", + " num_unique_objects mostly_tracked partially_tracked mostly_lost \\\n", + "set \n", + "control 86.0 43.0 5.0 38.0 \n", + "surprise 33.0 13.0 6.0 14.0 \n", + "\n", + " num_false_positives num_misses num_switches num_fragmentations \\\n", + "set \n", + "control 179.0 1763.0 7.0 22.0 \n", + "surprise 53.0 562.0 3.0 11.0 \n", + "\n", + " mota motp num_transfer num_ascend num_migrate \n", + "set \n", + "control 0.602001 0.059856 0.0 7.0 0.0 \n", + "surprise 0.608613 0.058177 0.0 3.0 0.0 " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "af[af.index == 'OVERALL'].groupby(['set']).mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gate Openings" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percept gate openings when visible: M: 0.061 , STD: 0.108, Count: 1246\n", + "Percept gate openings when occluded: M: 0.00026 , STD: 0.00481, Count: 342\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Percept gate openings when visible:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Percept gate openings when occluded:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl3.ipynb b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl3.ipynb new file mode 100644 index 0000000..f137a5f --- /dev/null +++ b/evaluation/adept_ablation/lambda/evaluation_loci_looped_abl3.ipynb @@ -0,0 +1,469 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import warnings\n", + "import scipy.stats as stats\n", + "import os\n", + "\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)\n", + "pd.options.mode.chained_assignment = None \n", + "plt.style.use('ggplot')\n", + "sns.color_palette(\"Paired\");\n", + "sns.set_theme();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# setting path to results folder\n", + "root_path = '../../../out/pretrained/adept_ablations/lambda'\n", + "\n", + "# list all folders in root path that don't stat with a dot\n", + "nets = ['adept_level1_ablation_lambda.run511']\n", + "\n", + "# read pickle file\n", + "tf = pd.DataFrame()\n", + "sf = pd.DataFrame()\n", + "af = pd.DataFrame()\n", + "\n", + "# load statistics files from nets\n", + "for net in nets:\n", + " path = os.path.join(root_path, net, 'results')\n", + " with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n", + " tf_temp = pd.read_csv(f, index_col=0)\n", + " tf_temp['net'] = net\n", + " tf = pd.concat([tf,tf_temp])\n", + "\n", + " with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n", + " sf_temp = pd.read_csv(f, index_col=0)\n", + " sf_temp['net'] = net\n", + " sf = pd.concat([sf,sf_temp])\n", + "\n", + " with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n", + " af_temp = pd.read_csv(f, index_col=0)\n", + " af_temp['net'] = net\n", + " af = pd.concat([af,af_temp])\n", + "\n", + "# cast variables\n", + "sf['visible'] = sf['visible'].astype(bool)\n", + "sf['bound'] = sf['bound'].astype(bool)\n", + "sf['occluder'] = sf['occluder'].astype(bool)\n", + "sf['inimage'] = sf['inimage'].astype(bool)\n", + "sf['vanishing'] = sf['vanishing'].astype(bool)\n", + "sf['alpha_pos'] = 1-sf['alpha_pos']\n", + "sf['alpha_ges'] = 1-sf['alpha_ges']\n", + "\n", + "# scale to percentage\n", + "sf['TE'] = sf['TE'] * 100\n", + "\n", + "# add surprise as dummy code\n", + "tf['control'] = [('control' in set) for set in tf['set']]\n", + "sf['control'] = [('control' in set) for set in sf['set']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Tracking Error (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tracking Error when visible: M: 3.34 , STD: 4.22, Count: 1873\n", + "Tracking Error when occluded: M: 2.78 , STD: 1.66, Count: 492\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "\n", + "def get_stats(col):\n", + " return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n", + "\n", + "# When Visible\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n", + "\n", + "# When Occluded\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Tracking Error when occluded:' + get_stats(temp['TE']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Succesfull Trackings (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
setevalmodetracked_postracked_negtracked_pos_protracked_neg_pro
0controlopen37110.7708330.229167
\n", + "
" + ], + "text/plain": [ + " set evalmode tracked_pos tracked_neg tracked_pos_pro \\\n", + "0 control open 37 11 0.770833 \n", + "\n", + " tracked_neg_pro \n", + "0 0.229167 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# succesfull trackings: In the last visible moment of the target, the slot was less than 10% away from the target\n", + "# determine last visible frame numeric\n", + "grouping_factors = ['net','set','evalmode','scene','slot']\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).max()\n", + "ff.rename(columns = {'frame':'last_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['last_visible']], on=grouping_factors, how='left')\n", + "\n", + "# same for first bound frame\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).min()\n", + "ff.rename(columns = {'frame':'first_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['first_visible']], on=grouping_factors, how='left')\n", + "\n", + "# add dummy variable to sf\n", + "sf['last_visible'] = (sf['last_visible'] == sf['frame'])\n", + "\n", + "# extract the trials where the target was last visible and threshold the TE\n", + "ff = sf[sf['last_visible']] \n", + "ff['tracked_pos'] = (ff['TE'] < 10)\n", + "ff['tracked_neg'] = (ff['TE'] >= 10)\n", + "\n", + "# fill NaN with 0\n", + "sf = sf.merge(ff[grouping_factors + ['tracked_pos', 'tracked_neg']], on=grouping_factors, how='left')\n", + "sf['tracked_pos'].fillna(False, inplace=True)\n", + "sf['tracked_neg'].fillna(False, inplace=True)\n", + "\n", + "# Aggreagte over all scenes\n", + "temp = sf[(sf['frame']== 1) & ~sf.occluder & sf.control & (sf.first_visible < 20)]\n", + "temp = temp.groupby(['set', 'evalmode']).sum()\n", + "temp = temp[['tracked_pos', 'tracked_neg']]\n", + "temp = temp.reset_index()\n", + "\n", + "temp['tracked_pos_pro'] = temp['tracked_pos'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "temp['tracked_neg_pro'] = temp['tracked_neg'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "\n", + "temp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mostly Tracked stats" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEmCAYAAAB20LuSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmL0lEQVR4nO3deUBU5eLG8e8AgksqiDPgbuZOoC0mkWma1+2nZmrmXlfD7SpJXa8abrkEleaSZUmZWVqZO10Dcq3UNL0pWe5W7jCKmigozMzvD29TXBdABwcOz+cvzpwzZx5heHw5c857TA6Hw4GIiBiGh7sDiIiIa6nYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIG4+XuAABnz17Ebtfp9K7g738XZ86kuTuGyDX03nQdDw8Tfn6lbri+QBS73e5QsbuQvpdSUOm9eWfoUIyIiMGo2EVEDKZAHIoRkVtns2Vx9qyVrKwr7o5yUykpHtjtdnfHKFS8vLzx8zPj6Zm3qlaxixRyZ89aKV68JKVKBWIymdwd54a8vDzIylKx55bD4eDixd85e9ZK+fIV8vTcXB2KSUtLo3379hw7duyadXv27KFz5860bt2aqKgosrKy8hRARG5PVtYVSpUqU6BLXfLOZDJRqlSZW/pLLMdi37VrFz169ODXX3+97voRI0Ywbtw4EhIScDgcLF68OM8hROT2qNSN6VZ/rjkW++LFixk/fjwWi+WadcePHycjI4OGDRsC0LlzZ+Lj428piIiIuEaOx9inTJlyw3UpKSmYzWbnstlsJjk5Oc8h/P3vyvNz7rQrtky8PYu5O0aumM2l3R0hR4Xp+1nQJaeY8PLKPkYrUdKH4j6u/wgt43IW6Zcu3/Lz/zdnbvz0027Wr1/L0KHP88UXq/jPf3YwbtzLedpHWtoFJk2awKuvTstxW4fDcdOR8uDB4Tz33EAeeODBPGUAiI19B4Dw8EG5fo6Hh0eef6dv6ydvt9uzfQNy+obcyJkzaQX+wgWzuTTdPhvs7hiGsfjpOVitF9wdwxBMmDiUeiTbYw/61aPDiytd/lpx055g97EDLt/vzWzd/T2/nTrCodQjpFw8w4Uradf8e3NiTU7mpz27c/W8e8pVvemHvA6HA5vNfksfBP/Rc3l5rt1uv+Z3xcPDdNMB8W0Ve2BgIFar1bl8+vTp6x6yEZGi4+ekH1n56RK8inmRciqZB0Ifwqd4cXZs2YoDB/96eRyH9h/g848W4rA7sAQG0H/YEMr6+bLwvQ/Y/cMuTCYTDzzcmDZPtGfpx5+QkZ7Bik8/x8+/HAC7d+5iyUefMGFaDAAbv1rLoX0H6Df0+iPhBe+8x9nUs0yfFE3vAf14dexESpcpg7e3N8+PGUnsjNmknj7D2TOphIWG8a9/jQFgzpw3+frrDXh5edKxY2e6devh3OfZs6lERAxiwIAhPProY3z00XzWr/8Km81O48ahDB4cgclkYtGiBaxatZyyZX0pXbo09eoF5fNP4DYvUKpUqRI+Pj7s2LEDgJUrV9K0aVOXBBORwuvgvv30GzqIyTOnkRi3mjJlyzJ51jSqVq/OmtXxzJs9hxfGjibm7ZnUrl+P+XPmYk1OYdf2/xD91gzGT43h+JGjFPP2pkvvHtwf2ohO3Z9y7j+oQQjnzp4l+eRJAL5du4GmLVvcME/fQc/hV86PyLGjATh57DhDRgxn9Csvs3PbdqrVuJuX33iVN957mx9+2MG+fXtZv34tP/64iwULPmXu3A9ZvTqOM2dOA3DxYhojRgynX78BPProY3z33Wb27dtDbOwCPvhgIVarlcTEL9m792f+/e9VzJu3kBkz3sZqTcnH7/qfbmnEHh4eTkREBMHBwUydOpUxY8aQlpZGUFAQffv2dXVGESlkqlSviv9/P38rXaY0QQ1DAChvMfPD1u+5p3YtzAEBALRo24pVi5dSrrw/3t7eTHhxFPc99CA9+vXF29v7uvs3mUw8+nhzvl23kWZ/e5zz585Rs27tXOcr41vW+fphjzXl0L79fLliFSeOHuP8+fOkp19i584dtGjxN7y9vfH29mb+/EXO57/+ejTlyvnTrNnV/0y2b9/Gzz/vpn//PgBcvpxBQEAgZ86cITT0EUqWLAlA8+YtsdlseflW3pJcF/u6deucX8fGxjq/rlu3LkuWLHFtKhEp1Ly8sleLp+efBwccjuyfp109Zm3D09OTidNfY8+PP7Fz+w4mvDCKMa9OvuFrNG3ZgtfGTcTb25tHH2+ep3x//Q8jYdUXbPt2Cy3atuLehg2wHkvG4XDg5eXFXz8yPHnyBL6+fgD06tWXLVs2sXz5Erp06YbdbqNbtx50794bgAsXLuDp6cnKlcuAP/+9np6ed6TYNVeMiNxR99SpzcG9+7H+9wy6dV8mUr9BML8eOsykkWOoGxxEr+f+TqWqlTl5/Dienp7Ybdd+2GgOsFDO3581//6SJi0eu+lrenh6YrvOPgB2/7CLFm1b8UjzZmReyeTAgX3Y7XYaNLifDRvWkZWVRUZGBi++OMx5KKVWrTq8+OIoPvggFqs1hfvvb0RCwmouXbpEVlYWo0e/yIYNa3nwwUZs2vQNaWlpXL58ma+/Xn9737xc0pQCInJHlfUtS/+IwUyfFENWVhblLWbChw/Fr1w5atWrw8jBEfj4+FC7fl0aPPgAKSdPsWzhp3z6wQIqVK6UbV8PN2vCtk1bnB+q3vg1fSlvKc/kUWMYGDks27o2T3Tgg7feYdXipZQsVYrg4AacPHmCDh06sXfvz/Tr1wu73cFTT/WgatVqzudVqVKVzp2f4o03XiM6eioHD+5nwIBnsdttNG4cRtu27TGZTDz1VA+ee64vpUuXJiAgb1MD3CqT43//LnIDne5Y9Oh0R9dJSTlKevHspxnXq1SDUsV9XP5aFzMus+f4YZfv91bYbDbmTJ1B4yZhNHrkYZftN6fTHe+0U6d+IzCwWrbH8vV0RxEpmApK+eYXh8PB0N79uPe+BjzwcGMA9u7+iQ/fib3u9v96eVyOo3ojUbGLSKFjMpmY88mH2R6re28Q0bNnuCdQAaMPT0VEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGB0VoyIAYVUqoZ38ZIu3++VjEskHf/N5fv9w6WLF3n3jVlEjh3N2TOpxM6czb8mjrvh9ks//gSALr170KtdJxauXpHn11zy8Sfc2zCEuvfe3qyLq1fH8cMPO4iKmpDn5548eYJhwwayZEncbWX4g4pdxIC8i5fk8JQuLt9vjailLt/nX11MS+PXQ1fPwffzL3fTUneVvT/upn7Ivfn+OneSil1EXOrnpB9Z8cliPL28sJ5KpkbtWoQPH8ryRZ+xe2cSFy+k4efvx7BRIyjr58ug7n25u1ZNzp1NxdevXLZ50yePHMPM+bEc/fU3PnwnlsvpGZw/d56O3brQ8v/aXPPadrudyH4DGTV5AhUqVyIjI4MRA/7BtPfmXHemyG/WrufwgUO8N/Mtho8ZxYJ3YrGUs3D48CEmTowmKWkn8fGrychIp1ixYkyYMIWqVavz/fdbmT17Bg6HncDACowfn32yspkzp5GaeoZx4yaxf/9eZs16g8uXMyhb1pcRI16iYsVK7N+/l5iYSQDUrJn7mSlzQ8fYRcTl9u/ZR5+Bz/H63LfIzMwkfkUcJ44eY8K0GKa99zb+ZjPfrt8AwIXff6fDU08SPXsG/YYOzDZv+h82JHxFp+5PMWnmVMbETOKTefOv+7oeHh482rIFm9ZvBOD7TVu476FGN5z+99HHm1Oj1j089/w/qHp3dQBq1qzFJ58so2LFSnz99UZmz36Xjz5aTFjYoyxdupgrV64wceJYxoyZwIIFn1GjRk2+/PIL5z7ff/9drNYUxo6diN1uJyZmMuPHT2HevIV0796bV1+9ervRyZPHM3jwMObNW0jFipWuF++WacQuIi5X9976VPzvhF1NWjzGui8TeXZIOBsSvuLEseMc2LsPS4VA5/Y169x8xNrrub+za8cPrPxsCUd//Y2M9Iwbbtvsby2Ifmk8Xfv05Js16+j2TJ88ZQ8KCgagVKm7mDBhMmvWJHL06BG2bt1MrVp1OHz4IGazmVq16gAwaNBQ4Oox9u++28y5c2eJjV2Al5cXhw8f5MSJY4wa9YJz/xcvXuTcuXOcPn2aRo1CAWjbtj1ffOG6Wxmq2EXE5Tw9PZ1fO+x2TCaIiXqZtk92pHGTMDw8POAv8w96+9x8wrJZ0a9T6q67uK9xIx5u9ihbNn5zw23NAQGUt5j5ftMWzp87n6cbcAD4/DdLcvIphg0bSJcu3QgNDaNcOX8OHNiHp6cX8Oeka2lpaVy6dBGAwMAKDBw4hDfeeJV33pmHzWanYsVKzpt02Gw2zp5NxWTKPi/91X26jg7FiIjL7ftpD6mnz2C32/lm3QbqBNWnXkgQLf+vDYGVKvLDtu3Y7dfOoHijedN//GEXXfv05MGHG7Nr+9VbcdpvcsOKZq1a8uE77+U4T/sfr3m9+d737v2ZypWr8PTTvahXrz5ff70eu91G1arVOHfuLL/8cvVD3oULP2TFiqsfKlevfjft23eiRIkSLFu2mGrVqvP777+za9cPAPz736uYMCGKsmV9CQwMZPPmbwH46qv4HHPmhUbsIgZ0JeNSvpzBciXjUq628/MvxzvTZpB6JpXg+xrwSPNmzJgcw8jBEQDUqFUTa/K19/+80bzpXXp15+URoylWrBjVatyNOcBCynWe/4dGYaG8N+utXBV7gwfuZ97sOQx6cXj2fTQKZfnyJfTu/RQOh4OGDe/n8OFD+Pj4MHbsRCZPHk9WViYVK1Zm7NiJbNiw1vncF18cxZAh/WnatDmTJsUwc+ZUrly5QsmSpRgz5mUAxo6dRHT0y8TGvk1QUEiOOfNC87HnkuZjdy3Nx+4615uP3Z1+TvqRZQs/Zcx/PyS80xwOB7u272Dt6gReHB+V5+drPnYRkQLm47nv85+t32c7B37yqDFcTEu7ZtvH27a57mmThZ1G7LmkEbtracTuOgVtxF7YGWHErg9PRUQMRsUuImIwKnYREYNRsYuIGIzOihExoHqVq1PKp4TL93vxcjp7jv3q8v3ezKF9+9m2aQs9+j3Dxq/WsufH3Qx64fk87eNWT8FMS0tjypQJREdPzdPz3E3FLmJApXxK5MtZXIufnuPyfebk2JGjnD937o6/LsCFC79z4MA+t7z27VCxi4hL/Zz0Iys/XYJXMS9STiXzQOhD+BQvzo4tW3Hg4F8vj+PQ/gN8/tFCHHYHlsAA+g8bQlk/Xxa+9wG7f9iFyWTigYcb0+aJ9iz9+BMy0jNY8enn+PmXA2D3zl0s+egTJkyLAWDjV2s5tO8A/YYOyjHfyWPHef/Nt0m7kIZP8eL0HfQc99Suxab1G/li6XJKFCtOhQoVGTt2EjNmvM7p01ZGj/5noRq16xi7iLjcwX376Td0EJNnTiMxbjVlypZl8qxpVK1enTWr45k3ew4vjB1NzNszqV2/HvPnzMWanMKu7f8h+q0ZjJ8aw/EjRynm7U2X3j24P7QRnbo/5dx/UIMQzp09S/LJkwB8u3YDTVu2yFW2t6fOoHXH9sS8PZPeA/oxc8prZGZm8vlHixg1eQIffriIChUqceTIrwwfPoLy5c2FqtRBxS4i+aBK9ar4m834FPehdJnSBDW8OhdKeYuZH7Z+zz21a2EOCACgRdtW/LQziXLl/fH29mbCi6OIXxlHj359bziPuslk4tHHm/Ptuo2cTrFy/ty5XM3imJGeTvKJkzR65GEAatWtw12l7+LksePc/1AjXv7naN58cwbNmrVwTstbGKnYRcTlvLyyH+X19Pyzav73YneHw4HNZsPT05OJ01/jqT49SbtwgQkvjOLkseM3fI2mLVvw3dffsmXjNzz6ePNc5bJf50J7B1dfv++g5xgeNZIyZcowadJYEhJW52qfBZGKXUTuqHvq1Obg3v1Yk5MBWPdlIvUbBPProcNMGjmGusFB9Hru71SqWpmTx4/jeYNpdc0BFsr5+7Pm31/mahZHgJIlS2IJDOD7TVsAOLB3H+fPnqNKtaq88NxgSpcpwzPP9KNNm/9j//59eHp6YrvJ9MAFVa4+PI2Li2POnDlkZWXxzDPP0KtXr2zrf/rpJ8aNG0dmZiYVKlTg9ddfp0yZMvkSWEQKt7K+ZekfMZjpk2LIysqivMVM+PCh+JUrR616dRg5OAIfHx9q169LgwcfIOXkKZYt/JRPP1hAhcrZbyH3cLMmbNu0xfmham4MGRHJvNlzWPLxJxQrVozhUaPwKlaMrr17EB01ntIl78LX14+oqAmULl2GgIBAhg0byJtvvuvqb0W+yXESsOTkZHr06MGyZcvw9vame/fuvPHGG9SsWdO5Tc+ePRk4cCDNmjUjJiYGHx8fIiMjcx1Ck4AVPZoEzHWuNwmYkc5jvxGbzcacqTNo3CTMeczcFYwwCViOI/bNmzcTGhqKr68vAK1btyY+Pp6hQ4c6t7Hb7Vy8ePXWUOnp6ZQtW/ZW8ouIixSU8s0vDoeDob37ce99DXjg4cYA7N39Ex++E3vd7f/18rg8jeoLuxyLPSUlBbPZ7Fy2WCwkJSVl22bUqFH069ePV155hRIlSrB48WLXJxUR+S+TycScTz7M9ljde4OInj3DPYEKmByL3W63YzL9+Weew+HItpyRkUFUVBTz588nJCSEDz74gJEjRzJ37txch7jZnxRiXGZzaXdHMISUG98hTm6Rl1fBOa/Ew8Mjz78rORZ7YGAg27dvdy5brVYsFotzef/+/fj4+BAScvU81aeffpqZM2fmKURhOcYurqVj7C7kcIBJN9twlYJyjN3hcGC326/5XbntG22EhYWxZcsWUlNTSU9PJzExkaZNmzrXV6tWjVOnTnH48NU7dq9du5bg4OBb/XeISB6VKFEc+5UrV8tdDMPhcHDx4u94eV3/Iq2byXHEHhAQQGRkJH379iUzM5OuXbsSEhJCeHg4ERERBAcHEx0dzfDhw3E4HPj7+/PKK6/c0j9ERPKuSpUqbP9uNWYfP0xo1H67TmRcHSUXBF5e3vj5mXPe8H/onqe5pNMdXUunO7qO3puuVRjem7rnqYhIEaNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBuPl7gCFxZWsKyx+eo67YxjGlawr7o4gYlgq9lzy9vLm8JQu7o5hGDWilgKX3R1DxJB0KEZExGBU7CIiBqNiFxExGBW7iIjBqNhFRAwmV8UeFxdHu3btaNWqFQsXLrxm/eHDh+nTpw8dO3akf//+nD9/3uVBRUQkd3Is9uTkZKZPn86iRYtYsWIFn332GQcPHnSudzgcDB48mPDwcFatWkW9evWYO3duvoYWEZEby7HYN2/eTGhoKL6+vpQsWZLWrVsTHx/vXP/TTz9RsmRJmjZtCsCgQYPo1atX/iUWEZGbyvECpZSUFMxms3PZYrGQlJTkXD5y5Ajly5fnpZdeYs+ePdSoUYOxY8fmT1oRuYauinYtI1wVnWOx2+12TCaTc9nhcGRbzsrKYtu2bXz88ccEBwczY8YMYmJiiImJyXUIf/+78hhbjMBsLu3uCIahq6Jdp0bUUsxmb3fHuC05FntgYCDbt293LlutViwWi3PZbDZTrVo1goODAWjfvj0RERF5CnHmTBp2uyNPz7nTVEKuZ7VecHcEQ9B70/UK+nvTw8N00wFxjsfYw8LC2LJlC6mpqaSnp5OYmOg8ng5w3333kZqayt69ewFYt24dQUFBLoguIiK3IscRe0BAAJGRkfTt25fMzEy6du1KSEgI4eHhREREEBwczFtvvcWYMWNIT08nMDCQ11577U5kFxGR68jV7I4dOnSgQ4cO2R6LjY11ft2gQQOWLFni2mQiInJLdOWpiIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMLm656mIFFz2zCvUiFrq7hiGYc+84u4It03Fnkv65XEtI/zyFBQexbzp8OJKd8cwjLhpTwCX3R3jtqjYc0m/PK5lhF8ekYJKx9hFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYPJVbHHxcXRrl07WrVqxcKFC2+43YYNG2jRooXLwomISN7lOAlYcnIy06dPZ9myZXh7e9O9e3caN25MzZo1s213+vRpXn311XwLKiIiuZPjiH3z5s2Ehobi6+tLyZIlad26NfHx8ddsN2bMGIYOHZovIUVEJPdyHLGnpKRgNpudyxaLhaSkpGzbLFiwgPr169OgQYNbCuHvf9ctPU8KN7O5tLsjiFxXYX9v5ljsdrsdk8nkXHY4HNmW9+/fT2JiIvPnz+fUqVO3FOLMmTTsdsctPfdOKew/6ILIar3g7giGoPem6xX096aHh+mmA+IcD8UEBgZitVqdy1arFYvF4lyOj4/HarXSpUsXBgwYQEpKCj179rzN2CIicqtyLPawsDC2bNlCamoq6enpJCYm0rRpU+f6iIgIEhISWLlyJXPnzsVisbBo0aJ8DS0iIjeWY7EHBAQQGRlJ37596dSpE+3btyckJITw8HB+/PHHO5FRRETyIFf3PO3QoQMdOnTI9lhsbOw121WuXJl169a5JpmIiNwSXXkqImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDCZXxR4XF0e7du1o1aoVCxcuvGb9mjVreOKJJ+jYsSNDhgzh/PnzLg8qIiK5k2OxJycnM336dBYtWsSKFSv47LPPOHjwoHN9WloaEyZMYO7cuaxatYo6derw5ptv5mtoERG5sRyLffPmzYSGhuLr60vJkiVp3bo18fHxzvWZmZmMHz+egIAAAOrUqcPJkyfzL7GIiNxUjsWekpKC2Wx2LlssFpKTk53Lfn5+/O1vfwMgIyODuXPn0rJly3yIKiIiueGV0wZ2ux2TyeRcdjgc2Zb/cOHCBf7xj39Qt25dnnzyyTyF8Pe/K0/bizGYzaXdHUHkugr7ezPHYg8MDGT79u3OZavVisViybZNSkoK/fv3JzQ0lJdeeinPIc6cScNud+T5eXdSYf9BF0RW6wV3RzAEvTddr6C/Nz08TDcdEOd4KCYsLIwtW7aQmppKeno6iYmJNG3a1LneZrMxaNAg2rZtS1RU1HVH8yIicufkOGIPCAggMjKSvn37kpmZSdeuXQkJCSE8PJyIiAhOnTrFzz//jM1mIyEhAYB7772XKVOm5Ht4ERG5Vo7FDtChQwc6dOiQ7bHY2FgAgoOD2bt3r+uTiYjILdGVpyIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgaTq2KPi4ujXbt2tGrVioULF16zfs+ePXTu3JnWrVsTFRVFVlaWy4OKiEju5FjsycnJTJ8+nUWLFrFixQo+++wzDh48mG2bESNGMG7cOBISEnA4HCxevDjfAouIyM155bTB5s2bCQ0NxdfXF4DWrVsTHx/P0KFDATh+/DgZGRk0bNgQgM6dOzNr1ix69uyZ6xAeHqa8J3cDi18Jd0cwlMLycy8M9N50rYL+3swpX47FnpKSgtlsdi5bLBaSkpJuuN5sNpOcnJynkH5+pfK0vbu8P6aVuyMYir//Xe6OYBh6b7pWYX9v5ngoxm63YzL9+b+Dw+HItpzTehERubNyLPbAwECsVqtz2Wq1YrFYbrj+9OnT2daLiMidlWOxh4WFsWXLFlJTU0lPTycxMZGmTZs611eqVAkfHx927NgBwMqVK7OtFxGRO8vkcDgcOW0UFxfHu+++S2ZmJl27diU8PJzw8HAiIiIIDg5m7969jBkzhrS0NIKCgoiOjsbb2/tO5BcRkf+Rq2IXEZHCQ1eeiogYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMJse5YqRgmj179k3X/zFJm4g7xcXFcfDgQQYNGkRCQgKdOnVyd6QiQSN2EckXU6dOZePGjSQmJmKz2Vi6dCkxMTHujlUk6AIlA0hNTWXXrl3YbDYaNmxI+fLl3R1JhE6dOrF8+XKefPJJVqxYQVZWFh07dmT16tXujmZ4GrEXct988w1PPPEEy5YtY/ny5XTs2JH169e7O5YIHh5X6+WP2V6vXLnifEzyl46xF3J/3N2qSpUqABw9epShQ4fSvHlzNyeToq5NmzYMHz6c8+fPM3/+fFauXEn79u3dHatIULEXcllZWc5SB6hSpQp2u92NiUSuGjBgAN988w0VK1bk5MmTPP/88zz22GPujlUk6O+iQq5ixYrMnz+ftLQ00tLSmD9/PpUqVXJ3LBGuXLmC2Wxm5MiRBAUFsXXrVlJTU90dq0jQh6eF3JkzZ5g0aRLfffcdDoeD0NBQoqKidLMTcbvnn3+eypUr07p1a0aMGEHHjh1JSkri3XffdXc0w9OhmEJuwYIFzJgxw90xRK5x7NgxZs6cyeuvv06XLl0YMGAAXbp0cXesIkGHYgq59evXoz+6pCCy2WykpqayZs0aHnvsMaxWK5cvX3Z3rCJBI/ZCztfXlzZt2hAUFISPj4/z8ejoaDemEoH+/fvTrVs3WrRoQe3atWndujXPP/+8u2MVCTrGXsgtX778msdMJpMu3ZYCx2az4enp6e4YRYJG7IVcSkoKAwcOzPbYG2+84aY0IjBw4EDeffddWrRo4bw46a/Wrl3rhlRFi0bshdTUqVM5c+YM69ato0WLFs7HbTYbu3btIiEhwY3ppChLSUnBYrGQlJSEv7//Net1Om7+U7EXUklJSRw6dIhZs2YRERHhfNzT05OQkBCqV6/uvnAiQNu2bfnyyy/dHaNIUrEXchcuXKB06dLujiFyjcjISJo1a0ZISAjFixd3Pl6xYkU3pioaVOyF3PLly4mJieH3338HwOFwYDKZ2LNnj5uTSVH310OEfzCZTDrGfgeo2Au5li1b8vbbb1O7dm13RxGRAkIXKBVyFotFpS4F0okTJxgyZAj3338/Dz30EP/85z81V8wdohF7ITdlyhSSk5N55JFHsl2gpPPYxd169uxJu3bt6NSpE3a7nWXLlrFp0yZiY2PdHc3wdB57IZeWlkapUqXYuXNntsdV7OJuaWlp9O7d27n87LPPsmzZMjcmKjpU7IVcdHQ0mZmZ/PLLL9hsNmrVqoWXl36s4n733XcfK1eu5IknngBgw4YN1K9f382pigYdiinkdu/eTUREBL6+vtjtdk6fPs1bb71FgwYN3B1NiriwsDBSU1MpXrw4JpOJ9PR05zqduZW/VOyFXPfu3Rk9erSzyHfu3MnkyZNZsmSJm5NJUbd3717q1q3r7hhFks6KKeQuXbqUbXTesGFDTY0qBUJkZKS7IxRZOhhbyJUtW5Y1a9bQsmVLANasWYOvr697Q4kANWvWZPbs2TRo0CDblaeNGjVyY6qiQYdiCrlff/2VgQMHcu7cOedjn376KXfffbf7QokAffr0ueYxk8nEggUL3JCmaNGIvZD7+uuvKVGiBMuXL+fIkSNERkaybds2Fbu43UcffeTuCEWWRuyFXPv27fn8888pUaIEAOnp6XTr1o24uDg3J5Oirk+fPtedj10j9vynEXshl5mZSbFixZzLf/1axJ2GDRvm/DorK4u1a9dSpkwZNyYqOjRiL+Ref/11du7cSdu2bTGZTCQkJHD//fczfPhwd0cTucZTTz3F559/7u4YhqcReyE3YsQI4uPj+f777/Hy8qJv377OM2RE3OnEiRPOrx0OBwcOHMj2Ib/kH43YRSRf/HHPU4fDgYeHB35+fgwbNoymTZu6O5rh6QIlEckX06dPp1evXsTHx1O1alWOHDni7khFhopdRPLFlClTqF27NomJiRQvXpwVK1Ywc+ZMd8cqElTsIpIv7HY7TZo0Yf369bRq1YoKFSpgs9ncHatIULGLSL4oUaIE8+bNY+vWrTRv3pwFCxZQqlQpd8cqElTsIpIvpk6dyqVLl5g1axZly5YlOTmZadOmuTtWkaCzYkREDEYjdhERg1Gxi4gYjIpdJA+OHj2abQ4UkYJIxS6SBydOnOCXX35xdwyRm9KHp1KkXbx4kdGjR/Pbb7/h4eFBUFAQEydOZMOGDcyZM4fMzEyKFy/OyJEjCQkJoU2bNiQnJ9OoUSPef/99d8cXuS6N2KVI++qrr7h48SIrV6503gD8yJEjTJ8+nblz57JixQomTZrEsGHDuHz5MpMnT6Zq1aoqdSnQNLujFGkPPPAA06dPp0+fPoSFhfHMM8+wadMmUlJSePbZZ53bmUwmzXUihYaKXYq0KlWq8NVXX7F161a+++47/v73vzNw4EAefvhhZsyY4dzu5MmTWCwWtm/f7r6wIrmkQzFSpC1atIjRo0fTpEkTRowYQZMmTTh//jybNm3i0KFDAGzcuJGOHTuSkZGBp6cnmZmZbk4tcnMasUuR1qlTJ7Zt20a7du0oUaIEFSpUoE+fPtxzzz288MILOBwOvLy8mDNnDqVKlaJmzZr4+PjQtWtXPv/88+ve01PE3XRWjIiIwehQjIiIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETGY/wd9OaIYPoF6DAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "temp = af[af.index == 'OVERALL']\n", + "temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n", + "temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n", + "temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n", + "g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MOTA " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idf1idpidrrecallprecisionnum_unique_objectsmostly_trackedpartially_trackedmostly_lostnum_false_positivesnum_missesnum_switchesnum_fragmentationsmotamotpnum_transfernum_ascendnum_migrate
set
control0.7738280.9438570.6557080.6734740.96943086.041.015.030.0104.01599.09.031.00.6503980.0393700.09.00.0
surprise0.7766200.9421860.6605450.6789110.96838333.016.04.013.035.0507.04.012.00.6542120.0394682.02.00.0
\n", + "
" + ], + "text/plain": [ + " idf1 idp idr recall precision \\\n", + "set \n", + "control 0.773828 0.943857 0.655708 0.673474 0.969430 \n", + "surprise 0.776620 0.942186 0.660545 0.678911 0.968383 \n", + "\n", + " num_unique_objects mostly_tracked partially_tracked mostly_lost \\\n", + "set \n", + "control 86.0 41.0 15.0 30.0 \n", + "surprise 33.0 16.0 4.0 13.0 \n", + "\n", + " num_false_positives num_misses num_switches num_fragmentations \\\n", + "set \n", + "control 104.0 1599.0 9.0 31.0 \n", + "surprise 35.0 507.0 4.0 12.0 \n", + "\n", + " mota motp num_transfer num_ascend num_migrate \n", + "set \n", + "control 0.650398 0.039370 0.0 9.0 0.0 \n", + "surprise 0.654212 0.039468 2.0 2.0 0.0 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "af[af.index == 'OVERALL'].groupby(['set']).mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gate Openings" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percept gate openings when visible: M: 0.132 , STD: 0.145, Count: 1873\n", + "Percept gate openings when occluded: M: 0.00809 , STD: 0.0444, Count: 492\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Percept gate openings when visible:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Percept gate openings when occluded:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/adept_ablation/recon/evaluation_loci_looped_norecon.ipynb b/evaluation/adept_ablation/recon/evaluation_loci_looped_norecon.ipynb new file mode 100644 index 0000000..a1bbc32 --- /dev/null +++ b/evaluation/adept_ablation/recon/evaluation_loci_looped_norecon.ipynb @@ -0,0 +1,464 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import warnings\n", + "import scipy.stats as stats\n", + "import os\n", + "\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)\n", + "pd.options.mode.chained_assignment = None \n", + "plt.style.use('ggplot')\n", + "sns.color_palette(\"Paired\");\n", + "sns.set_theme();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# setting path to results folder\n", + "root_path = '../../../out/pretrained/adept_ablations/recon'\n", + "\n", + "# list all folders in root path that don't stat with a dot\n", + "nets = ['norecon']\n", + "\n", + "# read pickle file\n", + "tf = pd.DataFrame()\n", + "sf = pd.DataFrame()\n", + "af = pd.DataFrame()\n", + "\n", + "# load statistics files from nets\n", + "for net in nets:\n", + " path = os.path.join(root_path, net, 'results')\n", + " with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n", + " tf_temp = pd.read_csv(f, index_col=0)\n", + " tf_temp['net'] = net\n", + " tf = pd.concat([tf,tf_temp])\n", + "\n", + " with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n", + " sf_temp = pd.read_csv(f, index_col=0)\n", + " sf_temp['net'] = net\n", + " sf = pd.concat([sf,sf_temp])\n", + "\n", + " with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n", + " af_temp = pd.read_csv(f, index_col=0)\n", + " af_temp['net'] = net\n", + " af = pd.concat([af,af_temp])\n", + "\n", + "# cast variables\n", + "sf['visible'] = sf['visible'].astype(bool)\n", + "sf['bound'] = sf['bound'].astype(bool)\n", + "sf['occluder'] = sf['occluder'].astype(bool)\n", + "sf['inimage'] = sf['inimage'].astype(bool)\n", + "sf['vanishing'] = sf['vanishing'].astype(bool)\n", + "sf['alpha_pos'] = 1-sf['alpha_pos']\n", + "sf['alpha_ges'] = 1-sf['alpha_ges']\n", + "\n", + "# scale to percentage\n", + "sf['TE'] = sf['TE'] * 100\n", + "\n", + "# add surprise as dummy code\n", + "tf['control'] = [('control' in set) for set in tf['set']]\n", + "sf['control'] = [('control' in set) for set in sf['set']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Tracking Error (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tracking Error when visible: M: 1.75 , STD: 1.23, Count: 1563\n", + "Tracking Error when occluded: M: 2.09 , STD: 1.39, Count: 486\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "\n", + "def get_stats(col):\n", + " return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n", + "\n", + "# When Visible\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n", + "\n", + "# When Occluded\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Tracking Error when occluded:' + get_stats(temp['TE']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Succesfull Trackings (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
setevalmodetracked_postracked_negtracked_pos_protracked_neg_pro
0controlopen4710.9791670.020833
\n", + "
" + ], + "text/plain": [ + " set evalmode tracked_pos tracked_neg tracked_pos_pro \\\n", + "0 control open 47 1 0.979167 \n", + "\n", + " tracked_neg_pro \n", + "0 0.020833 " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# succesfull trackings: In the last visible moment of the target, the slot was less than 10% away from the target\n", + "# determine last visible frame numeric\n", + "grouping_factors = ['net','set','evalmode','scene','slot']\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).max()\n", + "ff.rename(columns = {'frame':'last_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['last_visible']], on=grouping_factors, how='left')\n", + "\n", + "# same for first bound frame\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).min()\n", + "ff.rename(columns = {'frame':'first_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['first_visible']], on=grouping_factors, how='left')\n", + "\n", + "# add dummy variable to sf\n", + "sf['last_visible'] = (sf['last_visible'] == sf['frame'])\n", + "\n", + "# extract the trials where the target was last visible and threshold the TE\n", + "ff = sf[sf['last_visible']] \n", + "ff['tracked_pos'] = (ff['TE'] < 10)\n", + "ff['tracked_neg'] = (ff['TE'] >= 10)\n", + "\n", + "# fill NaN with 0\n", + "sf = sf.merge(ff[grouping_factors + ['tracked_pos', 'tracked_neg']], on=grouping_factors, how='left')\n", + "sf['tracked_pos'].fillna(False, inplace=True)\n", + "sf['tracked_neg'].fillna(False, inplace=True)\n", + "\n", + "# Aggreagte over all scenes\n", + "temp = sf[(sf['frame']== 1) & ~sf.occluder & sf.control & (sf.first_visible < 20)]\n", + "temp = temp.groupby(['set', 'evalmode']).sum()\n", + "temp = temp[['tracked_pos', 'tracked_neg']]\n", + "temp = temp.reset_index()\n", + "\n", + "temp['tracked_pos_pro'] = temp['tracked_pos'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "temp['tracked_neg_pro'] = temp['tracked_neg'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "\n", + "temp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mostly Tracked stats" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEmCAYAAAB20LuSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmQklEQVR4nO3deUCUdeLH8fcAgkcqiDPgbeZNoB0mkWm6rtfPKzXzpFYjj1WSWlcNrzwWKs0jy5Iys7Qyb3YNybNS03RTsrytvGEUNVEQmJnfH27Tsh6ADg48fF5/8czzzDMfYPz45Znn+T4mh8PhQEREDMPD3QFERMS1VOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYPxcncAgPPnL2O363R6V/D3v4dz59LcHUPkOnpvuo6Hhwk/vzI3XV8oit1ud6jYXUg/Syms9N68O3QoRkTEYFTsIiIGUygOxYjI7bPZsjl/3kp2dqa7o9xSSooHdrvd3TGKFC8vb/z8zHh65q+qVewiRdz581ZKlixNmTKBmEwmd8e5KS8vD7KzVex55XA4uHz5N86ft1KxYqV8PTdPh2LS0tLo2LEjJ06cuG7dvn376NatG23btiU6Oprs7Ox8BRCRO5OdnUmZMuUKdalL/plMJsqUKXdbf4nlWux79uyhd+/e/PLLLzdcP3LkSMaPH8/atWtxOBwsWbIk3yFE5M6o1I3pdn+vuRb7kiVLmDBhAhaL5bp1J0+eJCMjg8aNGwPQrVs3EhISbiuIiIi4Rq7H2KdOnXrTdSkpKZjNZuey2WwmOTk53yH8/e/J93PutkxbFt6eJdwdI0/M5rLujpCrovTzLOySU0x4eeUco5Uq7UNJH9d/hJZxNZv0K1dv+/n/mzMvfvxxLxs3rmfYsBf45z9X8+9/72L8+FfytY+0tEtMnjyRV1+dnuu2DofjliPlIUMieO65QTz00MP5ygAQF/cOABERg/P8HA8Pj3z/m76j37zdbs/xA8jtB3Iz586lFfoLF8zmsvT8bIi7YxjGkqfnYrVecncMQzBh4kjqsRyPPezXgE4vrXL5a8VP78LeE4dcvt9b2b73O349c4wjqcdIuXyOS5lp132/ubEmJ/Pjvr15et59Farf8kNeh8OBzWa/rQ+Cf++5/DzXbrdf92/Fw8N0ywHxHRV7YGAgVqvVuXz27NkbHrIRkeLjp6QfWPXpUrxKeJFyJpmHQh/Bp2RJdm3bjgMHf39lPEcOHuLzjxbhsDuwBAYwcPhQyvv5sui9D9j7/R5MJhMPPdqUdl06suzjT8hIz2Dlp5/j518BgL2797D0o0+YOD0WgM1frufIgUMMGHbjkfDCd97jfOp5ZkyOod/zA3h13CTKliuHt7c3L4wdRdzMOaSePcf5c6mEhYbx97+PBWDu3Df56qtNeHl50rlzN3r27O3c5/nzqURGDub554fy+ONP8NFHC9i48UtsNjtNm4YyZEgkJpOJxYsXsnr1CsqX96Vs2bI0aBBUwL+BO7xAqUqVKvj4+LBr1y4AVq1aRfPmzV0STESKrsMHDjJg2GCmzJpOYvwaypUvz5TZ06lesybr1iQwf85cXhw3hti3Z1G3YQMWzJ2HNTmFPTv/TcxbM5kwLZaTx45Twtub7v1682BoE7r2esq5/6BGIVw4f57k06cB+Gb9Jpq3bnXTPOGDn8Ovgh9R48YAcPrESYaOHMGYf7zC7h07qVHrXl5541XeeO9tvv9+FwcO7GfjxvX88MMeFi78lHnzPmTNmnjOnTsLwOXLaYwcOYIBA57n8cef4Ntvt3LgwD7i4hbywQeLsFqtJCZ+wf79P/Gvf61m/vxFzJz5NlZrSgH+1P9wWyP2iIgIIiMjCQ4OZtq0aYwdO5a0tDSCgoIIDw93dUYRKWKq1ayO/38+fytbrixBjUMAqGgx8/3277ivbh3MAQEAtGrfhtVLllGhoj/e3t5MfGk0DzzyML0HhOPt7X3D/ZtMJh7/U0u+2bCZFn/+ExcvXKB2/bp5zlfOt7zz9cOeaM6RAwf5YuVqTh0/wcWLF0lPv8Lu3bto1erPeHt74+3tzYIFi53Pf/31GCpU8KdFi2v/mezcuYOfftrLwIH9Abh6NYOAgEDOnTtHaOhjlC5dGoCWLVtjs9ny86O8LXku9g0bNji/jouLc35dv359li5d6tpUIlKkeXnlrBZPzz8ODjgcOT9Pu3bM2oanpyeTZrzGvh9+ZPfOXUx8cTRjX51y09do3roVr42fhLe3N4//qWW+8v33fxhrV/+THd9so1X7NtzfuBHWE8k4HA68vLz4748MT58+ha+vHwB9+4azbdsWVqxYSvfuPbHbbfTs2ZtevfoBcOnSJTw9PVm1ajnwx/fr6el5V4pdc8WIyF11X726HN5/EOt/zqDb8EUiDRsF88uRo0weNZb6wUH0fe4vVKleldMnT+Lp6Ynddv2HjeYACxX8/Vn3ry9o1uqJW76mh6cnthvsA2Dv93to1b4Nj7VsQVZmFocOHcBut9Oo0YNs2rSB7OxsMjIyeOml4c5DKXXq1OOll0bzwQdxWK0pPPhgE9auXcOVK1fIzs5mzJiX2LRpPQ8/3IQtW74mLS2Nq1ev8tVXG+/sh5dHmlJARO6q8r7lGRg5hBmTY8nOzqaixUzEiGH4VahAnQb1GDUkEh8fH+o2rE+jhx8i5fQZli/6lE8/WEilqlVy7OvRFs3YsWWb80PVm7+mLxUtFZkyeiyDoobnWNeuSyc+eOsdVi9ZRukyZQgObsTp06fo1Kkr+/f/xIABfbHbHTz1VG+qV6/hfF61atXp1u0p3njjNWJipnH48EGef/5Z7HYbTZuG0b59R0wmE0891ZvnngunbNmyBATkb2qA22Vy/O/fRW6g0x2LH53u6DopKcdJL5nzNOMGVWpRpqSPy1/rcsZV9p086vL93g6bzcbcaTNp2iyMJo896rL95na649125syvBAbWyPFYgZ7uKCKFU2Ep34LicDgY1m8A9z/QiIcebQrA/r0/8uE7cTfc/u+vjM91VG8kKnYRKXJMJhNzP/kwx2P17w8iZs5M9wQqZPThqYiIwajYRUQMRsUuImIwKnYREYNRsYuIGIzOihExoJAqNfAuWdrl+83MuELSyV9dvt/fXbl8mXffmE3UuDGcP5dK3Kw5/H3S+Jtuv+zjTwDo3q83fTt0ZdGalfl+zaUff8L9jUOof/+dzbq4Zk0833+/i+joifl+7unTpxg+fBBLl8bfUYbfqdhFDMi7ZGmOTu3u8v3Wil7m8n3+t8tpafxy5No5+H7+FW5Z6q6y/4e9NAy5v8Bf525SsYuIS/2U9AMrP1mCp5cX1jPJ1Kpbh4gRw1ix+DP27k7i8qU0/Pz9GD56JOX9fBncK5x769TmwvlUfP0q5Jg3fcqoscxaEMfxX37lw3fiuJqewcULF+ncszut/6/dda9tt9uJGjCI0VMmUqlqFTIyMhj5/F+Z/t7cG84U+fX6jRw9dIT3Zr3FiLGjWfhOHJYKFo4ePcKkSTEkJe0mIWENGRnplChRgokTp1K9ek2++247c+bMxOGwExhYiQkTck5WNmvWdFJTzzF+/GQOHtzP7NlvcPVqBuXL+zJy5MtUrlyFgwf3Exs7GYDatfM+M2Ve6Bi7iLjcwX0H6D/oOV6f9xZZWVkkrIzn1PETTJwey/T33sbfbOabjZsAuPTbb3R66kli5sxkwLBBOeZN/92mtV/StddTTJ41jbGxk/lk/oIbvq6HhwePt27Flo2bAfhuyzYeeKTJTaf/ffxPLalV5z6ee+GvVL+3JgC1a9fhk0+WU7lyFb76ajNz5rzLRx8tISzscZYtW0JmZiaTJo1j7NiJLFz4GbVq1eaLL/7p3Of777+L1ZrCuHGTsNvtxMZOYcKEqcyfv4hevfrx6qvXbjc6ZcoEhgwZzvz5i6hcucqN4t02jdhFxOXq39+Qyv+ZsKtZqyfY8EUizw6NYNPaLzl14iSH9h/AUinQuX3tercesfZ97i/s2fU9qz5byvFffiUjPeOm27b4cytiXp5Aj/59+HrdBno+0z9f2YOCggEoU+YeJk6cwrp1iRw/fozt27dSp049jh49jNlspk6degAMHjwMuHaM/dtvt3Lhwnni4hbi5eXF0aOHOXXqBKNHv+jc/+XLl7lw4QJnz56lSZNQANq378g//+m6Wxmq2EXE5Tw9PZ1fO+x2TCaIjX6F9k92pmmzMDw8POC/5h/09rn1hGWzY16nzD338EDTJjza4nG2bf76ptuaAwKoaDHz3ZZtXLxwMV834ADw+U+W5OQzDB8+iO7dexIaGkaFCv4cOnQAT08v4I9J19LS0rhy5TIAgYGVGDRoKG+88SrvvDMfm81O5cpVnDfpsNlsnD+fismUc176a/t0HR2KERGXO/DjPlLPnsNut/P1hk3UC2pIg5AgWv9fOwKrVOb7HTux26+fQfFm86b/8P0eevTvw8OPNmXPzmu34rTf4oYVLdq05sN33st1nvbfX/NG873v3/8TVatW4+mn+9KgQUO++mojdruN6tVrcOHCeX7++dqHvIsWfcjKldc+VK5Z8146duxKqVKlWL58CTVq1OS3335jz57vAfjXv1YzcWI05cv7EhgYyNat3wDw5ZcJuebMD43YRQwoM+NKgZzBkplxJU/b+flX4J3pM0k9l0rwA414rGULZk6JZdSQSABq1amNNfn6+3/ebN707n178crIMZQoUYIate7FHGAh5QbP/12TsFDem/1Wnoq90UMPMn/OXAa/NCLnPpqEsmLFUvr1ewqHw0Hjxg9y9OgRfHx8GDduElOmTCA7O4vKlasybtwkNm1a73zuSy+NZujQgTRv3pLJk2OZNWsamZmZlC5dhrFjXwFg3LjJxMS8Qlzc2wQFheSaMz80H3seaT5219J87K5zo/nY3emnpB9YvuhTxv7nQ8K7zeFwsGfnLtavWctLE6Lz/XzNxy4iUsh8PO99/r39uxznwE8ZPZbLaWnXbfun9u1ueNpkUacRex5pxO5aGrG7TmEbsRd1Rhix68NTERGDUbGLiBiMil1ExGBU7CIiBqOzYkQMqEHVmpTxKeXy/V6+ms6+E7+4fL+3cuTAQXZs2UbvAc+w+cv17PthL4NffCFf+7jdUzDT0tKYOnUiMTHT8vU8d1OxixhQGZ9SBXIW15Kn57p8n7k5cew4Fy9cuOuvC3Dp0m8cOnTALa99J1TsIuJSPyX9wKpPl+JVwouUM8k8FPoIPiVLsmvbdhw4+Psr4zly8BCff7QIh92BJTCAgcOHUt7Pl0XvfcDe7/dgMpl46NGmtOvSkWUff0JGegYrP/0cP/8KAOzdvYelH33CxOmxAGz+cj1HDhxiwLDBueY7feIk77/5NmmX0vApWZLwwc9xX906bNm4mX8uW0GpEiWpVKky48ZNZubM1zl71sqYMX8rUqN2HWMXEZc7fOAgA4YNZsqs6STGr6Fc+fJMmT2d6jVrsm5NAvPnzOXFcWOIfXsWdRs2YMHceViTU9iz89/EvDWTCdNiOXnsOCW8venerzcPhjaha6+nnPsPahTChfPnST59GoBv1m+ieetWecr29rSZtO3ckdi3Z9Hv+QHMmvoaWVlZfP7RYkZPmciHHy6mUqUqHDv2CyNGjKRiRXORKnVQsYtIAahWszr+ZjM+JX0oW64sQY2vzYVS0WLm++3fcV/dOpgDAgBo1b4NP+5OokJFf7y9vZn40mgSVsXTe0D4TedRN5lMPP6nlnyzYTNnU6xcvHAhT7M4ZqSnk3zqNE0eexSAOvXrcU/Zezh94iQPPtKEV/42hjffnEmLFq2c0/IWRSp2EXE5L6+cR3k9Pf+omv+92N3hcGCz2fD09GTSjNd4qn8f0i5dYuKLozl94uRNX6N561Z8+9U3bNv8NY//qWWectlvcKG9g2uvHz74OUZEj6JcuXJMnjyOtWvX5GmfhZGKXUTuqvvq1eXw/oNYk5MB2PBFIg0bBfPLkaNMHjWW+sFB9H3uL1SpXpXTJ0/ieZNpdc0BFir4+7PuX1/kaRZHgNKlS2MJDOC7LdsAOLT/ABfPX6Bajeq8+NwQypYrxzPPDKBdu//j4MEDeHp6YrvF9MCFVZ4+PI2Pj2fu3LlkZ2fzzDPP0Ldv3xzrf/zxR8aPH09WVhaVKlXi9ddfp1y5cgUSWESKtvK+5RkYOYQZk2PJzs6mosVMxIhh+FWoQJ0G9Rg1JBIfHx/qNqxPo4cfIuX0GZYv+pRPP1hIpao5byH3aItm7Niyzfmhal4MHRnF/DlzWfrxJ5QoUYIR0aPxKlGCHv16ExM9gbKl78HX14/o6ImULVuOgIBAhg8fxJtvvuvqH0WByXUSsOTkZHr37s3y5cvx9vamV69evPHGG9SuXdu5TZ8+fRg0aBAtWrQgNjYWHx8foqKi8hxCk4AVP5oEzHVuNAmYkc5jvxmbzcbcaTNp2izMeczcFYwwCViuI/atW7cSGhqKr68vAG3btiUhIYFhw4Y5t7Hb7Vy+fO3WUOnp6ZQvX/528ouIixSW8i0oDoeDYf0GcP8DjXjo0aYA7N/7Ix++E3fD7f/+yvh8jeqLulyLPSUlBbPZ7Fy2WCwkJSXl2Gb06NEMGDCAf/zjH5QqVYolS5a4PqmIyH+YTCbmfvJhjsfq3x9EzJyZ7glUyORa7Ha7HZPpjz/zHA5HjuWMjAyio6NZsGABISEhfPDBB4waNYp58+blOcSt/qQQ4zKby7o7giGk3PwOcXKbvLwKz3klHh4e+f63kmuxBwYGsnPnTuey1WrFYrE4lw8ePIiPjw8hIdfOU3366aeZNWtWvkIUlWPs4lo6xu5CDgeYdLMNVyksx9gdDgd2u/26fyt3fKONsLAwtm3bRmpqKunp6SQmJtK8eXPn+ho1anDmzBmOHr12x+7169cTHBx8u9+HiORTqVIlsWdmXit3MQyHw8Hly7/h5XXji7RuJdcRe0BAAFFRUYSHh5OVlUWPHj0ICQkhIiKCyMhIgoODiYmJYcSIETgcDvz9/fnHP/5xW9+IiORftWrV2PntGsw+fpjQqP1Oncq4NkouDLy8vPHzM+e+4f/QPU/zSKc7upZOd3QdvTddqyi8N3XPUxGRYkbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDMbL3QGKiszsTJY8PdfdMQwjMzvT3RFEDEvFnkfeXt4cndrd3TEMo1b0MuCqu2OIGJIOxYiIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMHkqdjj4+Pp0KEDbdq0YdGiRdetP3r0KP3796dz584MHDiQixcvujyoiIjkTa7FnpyczIwZM1i8eDErV67ks88+4/Dhw871DoeDIUOGEBERwerVq2nQoAHz5s0r0NAiInJzuRb71q1bCQ0NxdfXl9KlS9O2bVsSEhKc63/88UdKly5N8+bNARg8eDB9+/YtuMQiInJLuV6glJKSgtlsdi5bLBaSkpKcy8eOHaNixYq8/PLL7Nu3j1q1ajFu3LiCSSsi19FV0a5lhKuicy12u92OyWRyLjscjhzL2dnZ7Nixg48//pjg4GBmzpxJbGwssbGxeQ7h739PPmOLEZjNZd0dwTB0VbTr1Ipehtns7e4YdyTXYg8MDGTnzp3OZavVisVicS6bzWZq1KhBcHAwAB07diQyMjJfIc6dS8Nud+TrOXebSsj1rNZL7o5gCHpvul5hf296eJhuOSDO9Rh7WFgY27ZtIzU1lfT0dBITE53H0wEeeOABUlNT2b9/PwAbNmwgKCjIBdFFROR25DpiDwgIICoqivDwcLKysujRowchISFEREQQGRlJcHAwb731FmPHjiU9PZ3AwEBee+21u5FdRERuIE+zO3bq1IlOnTrleCwuLs75daNGjVi6dKlrk4mIyG3RlaciIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMHma3VHAnpVJrehl7o5hGPason/7MZHCSsWeRx4lvOn00ip3xzCM+OldgKvujiFiSDoUIyJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYPJU7HHx8fToUMH2rRpw6JFi2663aZNm2jVqpXLwomISP7les/T5ORkZsyYwfLly/H29qZXr140bdqU2rVr59ju7NmzvPrqqwUWVERE8ibXEfvWrVsJDQ3F19eX0qVL07ZtWxISEq7bbuzYsQwbNqxAQoqISN7lOmJPSUnBbDY7ly0WC0lJSTm2WbhwIQ0bNqRRo0a3FcLf/57bep4UbWZzWXdHELmhov7ezLXY7XY7JpPJuexwOHIsHzx4kMTERBYsWMCZM2duK8S5c2nY7Y7beu7dUtR/0YWR1XrJ3REMQe9N1yvs700PD9MtB8S5FntgYCA7d+50LlutViwWi3M5ISEBq9VK9+7dycrKIiUlhT59+rB48eI7jC4ieWHPyqRW9DJ3xzAMe1amuyPcsVyLPSwsjDfffJPU1FRKlSpFYmIikydPdq6PjIwkMjISgBMnThAeHq5SF7mLPEp40+mlVe6OYRjx07sAV90d447k+uFpQEAAUVFRhIeH07VrVzp27EhISAgRERH88MMPdyOjiIjkQ64jdoBOnTrRqVOnHI/FxcVdt13VqlXZsGGDa5KJiMht0ZWnIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGDyVOzx8fF06NCBNm3asGjRouvWr1u3ji5dutC5c2eGDh3KxYsXXR5URETyJtdiT05OZsaMGSxevJiVK1fy2WefcfjwYef6tLQ0Jk6cyLx581i9ejX16tXjzTffLNDQIiJyc7kW+9atWwkNDcXX15fSpUvTtm1bEhISnOuzsrKYMGECAQEBANSrV4/Tp08XXGIREbmlXIs9JSUFs9nsXLZYLCQnJzuX/fz8+POf/wxARkYG8+bNo3Xr1gUQVURE8sIrtw3sdjsmk8m57HA4ciz/7tKlS/z1r3+lfv36PPnkk/kK4e9/T762F2Mwm8u6O4LIDRX192auxR4YGMjOnTudy1arFYvFkmOblJQUBg4cSGhoKC+//HK+Q5w7l4bd7sj38+6mov6LLoys1kvujmAIem+6XmF/b3p4mG45IM71UExYWBjbtm0jNTWV9PR0EhMTad68uXO9zWZj8ODBtG/fnujo6BuO5kVE5O7JdcQeEBBAVFQU4eHhZGVl0aNHD0JCQoiIiCAyMpIzZ87w008/YbPZWLt2LQD3338/U6dOLfDwIiJyvVyLHaBTp0506tQpx2NxcXEABAcHs3//ftcnExGR26IrT0VEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYjIpdRMRgVOwiIgajYhcRMRgVu4iIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYFTsIiIGo2IXETEYFbuIiMGo2EVEDEbFLiJiMCp2ERGDUbGLiBiMil1ExGBU7CIiBqNiFxExGBW7iIjBqNhFRAwmT8UeHx9Phw4daNOmDYsWLbpu/b59++jWrRtt27YlOjqa7OxslwcVEZG8ybXYk5OTmTFjBosXL2blypV89tlnHD58OMc2I0eOZPz48axduxaHw8GSJUsKLLCIiNyaV24bbN26ldDQUHx9fQFo27YtCQkJDBs2DICTJ0+SkZFB48aNAejWrRuzZ8+mT58+eQ7h4WHKf3I3sPiVcncEQykqv/eiQO9N1yrs783c8uVa7CkpKZjNZueyxWIhKSnppuvNZjPJycn5CunnVyZf27vL+2PbuDuCofj73+PuCIah96ZrFfX3Zq6HYux2OybTH/87OByOHMu5rRcRkbsr12IPDAzEarU6l61WKxaL5abrz549m2O9iIjcXbkWe1hYGNu2bSM1NZX09HQSExNp3ry5c32VKlXw8fFh165dAKxatSrHehERubtMDofDkdtG8fHxvPvuu2RlZdGjRw8iIiKIiIggMjKS4OBg9u/fz9ixY0lLSyMoKIiYmBi8vb3vRn4REfkfeSp2EREpOnTlqYiIwajYRUQMRsUuImIwKnYREYNRsYuIGIyKXUTEYHKdK0YKpzlz5txy/e+TtIm4U3x8PIcPH2bw4MGsXbuWrl27ujtSsaARu4gUiGnTprF582YSExOx2WwsW7aM2NhYd8cqFnSBkgGkpqayZ88ebDYbjRs3pmLFiu6OJELXrl1ZsWIFTz75JCtXriQ7O5vOnTuzZs0ad0czPI3Yi7ivv/6aLl26sHz5clasWEHnzp3ZuHGju2OJ4OFxrV5+n+01MzPT+ZgULB1jL+J+v7tVtWrVADh+/DjDhg2jZcuWbk4mxV27du0YMWIEFy9eZMGCBaxatYqOHTu6O1axoGIv4rKzs52lDlCtWjXsdrsbE4lc8/zzz/P1119TuXJlTp8+zQsvvMATTzzh7ljFgv4uKuIqV67MggULSEtLIy0tjQULFlClShV3xxIhMzMTs9nMqFGjCAoKYvv27aSmpro7VrGgD0+LuHPnzjF58mS+/fZbHA4HoaGhREdH62Yn4nYvvPACVatWpW3btowcOZLOnTuTlJTEu+++6+5ohqdDMUXcwoULmTlzprtjiFznxIkTzJo1i9dff53u3bvz/PPP0717d3fHKhZ0KKaI27hxI/qjSwojm81Gamoq69at44knnsBqtXL16lV3xyoWNGIv4nx9fWnXrh1BQUH4+Pg4H4+JiXFjKhEYOHAgPXv2pFWrVtStW5e2bdvywgsvuDtWsaBj7EXcihUrrnvMZDLp0m0pdGw2G56enu6OUSxoxF7EpaSkMGjQoByPvfHGG25KIwKDBg3i3XffpVWrVs6Lk/7b+vXr3ZCqeNGIvYiaNm0a586dY8OGDbRq1cr5uM1mY8+ePaxdu9aN6aQ4S0lJwWKxkJSUhL+//3XrdTpuwVOxF1FJSUkcOXKE2bNnExkZ6Xzc09OTkJAQatas6b5wIkD79u354osv3B2jWFKxF3GXLl2ibNmy7o4hcp2oqChatGhBSEgIJUuWdD5euXJlN6YqHlTsRdyKFSuIjY3lt99+A8DhcGAymdi3b5+bk0lx99+HCH9nMpl0jP0uULEXca1bt+btt9+mbt267o4iIoWELlAq4iwWi0pdCqVTp04xdOhQHnzwQR555BH+9re/aa6Yu0Qj9iJu6tSpJCcn89hjj+W4QEnnsYu79enThw4dOtC1a1fsdjvLly9ny5YtxMXFuTua4ek89iIuLS2NMmXKsHv37hyPq9jF3dLS0ujXr59z+dlnn2X58uVuTFR8qNiLuJiYGLKysvj555+x2WzUqVMHLy/9WsX9HnjgAVatWkWXLl0A2LRpEw0bNnRzquJBh2KKuL179xIZGYmvry92u52zZ8/y1ltv0ahRI3dHk2IuLCyM1NRUSpYsiclkIj093blOZ24VLBV7EderVy/GjBnjLPLdu3czZcoUli5d6uZkUtzt37+f+vXruztGsaSzYoq4K1eu5BidN27cWFOjSqEQFRXl7gjFlg7GFnHly5dn3bp1tG7dGoB169bh6+vr3lAiQO3atZkzZw6NGjXKceVpkyZN3JiqeNChmCLul19+YdCgQVy4cMH52Keffsq9997rvlAiQP/+/a97zGQysXDhQjekKV40Yi/ivvrqK0qVKsWKFSs4duwYUVFR7NixQ8UubvfRRx+5O0KxpRF7EdexY0c+//xzSpUqBUB6ejo9e/YkPj7ezcmkuOvfv/8N52PXiL3gacRexGVlZVGiRAnn8n9/LeJOw4cPd36dnZ3N+vXrKVeunBsTFR8asRdxr7/+Ort376Z9+/aYTCbWrl3Lgw8+yIgRI9wdTeQ6Tz31FJ9//rm7YxieRuxF3MiRI0lISOC7777Dy8uL8PBw5xkyIu506tQp59cOh4NDhw7l+JBfCo5G7CJSIH6/56nD4cDDwwM/Pz+GDx9O8+bN3R3N8HSBkogUiBkzZtC3b18SEhKoXr06x44dc3ekYkPFLiIFYurUqdStW5fExERKlizJypUrmTVrlrtjFQsqdhEpEHa7nWbNmrFx40batGlDpUqVsNls7o5VLKjYRaRAlCpVivnz57N9+3ZatmzJwoULKVOmjLtjFQsqdhEpENOmTePKlSvMnj2b8uXLk5yczPTp090dq1jQWTEiIgajEbuIiMGo2EVEDEbFLpIPx48fzzEHikhhpGIXyYdTp07x888/uzuGyC3pw1Mp1i5fvsyYMWP49ddf8fDwICgoiEmTJrFp0ybmzp1LVlYWJUuWZNSoUYSEhNCuXTuSk5Np0qQJ77//vrvji9yQRuxSrH355ZdcvnyZVatWOW8AfuzYMWbMmMG8efNYuXIlkydPZvjw4Vy9epUpU6ZQvXp1lboUaprdUYq1hx56iBkzZtC/f3/CwsJ45pln2LJlCykpKTz77LPO7Uwmk+Y6kSJDxS7FWrVq1fjyyy/Zvn073377LX/5y18YNGgQjz76KDNnznRud/r0aSwWCzt37nRfWJE80qEYKdYWL17MmDFjaNasGSNHjqRZs2ZcvHiRLVu2cOTIEQA2b95M586dycjIwNPTk6ysLDenFrk1jdilWOvatSs7duygQ4cOlCpVikqVKtG/f3/uu+8+XnzxRRwOB15eXsydO5cyZcpQu3ZtfHx86NGjB59//vkN7+kp4m46K0ZExGB0KEZExGBU7CIiBqNiFxExGBW7iIjBqNhFRAxGxS4iYjAqdhERg1Gxi4gYzP8DK9ahmaov8k8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "temp = af[af.index == 'OVERALL']\n", + "temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n", + "temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n", + "temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n", + "g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MOTA " + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idf1idpidrrecallprecisionnum_unique_objectsmostly_trackedpartially_trackedmostly_lostnum_false_positivesnum_missesnum_switchesnum_fragmentationsmotamotpnum_transfernum_ascendnum_migrate
set
control0.7930200.9654260.6728610.6753110.96894286.044.012.030.0106.01590.02.037.00.6532570.0354030.02.00.0
surprise0.7819890.9594840.6599110.6662440.96869233.013.07.013.034.0527.02.011.00.6434450.0349900.02.00.0
\n", + "
" + ], + "text/plain": [ + " idf1 idp idr recall precision \\\n", + "set \n", + "control 0.793020 0.965426 0.672861 0.675311 0.968942 \n", + "surprise 0.781989 0.959484 0.659911 0.666244 0.968692 \n", + "\n", + " num_unique_objects mostly_tracked partially_tracked mostly_lost \\\n", + "set \n", + "control 86.0 44.0 12.0 30.0 \n", + "surprise 33.0 13.0 7.0 13.0 \n", + "\n", + " num_false_positives num_misses num_switches num_fragmentations \\\n", + "set \n", + "control 106.0 1590.0 2.0 37.0 \n", + "surprise 34.0 527.0 2.0 11.0 \n", + "\n", + " mota motp num_transfer num_ascend num_migrate \n", + "set \n", + "control 0.653257 0.035403 0.0 2.0 0.0 \n", + "surprise 0.643445 0.034990 0.0 2.0 0.0 " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "af[af.index == 'OVERALL'].groupby(['set']).mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gate Openings" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percept gate openings when visible: M: 0.257 , STD: 0.237, Count: 1563\n", + "Percept gate openings when occluded: M: 0.00921 , STD: 0.0524, Count: 486\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Percept gate openings when visible:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Percept gate openings when occluded:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/adept_ablation/recon/evaluation_loci_looped_recon.ipynb b/evaluation/adept_ablation/recon/evaluation_loci_looped_recon.ipynb new file mode 100644 index 0000000..a6422e0 --- /dev/null +++ b/evaluation/adept_ablation/recon/evaluation_loci_looped_recon.ipynb @@ -0,0 +1,466 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import warnings\n", + "import scipy.stats as stats\n", + "import os\n", + "\n", + "warnings.simplefilter(action='ignore', category=FutureWarning)\n", + "pd.options.mode.chained_assignment = None \n", + "plt.style.use('ggplot')\n", + "sns.color_palette(\"Paired\");\n", + "sns.set_theme();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# setting path to results folder\n", + "root_path = '../../../out/pretrained/adept_ablations/recon'\n", + "\n", + "# list all folders in root path that don't stat with a dot\n", + "nets = ['recon']\n", + "\n", + "# read pickle file\n", + "tf = pd.DataFrame()\n", + "sf = pd.DataFrame()\n", + "af = pd.DataFrame()\n", + "\n", + "# load statistics files from nets\n", + "for net in nets:\n", + " path = os.path.join(root_path, net, 'results')\n", + " with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n", + " tf_temp = pd.read_csv(f, index_col=0)\n", + " tf_temp['net'] = net\n", + " tf = pd.concat([tf,tf_temp])\n", + "\n", + " with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n", + " sf_temp = pd.read_csv(f, index_col=0)\n", + " sf_temp['net'] = net\n", + " sf = pd.concat([sf,sf_temp])\n", + "\n", + " with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n", + " af_temp = pd.read_csv(f, index_col=0)\n", + " af_temp['net'] = net\n", + " af = pd.concat([af,af_temp])\n", + "\n", + "# cast variables\n", + "sf['visible'] = sf['visible'].astype(bool)\n", + "sf['bound'] = sf['bound'].astype(bool)\n", + "sf['occluder'] = sf['occluder'].astype(bool)\n", + "sf['inimage'] = sf['inimage'].astype(bool)\n", + "sf['vanishing'] = sf['vanishing'].astype(bool)\n", + "sf['alpha_pos'] = 1-sf['alpha_pos']\n", + "sf['alpha_ges'] = 1-sf['alpha_ges']\n", + "\n", + "# scale to percentage\n", + "sf['TE'] = sf['TE'] * 100\n", + "\n", + "# add surprise as dummy code\n", + "tf['control'] = [('control' in set) for set in tf['set']]\n", + "sf['control'] = [('control' in set) for set in sf['set']]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Tracking Error (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tracking Error when visible: M: 2.95 , STD: 4.8, Count: 1686\n", + "Tracking Error when occluded: M: 2.54 , STD: 2.52, Count: 477\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "\n", + "def get_stats(col):\n", + " return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n", + "\n", + "# When Visible\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n", + "\n", + "# When Occluded\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Tracking Error when occluded:' + get_stats(temp['TE']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Calculate Succesfull Trackings (TE)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
setevalmodetracked_postracked_negtracked_pos_protracked_neg_pro
0controlopen3790.8043480.195652
\n", + "
" + ], + "text/plain": [ + " set evalmode tracked_pos tracked_neg tracked_pos_pro \\\n", + "0 control open 37 9 0.804348 \n", + "\n", + " tracked_neg_pro \n", + "0 0.195652 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# succesfull trackings: In the last visible moment of the target, the slot was less than 10% away from the target\n", + "# determine last visible frame numeric\n", + "grouping_factors = ['net','set','evalmode','scene','slot']\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).max()\n", + "ff.rename(columns = {'frame':'last_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['last_visible']], on=grouping_factors, how='left')\n", + "\n", + "# same for first bound frame\n", + "ff = sf[sf.visible & sf.bound & sf.inimage].groupby(grouping_factors).min()\n", + "ff.rename(columns = {'frame':'first_visible'}, inplace = True)\n", + "sf = sf.merge(ff[['first_visible']], on=grouping_factors, how='left')\n", + "\n", + "# add dummy variable to sf\n", + "sf['last_visible'] = (sf['last_visible'] == sf['frame'])\n", + "\n", + "# extract the trials where the target was last visible and threshold the TE\n", + "ff = sf[sf['last_visible']] \n", + "ff['tracked_pos'] = (ff['TE'] < 10)\n", + "ff['tracked_neg'] = (ff['TE'] >= 10)\n", + "\n", + "# fill NaN with 0\n", + "sf = sf.merge(ff[grouping_factors + ['tracked_pos', 'tracked_neg']], on=grouping_factors, how='left')\n", + "sf['tracked_pos'].fillna(False, inplace=True)\n", + "sf['tracked_neg'].fillna(False, inplace=True)\n", + "\n", + "# Aggreagte over all scenes\n", + "temp = sf[(sf['frame']== 1) & ~sf.occluder & sf.control & (sf.first_visible < 20)]\n", + "temp = temp.groupby(['set', 'evalmode']).sum()\n", + "temp = temp[['tracked_pos', 'tracked_neg']]\n", + "temp = temp.reset_index()\n", + "\n", + "temp['tracked_pos_pro'] = temp['tracked_pos'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "temp['tracked_neg_pro'] = temp['tracked_neg'] / (temp['tracked_pos'] + temp['tracked_neg'])\n", + "\n", + "temp" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mostly Tracked stats" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEmCAYAAAB20LuSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAioklEQVR4nO3dd3hUVf7H8fek0UsIoSoiCkdABUGKBVFkXeuqix10V0SKCy7KqvijN8FdUVAUFURBARfBLljoFqSJICoHBBsQIYagJBAgmfn9MQk7xEAmMOFmTj6v5+Fxbpl7v2YmHw7nnnuuLxAIICIi7ojxugAREYksBbuIiGMU7CIijlGwi4g4RsEuIuKYOI/PXwZoBaQAOR7XIiISLWKB2sBKYH/+jV4HeyvgY49rEBGJVu2AT/Kv9DrYUwDS0zPx+zWePhKSkiqSlpbhdRkif6DvZuTExPhITKwAuRman9fBngPg9wcU7BGkn6WUVPpuRlyBXdi6eCoi4hgFu4iIY7zuihGR45STk016eirZ2Qe8LuWodu6Mwe/3e11GVImLSyAxMZnY2KJFtYJdJMqlp6dStmx5KlSohc/n87qcI4qLiyE7W8EerkAgQGbm76Snp1K9eu0ivTesYDfGVAY+A6621v6Qb1tzYDJQGVgK9LTWZhepChE5ZtnZB0p8qEvR+Xw+KlSoTEbG7iK/t9A+dmNMG4LjJBsdYZdXgN7W2kaAD7i7yFWIyHFRqLvpWD/XcC6e3g38A9ief4Mx5hSgnLX289xVLwE3HlMlIiISEYV2xVhruwEYYwraXIfDB8inACcVtYikpIpFfcsJdyDnIAmx8V6XEZbk5Epel1CoaPp5lnQ7dvqIizu8jVaufBnKlon8JbSs/dns2/uHO9jDlr/OcHz99XoWLVpA797/5N133+aLL1YzePCwIh0jI2MPI0YM5dFHxxa6byAQOGpLuVevu+nWrQctW55bpBoAJk16FoC77+4Z9ntiYmKK/Dt9vJ98DBB6x4EPKPLVkbS0jBJ/40JyciVu+m8vr8twxqybJ5KausfrMpzgw8fmXT8dtu7cxMZc0++tiJ/rnbHXsn7rpogf92iWr1/Jj7/8xOZdP7EzM409BzL+8P9bmNQdO/j62/Vhve+0avWOepE3EAiQk+M/pgvBeTlXlPf6/f4//K7ExPiO2iA+3mDfSnAimjy1KKDLRkRKj2/WfcVbr84mLj6Onb/soGXb1pQpW5bVy5YTIMCDwwazeeMmXnt5OgF/gBq1anJXn3uokliV6ZNfZP2atfh8Plqe14bLr72aOa/MJGtfFm+++hqJSdUAWP/lWma/PJOhY8cAsOSjBWy2m+jau+CW8LRnJ5O+K50nRoymS/euPDpoOJUqVyYhIYF/DnyISeMmsOvXNNLTdnF+2/N58MGBAEyc+BRLly4mLi6Wv/zlr9x0062Hjpmevot77+1J9+730K7dxbz88kssWvQROTl+2rRpS69e9+Lz+ZgxYxpvv/0GVapUpVKlSjRu3LSYP4HjvEHJWvsjkGWMuSB31e3AvOOuSkSi2nd2I11792Tk+LF8+M5cKlepwsgnx1Kvfn3mz32fKRMmcv+ghxnzzHgaNWnMSxOfJ3XHTtau+oLRT49jyGNj2PbTz8QnJNCpy620aNuK62753+W7ps3OZnd6OjtSgj3BnyxYzEUdOxyxnjt6diOxWiL3DXoYgJSt27jngb48/MgwvlyxilManMqwxx/l8cnPsGbNaqzdwKJFC/jqq7VMm/Yqzz8/lblz3yEt7VcAMjMzeOCBvnTt2p127S7m888/w9pvmTRpGi++OJ3U1FQ+/HAeGzZ8w3vvvc2UKdMZN+4ZUlN3FuNP/X+OqcVujJkLDLbWrgI6A5Nyh0R+ATwZwfpEJAqdXL8eScnJAFSqXImmzc8GoHqNZNYsX8lpjRqSXLMmAB2uuIy3Z82hWvUkEhISGNqvP+e0Ppdbu95BQkJCgcf3+Xy0u/QSPlm4hPZ/upTfdu/m9DOONHDvjypXrXLo/OdffBGb7Ubmvfk223/eym+//ca+fXv58svVdOjwJxISEkhISOCll2Ycev9//jOaatWSaN8++JfJqlUr+Oab9dx11+0A7N+fRc2atUhLS6Nt2wsoX748AJdc0pGcnOKfoTzsYLfW1g95fWXI67VA68iWJSLRLC7u8GiJjf1f50AgcPj1tGCfdQ6xsbEMf+LffPvV13y5ajVD7+/PwEdHHvEcF3XswL8HDychIYF2l15SpPpC/8L44O13WfHJMjpccRlnNm9G6tYdBAIB4uLiCL2GmpKynapVEwHo3PkOli37lDfemE2nTjfh9+dw0023csstXQDYs2cPsbGxvPXW64RehoyNjT0hwa65YkTkhDrNNOK7DRtJ3bEDgIXzPqRJs7P4YfMWRjw0kDPOakrnbndSt95JpGzbRmxsLP6cP15sTK5Zg2pJScx/bx4Xdrj4qOeMiY0lp4BjAKxfs5YOV1zGBZe05+CBg2zaZPH7/TRr1oLFixeSnZ1NVlYW/fr1OdSV0rChoV+//rz44iRSU3fSokUrPvhgLnv37iU7O5uHH+7H4sULOPfcVnz66cdkZGSwf/9+li5ddHw/vDBpSgEROaGqVK3CXff24okRY8jOzqZ6jWTu7tubxGrVaNjY8FCveylTpgyNmpxBs3NbsjPlF16f/iqvvjiN2ifVPexY57W/kBWfLjt0UfXI56xK9RrVGdl/ID3u63PYtsuvvYYXn36Wt2fNoXyFCpx1VjNSUrZzzTXXsWHDN3Tt2hm/P8CNN95KvXqnHHrfySfX469/vZHHH/83o0c/xnffbaR797/j9+fQps35XHHF1fh8Pm688Va6dbuDSpUqUbNm0aYGOFa+/P8sOsHqA99ruGPpo+GOkbNz58/sK3v4uOvGdRtQoWyZiJ8rM2s/327bEvHjHoucnBwmPjaONheeT6sLzovYcQsb7nii/fLLj9Sqdcph60KGO54K/JD/PWqxiziopIRvcQkEAvTu0pUzz2lGy/PaALBh/ddMfXZSgfs/OGxwoa16lyjYRSTq+Hw+Js6ceti6M85syugJ47wpqITRxVMREcco2EVEHKNgFxFxjIJdRMQxCnYREcdoVIyIg86uewoJZctH/LgHsvaybtuPET9unr2ZmTz3+JPcN+hh0tN2MWn8BB4cPviI+895ZSYAnbrcSucrr2P63DeLfM7Zr8zkzOZnc8aZxzfr4ty577BmzWoGDBha5PempGynT58ezJ79znHVkEfBLuKghLLl2TKqU8SP22DAnIgfM1RmRgY/bA6OwU9MqnbUUI+UDV+tp8nZZxb7eU4kBbuIRNQ3677izZmziI2LI/WXHTRo1JC7+/bmjRn/Zf2X68jck0FiUiJ9+j9AlcSq9LzlDk5teDq703dRNbHaYfOmj3xoIONfmsTPP/zI1GcnsX9fFr/t/o2/3NSJjldd/odz+/1+7uvag/4jh1L7pLpkZWXxQPd/MHbyxAJnivx4wSK2bNrM5PFP03dgf6Y9O4ka1WqwZctmhg8fzbp1X/L++3PJytpHfHw8Q4eOol69+qxcuZwJE8YRCPipVas2Q4YcPlnZ+PFj2bUrjcGDR7Bx4waefPJx9u/PokqVqjzwwP9Rp05dNm7cwJgxIwA4/fTwZ6YMh/rYRSTiNn5rub1HN/7z/NMcPHiQ9998h+0/b2Xo2DGMnfwMScnJfLJoMQB7fv+da268ntETxtG1d4/D5k3Ps/iDj7julhsZMf4xBo4ZwcwpLxV43piYGNp17MCni5YAsPLTZZzTutURp/9td+klNGh4Gt3++Q/qnVofgNNPb8jMma9Tp05dli5dwoQJz/Hyy7M4//x2zJkziwMHDjB8+CAGDhzKtGn/pUGD05k3791Dx3zhhedITd3JoEHD8fv9jBkzkiFDRjFlynRuuaULjz46CoCRI4fQq1cfpkyZTp06dQsq75ipxS4iEXfGmU2okzth14UdLmbhvA/5+z13s/iDj9i+dRubNlhq1K51aP/TzdFbrJ273cna1Wt467+z+fmHH8nal3XEfdv/qQOj/28IN9x+Gx/PX8hNf7u9SLU3bXoWABUqVGTo0JHMn/8hP//8E8uXf0bDhoYtW74jOTmZhg2Dz4Hu2bM3EOxj//zzz9i9O51Jk6YRFxfHli3fsX37Vvr3v//Q8TMzM9m9eze//vorrVq1BeCKK67m3Xcj9yhDBbuIRFxsbOyh1wG/H58PxgwYxhXX/4U2F55PTEwMhExAmFDm6BOWPTn6P1SoWJFz2rTivPbtWLbk4yPum1yzJtVrJLPy02X8tvu3Ij2AA6BMbi07dvxCnz496NTpJtq2PZ9q1ZLYtMkSGxtH8PHOQRkZGezdmwlArVq16dHjHh5//FGefXYKOTl+6tSpe+ghHTk5OaSn78LnO3xe+uAxI0ddMSIScfbrb9n1axp+v5+PFy7GNG1C47Ob0vGqy6lVtw5rVqzC7//jDIpHmjf9qzVrueH22zj3vDasXbUaAP9RHljR/rKOTH12cqHztOeds6D53jds+IaTTjqZm2/uTOPGTVi6dBF+fw716p3C7t3pfP998CLv9OlTefPN4EXl+vVP5eqrr6NcuXK8/vosTjmlPr///jtr164B4L333mbo0AFUqVKVWrVq8dlnnwDw0UfvF1pnUajFLuKgA1l7i2UEy4GsvWHtl5hUjWfHjmNX2i7OOqcZF1zSnnEjx/BQr3sBaNDwdFJ3/PH5n0eaN71T51sY9sDDxMfHc0qDU0muWYOdBbw/T6vz2zL5yafDCvZmLVswZcJEevbre/gxWrXljTdm06XLjQQCAZo3b8GWLZspU6YMgwYNZ+TIIWRnH6ROnZMYNGg4ixcvOPTefv36c889d3HRRZcwYsQYxo9/jAMHDlC+fAUGDhwGwKBBIxg9ehiTJj1D06ZnF1pnUWg+9jBpPvbI0nzskVPQfOxe+mbdV7w+/VUG5l4kPNECgQBrV61mwdwP6DdkQJHfr/nYRURKmFeef4Evlq88bAz8yP4DyczI+MO+l15xeYHDJqOdgl1EIqrJ2WfR5OyzPDv/7T26cXuPboetGzjmyA/FdpEunoqIOEbBLiLiGAW7iIhjFOwiIo7RxVMRBzU+qT4VypSL+HEz9+/j260/RPy4R7PZbmTFp8u4tevfWPLRAr79aj097/9nkY5xrEMwMzIyGDVqKKNHP1ak93lNwS7ioAplyhXLfRezbp4Y8WMWZutPP/Pb7t0n/LwAe/b8zqZN1pNzHw8Fu4hE1DfrvuKtV2cTFx/Hzl920LJta8qULcvqZcsJEODBYYPZvHETr708nYA/QI1aNbmrzz1USazK9Mkvsn7NWnw+Hy3Pa8Pl117NnFdmkrUvizdffY3EpGoArP9yLbNfnsnQsWMAWPLRAjbbTXTt3bPQ+lK2buOFp54hY08GZcqW5Y6e3TitUUM+XbSEd+e8Qbn4stSuXYdBg0Ywbtx/+PXXVB5++F9R1WpXH7uIRNx3diNde/dk5PixfPjOXCpXqcLIJ8dSr3595s99nykTJnL/oIcZ88x4GjVpzEsTnyd1x07WrvqC0U+PY8hjY9j208/EJyTQqcuttGjbiutuufHQ8Zs2O5vd6ensSEkB4JMFi7moY4ewanvmsXH8+S9XM+aZ8XTp3pXxo/7NwYMHee3lGfQfOZSpU2dQu3ZdfvrpB/r2fYDq1ZOjKtRBwS4ixeDk+vVISk6mTNkyVKpciabNg3OhVK+RzJrlKzmtUUOSa9YEoMMVl/H1l+uoVj2JhIQEhvbrz/tvvcOtXe844jzqPp+PdpdewicLl/DrzlR+2707rFkcs/btY8f2FFpdcB4ADc8wVKxUkZSt22jRuhXD/vUwTz01jvbtOxyaljcaKdhFJOLi4g7v5Y2N/V/U5J+fKhAIkJOTQ2xsLMOf+Dc33n4bGXv2MPT+/qRs3XbEc1zUsQOfL/2EZUs+pt2ll4RVl7+AubECBM9/R89u9B3wEJUrV2bEiEF88MHcsI5ZEinYReSEOs004rsNG0ndsQOAhfM+pEmzs/hh8xZGPDSQM85qSudud1K33kmkbNtG7BGm1U2uWYNqSUnMf29eWLM4ApQvX54atWqy8tNlAGzaYPktfTcnn1KP+7v1olLlyvztb125/PKr2LjREhsbS85RpgcuqcK6eGqMuQ0YCMQD46y1T+fb3gJ4DkgAfga6WGt3R7ZUEXFBlapVuOveXjwxYgzZ2dlUr5HM3X17k1itGg0bGx7qdS9lypShUZMzaHZuS3am/MLr01/l1RenUfukwx8hd177C1nx6bJDF1XDcc8D9zFlwkRmvzKT+Ph4+g7oT1x8PDd0uZXRA4ZQqXxFqlZNZMCAoVSqVJmaNWvRp08PnnrquUj/KIpNodP2GmPqAp8ALYH9wGfArdbab0L2+Rh4xFo7zxgzFthnrR0Yxvnro2l7SyVN2xs5BU3b69I49iPJyclh4mPjaHPh+Yf6zCOhtEzb2xFYaK3dBWCMmQ3cAAwP2ScWqJz7ujywq4i1i0gElZTwLS6BQIDeXbpy5jnNaHleGwA2rP+aqc9OKnD/B4cNLlKrPtqFE+x1gJSQ5RSgdb597gc+NMaMAzKBNhGpTkSkAD6fj4kzpx627owzmzJ6wjhvCiphwgn2GCC0n8QHHPp3ijGmHPAC0NFau8IYcz8wDbgq3CJy/0khpUxyciWvS3DCziM/IU6OUVxcyRlXEhMTU+TflXCCfSvQLmS5FrA9ZPlMgn3qK3KXnwNGFKWIaOljl8hSH3sEBQLgKzmPx4t2JaWPPRAI4Pf7//C7EtLHXqBw/lqaD1xqjEk2xpQHOgGhj9T+DjjZGJM3mv9aYGVRiheRY1euXFn8Bw4Ew12cEQgEyMz8nbi4gm/SOppCW+zW2m3GmAHAIoLDGSfndrnMBQZba1cZY/4OzDLG+ICdwJ1FrkREjsnJJ5/Mqs/nklwmER9qtR+v7VnBVnJJEBeXQGJictHfF85O1toZwIx8664MeT0PmFfks4vIcYuPj2f29vcL31HC4sJQ3JJzhUBERCJCwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDhGwS4i4hgFu4iIYxTsIiKOUbCLiDgmzusCosWB7APMunmi12U440D2Aa9LEHGWgj1MCXEJbBnVyesynNFgwBxgv9dliDhJXTEiIo5Ri10kyqmbMLJc6CZUsItEOXUTRpYL3YTqihERcYyCXUTEMWF1xRhjbgMGAvHAOGvt0/m2G+A5IBH4BbjFWpse4VpFRCQMhbbYjTF1gVHAhUBzoLsxpknIdh/wNjDGWtsMWAP0L5ZqRUSkUOF0xXQEFlprd1lrM4HZwA0h21sAmdba93OXHwGeRkREPBFOV0wdICVkOQVoHbJ8OvCLMeYF4BzgW6BPxCoUEZEiCSfYY4BAyLIP8Oc7xsXARdbaVcaYEcDjwN/DLSIpqWK4u4pDkpMreV2CSIGi/bsZTrBvBdqFLNcCtocs/wJsstauyl2eSbC7JmxpaRn4/YHCd/RQtH/QJVFq6h6vS3CCvpuRV9K/mzExvqM2iMPpY58PXGqMSTbGlAc6Ae+HbP8MSDbGNMtdvgZYfYz1iojIcSo02K2124ABwCLgS2CGtXaFMWauMeZca+0+4HpgkjHma6AD0K8YaxYRkaMIaxy7tXYGMCPfuitDXi/n8AuqIiLiEd15KiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOCbO6wKihf/gARoMmON1Gc7wHzzgdQkizlKwhykmPoFr+r3ldRnOeGfstcB+r8sQcZK6YkREHKNgFxFxjIJdRMQxYQW7MeY2Y8w3xphNxph/HGW/q4wx30euPBERKapCg90YUxcYBVwINAe6G2OaFLBfTeAxwBfhGkVEpAjCabF3BBZaa3dZazOB2cANBew3GRgWyeJERKTowhnuWAdICVlOAVqH7mCMuRf4Avj8WIpISqp4LG+TKJecXMnrEkQKFO3fzXCCPQYIhCz7AH/egjHmTKATcClw0rEUkZaWgd8fKHxHD0X7B10Spabu8boEJ+i7GXkl/bsZE+M7aoM4nGDfCrQLWa4FbA9ZvhGoDawCEoA6xpiPrbWh7xGRYqK7oiPLhbuiwwn2+cBQY0wykEmwdd49b6O1dggwBMAYUx9YrFAXOXF0V3RkuXBXdKEXT62124ABwCLgS2CGtXaFMWauMebcYq5PRESKKKy5Yqy1M4AZ+dZdWcB+PwD1I1GYiIgcG915KiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOCYunJ2MMbcBA4F4YJy19ul8268FhgE+4HvgTmtteoRrFRGRMBTaYjfG1AVGARcCzYHuxpgmIdsrAxOBq6y1zYB1wNDiKFZERAoXTldMR2ChtXaXtTYTmA3cELI9HviHtXZb7vI6oF5kyxQRkXCF0xVTB0gJWU4BWuctWGvTgDcAjDHlgP7AUxGsUUREiiCcYI8BAiHLPsCffydjTBWCAb/WWju1KEUkJVUsyu7iiOTkSl6XIFKgaP9uhhPsW4F2Icu1gO2hOxhjagMfAAuB+4paRFpaBn5/oPAdPRTtH3RJlJq6x+sSnKDvZuSV9O9mTIzvqA3icIJ9PjDUGJMMZAKdgO55G40xscA7wCxr7cjjK1dERI5XocFurd1mjBkALAISgMnW2hXGmLnAYOBkoAUQZ4zJu6i6ylrbrbiKFhGRIwtrHLu1dgYwI9+6K3NfrkI3OomIlBgKZBERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERx8SFs5Mx5jZgIBAPjLPWPp1ve3NgMlAZWAr0tNZmR7ZUEREJR6EtdmNMXWAUcCHQHOhujGmSb7dXgN7W2kaAD7g7wnWKiEiYwmmxdwQWWmt3ARhjZgM3AMNzl08ByllrP8/d/yVgGDAxjGPHAsTE+IpWtUdqJJbzugSnRMvnHg303Yyskv7dDKkvtqDt4QR7HSAlZDkFaF3I9pPCrK82QGJihTB399YLAy/zugSnJCVV9LoEZ+i7GVlR9N2sDWzOvzKcYI8BAiHLPsBfhO1HsxJoR/Avg5ww3yMiUtrFEgz1lQVtDCfYtxIM3zy1gO35ttc+yvaj2Q98Eua+IiLyP39oqecJZ7jjfOBSY0yyMaY80Al4P2+jtfZHIMsYc0HuqtuBecdRrIiIHIdCg91auw0YACwCvgRmWGtXGGPmGmPOzd2tM/CEMWYDUBF4spjqFRGRQvgCgUDhe4mISNTQnaciIo5RsIuIOEbBLiLiGAW7iIhjFOwiIo5RsIuIOCasaXul5DHGDD7admvt8BNVi8iR5E753ZTgDLE3WGuneVxSqaAWe/TyFfJHxFPGmDHAlcBfCTYi7zTGjPW2qtJBNyg5wBiTDLQh+MuzzFq7w+OSRDDGrAFaAF9Ya88xxsQB66y1+Z/nIBGmFnuUM8b8meBUD3cCfwPWGWOu9rQokaC8WV7zWo9lCH/mVzkO6mOPfqOAC6213wMYYxoArwPvelqVCMwC/gtUM8b0Be4AZnhaUSmhFnv0i88LdQBr7Rb0uUoJYK19FHgBeA2oBwyy1j7ibVWlgwIg+v1kjOlrjKmU++c+4EevixIxxiQAKdbaB4DVwMXGmOoel1UqKNij313AecAW4Pvc1909rUgk6BWgszGmNTAE+J3gM5GlmKmPPfrda6292esiRApwqrX2JmPMo8AL1tpHjTEFPspNIkst9uh3jTFG49alJIrL7Xq5HnjPGFMLKOdxTaWCWuzRLw3YYIz5AtiXt9Ja29W7kkQA+A+wHHjbWrveGLMRGORxTaWCgj36TS1gne46E89Za2dw+PDGxtbaHK/qKU0U7NGvjrV2dOgKY4yGlIlnjDHvWmuvNsZ8T75GhjEGa20Dj0orNRTsUSp3Ho4awF+MMQ1DNsUBbYH/86QwEbg79783ATu9LKS0UrBHrzlAE+BSYEnI+mxghCcViQDW2pTcl9OstY09LaaU0iRgUc4YU8Va+5vXdYjkZ4x5FXgPWMHhF/Z/8qyoUkIt9uh3Xe5UqIm5yz4gYK2N9bAmEQjOONom37oAoD72YqYWe5QzxmwGrrXWrve6FhEpGdRij37bFepSEhlj6gFPAh0IXvuZC9xnrU31tLBSQMEe/VYbY2YDHwJZeSv1CDIpAaYTnLb3diCW4DMDphJ8qpIUIwV79KsC7CE4+VeeAKBgF69VttZOCFl+whjzd6+KKU0U7FHOWnunMSYeMAQ/z/XW2myPyxIB+MwY08Va+wqAMeYqYI3HNZUKunga5YwxLQmOaU8jOKlbTeB6a+1yTwuTUs8YswNIJjjU0Q9UCNmskVvFSC326PckcHNekBtj2gJPAa09rUoELrPWrvW6iNJI0/ZGv4qhrXNr7edAWQ/rEcnzqtcFlFZqsUe/XcaYa621bwEYY64j2C0j4rVvjDGDCU7dG3rn6VLvSiodFOzRrzvwrjHmBXLvOgXO97YkEQCqAZfk/skTIDiuXYqRgj36XQHsBVoApxEcN3wxsNHDmkSw1l5S+F5SHBTs0a870NpauxdYlztKZjnwvLdlSWlnjFlEAQ99sdaqxV7MFOzRLx44ELJ8AD1BSUqGoSGv44FrgXRvSildFOzR701goTFmFsFA7wS85WlFIoC1dkm+VfONMcuBwV7UU5roBiUHGGNuANoDB4Gl1to3va1I5NAkYHl8wJnAeGvt6R6VVGqoxe4Aa+1sYLbXdYjks4TgvyJ9BO88/RXo7WlFpYRuUBKR4nIL8DRwBrCZ4KgtOQEU7CJSXMYDXwF/JTgk9xz0PN4TQsEuIsUlxlr7IXA1MMda+zPq/j0hFOwiUlz2GmP6EbzT9F1jzL0Enx0gxUzBLiLFpTPBqXo7WWvTgbrAbd6WVDpouKOIiGPUYhcRcYyCXUTEMQp2kSIwxpxqjJnjdR0iR6NgFymaUwg+OFykxNLFUynVjDEVgReBhgRve18N9ACuAgYCCQRvrvkXsAKwBEd3LLXW/tmLmkUKoxa7lHbXA5Wstc2BVrnrTgMeAa601p5DcM771wk+S7YbsFmhLiWZ7gKT0u4T4BFjzGLgI2Ac8CegNrDAmEO9Ln5AsxJKVFCLXUo1a+33BAN7NFAZmA/EAgustc3z/gBtgfWeFSpSBAp2KdWMMb0I9rF/aK19CPiA4EOYLzPGnJG7z5XAOqAckE3waUAiJZYunkqpZoypAEwBziZ4kfQnoCvQERhAcC7xbKCvtfZjY0w1YBGwH2hjrdUvkJQ4CnYREceoK0ZExDEKdhERxyjYRUQco2AXEXGMgl1ExDEKdhERxyjYRUQco2AXEXHM/wMWsHwldsBw8QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "temp = af[af.index == 'OVERALL']\n", + "temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n", + "temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n", + "temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n", + "g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MOTA " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idf1idpidrrecallprecisionnum_unique_objectsmostly_trackedpartially_trackedmostly_lostnum_false_positivesnum_missesnum_switchesnum_fragmentationsmotamotpnum_transfernum_ascendnum_migrate
set
control0.7545760.9195770.6397790.6628550.95274486.038.018.030.0161.01651.013.028.00.6273230.0370940.012.00.0
surprise0.7904010.9687500.6675110.6675110.96875033.013.08.012.034.0525.00.011.00.6459780.0349790.00.00.0
\n", + "
" + ], + "text/plain": [ + " idf1 idp idr recall precision \\\n", + "set \n", + "control 0.754576 0.919577 0.639779 0.662855 0.952744 \n", + "surprise 0.790401 0.968750 0.667511 0.667511 0.968750 \n", + "\n", + " num_unique_objects mostly_tracked partially_tracked mostly_lost \\\n", + "set \n", + "control 86.0 38.0 18.0 30.0 \n", + "surprise 33.0 13.0 8.0 12.0 \n", + "\n", + " num_false_positives num_misses num_switches num_fragmentations \\\n", + "set \n", + "control 161.0 1651.0 13.0 28.0 \n", + "surprise 34.0 525.0 0.0 11.0 \n", + "\n", + " mota motp num_transfer num_ascend num_migrate \n", + "set \n", + "control 0.627323 0.037094 0.0 12.0 0.0 \n", + "surprise 0.645978 0.034979 0.0 0.0 0.0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "af[af.index == 'OVERALL'].groupby(['set']).mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gate Openings" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percept gate openings when visible: M: 0.223 , STD: 0.303, Count: 1686\n", + "Percept gate openings when occluded: M: 0.00995 , STD: 0.0471, Count: 477\n" + ] + } + ], + "source": [ + "grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n", + "temp = sf[grouping & sf.visible]\n", + "print(f'Percept gate openings when visible:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))\n", + "temp = sf[grouping & ~sf.visible]\n", + "print(f'Percept gate openings when occluded:' + get_stats(temp['alpha_pos'] + temp['alpha_ges']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- cgit v1.2.3