{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import seaborn as sns\n",
"import warnings\n",
"import scipy.stats as stats\n",
"import os\n",
"\n",
"warnings.simplefilter(action='ignore', category=FutureWarning)\n",
"pd.options.mode.chained_assignment = None \n",
"plt.style.use('ggplot')\n",
"sns.color_palette(\"Paired\");\n",
"sns.set_theme();"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Data Loading"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# setting path to results folder\n",
"root_path = '../../out/pretrained/adept/loci_v1/results'\n",
"\n",
"# list all folders in root path that don't stat with a dot\n",
"nets = [f for f in os.listdir(root_path) if not f.startswith('.')]\n",
"\n",
"# read pickle file\n",
"tf = pd.DataFrame()\n",
"sf = pd.DataFrame()\n",
"af = pd.DataFrame()\n",
"\n",
"# load statistics files from nets\n",
"for net in nets:\n",
" path = os.path.join(root_path, net, 'surprise', 'statistics',)\n",
" with open(os.path.join(path, 'trialframe.csv'), 'rb') as f:\n",
" tf_temp = pd.read_csv(f, index_col=0)\n",
" tf_temp['net'] = net\n",
" tf = pd.concat([tf,tf_temp])\n",
"\n",
" with open(os.path.join(path, 'slotframe.csv'), 'rb') as f:\n",
" sf_temp = pd.read_csv(f, index_col=0)\n",
" sf_temp['net'] = net\n",
" sf = pd.concat([sf,sf_temp])\n",
"\n",
" with open(os.path.join(path, 'accframe.csv'), 'rb') as f:\n",
" af_temp = pd.read_csv(f, index_col=0)\n",
" af_temp['net'] = net\n",
" af = pd.concat([af,af_temp])\n",
"\n",
"# cast variables\n",
"sf['visible'] = sf['visible'].astype(bool)\n",
"sf['bound'] = sf['bound'].astype(bool)\n",
"sf['occluder'] = sf['occluder'].astype(bool)\n",
"sf['inimage'] = sf['inimage'].astype(bool)\n",
"sf['vanishing'] = sf['vanishing'].astype(bool)\n",
"sf['alpha_pos'] = 1-sf['alpha_pos']\n",
"sf['alpha_ges'] = 1-sf['alpha_ges']\n",
"\n",
"# scale to percentage\n",
"sf['TE'] = sf['TE'] * 100\n",
"\n",
"# add surprise as dummy code\n",
"tf['control'] = [('control' in set) for set in tf['set']]\n",
"sf['control'] = [('control' in set) for set in sf['set']]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Calculate Tracking Error (TE)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tracking Error when visible: M: 12.5 , STD: 10.3, Count: 69\n",
"Tracking Error when occluded: M: 16.2 , STD: 7.48, Count: 34\n"
]
}
],
"source": [
"grouping = (sf.inimage & sf.bound & ~sf.occluder & sf.control)\n",
"\n",
"def get_stats(col):\n",
" return f' M: {col.mean():.3} , STD: {col.std():.3}, Count: {col.count()}'\n",
"\n",
"# When Visible\n",
"temp = sf[grouping & sf.visible]\n",
"temp = temp.groupby(['net', 'set', 'evalmode', 'scene', 'slot']).mean()\n",
"print(f'Tracking Error when visible:' + get_stats(temp['TE']))\n",
"\n",
"# When Occluded\n",
"temp = sf[grouping & ~sf.visible]\n",
"temp = temp.groupby(['net', 'set', 'evalmode', 'scene', 'slot']).mean()\n",
"print(f'Tracking Error when occluded:' + get_stats(temp['TE']))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Calculate Succesfull Trackings (TE)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" set | \n",
" evalmode | \n",
" tracked_pos | \n",
" tracked_neg | \n",
" tracked_pos_pro | \n",
" tracked_neg_pro | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" _control | \n",
" open | \n",
" 63 | \n",
" 101 | \n",
" 0.384146 | \n",
" 0.615854 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"temp = af[af.index == 'OVERALL']\n",
"temp['mostly_tracked'] = temp['mostly_tracked'] / temp['num_unique_objects']\n",
"temp['partially_tracked'] = temp['partially_tracked'] / temp['num_unique_objects']\n",
"temp['mostly_lost'] = temp['mostly_lost'] / temp['num_unique_objects']\n",
"g = temp[['mostly_tracked', 'partially_tracked', 'mostly_lost','set']].set_index(['set']).groupby(['set']).mean().plot(kind='bar', stacked=True);"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# MOTA "
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" idf1 | \n",
" idp | \n",
" idr | \n",
" recall | \n",
" precision | \n",
" num_unique_objects | \n",
" mostly_tracked | \n",
" partially_tracked | \n",
" mostly_lost | \n",
" num_false_positives | \n",
" num_misses | \n",
" num_switches | \n",
" num_fragmentations | \n",
" mota | \n",
" motp | \n",
" num_transfer | \n",
" num_ascend | \n",
" num_migrate | \n",
"
\n",
" \n",
" | set | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
" | \n",
"
\n",
" \n",
" \n",
" \n",
" | _control | \n",
" 0.395257 | \n",
" 0.264136 | \n",
" 0.804905 | \n",
" 0.901508 | \n",
" 0.298226 | \n",
" 86.0 | \n",
" 63.666667 | \n",
" 22.000000 | \n",
" 0.333333 | \n",
" 32836.000000 | \n",
" 1443.000000 | \n",
" 90.333333 | \n",
" 14.666667 | \n",
" -1.345869 | \n",
" 0.068371 | \n",
" 25.0 | \n",
" 65.666667 | \n",
" 1.000000 | \n",
"
\n",
" \n",
" | _surprise | \n",
" 0.390419 | \n",
" 0.257030 | \n",
" 0.823463 | \n",
" 0.896856 | \n",
" 0.281073 | \n",
" 33.0 | \n",
" 24.333333 | \n",
" 8.333333 | \n",
" 0.333333 | \n",
" 11238.333333 | \n",
" 487.666667 | \n",
" 24.000000 | \n",
" 6.333333 | \n",
" -1.485195 | \n",
" 0.068592 | \n",
" 9.0 | \n",
" 15.000000 | \n",
" 0.333333 | \n",
"
\n",
" \n",
"
\n",
"