diff options
Diffstat (limited to 'evaluation/clevrer')
| -rw-r--r-- | evaluation/clevrer/evaluation_loci_looped.ipynb | 164 | ||||
| -rw-r--r-- | evaluation/clevrer/evaluation_loci_unlooped.ipynb | 171 | ||||
| -rw-r--r-- | evaluation/clevrer/evaluation_savi.ipynb | 171 |
3 files changed, 506 insertions, 0 deletions
diff --git a/evaluation/clevrer/evaluation_loci_looped.ipynb b/evaluation/clevrer/evaluation_loci_looped.ipynb new file mode 100644 index 0000000..75ba075 --- /dev/null +++ b/evaluation/clevrer/evaluation_loci_looped.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# set path to resutls files\n", + "path = '../../out/pretrained/clevrer/loci_looped/results/net_loci_looped'\n", + "\n", + "# load pkl file with results\n", + "df = pd.read_pickle(os.path.join(path, 'blackout_metric_average.pkl'))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "msecomplete_average: 1.281\n", + "msecomplete_std: 1.445\n", + "\n", + "mseblackout_average: 1.724\n", + "mseblackout_std: 1.611\n", + "\n", + "msevisible_average: 1.172\n", + "msevisible_std: 1.379\n", + "\n", + "ssimcomplete_average: 0.963\n", + "ssimcomplete_std: 0.020\n", + "\n", + "ssimblackout_average: 0.954\n", + "ssimblackout_std: 0.023\n", + "\n", + "ssimvisible_average: 0.965\n", + "ssimvisible_std: 0.018\n", + "\n", + "psnrcomplete_average: 35.934\n", + "psnrcomplete_std: 2.344\n", + "\n", + "psnrblackout_average: 34.605\n", + "psnrblackout_std: 2.415\n", + "\n", + "psnrvisible_average: 36.260\n", + "psnrvisible_std: 2.208\n", + "\n", + "percept_distcomplete_average: 0.106\n", + "percept_distcomplete_std: 0.032\n", + "\n", + "percept_distblackout_average: 0.114\n", + "percept_distblackout_std: 0.033\n", + "\n", + "percept_distvisible_average: 0.104\n", + "percept_distvisible_std: 0.031\n", + "\n", + "aricomplete_average: 0.874\n", + "aricomplete_std: 0.065\n", + "\n", + "ariblackout_average: 0.861\n", + "ariblackout_std: 0.072\n", + "\n", + "arivisible_average: 0.878\n", + "arivisible_std: 0.062\n", + "\n", + "faricomplete_average: 0.803\n", + "faricomplete_std: 0.091\n", + "\n", + "fariblackout_average: 0.779\n", + "fariblackout_std: 0.106\n", + "\n", + "farivisible_average: 0.809\n", + "farivisible_std: 0.086\n", + "\n", + "mioucomplete_average: 0.429\n", + "mioucomplete_std: 0.048\n", + "\n", + "mioublackout_average: 0.420\n", + "mioublackout_std: 0.052\n", + "\n", + "miouvisible_average: 0.431\n", + "miouvisible_std: 0.047\n", + "\n", + "apcomplete_average: 0.647\n", + "apcomplete_std: 0.124\n", + "\n", + "apblackout_average: 0.621\n", + "apblackout_std: 0.136\n", + "\n", + "apvisible_average: 0.654\n", + "apvisible_std: 0.120\n", + "\n", + "arcomplete_average: 0.915\n", + "arcomplete_std: 0.140\n", + "\n", + "arblackout_average: 0.880\n", + "arblackout_std: 0.162\n", + "\n", + "arvisible_average: 0.924\n", + "arvisible_std: 0.132\n", + "\n", + "blackoutcomplete_average: 0.197\n", + "blackoutcomplete_std: 0.398\n", + "\n", + "blackoutblackout_average: 1.000\n", + "blackoutblackout_std: 0.000\n", + "\n", + "blackoutvisible_average: 0.000\n", + "blackoutvisible_std: 0.000\n", + "\n" + ] + } + ], + "source": [ + "for key,value in df.items():\n", + " print(f\"{key}: {value:.3f}\")\n", + " if 'std' in key:\n", + " print('')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/clevrer/evaluation_loci_unlooped.ipynb b/evaluation/clevrer/evaluation_loci_unlooped.ipynb new file mode 100644 index 0000000..51edd3c --- /dev/null +++ b/evaluation/clevrer/evaluation_loci_unlooped.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# set path to resutls files\n", + "path = '../../out/pretrained/clevrer/loci_unlooped/results/net_loci_unlooped'\n", + "\n", + "# load pkl file with results\n", + "df = pd.read_pickle(os.path.join(path, 'blackout_metric_average.pkl'))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "msecomplete_average: 13.490\n", + "msecomplete_std: 10.810\n", + "\n", + "mseblackout_average: 28.232\n", + "mseblackout_std: 7.041\n", + "\n", + "msevisible_average: 9.688\n", + "msevisible_std: 7.981\n", + "\n", + "ssimcomplete_average: 0.829\n", + "ssimcomplete_std: 0.096\n", + "\n", + "ssimblackout_average: 0.709\n", + "ssimblackout_std: 0.055\n", + "\n", + "ssimvisible_average: 0.860\n", + "ssimvisible_std: 0.078\n", + "\n", + "psnrcomplete_average: 26.774\n", + "psnrcomplete_std: 4.724\n", + "\n", + "psnrblackout_average: 21.757\n", + "psnrblackout_std: 1.126\n", + "\n", + "psnrvisible_average: 28.068\n", + "psnrvisible_std: 4.425\n", + "\n", + "percept_distcomplete_average: 0.300\n", + "percept_distcomplete_std: 0.138\n", + "\n", + "percept_distblackout_average: 0.474\n", + "percept_distblackout_std: 0.053\n", + "\n", + "percept_distvisible_average: 0.255\n", + "percept_distvisible_std: 0.116\n", + "\n", + "aricomplete_average: 0.365\n", + "aricomplete_std: 0.334\n", + "\n", + "ariblackout_average: -0.022\n", + "ariblackout_std: 0.038\n", + "\n", + "arivisible_average: 0.465\n", + "arivisible_std: 0.302\n", + "\n", + "faricomplete_average: 0.305\n", + "faricomplete_std: 0.281\n", + "\n", + "fariblackout_average: 0.018\n", + "fariblackout_std: 0.049\n", + "\n", + "farivisible_average: 0.378\n", + "farivisible_std: 0.269\n", + "\n", + "mioucomplete_average: 0.163\n", + "mioucomplete_std: 0.142\n", + "\n", + "mioublackout_average: 0.012\n", + "mioublackout_std: 0.015\n", + "\n", + "miouvisible_average: 0.202\n", + "miouvisible_std: 0.134\n", + "\n", + "apcomplete_average: 0.289\n", + "apcomplete_std: 0.295\n", + "\n", + "apblackout_average: 0.000\n", + "apblackout_std: 0.006\n", + "\n", + "apvisible_average: 0.363\n", + "apvisible_std: 0.287\n", + "\n", + "arcomplete_average: 0.301\n", + "arcomplete_std: 0.329\n", + "\n", + "arblackout_average: 0.000\n", + "arblackout_std: 0.007\n", + "\n", + "arvisible_average: 0.378\n", + "arvisible_std: 0.327\n", + "\n", + "blackoutcomplete_average: 0.205\n", + "blackoutcomplete_std: 0.404\n", + "\n", + "blackoutblackout_average: 1.000\n", + "blackoutblackout_std: 0.000\n", + "\n", + "blackoutvisible_average: 0.000\n", + "blackoutvisible_std: 0.000\n", + "\n" + ] + } + ], + "source": [ + "for key,value in df.items():\n", + " print(f\"{key}: {value:.3f}\")\n", + " if 'std' in key:\n", + " print('')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/evaluation/clevrer/evaluation_savi.ipynb b/evaluation/clevrer/evaluation_savi.ipynb new file mode 100644 index 0000000..c802408 --- /dev/null +++ b/evaluation/clevrer/evaluation_savi.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# set path to resutls files\n", + "path = '../../out/pretrained/clevrer/savi/results/net_savi'\n", + "\n", + "# load pkl file with results\n", + "df = pd.read_pickle(os.path.join(path, 'blackout_metric_average.pkl'))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "msecomplete_average: 3.352\n", + "msecomplete_std: 5.657\n", + "\n", + "mseblackout_average: 13.360\n", + "mseblackout_std: 5.790\n", + "\n", + "msevisible_average: 0.849\n", + "msevisible_std: 0.543\n", + "\n", + "ssimcomplete_average: 0.934\n", + "ssimcomplete_std: 0.065\n", + "\n", + "ssimblackout_average: 0.814\n", + "ssimblackout_std: 0.052\n", + "\n", + "ssimvisible_average: 0.964\n", + "ssimvisible_std: 0.009\n", + "\n", + "psnrcomplete_average: 35.079\n", + "psnrcomplete_std: 5.418\n", + "\n", + "psnrblackout_average: 25.306\n", + "psnrblackout_std: 2.048\n", + "\n", + "psnrvisible_average: 37.524\n", + "psnrvisible_std: 2.404\n", + "\n", + "percept_distcomplete_average: 0.249\n", + "percept_distcomplete_std: 0.114\n", + "\n", + "percept_distblackout_average: 0.468\n", + "percept_distblackout_std: 0.035\n", + "\n", + "percept_distvisible_average: 0.194\n", + "percept_distvisible_std: 0.029\n", + "\n", + "aricomplete_average: 0.456\n", + "aricomplete_std: 0.266\n", + "\n", + "ariblackout_average: -0.002\n", + "ariblackout_std: 0.009\n", + "\n", + "arivisible_average: 0.571\n", + "arivisible_std: 0.152\n", + "\n", + "faricomplete_average: 0.726\n", + "faricomplete_std: 0.377\n", + "\n", + "fariblackout_average: 0.009\n", + "fariblackout_std: 0.094\n", + "\n", + "farivisible_average: 0.905\n", + "farivisible_std: 0.120\n", + "\n", + "mioucomplete_average: 0.285\n", + "mioucomplete_std: 0.144\n", + "\n", + "mioublackout_average: 0.005\n", + "mioublackout_std: 0.003\n", + "\n", + "miouvisible_average: 0.355\n", + "miouvisible_std: 0.038\n", + "\n", + "apcomplete_average: 0.248\n", + "apcomplete_std: 0.196\n", + "\n", + "apblackout_average: 0.000\n", + "apblackout_std: 0.011\n", + "\n", + "apvisible_average: 0.309\n", + "apvisible_std: 0.170\n", + "\n", + "arcomplete_average: 0.374\n", + "arcomplete_std: 0.302\n", + "\n", + "arblackout_average: 0.000\n", + "arblackout_std: 0.005\n", + "\n", + "arvisible_average: 0.467\n", + "arvisible_std: 0.265\n", + "\n", + "blackoutcomplete_average: 0.200\n", + "blackoutcomplete_std: 0.400\n", + "\n", + "blackoutblackout_average: 1.000\n", + "blackoutblackout_std: 0.000\n", + "\n", + "blackoutvisible_average: 0.000\n", + "blackoutvisible_std: 0.000\n", + "\n" + ] + } + ], + "source": [ + "for key,value in df.items():\n", + " print(f\"{key}: {value:.3f}\")\n", + " if 'std' in key:\n", + " print('')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "loci23", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} |