jwuphysics/Neural network initialization and critical layers.ipynb

## Neural network initialization and critical layers.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments on critical layers and CNN initialization\n",
    "\n",
    "Inspired by the paper [\"Are All Layers Created Equal?\"](https://arxiv.org/pdf/1902.01996.pdf) by Chiyuan Zhang et al. (2019, ICML)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assume each `r1` is like the activations of a neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_examples = 1000\n",
    "\n",
    "r1 = np.random.randn(N_examples, 16, 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAW00lEQVR4nO3dfZBV9Z3n8fc3gIAxCAhNWqTToriKZSSx1+iOu4Ggi/hsMm4IbqCMlk75RKosS3ASIVXRMGPG2WwgRDQqSRkVV0cxGlbG1XXNmDiYQoMSEAPBhi4e1Gh2NFkevvvHvZIGGvr24+3mvF9VXX3v757T93O7+3z69O/ec25kJpKk4vhYtQNIkrqXxS9JBWPxS1LBWPySVDAWvyQVTN9qBwAYNmxY1tfXVzuGJPUqL7/88rbMHN7W9XpE8dfX17N8+fJqx5CkXiUift+e9ZzqkaSCsfglqWAsfkkqGItfkgrG4pekgrH4JalgLH5JKhiLX5IKxuKXpILpEUfuHqzqZz7Z6jLr557bDUkk6S9a3eOPiFER8WxErIqI1yJiRnl8TkRsjIgV5Y9zmq0zKyLWRsTqiJjUlQ9AktQ2lezx7wBuyMxfR8QngJcjYln5tn/MzO82XzgixgJTgBOBI4F/jojjMnNnZwaXJLVPq3v8mdmUmb8uX/4jsAoYeYBVLgQezMw/Z+Y6YC1wameElSR1XJue3I2IeuAzwK/KQ9dGxKsRcU9EDCmPjQTearZaIwf+QyFJ6kYVF39EHAY8Anw9M98HFgDHAOOAJuAfPlq0hdWzha93ZUQsj4jlW7dubXNwSVL7VFT8EdGPUunfn5mPAmTm5szcmZm7gLv4y3ROIzCq2epHAZv2/pqZuTAzGzKzYfjwNr+PgCSpnSp5VU8APwJWZeYdzcZrmy12MbCyfHkJMCUi+kfE0cAY4KXOiyxJ6ohKXtXzV8BXgd9ExIry2M3AVyJiHKVpnPXAVQCZ+VpELAZep/SKoGt8RY8k9RytFn9mvkDL8/ZPHWCdW4FbO5BLktRFPGWDJBWMxS9JBWPxS1LBWPySVDAWvyQVjMUvSQXj+firrJJz9oPn7ZfUedzjl6SCsfglqWAsfkkqGItfkgrG4pekgrH4JalgLH5JKhiLX5IKxuKXpIKx+CWpYCx+SSoYi1+SCsaTtEkqjjmH72f8ve7NUWXu8UtSwVj8klQwFr8kFYzFL0kFY/FLUsFY/JJUMBa/JBWMxS9JBWPxS1LBWPySVDCeskGSCnYqB/f4JalgWi3+iBgVEc9GxKqIeC0iZpTHh0bEsoh4o/x5SLN1ZkXE2ohYHRGTuvIBSJLappI9/h3ADZl5AnAacE1EjAVmAs9k5hjgmfJ1yrdNAU4EzgZ+EBF9uiK8JKntWi3+zGzKzF+XL/8RWAWMBC4EFpUXWwRcVL58IfBgZv45M9cBa4FTOzu4JKl92jTHHxH1wGeAXwEjMrMJSn8cgJryYiOBt5qt1lge2/trXRkRyyNi+datW9ueXJLULhUXf0QcBjwCfD0z3z/Qoi2M5T4DmQszsyEzG4YPH15pDElSB1VU/BHRj1Lp35+Zj5aHN0dEbfn2WmBLebwRGNVs9aOATZ0TV5LUUZW8qieAHwGrMvOOZjctAaaXL08HHm82PiUi+kfE0cAY4KXOiyxJ6ohKDuD6K+CrwG8iYkV57GZgLrA4Ii4HNgCXAGTmaxGxGHid0iuCrsnMnZ2eXJLULq0Wf2a+QMvz9gAT97POrcCtHcglSeoiHrkrSQVj8UtSwVj8klQwnp2zHepnPlntCJLUbu7xS1LBWPySVDAWvyQVjMUvSQVj8UtSwVj8klQwFr8kFYzFL0kFY/FLUsFY/JJUMBa/JBWMxS9JBWPxS1LBeHZOSQeXOYdXO0GP5x6/JBWMxS9JBWPxS1LBWPySVDAWvyQVjMUvSQVj8UtSwVj8klQwHsAlSfuzv4PB5rzXvTk6mXv8klQwFr8kFYzFL0kFY/FLUsG0WvwRcU9EbImIlc3G5kTExohYUf44p9ltsyJibUSsjohJXRVcktQ+lezx3wec3cL4P2bmuPLHUwARMRaYApxYXucHEdGns8JKkjqu1eLPzOeBdyr8ehcCD2bmnzNzHbAWOLUD+SRJnawjc/zXRsSr5amgIeWxkcBbzZZpLI/tIyKujIjlEbF869atHYghSWqL9hb/AuAYYBzQBPxDeTxaWDZb+gKZuTAzGzKzYfjw4e2MIUlqq3YVf2ZuzsydmbkLuIu/TOc0AqOaLXoUsKljESVJnaldxR8Rtc2uXgx89IqfJcCUiOgfEUcDY4CXOhZRktSZWj1XT0Q8AIwHhkVEIzAbGB8R4yhN46wHrgLIzNciYjHwOrADuCYzd3ZNdElSe7Ra/Jn5lRaGf3SA5W8Fbu1IKElS1/HIXUkqGItfkgrG4pekgrH4JalgLH5JKhiLX5IKxuKXpIKx+CWpYFo9gEs9Q/3MJ1tdZv3cc7shiaTezj1+SSoYi1+SCsbil6SCsfglqWAsfkkqGItfkgrG4pekgrH4JalgLH5JKhiLX5IKxuKXpIKx+CWpYCx+SSoYi1+SCsbil6SCsfglqWAsfkkqGItfkgrG4pekgrH4JalgLH5JKhiLX5IKxuKXpIJptfgj4p6I2BIRK5uNDY2IZRHxRvnzkGa3zYqItRGxOiImdVVwSVL7VLLHfx9w9l5jM4FnMnMM8Ez5OhExFpgCnFhe5wcR0afT0kqSOqzV4s/M54F39hq+EFhUvrwIuKjZ+IOZ+efMXAesBU7tpKySpE7Q3jn+EZnZBFD+XFMeHwm81Wy5xvLYPiLiyohYHhHLt27d2s4YkqS26uwnd6OFsWxpwcxcmJkNmdkwfPjwTo4hSdqfvu1cb3NE1GZmU0TUAlvK443AqGbLHQVs6khASWrRnMOrnaDXau8e/xJgevnydODxZuNTIqJ/RBwNjAFe6lhESVJnanWPPyIeAMYDwyKiEZgNzAUWR8TlwAbgEoDMfC0iFgOvAzuAazJzZxdllyS1Q6vFn5lf2c9NE/ez/K3ArR0JVU31M5+sdgRJ6lIeuStJBWPxS1LBWPySVDAWvyQVjMUvSQVj8UtSwVj8klQwFr8kFYzFL0kFY/FLUsFY/JJUMBa/JBWMxS9JBdPeN2KRpOLa35vAzHmve3O0k3v8klQwFr8kFYxTPeoS77//Plu2bGH79u3VjiKgX79+1NTUMGjQoGpHUQ9g8avTvf/++2zevJmRI0cycOBAIqLakQotM/nwww/ZuHEjgOUvp3rU+bZs2cLIkSM59NBDLf0eICI49NBDGTlyJFu2bKl2HPUAFr863fbt2xk4cGC1Y2gvAwcOdOpNgMWvLuKefs/jz0QfsfglqWAsfkkqGF/Vo25VP/PJqt7/+rnnVvX+pZ7APX5JKhiLX5IKxuKXKjBnzhwigt/+9rdMmjSJj3/849TV1XHvvfcC8JOf/ITjjz+eww47jAkTJvDmm2/usf5dd93FySefzIABAxg2bBiXX34577zzzh7LzJs3j9NPP52hQ4cyePBgTjvtNJ58cs+psfXr1xMR3Hnnndxyyy3U1tYyePBgzj//fBobG7v2m6CDhsUvtcEll1zCueeey2OPPcYpp5zC1772NW6++WYWLFjA3Llzuffee1m9ejVTp07dvc7MmTO5+uqrOfPMM1myZAm33347S5cuZfLkyezcuXP3cuvXr+eKK67g4Ycf5qGHHqKhoYHzzjuPn//85/vk+M53vsPatWu55557+N73vseLL77IpZde2i3fA/V+PrkrtcGNN97ItGnTAGhoaOCJJ57gzjvvZN26dbtPhdDU1MSMGTP4/e9/T2Zy++23M3v2bG655ZbdX+e4447jjDPO4IknnuCiiy4C4Lvf/e7u23ft2sXEiRNZs2YNP/zhD5k8efIeOT71qU/x05/+dPf1rVu3cuONN7Jp0yaOPPLILnv8Oji4xy+1QfMCHjJkCDU1NZx22ml7nP/m+OOPB+Ctt95i2bJl7Nq1i0svvZQdO3bs/vjc5z7HoEGDeP7553ev9/LLL3PeeecxYsQI+vbtS79+/Vi2bBmrV6/eJ8e55+756qSTTjoJgA0bNnTq49XByT1+qQ2GDBmyx/VDDjmkxTGAP/3pT7vPjXPssce2+PXefvttoPRHYuLEiYwdO5bvf//71NXV0bdvX775zW+yatWqfdYbOnToHtf79++/+z6l1nSo+CNiPfBHYCewIzMbImIo8BBQD6wH/ktmvtuxmFLvdMQRRwDw9NNP7/MHovntS5cu5b333mPx4sUcddRRu2//4IMPuieoCqUz9vgnZOa2ZtdnAs9k5tyImFm+flMn3I/U65x11ll87GMfY8OGDZx11ln7Xe6jgu/Xr9/usTVr1vCLX/xijz8EUmfoiqmeC4Hx5cuLgOew+FVQxxxzDDfddBPXXnstq1ev5vOf/zwDBgzYPf9/xRVXMGHCBM4880z69u3LtGnTuOGGG2hqamL27NnU1dWxa9euaj8MHWQ6WvwJPB0RCdyZmQuBEZnZBJCZTRFR09KKEXElcCVAXV1dB2OotyjiKRNuu+02TjjhBObPn8/8+fOJCEaNGsXEiRMZM2YMACeeeCL3338/t9xyCxdccAHHHHMMc+fOZenSpTz33HPVfQA66ERmtn/liCMzc1O53JcB1wFLMnNws2Xezcx9JzebaWhoyOXLl7c7R2eq9rlkOqKnlOqqVas44YQTqh1DLTiofjZzDq92gn3Nea9b7y4iXs7Mhrau16GXc2bmpvLnLcA/AacCmyOithyqFvAtfySpB2l38UfExyPiEx9dBv4zsBJYAkwvLzYdeLyjISVJnacjc/wjgH8qv6tPX+Cnmbk0Iv4VWBwRlwMbgEs6HlOVqHSaqqdMCUmqjnYXf2b+Dji5hfG3gYkdCSVJ6jqeskGSCsbil6SCsfglqWAsfkkqGItfkgrG4pekgvF8/Ope1T7MvgsOqV+xYgWPPfYY119//T7nyW/Jc889x4QJE3j22WcZP358p+eRWuMev9RBK1as4Fvf+tY+b54u9VQWvyQVjMUvVWDNmjVcfPHF1NTUMGDAAOrq6rjkkku4++67ueyyywAYM2YMEUFEsH79eqD0JuhTp05l0KBBDB48mGnTpvGHP/yhio9Eco5fqsh5553H4MGDWbBgAcOGDWPjxo089dRTnH/++XzjG9/g29/+Ng8//PDud8uqra0F4Itf/CKvvPIKt912G2PGjOGhhx7iuuuuq+ZDkSx+qTXbtm3jjTfe4PHHH+eCCy7YPT516lSg9C5bAOPGjdvjTdWXLVvGCy+8wAMPPMCUKVMAmDRpEpMnT6axsbEbH4G0J4tfasURRxzB6NGjmTlzJps3b2b8+PG73znrQF588UX69OnDl770pT3Gp0yZwtKlS7sqrqrpQK9a6+Y3aTkQ5/ilVkQEy5Yto6GhgVmzZnHccccxevRoFixYcMD1mpqaGDJkyB5voA4wYsSIrowrtco9fqkCo0eP5sc//jGZySuvvMK8efO4+uqrqa+v3+86tbW1vPvuu2zfvn2P8t+8eXM3JD6IVPvYj4OQe/xSG0QE48aN44477gBg5cqV9O/fH4APP/xwj2VPP/10du7cySOPPLLH+IMPPtg9YaX9cI9fasWrr77KjBkz+PKXv8yxxx7Lzp07ue++++jbty9f+MIX6Nu3tBnNnz+f6dOn069fPz796U9z1llnccYZZ3DVVVexbdu23a/qWblyZZUfkYrO4lf36kFPcFXqk5/8JHV1ddxxxx00NjYyYMAATjrpJH72s59xyimnADBnzhwWLlzIXXfdxa5du1i3bh319fU8+uijXH/99cyaNYs+ffpwwQUXMG/ePC666KIqPyoVmcUvtaKmpoZFixYdcJnZs2cze/bsfcaHDx/OAw88sM94ZnZaPqmtClP8lb4RuSQd7HxyV5IKxuKXpIKx+CWpYAozx6+/qOT5jvVzz+2GJJKqwT1+dQlftdLz+DPRRyx+dbp+/frtcxSrqu/DDz/c57xBKiaLX52upqaGjRs38sEHH7iX2QNkJh988AEbN26kpqam2nHUAzjHr043aNAgADZt2sT27durnEZQ+i9sxIgRu382KraDovg9OKvnGTRokCUj9VAHRfFLOggc7Kdf3t/jq8L5q5zjl6SCsfglqWC6bKonIs4Gvgf0Ae7OzLlddV/qfJU+b+KBXlLv0yV7/BHRB5gPTAbGAl+JiLFdcV+SpLbpqqmeU4G1mfm7zPx/wIPAhV10X5KkNuiqqZ6RwFvNrjcCn2u+QERcCVxZvvp/I2J1B+5vGLCtA+tXQ2/MDHvljr+rYpLKHRTf616iN2aGaub+VnRk7X/XnpW6qvhbeiR7HMKZmQuBhZ1yZxHLM7OhM75Wd+mNmaF35u6NmaF35u6NmaF3527Pel011dMIjGp2/ShgUxfdlySpDbqq+P8VGBMRR0fEIcAUYEkX3ZckqQ26ZKonM3dExLXA/6T0cs57MvO1rrivsk6ZMupmvTEz9M7cvTEz9M7cvTEzFCx3ePZESSoWj9yVpIKx+CWpYHpl8UfE0IhYFhFvlD8PaWGZURHxbESsiojXImJGlbKeHRGrI2JtRMxs4faIiP9evv3ViPhsNXLulam1zJeWs74aEf8SESdXI+feWsvdbLl/HxE7I+KvuzPffrK0mjkixkfEivLv8f/u7owtqeB35PCIeCIiXinnvqwaOffKdE9EbImIlfu5vcdti1BR7rZvj5nZ6z6Avwdmli/PBP6uhWVqgc+WL38CWAOM7eacfYA3gdHAIcAre2cAzgF+TunYh9OAX1X5e1tJ5v8ADClfnlztzJXmbrbc/wKeAv66p2cGBgOvA3Xl6zW94XsN3PzRdgkMB94BDqly7v8EfBZYuZ/be9S22Ibcbd4ee+UeP6XTPywqX14EXLT3ApnZlJm/Ll/+I7CK0hHF3amSU1dcCPw4S34JDI6I2m7O2VyrmTPzXzLz3fLVX1I6TqPaKj1NyHXAI8CW7gy3H5Vkngo8mpkbADKzt+RO4BMREcBhlIp/R/fG3CtQ5vPlHPvT07ZFoPXc7dkee2vxj8jMJigVPHDANxKNiHrgM8CvujzZnlo6dcXef3wqWaY7tTXP5ZT2kqqt1dwRMRK4GPhhN+Y6kEq+18cBQyLiuYh4OSKmdVu6/ask9zzgBEoHbv4GmJGZu7onXrv1tG2xPSraHnvsO3BFxD8Dn2zhpr9t49c5jNIe3tcz8/3OyNaWu29hbO/Xz1ayTHeqOE9ETKD0i3ZGlyaqTCW5/xtwU2buLO2IVl0lmfsCpwATgYHAixHxy8xc09XhDqCS3JOAFcAXgGOAZRHxf6qwDbZFT9sW26Qt22OPLf7MPHN/t0XE5oiozcym8r9iLf77GxH9KJX+/Zn5aBdFPZBKTl3R005vUVGeiPg0cDcwOTPf7qZsB1JJ7gbgwXLpDwPOiYgdmflY90TcR6W/H9sy89+Af4uI54GTKT1nVS2V5L4MmJuliee1EbEOOB54qXsitktP2xYr1tbtsbdO9SwBppcvTwce33uB8tzij4BVmXlHN2ZrrpJTVywBppVfUXAa8N5H01hV0mrmiKgDHgW+WuU9z+ZazZ2ZR2dmfWbWA/8DuLqKpQ+V/X48DvzHiOgbEYdSOsvtqm7OubdKcm+g9F8KETGC0lkkf9etKduup22LFWnX9ljtZ6zb+Sz3EcAzwBvlz0PL40cCT5Uvn0Hp37RXKf3LuQI4pwpZz6G0d/Ym8Lflsb8B/qZ8OSi9ac2blOZCG3rA97e1zHcD7zb7vi6vduZKcu+17H1U+VU9lWYGbqT0yp6VlKYse/z3urwtPl3+nV4J/NcekPkBoAnYTmnv/vKevi1WmLvN26OnbJCkgumtUz2SpHay+CWpYCx+SSoYi1+SCsbil6SCsfglqWAsfkkqmP8PVpOGLJ1xI1YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "means1, stds1 = r1.mean((1,2)), r1.std((1,2))\n",
    "\n",
    "plt.hist(means1, label='mean')\n",
    "plt.hist(stds1, label='std')\n",
    "\n",
    "plt.legend(fontsize=16);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Okay now what are the activation statistics of a resnet-like initialization?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD8CAYAAAB6iWHJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANTklEQVR4nO3df4xddZnH8c/HMhUtLMVU7NB2AWOjQRN+bDNC2N1UVza0ISnJElP+EENMJpKyQbNNNG6WzSb7x2b/MBGLsE1kl2aNrAmKjQ6aLsEAapXatJVSWWdZI5M2li04pZTITnn2j3Nobobn9sec7zn3Tuf9Sm7mnHu+Pc/3pp1P7z3n3PM4IgQAs71j0BMAMJwIBwApwgFAinAAkCIcAKQIBwCp85r8YdvvkfQfki6X9BtJn4yIV5Jxv5H0qqQTkmYiYk2TugDa1/SdwxclPR4RqyU9Xq/387GIuJpgAOaHpuGwQdJD9fJDkm5puD8AQ8JNrpC0/fuIWNqz/kpEXJyM+x9Jr0gKSf8SEVtPsc9xSeOS5Hcu/pORS5fNeX7DavF5JwY9hda8Y+rcPIx1YuWbg55CK/7wu6P6v+njzrad9piD7f+UtDzZ9LdnMYcbIuKg7Usk7bD9q4h4MhtYB8dWSXrn+1fEpf+46SzKzA+rLnnbYZlzxrs2nz/oKbTi6D+/MegptOKXdz3Ud9tpwyEiPtFvm+3f2R6NiEO2RyUd7rOPg/XPw7a/I2lMUhoOAIZD0/eA2yV9ul7+tKTvzh5ge4ntC99alvSXkp5tWBdAy5qGwz9JutH2ryXdWK/L9qW2J+ox75P0tO29kn4u6fsR8YOGdQG0rNF1DhFxRNJfJM8flLS+Xn5B0lVN6gDo3rl5aBlAY4QDgBThACBFOABIEQ4AUoQDgBThACBFOABIEQ4AUoQDgBThACBFOABIEQ4AUoQDgBThACBFOABIEQ4AUoQDgFSRcLB9k+3nbU/aflvXK1furbfvs31tiboA2tM4HGwvknSfpHWSrpR0m+0rZw1bJ2l1/RiXdH/TugDaVeKdw5ikyYh4ISLekPSwqjZ5vTZI2haVnZKW1n0uAAypEuGwQtKLPetT9XNnOwbAECkRDlmfvdkNOM9kTDXQHre9y/auE6++1nhyAOamRDhMSVrVs75S0sE5jJFU9cqMiDURsWbRhUsKTA/AXJQIh2ckrbZ9he3FkjaqapPXa7uk2+uzFtdJmo6IQwVqA2hJo45XkhQRM7bvkvRDSYskPRgR+21/tt7+gKQJVR2wJiUdl3RH07oA2tU4HCQpIiZUBUDvcw/0LIekTSVqAegGV0gCSBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4BUV70y19qetr2nftxToi6A9jS+wWxPr8wbVfWneMb29oh4btbQpyLi5qb1AHSjxN2nT/bKlCTbb/XKnB0OZ+0DF7ykh//0a013M3Tue+ljg55Cax77/IcHPYVWvO/flw56Cq2IIyN9t3XVK1OSrre91/Zjtvv+C+pth/fKy28WmB6AueiqV+ZuSZdFxFWSvirp0X47622Hd/F7OF4KDEonvTIj4mhEHKuXJySN2F5WoDaAlnTSK9P2ctuul8fqukcK1AbQkq56Zd4q6U7bM5Jel7SxbpEHYEh11Stzi6QtJWoB6AZH/ACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAqVLt8B60fdj2s3222/a9dbu8fbavLVEXQHtKvXP4N0k3nWL7Okmr68e4pPsL1QXQkiLhEBFPSnr5FEM2SNoWlZ2SltoeLVEbQDu6OuZwpi3zaIcHDImuwuFMWuZVT9IODxgKXf32nbZlHoDh0lU4bJd0e33W4jpJ0xFxqKPaAOagSMcr29+UtFbSMttTkv5e0oh0svPVhKT1kiYlHZd0R4m6ANpTqh3ebafZHpI2lagFoBsc8QOQIhwApAgHACnCAUCKcACQIhwApAgHACnCAUCKcACQIhwApAgHACnCAUCKcACQIhwApAgHACnCAUCKcACQIhwApLpqh7fW9rTtPfXjnhJ1AbSnyD0kVbXD2yJp2ynGPBURNxeqB6BlXbXDAzDPlHrncCaut71XVTObzRGxPxtke1xVs12dP3KRNt/ymQ6n2I3f/l3WAOzc8Gcf+vWgp9CKnxy5ctBTaMXME/23dRUOuyVdFhHHbK+X9KiqjttvExFbJW2VpIvefWnaMg9A+zo5WxERRyPiWL08IWnE9rIuagOYm07CwfZy266Xx+q6R7qoDWBuumqHd6ukO23PSHpd0sa6CxaAIdVVO7wtqk51ApgnuEISQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJBqHA62V9l+wvYB2/tt352Mse17bU/a3mf72qZ1AbSrxD0kZyT9TUTstn2hpF/Y3hERz/WMWaeqT8VqSR+VdH/9E8CQavzOISIORcTuevlVSQckrZg1bIOkbVHZKWmp7dGmtQG0p+gxB9uXS7pG0s9mbVoh6cWe9Sm9PUDe2se47V22d70xc7zk9ACchWLhYPsCSY9I+lxEHJ29Ofkjad+KiNgaEWsiYs3i895danoAzlKRcLA9oioYvhER306GTEla1bO+UlVDXQBDqsTZCkv6uqQDEfHlPsO2S7q9PmtxnaTpiDjUtDaA9pQ4W3GDpE9J+qXtPfVzX5L0x9LJdngTktZLmpR0XNIdBeoCaFHjcIiIp5UfU+gdE5I2Na0FoDtcIQkgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASBEOAFKEA4AU4QAgRTgASHXVDm+t7Wnbe+rHPU3rAmhXV+3wJOmpiLi5QD0AHeiqHR6AeabEO4eTTtEOT5Kut71XVTObzRGxv88+xiWNS9J5Sy/Wf3/yopJTHApX/NVPBz2F1mw7uOf0g+ahxy/58aCn0IpN//pS321dtcPbLemyiLhK0lclPdpvP73t8BYtWVJqegDOUift8CLiaEQcq5cnJI3YXlaiNoB2dNIOz/byepxsj9V1jzStDaA9XbXDu1XSnbZnJL0uaWPdBQvAkOqqHd4WSVua1gLQHa6QBJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkCAcAKcIBQIpwAJAiHACkStxg9nzbP7e9t26H9w/JGNu+1/ak7X22r21aF0C7Stxg9g+SPh4Rx+pb1D9t+7GI2NkzZp2k1fXjo5Lur38CGFIl2uHFWz0pJI3Uj9l3lt4gaVs9dqekpbZHm9YG0J5STW0W1belPyxpR0TMboe3QtKLPetTop8mMNSKhENEnIiIqyWtlDRm+yOzhmS3rk/7Vtget73L9q4Tr71WYnoA5qDo2YqI+L2kH0m6adamKUmretZXqmqom+2DXpnAEChxtuK9tpfWy++S9AlJv5o1bLuk2+uzFtdJmo6IQ01rA2hPibMVo5Iesr1IVdh8KyK+Z/uz0sl2eBOS1kualHRc0h0F6gJoUYl2ePskXZM8/0DPckja1LQWgO5whSSAFOEAIEU4AEgRDgBShAOAFOEAIEU4AEgRDgBShAOAFOEAIEU4AEgRDgBShAOAFOEAIEU4AEgRDgBShAOAFOEAIEU4AEh11Stzre1p23vqxz1N6wJoV1e9MiXpqYi4uUA9AB0ocffpkHS6XpkA5hlXv9sNd1L1rPiFpA9Iui8ivjBr+1pJj6jqfHVQ0uaI2N9nX+OSxuvVD0p6vvEEz8wySf/bUa0u8brmny5f22UR8d5sQ5FwOLmzqvPVdyT9dUQ82/P8H0l6s/7osV7SVyJidbHCBdjeFRFrBj2P0nhd88+wvLZOemVGxNGIOFYvT0gasb2sZG0AZXXSK9P2ctuul8fqukea1gbQnq56Zd4q6U7bM5Jel7QxSn6eKWProCfQEl7X/DMUr63oMQcA5w6ukASQIhwApBZ8ONi+yfbztidtf3HQ8ynF9oO2D9t+9vSj5w/bq2w/YftAfbn+3YOeUwln8jWEzue0kI851AdR/0vSjaou0HpG0m0R8dxAJ1aA7T9XdeXqtoj4yKDnU4rtUUmjEbHb9oWqLr67Zb7/ndVn85b0fg1B0t3J1xA6s9DfOYxJmoyIFyLiDUkPS9ow4DkVERFPSnp50PMoLSIORcTuevlVSQckrRjsrJqLylB9DWGhh8MKSS/2rE/pHPiHtlDYvlzSNZJ+NtiZlGF7ke09kg5L2hERA31dCz0cnDy3cD9nzSO2L1D1fZ3PRcTRQc+nhIg4ERFXS1opacz2QD8OLvRwmJK0qmd9paovhmGI1Z/JH5H0jYj49qDnU1q/ryF0baGHwzOSVtu+wvZiSRslbR/wnHAK9YG7r0s6EBFfHvR8SjmTryF0bUGHQ0TMSLpL0g9VHdj6Vr+vks83tr8p6aeSPmh7yvZnBj2nQm6Q9ClJH++5s9j6QU+qgFFJT9jep+o/rR0R8b1BTmhBn8oE0N+CfucAoD/CAUCKcACQIhwApAgHACnCAUCKcACQ+n8KWySmwrgoHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAKmElEQVR4nO3dXYhc9RnH8d/P3QRNjKbWKJoNjVJJsYUaWQKSVjC2JVZRoaUkoFARvFHRtiJa6EWhV70Qe2GFELUWU0MbFUR8qVRFba3VvLSabGxjsM02auJbEyO4TXx6sZOy6qZ75sz575l9+v3A4s7OMDxD/O6ZOTP7/zsiBCCPo9oeAECziBpIhqiBZIgaSIaogWQGS9zpwLy5Mbhgfom7btXswUNtj1DMUaM5f78fGvqo7RGK+PDNffr3vz7wZNcViXpwwXyd+pOrS9x1qxad9G7bIxRzzA1Htz1CEft+Otb2CEW8dM3dR7wu569n4P8YUQPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQTKWoba+0/YrtHbZvKj0UgPqmjNr2gKTbJF0g6UxJq22fWXowAPVUOVIvk7QjInZGxJik9ZIuKTsWgLqqRL1Q0q4Jl0c7P/sY21fZftH2i4f2H2hqPgBdqhL1ZH/e9anVCiNiTUQMR8TwwLy5vU8GoJYqUY9KWjTh8pCk3WXGAdCrKlG/IOkM26fZni1plaQHy44FoK4pF0mIiIO2r5H0mKQBSXdGxNbikwGopdLKJxHxsKSHC88CoAF8ogxIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkquzQcaftPbZfno6BAPSmypH6F5JWFp4DQEOmjDoinpb0zjTMAqABjb2mZtsdoD80FjXb7gD9gbPfQDJEDSRT5S2teyU9J2mJ7VHbV5YfC0BdVfbSWj0dgwBoBk+/gWSIGkiGqIFkiBpIhqiBZIgaSIaogWSmfJ+6js8fu1frv/LzEnfdqtv2ntf2CMU88r0vtj1CESffM7/tEYqIt2cd8TqO1EAyRA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJFNljbJFtp+0PWJ7q+3rpmMwAPVU+ez3QUk/iIhNtudJ2mj78YjYVng2ADVU2Xbn9YjY1Pl+v6QRSQtLDwagnq5eU9teLGmppOcnue6/2+68+85HzUwHoGuVo7Z9rKT7JF0fEfs+ef3EbXc+cwLn34C2VKrP9iyNB70uIu4vOxKAXlQ5+21Jd0gaiYhbyo8EoBdVjtTLJV0uaYXtLZ2vbxaeC0BNVbbdeVaSp2EWAA3gjBaQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJVFl48Gjbf7L95862Oz+ejsEA1FNl250PJa2IiPc7SwU/a/uRiPhj4dkA1FBl4cGQ9H7n4qzOV5QcCkB9VRfzH7C9RdIeSY9HBNvuAH2qUtQRcSgizpI0JGmZ7S9Nchu23QH6QFf1RcR7kp6StLLINAB6VuXs9wLb8zvfHyPpa5K2lx4MQD1Vzn6fIulu2wMa/yXw64h4qOxYAOqqcvb7LxrfkxrADMAZLSAZogaSIWogGaIGkiFqIBmiBpIhaiAZogaSqfKJsq6N7jhRN1x6ZYm7btU/fuS2Ryjmq1/4W9sjFPGHt89se4QiDj555Os4UgPJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kEzlqDsL+m+2zaKDQB/r5kh9naSRUoMAaEbVbXeGJF0oaW3ZcQD0quqR+lZJN0o64iZZE/fSGjv4QSPDAehelR06LpK0JyI2/q/bTdxLa/bgnMYGBNCdKkfq5ZIutv2apPWSVti+p+hUAGqbMuqIuDkihiJisaRVkp6IiMuKTwagFt6nBpLpajmjiHhK41vZAuhTHKmBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIptJyRp2VRPdLOiTpYEQMlxwKQH3drFF2XkS8VWwSAI3g6TeQTNWoQ9JvbW+0fdVkN2DbHaA/VH36vTwidts+SdLjtrdHxNMTbxARayStkaTj55waDc8JoKJKR+qI2N357x5JD0haVnIoAPVV2SBvru15h7+X9A1JL5ceDEA9VZ5+nyzpAduHb/+riHi06FQAapsy6ojYKenL0zALgAbwlhaQDFEDyRA1kAxRA8kQNZAMUQPJEDWQTDd/elnZhycM6NXvHF/irlt12reea3uEYn65e0vbIxTxu5N+3/YIRVx9194jXseRGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZCpFbXu+7Q22t9sesX1O6cEA1FP1s98/k/RoRHzb9mxJcwrOBKAHU0Zt+zhJ50r6riRFxJiksbJjAairytPv0yXtlXSX7c2213bW//6YidvuHDpwoPFBAVRTJepBSWdLuj0ilko6IOmmT94oItZExHBEDA/M/VTzAKZJlahHJY1GxPOdyxs0HjmAPjRl1BHxhqRdtpd0fnS+pG1FpwJQW9Wz39dKWtc5871T0hXlRgLQi0pRR8QWScOFZwHQAD5RBiRD1EAyRA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJTRm17ie0tE7722b5+OoYD0L0p1yiLiFcknSVJtgck/VPSA4XnAlBTt0+/z5f0akT8vcQwAHrXbdSrJN072RVsuwP0h8pRd9b8vljSbya7nm13gP7QzZH6AkmbIuLNUsMA6F03Ua/WEZ56A+gflaK2PUfS1yXdX3YcAL2quu3OB5I+W3gWAA3gE2VAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJOOIaP5O7b2Suv3zzBMlvdX4MP0h62PjcbXncxGxYLIrikRdh+0XI2K47TlKyPrYeFz9iaffQDJEDSTTT1GvaXuAgrI+Nh5XH+qb19QAmtFPR2oADSBqIJm+iNr2Stuv2N5h+6a252mC7UW2n7Q9Ynur7evanqlJtgdsb7b9UNuzNMn2fNsbbG/v/Nud0/ZM3Wr9NXVng4C/any5pFFJL0haHRHbWh2sR7ZPkXRKRGyyPU/SRkmXzvTHdZjt70salnRcRFzU9jxNsX23pGciYm1nBd05EfFe23N1ox+O1Msk7YiInRExJmm9pEtanqlnEfF6RGzqfL9f0oikhe1O1QzbQ5IulLS27VmaZPs4SedKukOSImJspgUt9UfUCyXtmnB5VEn+5z/M9mJJSyU93+4kjblV0o2SPmp7kIadLmmvpLs6Ly3W2p5xi9j3Q9Se5Gdp3mezfayk+yRdHxH72p6nV7YvkrQnIja2PUsBg5LOlnR7RCyVdEDSjDvH0w9Rj0paNOHykKTdLc3SKNuzNB70uojIsrzyckkX235N4y+VVti+p92RGjMqaTQiDj+j2qDxyGeUfoj6BUln2D6tc2JilaQHW56pZ7at8ddmIxFxS9vzNCUibo6IoYhYrPF/qyci4rKWx2pERLwhaZftJZ0fnS9pxp3YrLTud0kRcdD2NZIekzQg6c6I2NryWE1YLulySS/Z3tL52Q8j4uEWZ8LUrpW0rnOA2Snpipbn6Vrrb2kBaFY/PP0G0CCiBpIhaiAZogaSIWogGaIGkiFqIJn/ADPYkI5cmkQYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "im = np.random.randn(4, 4)\n",
    "plt.imshow(im)\n",
    "plt.show()\n",
    "plt.imshow(im.repeat(2, axis=0).repeat(2, axis=1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def upsample(a, b):\n",
    "    return np.array(a).repeat(b, axis=1).repeat(b, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "r2 = (\n",
    "    np.random.randn(N_examples, 16, 16) +\n",
    "    upsample(np.random.randn(N_examples, 8, 8), 2) +\n",
    "    upsample(np.random.randn(N_examples, 4, 4), 4) +\n",
    "    upsample(np.random.randn(N_examples, 2, 2), 8) +\n",
    "    upsample(np.random.randn(N_examples, 1, 1), 16)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f32d4e89a90>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAUmUlEQVR4nO3de5CV9Z3n8fc33HpLRRobCCqdjorrZUyYsstojbVRkVFGRE3KjcFarUTXTBmjU5W1RCcBshsNU2bdTUVChMRLUt5L4yUqI+tquabMJDghLgZBMrbYkRKUeElFXS7f/aOPpIFu+87T58f7VdXV5/md5znnc7rhw4/nec5zIjORJJXlY1UHkCQNPstdkgpkuUtSgSx3SSqQ5S5JBRpZdQCApqambGlpqTqGJNWV55577o3MnNDVfcOi3FtaWlixYkXVMSSprkTEK93d524ZSSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVKBhcSpkb3zwwQds3ryZd999l23btlUdZ683YsQI9ttvP8aPH8+YMWOqjiNpF3VR7h988AHr16+nsbGRlpYWRo0aRURUHWuvlZls2bKFd955h/Xr19Pc3GzBS8NMXeyW2bx5M42NjTQ1NTF69GiLvWIRwejRo2lqaqKxsZHNmzdXHUnSLupi5v7uu+/i5QmGp7Fjx9LW1sbkyZOrjqLhZsH+3Yy/vWdz7KXqYua+bds2Ro0aVXUMdWHUqFEeA5GGobqYuQPuihmm/L2oz7qb0YOz+kFUFzN3SVLfWO6SVKC62S3TnZa5j1T6/G0Lz6j0+SWpK87cJalAlrskFchyr8iCBQuICF588UVOO+009tlnH5qbm7nlllsA+OlPf8oRRxzBvvvuy8knn8zvf//7nbZfunQpn/70p2loaKCpqYmLLrpotzcT3XjjjZxwwgmMHz+ecePGcfzxx/PIIzvvxmprayMiuOmmm5g3bx6TJ09m3LhxnHnmmbS3tw/tD0HSkLHcK3buuedyxhln8MADD3Dsscfy5S9/mWuuuYbFixezcOFCbrnlFtasWcOcOXN2bDN37lwuvfRSTj31VB566CGuv/56li1bxsyZM3c657ytrY2LL76Ye++9l7vvvpvW1lZmzZrFY489tluO73znO6xbt46bb76Z733vezz77LOcf/75e+RnIGnw1f0B1Xp35ZVXcsEFFwDQ2trKww8/zE033cTLL7/M2LFjAdiwYQNXXHEFr7zyCpnJ9ddfz/z585k3b96Oxzn88MM58cQTefjhhzn77LMB+O53v7vj/u3btzN9+nTWrl3LD3/4Q2bOnLlTjk984hPccccdO5Y3bdrElVdeyWuvvcaBBx44ZK9f0tBw5l6xziXb2NjIxIkTOf7443cUO8ARRxwBwKuvvsry5cvZvn07559/Plu3bt3x9ZnPfIaxY8fy9NNP79juueeeY9asWUyaNImRI0cyatQoli9fzpo1a3bLccYZO5/1c8wxxwCwfv36QX29kvYMZ+4Va2xs3Gl59OjRXY4BvP/++2zcuBGAww47rMvHe/PNN4GOfwimT5/OUUcdxfe//32am5sZOXIk3/zmN1m9evVu240fP36n5Q+v8vj+++/341WVparTbT3NVgNhudeZAw44AIDHH398t38EOt+/bNky3n77be655x4OPvjgHff/+c9/3jNBJVXKcq8zM2bM4GMf+xjr169nxowZ3a73YYl3vuDa2rVr+cUvfrFT2Usqk+VeZw499FCuuuoqLrvsMtasWcNnP/tZGhoaduyPv/jiizn55JM59dRTGTlyJBdccAFf//rX2bBhA/Pnz6e5uZnt27dX/TL6rOp3Ikv1pu7LfW/cL3nddddx5JFHsmjRIhYtWkREMGXKFKZPn87UqVMBOProo7n99tuZN28es2fP5tBDD2XhwoUsW7aMp556qtoXIGnIRWZWnYHW1tZcsWJFt/evXr2aI488cg8mUl/sid/P3jhzr/uJy0dd2rfbbbzkb19ExHOZ2drVfZ4KKUkFstwlqUCWuyQVyHKXpAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAdX9tmX69xXlQn3/w3y69cuVKHnjgAS6//PLdrrPelaeeeoqTTz6ZJ598kpNOOmnQ80iqP87ch6GVK1fyrW99a7cPvJak3uqx3CNiSkQ8GRGrI+KFiLiiNj4+IpZHxEu1742dtrk6ItZFxJqIOG0oX4AkaXe9mblvBb6emUcCxwNfjYijgLnAE5k5FXiitkztvvOAo4HTgR9ExIihCF/P1q5dyznnnMPEiRNpaGigubmZc889lx/96Ed86UtfAmDq1KlEBBFBW1sb0PHB1XPmzGHs2LGMGzeOCy64gLfeeqvCVyJpOOpxn3tmbgA21G6/GxGrgYOAs4CTaqvdBjwFXFUbvyszPwBejoh1wHHAs4Mdvp7NmjWLcePGsXjxYpqamvjDH/7Ao48+yplnnsk3vvENvv3tb3Pvvffu+NSkyZMnA/C5z32O3/72t1x33XVMnTqVu+++m6997WtVvhRJw1CfDqhGRAvw18C/AJNqxU9mboiIibXVDgJ+2Wmz9trYro91CXAJQHNzc19z17U33niDl156iQcffJDZs2fvGJ8zZw7Q8WlLANOmTdvpg7CXL1/OM888w5133sl5550HwGmnncbMmTNpb2/fg69A0nDX6wOqEbEvcB/wD5n5zket2sXYbp8IkplLMrM1M1snTJjQ2xhFOOCAAzjkkEOYO3cuS5cu5aWXXurVds8++ywjRozg85///E7jHxa9JH2oV+UeEaPoKPbbM/P+2vDrETG5dv9kYGNtvB2Y0mnzg4HXBiduGSKC5cuX09raytVXX83hhx/OIYccwuLFiz9yuw0bNtDY2LjTh14DTJo0aSjjSqpDvTlbJoAfA6sz84ZOdz0EXFi7fSHwYKfx8yJiTER8EpgK/GrwIpfhkEMO4Sc/+QmbNm3iN7/5DaeccgqXXnopjz32WLfbTJ48mT/+8Y9s2bJlp/HXX399qONKqjO9mbn/DfCfgFMiYmXt6++AhcCMiHgJmFFbJjNfAO4BfgcsA76amduGJH0BIoJp06Zxww0d/26uWrWKMWPGAPDee+/ttO4JJ5zAtm3buO+++3Yav+uuu/ZMWEl1ozdnyzxD1/vRAaZ3s821wLUDyFW0559/niuuuIIvfOELHHbYYWzbto1bb72VkSNHcsoppzByZMevZdGiRVx44YWMGjWKT33qU8yYMYMTTzyRr3zlK7zxxhs7zpZZtWpVxa9I0nBTwOUH6u/T0j/+8Y/T3NzMDTfcQHt7Ow0NDRxzzDH8/Oc/59hjjwVgwYIFLFmyhKVLl7J9+3ZefvllWlpauP/++7n88su5+uqrGTFiBLNnz+bGG2/k7LPPrvhVSRpO6r/c69DEiRO57bbbPnKd+fPnM3/+/N3GJ0yYwJ133rnbeOZuJyRJ2ot5bRlJKpDlLkkFstwlqUCWuyQVqG7K3QOGw5O/F2l4qotyHz169G5v6NHw8N577+1405Wk4aMuyr2pqYn29nY2b97Mli1bnC1WLDPZsmULmzdvpr29nQMOOKDqSJJ2URfnue+///6MGTOGTZs28eabb7J169aqI+31Ro4cueNDRhoaGqqOI2kXdVHuAA0NDUyZMqXnFSVJ9bFbRpLUN5a7JBWobnbLSNoLLNi/m/H6u0Bg1Zy5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKXpAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCCv5y4NUy1zH6nsudsWnlHZc2twOHOXpAJZ7pJUIMtdkgpkuUtSgXo8oBoRNwOzgI2Z+Ve1sQXAfwY21Va7JjMfrd13NXARsA24PDP/eQhyqyJVHuST1Hu9mbnfCpzexfj/yMxpta8Pi/0o4Dzg6No2P4iIEYMVVpLUOz2We2Y+DWzu5eOdBdyVmR9k5svAOuC4AeSTJPXDQPa5XxYRz0fEzRHRWBs7CHi10zrttbHdRMQlEbEiIlZs2rSpq1UkSf3U33JfDBwKTAM2AP+9Nh5drJtdPUBmLsnM1sxsnTBhQj9jSJK60q9yz8zXM3NbZm4HlvKXXS/twJROqx4MvDawiJKkvupXuUfE5E6L5wCrarcfAs6LiDER8UlgKvCrgUWUJPVVb06FvBM4CWiKiHZgPnBSREyjY5dLG/AVgMx8ISLuAX4HbAW+mpnbhia6JKk7PZZ7Zn6xi+Eff8T61wLXDiSUJGlgfIeqJBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKXpAJZ7pJUIMtdkgrU44XDJOkjLdi/6gTqgjN3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kqkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKXpAJZ7pJUoB7LPSJujoiNEbGq09j4iFgeES/Vvjd2uu/qiFgXEWsi4rShCi5J6l5vZu63AqfvMjYXeCIzpwJP1JaJiKOA84Cja9v8ICJGDFpaSVKv9Fjumfk0sHmX4bOA22q3bwPO7jR+V2Z+kJkvA+uA4wYpqySpl/q7z31SZm4AqH2fWBs/CHi103rttbHdRMQlEbEiIlZs2rSpnzEkSV0Z7AOq0cVYdrViZi7JzNbMbJ0wYcIgx5CkvVt/y/31iJgMUPu+sTbeDkzptN7BwGv9jydJ6o+R/dzuIeBCYGHt+4Odxu+IiBuAA4GpwK8GGlLSXm7B/t2Mv71nc9SRHss9Iu4ETgKaIqIdmE9Hqd8TERcB64FzATLzhYi4B/gdsBX4amZuG6LskqRu9FjumfnFbu6a3s361wLXDiSUJGlgfIeqJBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKXpAL198JhqlDL3EeqjiBpmHPmLkkFstwlqUDulpG0m77s+mtrGMIg6jdn7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kqkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKtCAPiA7ItqAd4FtwNbMbI2I8cDdQAvQBvzHzPzjwGJKkvpiQOVec3JmvtFpeS7wRGYujIi5teWrBuF5JGlnC/bvZvztPZtjGBqK3TJnAbfVbt8GnD0EzyFJ+ggDLfcEHo+I5yLiktrYpMzcAFD7PrGrDSPikohYERErNm3aNMAYkqTOBrpb5m8y87WImAgsj4gXe7thZi4BlgC0trbmAHNIkjoZ0Mw9M1+rfd8I/Aw4Dng9IiYD1L5vHGhISVLf9LvcI2KfiNjvw9vA3wKrgIeAC2urXQg8ONCQkqS+GchumUnAzyLiw8e5IzOXRcSvgXsi4iJgPXDuwGNKkvqi3+Wemf8GfLqL8TeB6QMJJUkaGN+hKkkFstwlqUCWuyQVyHKXpAINxrVlJGlYaZn7SJ/Wb1t4xhAlqY4zd0kqkOUuSQWy3CWpQJa7JBXIcpekAnm2jKReaWuYU3UE9YEzd0kqkOUuSQWy3CWpQJa7JBXIA6qSitPdwd+W9+/Yw0mq48xdkgpkuUtSgdwtMwB9vfKcJO0pztwlqUCWuyQVyHKXpAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBSriTUy+mUiSdubMXZIKVMTMXZIGosr//bctPGNIHtdyl7TX2JsuBexuGUkqkOUuSQWy3CWpQJa7JBVoyA6oRsTpwPeAEcCPMnPhUD2XJA1EdwdaoX4Ptg7JzD0iRgCLgJnAUcAXI+KooXguSdLuhmq3zHHAusz8t8z8f8BdwFlD9FySpF0M1W6Zg4BXOy23A5/pvEJEXAJcUlv8U0SsGaIsvdUEvFFxhv6ox9z1mBnqM/egZY7BeJDeG0Y/61m9XbFfmeOf+rrFTj7R3R1DVe5d/TnInRYylwBLhuj5+ywiVmRma9U5+qoec9djZqjP3PWYGeoz93DLPFS7ZdqBKZ2WDwZeG6LnkiTtYqjK/dfA1Ij4ZESMBs4DHhqi55Ik7WJIdstk5taIuAz4ZzpOhbw5M18YiucaRMNmF1Ef1WPueswM9Zm7HjNDfeYeVpkjM3teS5JUV3yHqiQVyHKXpAJZ7p1ExH+LiOcjYmVEPB4RB1adqTci4vqIeLGW/WcRMa7qTD2JiHMj4oWI2B4Rw+b0sa5ExOkRsSYi1kXE3Krz9EZE3BwRGyNiVdVZeisipkTEkxGxuvZn44qqM/VGRDRExK8i4re13N+qOhO4z30nETE2M9+p3b4cOCoz/77iWD2KiL8F/nftQPY/AWTmVRXH+kgRcSSwHbgJ+C+ZuaLiSF2qXUpjLTCDjlN8fw18MTN/V2mwHkTEfwD+BPwkM/+q6jy9ERGTgcmZ+a8RsR/wHHB2HfysA9gnM/8UEaOAZ4ArMvOXVeZy5t7Jh8Vesw+7vPFquMrMxzNza23xl3S8r2BYy8zVmVn1u5J7oy4vpZGZTwObq87RF5m5ITP/tXb7XWA1He92H9ayw59qi6NqX5V3h+W+i4i4NiJeBc4H5lWdpx++DDxWdYiCdHUpjWFfOPUuIlqAvwb+pdokvRMRIyJiJbARWJ6Zlefe68o9Iv5XRKzq4ussgMz8x8ycAtwOXFZt2r/oKXdtnX8EttKRvXK9yVwHeryUhgZXROwL3Af8wy7/mx62MnNbZk6j43/Nx0VE5bvC9roPyM7MU3u56h3AI8D8IYzTaz3ljogL6bjC0fQcJgdS+vCzHs68lMYeVNtnfR9we2beX3WevsrMtyLiKeB0oNKD2XvdzP2jRMTUTouzgRerytIXtQ9GuQqYnZl/rjpPYbyUxh5SOzD5Y2B1Zt5QdZ7eiogJH56hFhH/DjiVYdAdni3TSUTcB/x7Os7ieAX4+8z8Q7WpehYR64AxwJu1oV8O97N8IuIc4PvABOAtYGVmnlZtqq5FxN8B/5O/XErj2ooj9Sgi7gROouMytK8D8zPzx5WG6kFEnAj8H+D/0vF3EOCazHy0ulQ9i4hPAbfR8efjY8A9mflfq01luUtSkdwtI0kFstwlqUCWuyQVyHKXpAJZ7pJUIMtdkgpkuUtSgf4/iAl5Trieo0cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "means2, stds2 = r2.mean((1,2)), r2.std((1,2))\n",
    "\n",
    "plt.hist(means2, label='mean')\n",
    "plt.hist(stds2, label='std')\n",
    "\n",
    "plt.legend(fontsize=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How does this change Kaiming initialization?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
No results found