notebooks/HyperparameterSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jax\n",
    "import jax.numpy as jnp\n",
    "import numpy as np # get rid of this eventually\n",
    "import argparse\n",
    "from jax import jit\n",
    "from jax.experimental.ode import odeint\n",
    "from functools import partial # reduces arguments to function by making some subset implicit\n",
    "\n",
    "from jax.experimental import stax\n",
    "from jax.experimental import optimizers\n",
    "\n",
    "import os, sys, time\n",
    "sys.path.append('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../experiment_dblpend/')\n",
    "\n",
    "from lnn import lagrangian_eom_rk4, lagrangian_eom, unconstrained_eom\n",
    "from data import get_dataset\n",
    "from models import mlp as make_mlp\n",
    "from utils import wrap_coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../hyperopt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HyperparameterSearch import extended_mlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ObjectView(object):\n",
    "    def __init__(self, d): self.__dict__ = d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import get_trajectory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import get_trajectory_analytic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from physics import analytical_fn\n",
    "\n",
    "vfnc = jax.jit(jax.vmap(analytical_fn))\n",
    "vget = partial(jax.jit, backend='cpu')(jax.vmap(partial(get_trajectory_analytic, mxsteps=100), (0, None), 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0.29830917716026306 {'act': [4],\n",
    "# 'batch_size': [27.0], 'dt': [0.09609870774790222],\n",
    "# 'hidden_dim': [596.0], 'l2reg': [0.24927677946969878],\n",
    "# 'layers': [4.0], 'lr': [0.005516656601005163],\n",
    "# 'lr2': [1.897157209816416e-05], 'n_updates': [4.0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = ObjectView(dict(\n",
    "    dataset_size=200,\n",
    "    fps=10,\n",
    "    samples=100,\n",
    "    num_epochs=80000,\n",
    "    seed=0,\n",
    "    loss='l1',\n",
    "    act='softplus',\n",
    "    hidden_dim=500,\n",
    "    output_dim=1,\n",
    "    layers=3,\n",
    "    n_updates=1,#6,#4,\n",
    "    lr=1e-3,#5.5e-3,\n",
    "    lr2=2e-5,\n",
    "    dt=0.1,\n",
    "    model='gln',\n",
    "    batch_size=68,\n",
    "    l2reg=5.7e-7\n",
    "))\n",
    "rng = jax.random.PRNGKey(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jax.experimental.ode import odeint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HyperparameterSearch import new_get_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = new_get_dataset(rng+2, t_span=[0, args.dataset_size], fps=args.fps, samples=args.samples, test_split=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = None\n",
    "best_loss = np.inf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HyperparameterSearch import make_loss, train\n",
    "loss = make_loss(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "opti = optimizers.adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "init_random_params, nn_forward_fn = extended_mlp(args)\n",
    "_, init_params = init_random_params(rng+1, (-1, 4))\n",
    "import HyperparameterSearch\n",
    "HyperparameterSearch.nn_forward_fn = nn_forward_fn\n",
    "rng += 1\n",
    "model = (nn_forward_fn, init_params)\n",
    "opt_init, opt_update, get_params = opti(3e-4)##lambda i: jnp.select([i<10000, i>= 10000], [args.lr, args.lr2]))\n",
    "opt_state = opt_init(init_params)\n",
    "from jax.tree_util import tree_flatten\n",
    "from copy import deepcopy as copy\n",
    "train(args, model, data, rng);\n",
    "from jax.tree_util import tree_flatten\n",
    "\n",
    "@jax.jit\n",
    "def update_derivative(i, opt_state, batch, l2reg, params): #iteration+offset, opt_state, batch, args.l2reg\n",
    "    param_update = jax.grad(loss, 0)(params, batch, l2reg)\n",
    "    new_state = opt_update(i, param_update, opt_state)\n",
    "    leaves, _ = tree_flatten(get_params(new_state))\n",
    "    infinities = sum((~jnp.isfinite(param)).sum() for param in leaves)\n",
    "    \n",
    "    def true_fun(x):\n",
    "        #No introducing NaNs.\n",
    "        return new_state, params\n",
    "\n",
    "    def false_fun(x):\n",
    "        #No introducing NaNs.\n",
    "        return opt_state, params\n",
    "\n",
    "    return jax.lax.cond(infinities==0, 0, true_fun, 0, false_fun)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "(nn_forward_fn, init_params) = model\n",
    "data = {k: jax.device_put(v) for k,v in data.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_new_params(params):\n",
    "    rng = jax.random.PRNGKey(0)\n",
    "    all_new_params = []\n",
    "    for i in range(len(params)):\n",
    "        new_params = []\n",
    "        for j in range(len(params[i])):\n",
    "            p = params[i][j]\n",
    "            n_in = p.shape[0]\n",
    "            n_out = 0 if len(p.shape) == 1 else p.shape[1]\n",
    "\n",
    "            scaling = np.sqrt(6)/np.sqrt(n_in + n_out)\n",
    "            new_p = jax.random.normal(rng, p.shape)\n",
    "            \n",
    "            if n_out > 0:\n",
    "                if n_in >= n_out:\n",
    "                    new_p = jnp.linalg.qr(new_p)[0]\n",
    "                else:\n",
    "                    new_p = jnp.linalg.qr(new_p.T)[0].T\n",
    "            \n",
    "            new_p *= scaling\n",
    "            rng += 1\n",
    "            \n",
    "            new_params.append(new_p)\n",
    "        new_params = tuple(new_params)\n",
    "        all_new_params.append(new_params)\n",
    "    return all_new_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running 0\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.718885, test_loss=13.740966\n",
      "iteration=5, train_loss=11.454437, test_loss=11.545874\n",
      "iteration=6, train_loss=9.062446, test_loss=9.131317\n",
      "iteration=7, train_loss=7.106303, test_loss=7.154757\n",
      "iteration=8, train_loss=5.397975, test_loss=5.447294\n",
      "iteration=9, train_loss=3.966626, test_loss=4.046118\n",
      "iteration=10, train_loss=3.184337, test_loss=3.308728\n",
      "iteration=17, train_loss=2.965082, test_loss=3.091791\n",
      "iteration=23, train_loss=2.551312, test_loss=2.707976\n",
      "iteration=24, train_loss=2.500699, test_loss=2.663209\n",
      "iteration=100, train_loss=2.415506, test_loss=2.595785\n",
      "iteration=200, train_loss=2.401528, test_loss=2.582589\n",
      "iteration=204, train_loss=2.409171, test_loss=2.587458\n",
      "iteration=250, train_loss=2.397993, test_loss=2.579034\n",
      "iteration=300, train_loss=2.398918, test_loss=2.579388\n",
      "iteration=357, train_loss=2.409793, test_loss=2.587962\n",
      "iteration=400, train_loss=2.395849, test_loss=2.575941\n",
      "iteration=500, train_loss=2.389503, test_loss=2.569693\n",
      "iteration=600, train_loss=2.386894, test_loss=2.564442\n",
      "iteration=700, train_loss=2.378026, test_loss=2.557636\n",
      "iteration=745, train_loss=2.382586, test_loss=2.561190\n",
      "iteration=800, train_loss=2.387663, test_loss=2.562723\n",
      "iteration=900, train_loss=2.377737, test_loss=2.555969\n",
      "Running 1\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.848234, test_loss=12.907997\n",
      "iteration=5, train_loss=9.138891, test_loss=9.210113\n",
      "iteration=6, train_loss=6.987287, test_loss=7.011904\n",
      "iteration=7, train_loss=5.601937, test_loss=5.647450\n",
      "iteration=8, train_loss=5.197227, test_loss=5.288677\n",
      "iteration=11, train_loss=4.687633, test_loss=4.763722\n",
      "iteration=12, train_loss=4.010548, test_loss=4.104530\n",
      "iteration=13, train_loss=3.269433, test_loss=3.392805\n",
      "iteration=14, train_loss=2.701051, test_loss=2.854905\n",
      "iteration=15, train_loss=2.459356, test_loss=2.641931\n",
      "iteration=16, train_loss=2.453800, test_loss=2.632758\n",
      "iteration=44, train_loss=2.441797, test_loss=2.624241\n",
      "iteration=55, train_loss=2.437059, test_loss=2.622451\n",
      "iteration=63, train_loss=2.430655, test_loss=2.613181\n",
      "iteration=91, train_loss=2.425890, test_loss=2.610370\n",
      "iteration=100, train_loss=2.425451, test_loss=2.609416\n",
      "iteration=128, train_loss=2.431713, test_loss=2.614564\n",
      "iteration=151, train_loss=2.430420, test_loss=2.611293\n",
      "iteration=200, train_loss=2.431550, test_loss=2.610757\n",
      "iteration=300, train_loss=2.424230, test_loss=2.605148\n",
      "iteration=400, train_loss=2.413634, test_loss=2.594948\n",
      "iteration=463, train_loss=2.400378, test_loss=2.580051\n",
      "iteration=500, train_loss=2.394181, test_loss=2.575795\n",
      "iteration=600, train_loss=2.391568, test_loss=2.573117\n",
      "iteration=636, train_loss=2.416558, test_loss=2.590357\n",
      "iteration=700, train_loss=2.383452, test_loss=2.564287\n",
      "iteration=800, train_loss=2.381444, test_loss=2.562903\n",
      "iteration=812, train_loss=2.383180, test_loss=2.564641\n",
      "iteration=900, train_loss=2.371460, test_loss=2.552004\n",
      "Running 2\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.818771, test_loss=12.852744\n",
      "iteration=5, train_loss=9.251243, test_loss=9.327949\n",
      "iteration=6, train_loss=7.126527, test_loss=7.156464\n",
      "iteration=7, train_loss=5.647215, test_loss=5.685051\n",
      "iteration=8, train_loss=4.955046, test_loss=5.040493\n",
      "iteration=9, train_loss=4.919091, test_loss=5.014660\n",
      "iteration=10, train_loss=4.809366, test_loss=4.896026\n",
      "iteration=12, train_loss=3.787352, test_loss=3.891600\n",
      "iteration=13, train_loss=3.147958, test_loss=3.276821\n",
      "iteration=14, train_loss=2.639018, test_loss=2.798629\n",
      "iteration=15, train_loss=2.450061, test_loss=2.633556\n",
      "iteration=20, train_loss=2.687837, test_loss=2.829185\n",
      "iteration=22, train_loss=2.622851, test_loss=2.771850\n",
      "iteration=32, train_loss=2.432078, test_loss=2.614863\n",
      "iteration=40, train_loss=2.455782, test_loss=2.635851\n",
      "iteration=100, train_loss=2.427098, test_loss=2.611501\n",
      "iteration=130, train_loss=2.427304, test_loss=2.609870\n",
      "iteration=200, train_loss=2.424355, test_loss=2.607951\n",
      "iteration=300, train_loss=2.411262, test_loss=2.592015\n",
      "iteration=334, train_loss=2.413097, test_loss=2.594273\n",
      "iteration=346, train_loss=2.407392, test_loss=2.588041\n",
      "iteration=400, train_loss=2.404447, test_loss=2.586410\n",
      "iteration=500, train_loss=2.396346, test_loss=2.577250\n",
      "iteration=600, train_loss=2.415645, test_loss=2.597130\n",
      "iteration=700, train_loss=2.459907, test_loss=2.628524\n",
      "iteration=800, train_loss=2.375995, test_loss=2.555554\n",
      "iteration=900, train_loss=2.368971, test_loss=2.549193\n",
      "Running 3\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.632855, test_loss=10.726792\n",
      "iteration=5, train_loss=10.468384, test_loss=10.555969\n",
      "iteration=6, train_loss=8.187521, test_loss=8.230478\n",
      "iteration=7, train_loss=6.415595, test_loss=6.444987\n",
      "iteration=8, train_loss=5.031789, test_loss=5.082609\n",
      "iteration=9, train_loss=4.307581, test_loss=4.401721\n",
      "iteration=10, train_loss=4.157310, test_loss=4.260509\n",
      "iteration=11, train_loss=4.063603, test_loss=4.156678\n",
      "iteration=12, train_loss=3.794803, test_loss=3.887281\n",
      "iteration=13, train_loss=3.348495, test_loss=3.455379\n",
      "iteration=14, train_loss=2.901171, test_loss=3.034451\n",
      "iteration=15, train_loss=2.638794, test_loss=2.805393\n",
      "iteration=16, train_loss=2.544724, test_loss=2.719201\n",
      "iteration=17, train_loss=2.566383, test_loss=2.728317\n",
      "iteration=27, train_loss=2.452332, test_loss=2.625476\n",
      "iteration=29, train_loss=2.417514, test_loss=2.600015\n",
      "iteration=100, train_loss=2.415819, test_loss=2.598290\n",
      "iteration=179, train_loss=2.409534, test_loss=2.591806\n",
      "iteration=200, train_loss=2.407841, test_loss=2.588886\n",
      "iteration=300, train_loss=2.400568, test_loss=2.582919\n",
      "iteration=400, train_loss=2.420266, test_loss=2.600818\n",
      "iteration=500, train_loss=2.394123, test_loss=2.573447\n",
      "iteration=600, train_loss=2.390303, test_loss=2.570340\n",
      "iteration=619, train_loss=2.381768, test_loss=2.562209\n",
      "iteration=700, train_loss=2.400543, test_loss=2.581368\n",
      "iteration=800, train_loss=2.377065, test_loss=2.555528\n",
      "iteration=900, train_loss=2.365952, test_loss=2.545392\n",
      "Running 4\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.061084, test_loss=11.153290\n",
      "iteration=6, train_loss=9.523594, test_loss=9.580461\n",
      "iteration=7, train_loss=7.639211, test_loss=7.676693\n",
      "iteration=8, train_loss=5.940866, test_loss=5.977545\n",
      "iteration=9, train_loss=4.497884, test_loss=4.561485\n",
      "iteration=10, train_loss=3.657466, test_loss=3.765125\n",
      "iteration=11, train_loss=3.407699, test_loss=3.528522\n",
      "iteration=12, train_loss=3.313806, test_loss=3.427069\n",
      "iteration=13, train_loss=3.206478, test_loss=3.318645\n",
      "iteration=15, train_loss=2.906411, test_loss=3.054374\n",
      "iteration=16, train_loss=2.826977, test_loss=2.976377\n",
      "iteration=21, train_loss=2.778980, test_loss=2.914484\n",
      "iteration=22, train_loss=2.730094, test_loss=2.867698\n",
      "iteration=27, train_loss=2.435691, test_loss=2.610266\n",
      "iteration=28, train_loss=2.418148, test_loss=2.597313\n",
      "iteration=31, train_loss=2.420256, test_loss=2.601003\n",
      "iteration=91, train_loss=2.409730, test_loss=2.591474\n",
      "iteration=100, train_loss=2.409383, test_loss=2.592144\n",
      "iteration=105, train_loss=2.418056, test_loss=2.601139\n",
      "iteration=112, train_loss=2.409287, test_loss=2.590768\n",
      "iteration=200, train_loss=2.400597, test_loss=2.582205\n",
      "iteration=271, train_loss=2.395134, test_loss=2.576190\n",
      "iteration=278, train_loss=2.405815, test_loss=2.586364\n",
      "iteration=300, train_loss=2.395004, test_loss=2.577023\n",
      "iteration=400, train_loss=2.394810, test_loss=2.573499\n",
      "iteration=500, train_loss=2.379334, test_loss=2.558286\n",
      "iteration=600, train_loss=2.371425, test_loss=2.551520\n",
      "iteration=653, train_loss=2.392010, test_loss=2.569150\n",
      "iteration=700, train_loss=2.367377, test_loss=2.548740\n",
      "iteration=800, train_loss=2.364863, test_loss=2.545068\n",
      "iteration=900, train_loss=2.370006, test_loss=2.549572\n",
      "iteration=904, train_loss=2.359295, test_loss=2.539868\n",
      "Running 5\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.368523, test_loss=12.439578\n",
      "iteration=7, train_loss=11.359226, test_loss=11.413287\n",
      "iteration=8, train_loss=10.408591, test_loss=10.492361\n",
      "iteration=9, train_loss=9.444743, test_loss=9.534170\n",
      "iteration=10, train_loss=8.414517, test_loss=8.499562\n",
      "iteration=11, train_loss=7.389285, test_loss=7.463596\n",
      "iteration=12, train_loss=6.426681, test_loss=6.492372\n",
      "iteration=13, train_loss=5.531446, test_loss=5.594750\n",
      "iteration=14, train_loss=4.711023, test_loss=4.783000\n",
      "iteration=15, train_loss=3.988400, test_loss=4.079068\n",
      "iteration=16, train_loss=3.383212, test_loss=3.499904\n",
      "iteration=17, train_loss=2.963343, test_loss=3.111150\n",
      "iteration=19, train_loss=2.637873, test_loss=2.821039\n",
      "iteration=20, train_loss=2.624270, test_loss=2.802641\n",
      "iteration=22, train_loss=2.694953, test_loss=2.860702\n",
      "iteration=61, train_loss=2.541072, test_loss=2.726829\n",
      "iteration=95, train_loss=2.510333, test_loss=2.697455\n",
      "iteration=100, train_loss=2.508179, test_loss=2.695256\n",
      "iteration=200, train_loss=2.463689, test_loss=2.648346\n",
      "iteration=232, train_loss=2.466413, test_loss=2.646597\n",
      "iteration=300, train_loss=2.435335, test_loss=2.617802\n",
      "iteration=346, train_loss=2.425177, test_loss=2.609104\n",
      "iteration=394, train_loss=2.417503, test_loss=2.599333\n",
      "iteration=400, train_loss=2.421106, test_loss=2.605088\n",
      "iteration=470, train_loss=2.406487, test_loss=2.589654\n",
      "iteration=500, train_loss=2.406475, test_loss=2.588228\n",
      "iteration=600, train_loss=2.400751, test_loss=2.581700\n",
      "iteration=700, train_loss=2.384063, test_loss=2.564985\n",
      "iteration=800, train_loss=2.375316, test_loss=2.556720\n",
      "iteration=900, train_loss=2.364359, test_loss=2.543905\n",
      "Running 6\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.410131, test_loss=12.465072\n",
      "iteration=5, train_loss=9.643529, test_loss=9.726731\n",
      "iteration=6, train_loss=7.343414, test_loss=7.376391\n",
      "iteration=7, train_loss=5.720562, test_loss=5.755134\n",
      "iteration=8, train_loss=4.806473, test_loss=4.887517\n",
      "iteration=9, train_loss=4.729876, test_loss=4.826451\n",
      "iteration=11, train_loss=4.264972, test_loss=4.350142\n",
      "iteration=12, train_loss=3.695663, test_loss=3.796241\n",
      "iteration=13, train_loss=3.070985, test_loss=3.199392\n",
      "iteration=15, train_loss=2.456888, test_loss=2.641778\n",
      "iteration=16, train_loss=2.502316, test_loss=2.669615\n",
      "iteration=53, train_loss=2.425180, test_loss=2.608970\n",
      "iteration=100, train_loss=2.429541, test_loss=2.613903\n",
      "iteration=134, train_loss=2.420822, test_loss=2.604668\n",
      "iteration=200, train_loss=2.420493, test_loss=2.604333\n",
      "iteration=300, train_loss=2.406976, test_loss=2.588835\n",
      "iteration=400, train_loss=2.400202, test_loss=2.581622\n",
      "iteration=500, train_loss=2.391959, test_loss=2.572978\n",
      "iteration=600, train_loss=2.388130, test_loss=2.568788\n",
      "iteration=651, train_loss=2.384032, test_loss=2.564456\n",
      "iteration=700, train_loss=2.385754, test_loss=2.567390\n",
      "iteration=800, train_loss=2.377269, test_loss=2.556990\n",
      "iteration=900, train_loss=2.385175, test_loss=2.559760\n",
      "Running 7\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.201068, test_loss=11.294499\n",
      "iteration=6, train_loss=9.343706, test_loss=9.398448\n",
      "iteration=7, train_loss=7.491283, test_loss=7.523981\n",
      "iteration=8, train_loss=5.894672, test_loss=5.927765\n",
      "iteration=9, train_loss=4.634439, test_loss=4.698241\n",
      "iteration=10, train_loss=4.063595, test_loss=4.165592\n",
      "iteration=12, train_loss=3.843224, test_loss=3.938372\n",
      "iteration=13, train_loss=3.617240, test_loss=3.713107\n",
      "iteration=14, train_loss=3.234862, test_loss=3.346164\n",
      "iteration=16, train_loss=2.689679, test_loss=2.852879\n",
      "iteration=17, train_loss=2.623805, test_loss=2.788517\n",
      "iteration=19, train_loss=2.681878, test_loss=2.827689\n",
      "iteration=30, train_loss=2.423025, test_loss=2.607008\n",
      "iteration=31, train_loss=2.425388, test_loss=2.609266\n",
      "iteration=33, train_loss=2.434674, test_loss=2.616888\n",
      "iteration=42, train_loss=2.425035, test_loss=2.607214\n",
      "iteration=100, train_loss=2.421773, test_loss=2.605862\n",
      "iteration=200, train_loss=2.420693, test_loss=2.600158\n",
      "iteration=300, train_loss=2.430557, test_loss=2.605476\n",
      "iteration=400, train_loss=2.402611, test_loss=2.583363\n",
      "iteration=500, train_loss=2.392224, test_loss=2.573855\n",
      "iteration=600, train_loss=2.414262, test_loss=2.592572\n",
      "iteration=700, train_loss=2.404754, test_loss=2.585346\n",
      "iteration=800, train_loss=2.376320, test_loss=2.556336\n",
      "iteration=900, train_loss=2.387135, test_loss=2.562626\n",
      "Running 8\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.721208, test_loss=12.773131\n",
      "iteration=6, train_loss=11.196974, test_loss=11.204529\n",
      "iteration=7, train_loss=9.079095, test_loss=9.081933\n",
      "iteration=8, train_loss=7.365838, test_loss=7.398904\n",
      "iteration=9, train_loss=6.592470, test_loss=6.684232\n",
      "iteration=10, train_loss=6.628090, test_loss=6.725513\n",
      "iteration=12, train_loss=6.277741, test_loss=6.344368\n",
      "iteration=13, train_loss=5.606072, test_loss=5.677941\n",
      "iteration=14, train_loss=4.747112, test_loss=4.838281\n",
      "iteration=15, train_loss=3.866527, test_loss=3.986456\n",
      "iteration=16, train_loss=3.142917, test_loss=3.283013\n",
      "iteration=17, train_loss=2.751697, test_loss=2.916130\n",
      "iteration=18, train_loss=2.694024, test_loss=2.872385\n",
      "iteration=19, train_loss=2.780612, test_loss=2.945572\n",
      "iteration=44, train_loss=2.581418, test_loss=2.769681\n",
      "iteration=47, train_loss=2.555792, test_loss=2.742980\n",
      "iteration=55, train_loss=2.542004, test_loss=2.730989\n",
      "iteration=56, train_loss=2.542640, test_loss=2.731544\n",
      "iteration=100, train_loss=2.532124, test_loss=2.719016\n",
      "iteration=103, train_loss=2.534479, test_loss=2.723366\n",
      "iteration=143, train_loss=2.538691, test_loss=2.726803\n",
      "iteration=200, train_loss=2.506342, test_loss=2.693167\n",
      "iteration=300, train_loss=2.488029, test_loss=2.674575\n",
      "iteration=313, train_loss=2.492268, test_loss=2.678542\n",
      "iteration=400, train_loss=2.468217, test_loss=2.650953\n",
      "iteration=500, train_loss=2.445380, test_loss=2.629420\n",
      "iteration=558, train_loss=2.445366, test_loss=2.631306\n",
      "iteration=600, train_loss=2.434293, test_loss=2.618207\n",
      "iteration=700, train_loss=2.430603, test_loss=2.613455\n",
      "iteration=800, train_loss=2.412244, test_loss=2.595571\n",
      "iteration=871, train_loss=2.406654, test_loss=2.589919\n",
      "iteration=900, train_loss=2.409602, test_loss=2.590832\n",
      "Running 9\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.314607, test_loss=13.338156\n",
      "iteration=5, train_loss=9.667274, test_loss=9.758630\n",
      "iteration=6, train_loss=7.194166, test_loss=7.237838\n",
      "iteration=7, train_loss=5.453025, test_loss=5.493741\n",
      "iteration=8, train_loss=4.397953, test_loss=4.478086\n",
      "iteration=10, train_loss=4.271118, test_loss=4.361649\n",
      "iteration=11, train_loss=4.046526, test_loss=4.133562\n",
      "iteration=12, train_loss=3.569545, test_loss=3.669577\n",
      "iteration=13, train_loss=3.001092, test_loss=3.128452\n",
      "iteration=14, train_loss=2.628902, test_loss=2.795944\n",
      "iteration=24, train_loss=2.480736, test_loss=2.651617\n",
      "iteration=25, train_loss=2.448757, test_loss=2.627906\n",
      "iteration=27, train_loss=2.428632, test_loss=2.612571\n",
      "iteration=31, train_loss=2.436100, test_loss=2.617903\n",
      "iteration=33, train_loss=2.443820, test_loss=2.624279\n",
      "iteration=36, train_loss=2.436361, test_loss=2.618012\n",
      "iteration=48, train_loss=2.432513, test_loss=2.617013\n",
      "iteration=57, train_loss=2.424504, test_loss=2.606765\n",
      "iteration=100, train_loss=2.420667, test_loss=2.603374\n",
      "iteration=200, train_loss=2.424996, test_loss=2.608013\n",
      "iteration=262, train_loss=2.407112, test_loss=2.589794\n",
      "iteration=300, train_loss=2.410291, test_loss=2.591468\n",
      "iteration=400, train_loss=2.398719, test_loss=2.579152\n",
      "iteration=451, train_loss=2.394831, test_loss=2.576213\n",
      "iteration=500, train_loss=2.392321, test_loss=2.574316\n",
      "iteration=600, train_loss=2.384383, test_loss=2.565151\n",
      "iteration=700, train_loss=2.386328, test_loss=2.564132\n",
      "iteration=800, train_loss=2.375499, test_loss=2.554970\n",
      "iteration=900, train_loss=2.379861, test_loss=2.560826\n",
      "Running 10\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.746146, test_loss=10.839652\n",
      "iteration=6, train_loss=9.198709, test_loss=9.251972\n",
      "iteration=7, train_loss=7.406824, test_loss=7.439718\n",
      "iteration=8, train_loss=5.821659, test_loss=5.856276\n",
      "iteration=9, train_loss=4.567382, test_loss=4.632342\n",
      "iteration=10, train_loss=3.972180, test_loss=4.075824\n",
      "iteration=11, train_loss=3.793082, test_loss=3.904458\n",
      "iteration=12, train_loss=3.625256, test_loss=3.731015\n",
      "iteration=13, train_loss=3.348729, test_loss=3.457608\n",
      "iteration=14, train_loss=3.012118, test_loss=3.137235\n",
      "iteration=15, train_loss=2.803802, test_loss=2.950828\n",
      "iteration=17, train_loss=2.623058, test_loss=2.788473\n",
      "iteration=27, train_loss=2.454952, test_loss=2.631148\n",
      "iteration=28, train_loss=2.434337, test_loss=2.615675\n",
      "iteration=35, train_loss=2.438234, test_loss=2.617950\n",
      "iteration=53, train_loss=2.421531, test_loss=2.605042\n",
      "iteration=100, train_loss=2.414855, test_loss=2.598188\n",
      "iteration=127, train_loss=2.423009, test_loss=2.604322\n",
      "iteration=200, train_loss=2.408864, test_loss=2.591730\n",
      "iteration=300, train_loss=2.427558, test_loss=2.608967\n",
      "iteration=400, train_loss=2.396786, test_loss=2.578028\n",
      "iteration=401, train_loss=2.400758, test_loss=2.581073\n",
      "iteration=500, train_loss=2.390788, test_loss=2.571251\n",
      "iteration=600, train_loss=2.417820, test_loss=2.600067\n",
      "iteration=700, train_loss=2.382271, test_loss=2.561759\n",
      "iteration=800, train_loss=2.387344, test_loss=2.563989\n",
      "iteration=900, train_loss=2.374682, test_loss=2.553835\n",
      "Running 11\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.450608, test_loss=12.492636\n",
      "iteration=5, train_loss=10.953157, test_loss=11.044152\n",
      "iteration=6, train_loss=8.619108, test_loss=8.668074\n",
      "iteration=7, train_loss=6.803394, test_loss=6.834778\n",
      "iteration=8, train_loss=5.267127, test_loss=5.311120\n",
      "iteration=9, train_loss=4.266599, test_loss=4.349000\n",
      "iteration=10, train_loss=3.959252, test_loss=4.065735\n",
      "iteration=11, train_loss=3.855531, test_loss=3.960429\n",
      "iteration=13, train_loss=3.243219, test_loss=3.355809\n",
      "iteration=14, train_loss=2.872195, test_loss=3.009070\n",
      "iteration=25, train_loss=2.523139, test_loss=2.682839\n",
      "iteration=34, train_loss=2.446085, test_loss=2.624364\n",
      "iteration=40, train_loss=2.436581, test_loss=2.617581\n",
      "iteration=48, train_loss=2.415952, test_loss=2.598033\n",
      "iteration=100, train_loss=2.418649, test_loss=2.600094\n",
      "iteration=194, train_loss=2.418964, test_loss=2.596794\n",
      "iteration=200, train_loss=2.406873, test_loss=2.588368\n",
      "iteration=299, train_loss=2.402771, test_loss=2.584517\n",
      "iteration=300, train_loss=2.400684, test_loss=2.581844\n",
      "iteration=400, train_loss=2.399387, test_loss=2.580570\n",
      "iteration=445, train_loss=2.393482, test_loss=2.575023\n",
      "iteration=500, train_loss=2.401207, test_loss=2.582772\n",
      "iteration=600, train_loss=2.397418, test_loss=2.574988\n",
      "iteration=700, train_loss=2.387357, test_loss=2.564418\n",
      "iteration=800, train_loss=2.375774, test_loss=2.555237\n",
      "iteration=807, train_loss=2.384759, test_loss=2.560869\n",
      "iteration=884, train_loss=2.382144, test_loss=2.560317\n",
      "iteration=900, train_loss=2.376284, test_loss=2.553474\n",
      "Running 12\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.054958, test_loss=13.109487\n",
      "iteration=5, train_loss=9.681135, test_loss=9.773814\n",
      "iteration=6, train_loss=6.888355, test_loss=6.935493\n",
      "iteration=7, train_loss=5.041870, test_loss=5.092198\n",
      "iteration=8, train_loss=4.156286, test_loss=4.251119\n",
      "iteration=12, train_loss=3.263553, test_loss=3.378702\n",
      "iteration=13, train_loss=2.824945, test_loss=2.967281\n",
      "iteration=14, train_loss=2.553076, test_loss=2.728707\n",
      "iteration=16, train_loss=2.523012, test_loss=2.685896\n",
      "iteration=17, train_loss=2.640962, test_loss=2.785572\n",
      "iteration=18, train_loss=2.748879, test_loss=2.884354\n",
      "iteration=23, train_loss=2.577640, test_loss=2.730537\n",
      "iteration=25, train_loss=2.491354, test_loss=2.658458\n",
      "iteration=27, train_loss=2.442017, test_loss=2.619924\n",
      "iteration=28, train_loss=2.431666, test_loss=2.612186\n",
      "iteration=48, train_loss=2.423872, test_loss=2.606742\n",
      "iteration=71, train_loss=2.420381, test_loss=2.603869\n",
      "iteration=100, train_loss=2.425361, test_loss=2.608451\n",
      "iteration=200, train_loss=2.411875, test_loss=2.593813\n",
      "iteration=217, train_loss=2.415668, test_loss=2.595814\n",
      "iteration=296, train_loss=2.412200, test_loss=2.592204\n",
      "iteration=300, train_loss=2.405007, test_loss=2.587917\n",
      "iteration=400, train_loss=2.402539, test_loss=2.584927\n",
      "iteration=500, train_loss=2.393686, test_loss=2.574255\n",
      "iteration=528, train_loss=2.389242, test_loss=2.570968\n",
      "iteration=600, train_loss=2.384613, test_loss=2.565162\n",
      "iteration=700, train_loss=2.378613, test_loss=2.559063\n",
      "iteration=709, train_loss=2.387261, test_loss=2.563894\n",
      "iteration=755, train_loss=2.383766, test_loss=2.562760\n",
      "iteration=763, train_loss=2.388963, test_loss=2.570020\n",
      "iteration=800, train_loss=2.382649, test_loss=2.560727\n",
      "iteration=900, train_loss=2.367778, test_loss=2.546293\n",
      "Running 13\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=14.050847, test_loss=14.092979\n",
      "iteration=5, train_loss=13.608040, test_loss=13.615337\n",
      "iteration=6, train_loss=9.312135, test_loss=9.371268\n",
      "iteration=7, train_loss=6.637602, test_loss=6.717304\n",
      "iteration=8, train_loss=5.182217, test_loss=5.235209\n",
      "iteration=9, train_loss=4.452972, test_loss=4.530675\n",
      "iteration=10, train_loss=4.576710, test_loss=4.678331\n",
      "iteration=15, train_loss=4.514338, test_loss=4.593413\n",
      "iteration=17, train_loss=3.320185, test_loss=3.451875\n",
      "iteration=18, train_loss=2.910166, test_loss=3.058219\n",
      "iteration=19, train_loss=2.680604, test_loss=2.844771\n",
      "iteration=20, train_loss=2.606539, test_loss=2.786215\n",
      "iteration=26, train_loss=2.635277, test_loss=2.805792\n",
      "iteration=33, train_loss=2.555176, test_loss=2.743518\n",
      "iteration=37, train_loss=2.535978, test_loss=2.723237\n",
      "iteration=86, train_loss=2.498394, test_loss=2.685985\n",
      "iteration=100, train_loss=2.495449, test_loss=2.683661\n",
      "iteration=104, train_loss=2.496133, test_loss=2.684450\n",
      "iteration=175, train_loss=2.480793, test_loss=2.666097\n",
      "iteration=200, train_loss=2.486121, test_loss=2.674274\n",
      "iteration=300, train_loss=2.462585, test_loss=2.647742\n",
      "iteration=400, train_loss=2.440022, test_loss=2.625955\n",
      "iteration=500, train_loss=2.439235, test_loss=2.619638\n",
      "iteration=600, train_loss=2.413371, test_loss=2.597404\n",
      "iteration=700, train_loss=2.407227, test_loss=2.591320\n",
      "iteration=722, train_loss=2.407005, test_loss=2.591123\n",
      "iteration=800, train_loss=2.413541, test_loss=2.595226\n",
      "iteration=827, train_loss=2.397358, test_loss=2.581139\n",
      "iteration=900, train_loss=2.398783, test_loss=2.582506\n",
      "Running 14\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=9.912437, test_loss=9.998125\n",
      "iteration=7, train_loss=9.108888, test_loss=9.147411\n",
      "iteration=8, train_loss=7.469751, test_loss=7.499760\n",
      "iteration=9, train_loss=5.946938, test_loss=5.981630\n",
      "iteration=10, train_loss=4.620055, test_loss=4.680036\n",
      "iteration=11, train_loss=3.711021, test_loss=3.810468\n",
      "iteration=12, train_loss=3.326289, test_loss=3.447938\n",
      "iteration=13, train_loss=3.197095, test_loss=3.321393\n",
      "iteration=16, train_loss=3.046067, test_loss=3.183522\n",
      "iteration=17, train_loss=2.983911, test_loss=3.128915\n",
      "iteration=18, train_loss=2.935365, test_loss=3.072823\n",
      "iteration=20, train_loss=2.904996, test_loss=3.027864\n",
      "iteration=21, train_loss=2.900320, test_loss=3.026279\n",
      "iteration=23, train_loss=2.817155, test_loss=2.950731\n",
      "iteration=28, train_loss=2.460878, test_loss=2.632900\n",
      "iteration=30, train_loss=2.422129, test_loss=2.601692\n",
      "iteration=50, train_loss=2.415040, test_loss=2.596160\n",
      "iteration=100, train_loss=2.413576, test_loss=2.594674\n",
      "iteration=142, train_loss=2.421398, test_loss=2.602389\n",
      "iteration=157, train_loss=2.408593, test_loss=2.589614\n",
      "iteration=200, train_loss=2.454870, test_loss=2.634988\n",
      "iteration=231, train_loss=2.405514, test_loss=2.587824\n",
      "iteration=260, train_loss=2.410936, test_loss=2.589396\n",
      "iteration=300, train_loss=2.406431, test_loss=2.586789\n",
      "iteration=393, train_loss=2.394063, test_loss=2.575168\n",
      "iteration=400, train_loss=2.394560, test_loss=2.575136\n",
      "iteration=500, train_loss=2.389023, test_loss=2.569285\n",
      "iteration=600, train_loss=2.392546, test_loss=2.572837\n",
      "iteration=700, train_loss=2.384112, test_loss=2.561965\n",
      "iteration=800, train_loss=2.407677, test_loss=2.577909\n",
      "iteration=900, train_loss=2.381027, test_loss=2.558992\n",
      "Running 15\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=14.296177, test_loss=14.308158\n",
      "iteration=4, train_loss=14.112427, test_loss=14.147127\n",
      "iteration=5, train_loss=10.698531, test_loss=10.787857\n",
      "iteration=6, train_loss=8.160395, test_loss=8.232709\n",
      "iteration=7, train_loss=6.148606, test_loss=6.200815\n",
      "iteration=8, train_loss=4.487638, test_loss=4.553454\n",
      "iteration=9, train_loss=3.397815, test_loss=3.504816\n",
      "iteration=11, train_loss=3.105880, test_loss=3.231374\n",
      "iteration=12, train_loss=3.146482, test_loss=3.263037\n",
      "iteration=20, train_loss=2.617918, test_loss=2.767913\n",
      "iteration=21, train_loss=2.579162, test_loss=2.731238\n",
      "iteration=22, train_loss=2.541096, test_loss=2.697185\n",
      "iteration=26, train_loss=2.440906, test_loss=2.618819\n",
      "iteration=27, train_loss=2.431391, test_loss=2.612269\n",
      "iteration=28, train_loss=2.432006, test_loss=2.612826\n",
      "iteration=33, train_loss=2.448160, test_loss=2.625366\n",
      "iteration=100, train_loss=2.419618, test_loss=2.599366\n",
      "iteration=200, train_loss=2.415635, test_loss=2.598131\n",
      "iteration=300, train_loss=2.398773, test_loss=2.580238\n",
      "iteration=322, train_loss=2.401333, test_loss=2.580781\n",
      "iteration=337, train_loss=2.406473, test_loss=2.589195\n",
      "iteration=341, train_loss=2.404866, test_loss=2.587498\n",
      "iteration=400, train_loss=2.412080, test_loss=2.594135\n",
      "iteration=500, train_loss=2.387898, test_loss=2.569069\n",
      "iteration=600, train_loss=2.384448, test_loss=2.565030\n",
      "iteration=683, train_loss=2.389764, test_loss=2.571083\n",
      "iteration=700, train_loss=2.398934, test_loss=2.572993\n",
      "iteration=800, train_loss=2.418338, test_loss=2.594626\n",
      "iteration=900, train_loss=2.374022, test_loss=2.554164\n",
      "Running 16\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.240361, test_loss=13.296300\n",
      "iteration=5, train_loss=9.074054, test_loss=9.153015\n",
      "iteration=6, train_loss=6.918292, test_loss=6.944224\n",
      "iteration=7, train_loss=5.569597, test_loss=5.617812\n",
      "iteration=10, train_loss=5.098899, test_loss=5.175790\n",
      "iteration=11, train_loss=4.560720, test_loss=4.643272\n",
      "iteration=12, train_loss=3.872571, test_loss=3.976468\n",
      "iteration=13, train_loss=3.173956, test_loss=3.305085\n",
      "iteration=14, train_loss=2.679651, test_loss=2.837502\n",
      "iteration=15, train_loss=2.478713, test_loss=2.662420\n",
      "iteration=75, train_loss=2.444898, test_loss=2.630829\n",
      "iteration=81, train_loss=2.444578, test_loss=2.630331\n",
      "iteration=86, train_loss=2.454752, test_loss=2.641084\n",
      "iteration=100, train_loss=2.449090, test_loss=2.632338\n",
      "iteration=200, train_loss=2.434319, test_loss=2.617708\n",
      "iteration=230, train_loss=2.434327, test_loss=2.620073\n",
      "iteration=300, train_loss=2.424775, test_loss=2.608156\n",
      "iteration=400, train_loss=2.409915, test_loss=2.593572\n",
      "iteration=500, train_loss=2.402207, test_loss=2.584947\n",
      "iteration=600, train_loss=2.409549, test_loss=2.591057\n",
      "iteration=700, train_loss=2.400181, test_loss=2.581859\n",
      "iteration=800, train_loss=2.383698, test_loss=2.565445\n",
      "iteration=900, train_loss=2.379489, test_loss=2.560956\n",
      "Running 17\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=14.548245, test_loss=14.562477\n",
      "iteration=4, train_loss=14.579824, test_loss=14.613599\n",
      "iteration=5, train_loss=11.059396, test_loss=11.150630\n",
      "iteration=6, train_loss=8.542323, test_loss=8.610959\n",
      "iteration=7, train_loss=6.501469, test_loss=6.551985\n",
      "iteration=8, train_loss=4.775158, test_loss=4.834453\n",
      "iteration=9, train_loss=3.495550, test_loss=3.593116\n",
      "iteration=10, train_loss=3.025774, test_loss=3.157802\n",
      "iteration=11, train_loss=2.960466, test_loss=3.094171\n",
      "iteration=20, train_loss=2.668411, test_loss=2.815667\n",
      "iteration=22, train_loss=2.543919, test_loss=2.702919\n",
      "iteration=23, train_loss=2.486780, test_loss=2.653579\n",
      "iteration=25, train_loss=2.424486, test_loss=2.603646\n",
      "iteration=37, train_loss=2.431106, test_loss=2.610837\n",
      "iteration=50, train_loss=2.411378, test_loss=2.593991\n",
      "iteration=100, train_loss=2.409507, test_loss=2.590598\n",
      "iteration=200, train_loss=2.414159, test_loss=2.591462\n",
      "iteration=300, train_loss=2.405201, test_loss=2.584904\n",
      "iteration=370, train_loss=2.391828, test_loss=2.572779\n",
      "iteration=400, train_loss=2.408601, test_loss=2.584295\n",
      "iteration=446, train_loss=2.389765, test_loss=2.570872\n",
      "iteration=500, train_loss=2.392027, test_loss=2.570082\n",
      "iteration=600, train_loss=2.381091, test_loss=2.561845\n",
      "iteration=700, train_loss=2.375858, test_loss=2.555824\n",
      "iteration=800, train_loss=2.379071, test_loss=2.558306\n",
      "iteration=827, train_loss=2.369063, test_loss=2.548839\n",
      "iteration=900, train_loss=2.363753, test_loss=2.542573\n",
      "Running 18\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.277831, test_loss=11.351416\n",
      "iteration=5, train_loss=11.542801, test_loss=11.633050\n",
      "iteration=6, train_loss=9.329547, test_loss=9.380530\n",
      "iteration=7, train_loss=7.558184, test_loss=7.589828\n",
      "iteration=8, train_loss=5.991046, test_loss=6.023345\n",
      "iteration=9, train_loss=4.691273, test_loss=4.751252\n",
      "iteration=10, train_loss=3.957534, test_loss=4.059126\n",
      "iteration=11, train_loss=3.702689, test_loss=3.816255\n",
      "iteration=12, train_loss=3.519388, test_loss=3.630756\n",
      "iteration=13, train_loss=3.246239, test_loss=3.359625\n",
      "iteration=14, train_loss=2.953640, test_loss=3.082750\n",
      "iteration=16, train_loss=2.651262, test_loss=2.814823\n",
      "iteration=25, train_loss=2.506532, test_loss=2.674314\n",
      "iteration=34, train_loss=2.450726, test_loss=2.627416\n",
      "iteration=75, train_loss=2.419522, test_loss=2.600156\n",
      "iteration=100, train_loss=2.412492, test_loss=2.593794\n",
      "iteration=126, train_loss=2.416977, test_loss=2.597981\n",
      "iteration=174, train_loss=2.410953, test_loss=2.591798\n",
      "iteration=200, train_loss=2.405941, test_loss=2.587394\n",
      "iteration=300, train_loss=2.404690, test_loss=2.586858\n",
      "iteration=400, train_loss=2.394324, test_loss=2.575149\n",
      "iteration=500, train_loss=2.394479, test_loss=2.573932\n",
      "iteration=600, train_loss=2.415015, test_loss=2.586097\n",
      "iteration=617, train_loss=2.385754, test_loss=2.566853\n",
      "iteration=700, train_loss=2.386721, test_loss=2.567743\n",
      "iteration=800, train_loss=2.377312, test_loss=2.555600\n",
      "iteration=900, train_loss=2.377065, test_loss=2.554265\n",
      "iteration=950, train_loss=2.371922, test_loss=2.549622\n",
      "Running 19\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.227495, test_loss=11.316231\n",
      "iteration=5, train_loss=9.560776, test_loss=9.646337\n",
      "iteration=6, train_loss=7.130726, test_loss=7.167501\n",
      "iteration=7, train_loss=5.417593, test_loss=5.458771\n",
      "iteration=8, train_loss=4.551360, test_loss=4.638113\n",
      "iteration=9, train_loss=4.480790, test_loss=4.579831\n",
      "iteration=11, train_loss=4.148972, test_loss=4.235780\n",
      "iteration=12, train_loss=3.652022, test_loss=3.752488\n",
      "iteration=13, train_loss=3.102367, test_loss=3.227904\n",
      "iteration=14, train_loss=2.694357, test_loss=2.850048\n",
      "iteration=18, train_loss=2.631644, test_loss=2.777463\n",
      "iteration=33, train_loss=2.433637, test_loss=2.617570\n",
      "iteration=76, train_loss=2.433902, test_loss=2.618657\n",
      "iteration=80, train_loss=2.444161, test_loss=2.629242\n",
      "iteration=100, train_loss=2.423584, test_loss=2.607169\n",
      "iteration=103, train_loss=2.425067, test_loss=2.608178\n",
      "iteration=200, train_loss=2.415867, test_loss=2.598321\n",
      "iteration=300, train_loss=2.413051, test_loss=2.593481\n",
      "iteration=350, train_loss=2.407131, test_loss=2.589891\n",
      "iteration=400, train_loss=2.409709, test_loss=2.591911\n",
      "iteration=448, train_loss=2.396626, test_loss=2.578058\n",
      "iteration=500, train_loss=2.392414, test_loss=2.573986\n",
      "iteration=600, train_loss=2.387326, test_loss=2.568667\n",
      "iteration=700, train_loss=2.381335, test_loss=2.562356\n",
      "iteration=751, train_loss=2.386411, test_loss=2.565810\n",
      "iteration=800, train_loss=2.384032, test_loss=2.565421\n",
      "iteration=900, train_loss=2.373323, test_loss=2.553831\n",
      "Running 20\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.678930, test_loss=11.761779\n",
      "iteration=6, train_loss=9.536934, test_loss=9.606233\n",
      "iteration=7, train_loss=7.524251, test_loss=7.574708\n",
      "iteration=8, train_loss=5.734658, test_loss=5.782845\n",
      "iteration=9, train_loss=4.204273, test_loss=4.276391\n",
      "iteration=10, train_loss=3.220342, test_loss=3.337470\n",
      "iteration=11, train_loss=2.940499, test_loss=3.077343\n",
      "iteration=12, train_loss=2.927260, test_loss=3.059624\n",
      "iteration=15, train_loss=2.888232, test_loss=3.037020\n",
      "iteration=19, train_loss=2.761757, test_loss=2.898896\n",
      "iteration=23, train_loss=2.535689, test_loss=2.694637\n",
      "iteration=25, train_loss=2.447183, test_loss=2.619956\n",
      "iteration=27, train_loss=2.411985, test_loss=2.592778\n",
      "iteration=48, train_loss=2.409425, test_loss=2.590905\n",
      "iteration=52, train_loss=2.407634, test_loss=2.589324\n",
      "iteration=70, train_loss=2.411139, test_loss=2.592653\n",
      "iteration=100, train_loss=2.411502, test_loss=2.592693\n",
      "iteration=200, train_loss=2.399756, test_loss=2.581701\n",
      "iteration=300, train_loss=2.399705, test_loss=2.580232\n",
      "iteration=400, train_loss=2.397992, test_loss=2.577743\n",
      "iteration=495, train_loss=2.409362, test_loss=2.588330\n",
      "iteration=500, train_loss=2.391412, test_loss=2.569472\n",
      "iteration=600, train_loss=2.397818, test_loss=2.572312\n",
      "iteration=602, train_loss=2.381378, test_loss=2.561455\n",
      "iteration=700, train_loss=2.397467, test_loss=2.574700\n",
      "iteration=800, train_loss=2.377382, test_loss=2.554390\n",
      "iteration=900, train_loss=2.360443, test_loss=2.539626\n",
      "Running 21\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.186296, test_loss=12.242901\n",
      "iteration=6, train_loss=11.452955, test_loss=11.475160\n",
      "iteration=7, train_loss=8.695655, test_loss=8.766381\n",
      "iteration=8, train_loss=6.828106, test_loss=6.917915\n",
      "iteration=9, train_loss=5.585907, test_loss=5.647357\n",
      "iteration=10, train_loss=4.783210, test_loss=4.844387\n",
      "iteration=11, train_loss=4.287943, test_loss=4.368144\n",
      "iteration=12, train_loss=4.216197, test_loss=4.321063\n",
      "iteration=21, train_loss=3.372298, test_loss=3.503189\n",
      "iteration=22, train_loss=3.073607, test_loss=3.215549\n",
      "iteration=23, train_loss=2.849580, test_loss=3.000193\n",
      "iteration=24, train_loss=2.713432, test_loss=2.874753\n",
      "iteration=25, train_loss=2.650024, test_loss=2.822806\n",
      "iteration=36, train_loss=2.552196, test_loss=2.734827\n",
      "iteration=50, train_loss=2.539546, test_loss=2.729783\n",
      "iteration=100, train_loss=2.505449, test_loss=2.693842\n",
      "iteration=130, train_loss=2.508171, test_loss=2.697146\n",
      "iteration=200, train_loss=2.489512, test_loss=2.676313\n",
      "iteration=300, train_loss=2.473890, test_loss=2.660334\n",
      "iteration=389, train_loss=2.462296, test_loss=2.646065\n",
      "iteration=400, train_loss=2.460891, test_loss=2.646558\n",
      "iteration=500, train_loss=2.449772, test_loss=2.630820\n",
      "iteration=600, train_loss=2.433186, test_loss=2.619356\n",
      "iteration=700, train_loss=2.417408, test_loss=2.602118\n",
      "iteration=800, train_loss=2.410572, test_loss=2.595824\n",
      "iteration=900, train_loss=2.401275, test_loss=2.586089\n",
      "Running 22\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=16.951490, test_loss=16.963968\n",
      "iteration=6, train_loss=14.776170, test_loss=14.777596\n",
      "iteration=7, train_loss=8.742458, test_loss=8.783796\n",
      "iteration=8, train_loss=5.123584, test_loss=5.220251\n",
      "iteration=9, train_loss=3.396913, test_loss=3.520808\n",
      "iteration=10, train_loss=2.649119, test_loss=2.800017\n",
      "iteration=11, train_loss=2.686893, test_loss=2.829917\n",
      "iteration=21, train_loss=2.596082, test_loss=2.756708\n",
      "iteration=29, train_loss=2.460396, test_loss=2.634653\n",
      "iteration=45, train_loss=2.411355, test_loss=2.591544\n",
      "iteration=100, train_loss=2.400922, test_loss=2.580985\n",
      "iteration=105, train_loss=2.406709, test_loss=2.583762\n",
      "iteration=169, train_loss=2.391033, test_loss=2.570344\n",
      "iteration=200, train_loss=2.385982, test_loss=2.564994\n",
      "iteration=300, train_loss=2.384080, test_loss=2.563703\n",
      "iteration=400, train_loss=2.362680, test_loss=2.539050\n",
      "iteration=500, train_loss=2.496584, test_loss=2.648267\n",
      "iteration=565, train_loss=2.339861, test_loss=2.515544\n",
      "iteration=600, train_loss=3.521893, test_loss=3.614716\n",
      "iteration=700, train_loss=4.150445, test_loss=4.219110\n",
      "iteration=800, train_loss=3.466068, test_loss=3.554647\n",
      "iteration=900, train_loss=2.891826, test_loss=2.994435\n",
      "iteration=925, train_loss=2.156915, test_loss=2.325641\n",
      "Running 23\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.144101, test_loss=10.231105\n",
      "iteration=6, train_loss=8.775746, test_loss=8.820419\n",
      "iteration=7, train_loss=7.052770, test_loss=7.080475\n",
      "iteration=8, train_loss=5.585125, test_loss=5.621638\n",
      "iteration=9, train_loss=4.553397, test_loss=4.625069\n",
      "iteration=10, train_loss=4.221555, test_loss=4.321754\n",
      "iteration=12, train_loss=3.879957, test_loss=3.975417\n",
      "iteration=13, train_loss=3.545758, test_loss=3.646872\n",
      "iteration=15, train_loss=2.828221, test_loss=2.969784\n",
      "iteration=18, train_loss=2.590314, test_loss=2.750429\n",
      "iteration=20, train_loss=2.702689, test_loss=2.843754\n",
      "iteration=28, train_loss=2.432635, test_loss=2.611815\n",
      "iteration=35, train_loss=2.453220, test_loss=2.631876\n",
      "iteration=42, train_loss=2.427475, test_loss=2.609052\n",
      "iteration=100, train_loss=2.413931, test_loss=2.596580\n",
      "iteration=200, train_loss=2.415637, test_loss=2.597212\n",
      "iteration=300, train_loss=2.407143, test_loss=2.587716\n",
      "iteration=400, train_loss=2.398073, test_loss=2.580315\n",
      "iteration=500, train_loss=2.415010, test_loss=2.592727\n",
      "iteration=600, train_loss=2.410068, test_loss=2.588573\n",
      "iteration=700, train_loss=2.404203, test_loss=2.582597\n",
      "iteration=800, train_loss=2.378891, test_loss=2.557972\n",
      "iteration=900, train_loss=2.374936, test_loss=2.554933\n",
      "Running 24\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.126869, test_loss=11.220910\n",
      "iteration=6, train_loss=9.583055, test_loss=9.640137\n",
      "iteration=7, train_loss=7.702722, test_loss=7.740479\n",
      "iteration=8, train_loss=6.035113, test_loss=6.070464\n",
      "iteration=9, train_loss=4.650710, test_loss=4.710141\n",
      "iteration=10, train_loss=3.802325, test_loss=3.903481\n",
      "iteration=11, train_loss=3.498375, test_loss=3.616155\n",
      "iteration=12, train_loss=3.345283, test_loss=3.464680\n",
      "iteration=13, train_loss=3.117131, test_loss=3.238189\n",
      "iteration=14, train_loss=2.889613, test_loss=3.023801\n",
      "iteration=15, train_loss=2.736007, test_loss=2.892409\n",
      "iteration=19, train_loss=2.739996, test_loss=2.879556\n",
      "iteration=22, train_loss=2.722107, test_loss=2.861831\n",
      "iteration=24, train_loss=2.603239, test_loss=2.752970\n",
      "iteration=32, train_loss=2.441729, test_loss=2.620048\n",
      "iteration=44, train_loss=2.416009, test_loss=2.597691\n",
      "iteration=65, train_loss=2.416122, test_loss=2.598167\n",
      "iteration=73, train_loss=2.428894, test_loss=2.611462\n",
      "iteration=83, train_loss=2.425023, test_loss=2.602829\n",
      "iteration=100, train_loss=2.417037, test_loss=2.598701\n",
      "iteration=200, train_loss=2.414511, test_loss=2.597885\n",
      "iteration=300, train_loss=2.406257, test_loss=2.588618\n",
      "iteration=366, train_loss=2.416478, test_loss=2.595114\n",
      "iteration=400, train_loss=2.397580, test_loss=2.579386\n",
      "iteration=500, train_loss=2.388252, test_loss=2.568676\n",
      "iteration=600, train_loss=2.395573, test_loss=2.571086\n",
      "iteration=700, train_loss=2.392041, test_loss=2.571933\n",
      "iteration=800, train_loss=2.366917, test_loss=2.545495\n",
      "iteration=870, train_loss=2.345422, test_loss=2.524566\n",
      "iteration=900, train_loss=2.339404, test_loss=2.518729\n",
      "iteration=987, train_loss=2.155347, test_loss=2.330320\n",
      "Running 25\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.994577, test_loss=11.088820\n",
      "iteration=6, train_loss=9.335462, test_loss=9.389329\n",
      "iteration=7, train_loss=7.483647, test_loss=7.518122\n",
      "iteration=8, train_loss=5.827081, test_loss=5.863116\n",
      "iteration=9, train_loss=4.473619, test_loss=4.539587\n",
      "iteration=10, train_loss=3.787130, test_loss=3.893947\n",
      "iteration=11, train_loss=3.585425, test_loss=3.701873\n",
      "iteration=12, train_loss=3.449678, test_loss=3.560122\n",
      "iteration=13, train_loss=3.252342, test_loss=3.363824\n",
      "iteration=14, train_loss=3.010577, test_loss=3.135985\n",
      "iteration=15, train_loss=2.803726, test_loss=2.954998\n",
      "iteration=16, train_loss=2.685756, test_loss=2.847441\n",
      "iteration=17, train_loss=2.655619, test_loss=2.811899\n",
      "iteration=25, train_loss=2.564002, test_loss=2.715936\n",
      "iteration=28, train_loss=2.423129, test_loss=2.602472\n",
      "iteration=30, train_loss=2.416300, test_loss=2.599061\n",
      "iteration=38, train_loss=2.436070, test_loss=2.614902\n",
      "iteration=44, train_loss=2.412283, test_loss=2.594140\n",
      "iteration=57, train_loss=2.412171, test_loss=2.595036\n",
      "iteration=83, train_loss=2.410861, test_loss=2.593025\n",
      "iteration=100, train_loss=2.414926, test_loss=2.595594\n",
      "iteration=200, train_loss=2.407823, test_loss=2.590524\n",
      "iteration=300, train_loss=2.399701, test_loss=2.582015\n",
      "iteration=400, train_loss=2.400882, test_loss=2.583109\n",
      "iteration=470, train_loss=2.405628, test_loss=2.582493\n",
      "iteration=500, train_loss=2.397474, test_loss=2.576235\n",
      "iteration=600, train_loss=2.389482, test_loss=2.567230\n",
      "iteration=700, train_loss=2.388307, test_loss=2.566962\n",
      "iteration=800, train_loss=2.376725, test_loss=2.555563\n",
      "iteration=900, train_loss=2.367204, test_loss=2.547183\n",
      "Running 26\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.971849, test_loss=13.038823\n",
      "iteration=13, train_loss=12.394295, test_loss=12.393297\n",
      "iteration=14, train_loss=11.787224, test_loss=11.787037\n",
      "iteration=15, train_loss=11.253886, test_loss=11.254256\n",
      "iteration=16, train_loss=10.782933, test_loss=10.783613\n",
      "iteration=17, train_loss=10.365269, test_loss=10.366108\n",
      "iteration=18, train_loss=9.993035, test_loss=9.994240\n",
      "iteration=21, train_loss=9.090471, test_loss=9.093159\n",
      "iteration=22, train_loss=8.846110, test_loss=8.849423\n",
      "iteration=24, train_loss=8.421899, test_loss=8.426602\n",
      "iteration=26, train_loss=8.067106, test_loss=8.073754\n",
      "iteration=27, train_loss=7.910848, test_loss=7.918304\n",
      "iteration=33, train_loss=7.183150, test_loss=7.195513\n",
      "iteration=35, train_loss=6.997433, test_loss=7.011414\n",
      "iteration=36, train_loss=6.911933, test_loss=6.926711\n",
      "iteration=43, train_loss=6.411360, test_loss=6.431741\n",
      "iteration=47, train_loss=6.176955, test_loss=6.200452\n",
      "iteration=50, train_loss=6.016750, test_loss=6.042593\n",
      "iteration=51, train_loss=5.965596, test_loss=5.992290\n",
      "iteration=56, train_loss=5.721368, test_loss=5.752281\n",
      "iteration=60, train_loss=5.537045, test_loss=5.571542\n",
      "iteration=62, train_loss=5.446122, test_loss=5.482563\n",
      "iteration=71, train_loss=5.050756, test_loss=5.096193\n",
      "iteration=76, train_loss=4.840001, test_loss=4.890845\n",
      "iteration=77, train_loss=4.798332, test_loss=4.850338\n",
      "iteration=86, train_loss=4.435461, test_loss=4.498334\n",
      "iteration=90, train_loss=4.277449, test_loss=4.345681\n",
      "iteration=91, train_loss=4.239327, test_loss=4.308823\n",
      "iteration=100, train_loss=3.914885, test_loss=3.996270\n",
      "iteration=101, train_loss=3.880617, test_loss=3.963311\n",
      "iteration=103, train_loss=3.812955, test_loss=3.898436\n",
      "iteration=111, train_loss=3.571390, test_loss=3.668857\n",
      "iteration=125, train_loss=3.269207, test_loss=3.387553\n",
      "iteration=127, train_loss=3.235454, test_loss=3.356338\n",
      "iteration=129, train_loss=3.203479, test_loss=3.326728\n",
      "iteration=143, train_loss=3.024825, test_loss=3.160318\n",
      "iteration=144, train_loss=3.014773, test_loss=3.150863\n",
      "iteration=149, train_loss=2.968304, test_loss=3.107261\n",
      "iteration=166, train_loss=2.860930, test_loss=3.006363\n",
      "iteration=170, train_loss=2.844634, test_loss=2.991207\n",
      "iteration=171, train_loss=2.840749, test_loss=2.987615\n",
      "iteration=184, train_loss=2.800165, test_loss=2.950744\n",
      "iteration=200, train_loss=2.766614, test_loss=2.921536\n",
      "iteration=211, train_loss=2.747503, test_loss=2.905709\n",
      "iteration=241, train_loss=2.714705, test_loss=2.878934\n",
      "iteration=249, train_loss=2.708113, test_loss=2.873362\n",
      "iteration=300, train_loss=2.671193, test_loss=2.842254\n",
      "iteration=374, train_loss=2.626853, test_loss=2.805003\n",
      "iteration=400, train_loss=2.614143, test_loss=2.793158\n",
      "iteration=422, train_loss=2.604534, test_loss=2.782998\n",
      "iteration=451, train_loss=2.593193, test_loss=2.772059\n",
      "iteration=500, train_loss=2.574797, test_loss=2.758222\n",
      "iteration=600, train_loss=2.545756, test_loss=2.730521\n",
      "iteration=700, train_loss=2.527385, test_loss=2.714173\n",
      "iteration=800, train_loss=2.514122, test_loss=2.701919\n",
      "iteration=900, train_loss=2.504037, test_loss=2.692361\n",
      "iteration=903, train_loss=2.503740, test_loss=2.692066\n",
      "Running 27\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.662343, test_loss=11.751596\n",
      "iteration=5, train_loss=10.379432, test_loss=10.469355\n",
      "iteration=6, train_loss=7.931110, test_loss=7.970491\n",
      "iteration=7, train_loss=6.161123, test_loss=6.189738\n",
      "iteration=8, train_loss=4.905289, test_loss=4.966385\n",
      "iteration=9, train_loss=4.574538, test_loss=4.670161\n",
      "iteration=10, train_loss=4.642522, test_loss=4.731626\n",
      "iteration=12, train_loss=4.184286, test_loss=4.265310\n",
      "iteration=13, train_loss=3.643568, test_loss=3.740795\n",
      "iteration=15, train_loss=2.665253, test_loss=2.824558\n",
      "iteration=16, train_loss=2.486135, test_loss=2.669528\n",
      "iteration=17, train_loss=2.465878, test_loss=2.642303\n",
      "iteration=18, train_loss=2.548331, test_loss=2.705287\n",
      "iteration=24, train_loss=2.591572, test_loss=2.742321\n",
      "iteration=32, train_loss=2.435453, test_loss=2.619576\n",
      "iteration=43, train_loss=2.427885, test_loss=2.609569\n",
      "iteration=53, train_loss=2.445408, test_loss=2.630079\n",
      "iteration=95, train_loss=2.421313, test_loss=2.603029\n",
      "iteration=100, train_loss=2.423120, test_loss=2.607342\n",
      "iteration=135, train_loss=2.418103, test_loss=2.601613\n",
      "iteration=172, train_loss=2.414912, test_loss=2.597455\n",
      "iteration=183, train_loss=2.423722, test_loss=2.602187\n",
      "iteration=200, train_loss=2.416889, test_loss=2.597236\n",
      "iteration=300, train_loss=2.410059, test_loss=2.591571\n",
      "iteration=318, train_loss=2.401789, test_loss=2.584110\n",
      "iteration=400, train_loss=2.396452, test_loss=2.578570\n",
      "iteration=500, train_loss=2.393610, test_loss=2.573902\n",
      "iteration=600, train_loss=2.384900, test_loss=2.565961\n",
      "iteration=700, train_loss=2.381565, test_loss=2.561394\n",
      "iteration=781, train_loss=2.375759, test_loss=2.556271\n",
      "iteration=800, train_loss=2.375140, test_loss=2.555628\n",
      "iteration=900, train_loss=2.376341, test_loss=2.557213\n",
      "Running 28\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.984464, test_loss=14.019958\n",
      "iteration=6, train_loss=13.328698, test_loss=13.334160\n",
      "iteration=7, train_loss=7.917772, test_loss=7.959701\n",
      "iteration=8, train_loss=6.239932, test_loss=6.272750\n",
      "iteration=9, train_loss=6.373558, test_loss=6.392668\n",
      "iteration=10, train_loss=4.828526, test_loss=4.881982\n",
      "iteration=11, train_loss=4.569597, test_loss=4.645125\n",
      "iteration=12, train_loss=4.352071, test_loss=4.446782\n",
      "iteration=14, train_loss=3.992494, test_loss=4.076878\n",
      "iteration=15, train_loss=3.824806, test_loss=3.903298\n",
      "iteration=16, train_loss=3.631175, test_loss=3.714872\n",
      "iteration=18, train_loss=3.106736, test_loss=3.220814\n",
      "iteration=19, train_loss=2.869964, test_loss=3.006212\n",
      "iteration=20, train_loss=2.691434, test_loss=2.849505\n",
      "iteration=22, train_loss=2.464915, test_loss=2.641497\n",
      "iteration=24, train_loss=2.471223, test_loss=2.649564\n",
      "iteration=31, train_loss=2.560491, test_loss=2.729421\n",
      "iteration=58, train_loss=2.433036, test_loss=2.612116\n",
      "iteration=84, train_loss=2.418536, test_loss=2.599484\n",
      "iteration=100, train_loss=2.414842, test_loss=2.593941\n",
      "iteration=109, train_loss=2.430055, test_loss=2.604260\n",
      "iteration=112, train_loss=2.410643, test_loss=2.590063\n",
      "iteration=118, train_loss=2.415245, test_loss=2.592472\n",
      "iteration=200, train_loss=2.393852, test_loss=2.573588\n",
      "iteration=252, train_loss=2.398221, test_loss=2.573379\n",
      "iteration=300, train_loss=2.427824, test_loss=2.599353\n",
      "iteration=400, train_loss=2.574563, test_loss=2.726988\n",
      "iteration=500, train_loss=3.121053, test_loss=3.232846\n",
      "iteration=600, train_loss=3.300326, test_loss=3.400384\n",
      "iteration=700, train_loss=4.022172, test_loss=4.091594\n",
      "iteration=800, train_loss=3.003404, test_loss=3.107499\n",
      "iteration=900, train_loss=4.628347, test_loss=4.671073\n",
      "Running 29\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.039157, test_loss=11.130931\n",
      "iteration=6, train_loss=9.537361, test_loss=9.599998\n",
      "iteration=7, train_loss=7.633371, test_loss=7.674471\n",
      "iteration=8, train_loss=5.948967, test_loss=5.986752\n",
      "iteration=9, train_loss=4.525985, test_loss=4.588778\n",
      "iteration=10, train_loss=3.704181, test_loss=3.810110\n",
      "iteration=12, train_loss=3.424617, test_loss=3.534203\n",
      "iteration=13, train_loss=3.288049, test_loss=3.396762\n",
      "iteration=15, train_loss=2.880755, test_loss=3.028529\n",
      "iteration=16, train_loss=2.777192, test_loss=2.931694\n",
      "iteration=20, train_loss=2.848395, test_loss=2.979825\n",
      "iteration=24, train_loss=2.629097, test_loss=2.772662\n",
      "iteration=30, train_loss=2.431709, test_loss=2.611885\n",
      "iteration=54, train_loss=2.416861, test_loss=2.597255\n",
      "iteration=55, train_loss=2.417473, test_loss=2.597020\n",
      "iteration=100, train_loss=2.414165, test_loss=2.595069\n",
      "iteration=200, train_loss=2.413373, test_loss=2.593817\n",
      "iteration=298, train_loss=2.402611, test_loss=2.585075\n",
      "iteration=300, train_loss=2.399817, test_loss=2.579628\n",
      "iteration=326, train_loss=2.395678, test_loss=2.577435\n",
      "iteration=400, train_loss=2.410430, test_loss=2.592702\n",
      "iteration=418, train_loss=2.390481, test_loss=2.570980\n",
      "iteration=500, train_loss=2.387197, test_loss=2.567006\n",
      "iteration=600, train_loss=2.383601, test_loss=2.563006\n",
      "iteration=700, train_loss=2.381395, test_loss=2.561865\n",
      "iteration=800, train_loss=2.371567, test_loss=2.550655\n",
      "iteration=900, train_loss=2.353370, test_loss=2.530817\n",
      "Running 30\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.129021, test_loss=13.160815\n",
      "iteration=5, train_loss=9.713631, test_loss=9.804708\n",
      "iteration=6, train_loss=7.228147, test_loss=7.273299\n",
      "iteration=7, train_loss=5.449048, test_loss=5.490242\n",
      "iteration=8, train_loss=4.367908, test_loss=4.444538\n",
      "iteration=10, train_loss=4.065142, test_loss=4.165619\n",
      "iteration=12, train_loss=3.457761, test_loss=3.563156\n",
      "iteration=13, train_loss=2.953009, test_loss=3.084352\n",
      "iteration=14, train_loss=2.633516, test_loss=2.798887\n",
      "iteration=16, train_loss=2.502241, test_loss=2.675266\n",
      "iteration=27, train_loss=2.437539, test_loss=2.617725\n",
      "iteration=42, train_loss=2.429541, test_loss=2.611629\n",
      "iteration=80, train_loss=2.438415, test_loss=2.618134\n",
      "iteration=100, train_loss=2.425063, test_loss=2.605665\n",
      "iteration=110, train_loss=2.420153, test_loss=2.601698\n",
      "iteration=126, train_loss=2.417093, test_loss=2.598939\n",
      "iteration=200, train_loss=2.411599, test_loss=2.592964\n",
      "iteration=281, train_loss=2.403842, test_loss=2.586417\n",
      "iteration=300, train_loss=2.408783, test_loss=2.588585\n",
      "iteration=400, train_loss=2.398167, test_loss=2.578545\n",
      "iteration=500, train_loss=2.389844, test_loss=2.571201\n",
      "iteration=600, train_loss=2.388429, test_loss=2.567009\n",
      "iteration=700, train_loss=2.381061, test_loss=2.561932\n",
      "iteration=800, train_loss=2.375238, test_loss=2.554138\n",
      "iteration=900, train_loss=2.392002, test_loss=2.566276\n",
      "Running 31\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.845211, test_loss=10.938331\n",
      "iteration=5, train_loss=10.358179, test_loss=10.444927\n",
      "iteration=6, train_loss=8.049035, test_loss=8.088852\n",
      "iteration=7, train_loss=6.303874, test_loss=6.331738\n",
      "iteration=8, train_loss=4.987300, test_loss=5.043530\n",
      "iteration=9, train_loss=4.488983, test_loss=4.584450\n",
      "iteration=10, train_loss=4.460586, test_loss=4.556609\n",
      "iteration=12, train_loss=3.980043, test_loss=4.067809\n",
      "iteration=13, train_loss=3.442742, test_loss=3.547639\n",
      "iteration=14, train_loss=2.894291, test_loss=3.029392\n",
      "iteration=15, train_loss=2.565686, test_loss=2.740569\n",
      "iteration=16, train_loss=2.478981, test_loss=2.657804\n",
      "iteration=38, train_loss=2.467683, test_loss=2.648417\n",
      "iteration=100, train_loss=2.453521, test_loss=2.629147\n",
      "iteration=161, train_loss=2.414293, test_loss=2.597334\n",
      "iteration=200, train_loss=2.421905, test_loss=2.603689\n",
      "iteration=300, train_loss=2.411180, test_loss=2.592592\n",
      "iteration=400, train_loss=2.407021, test_loss=2.588615\n",
      "iteration=460, train_loss=2.396749, test_loss=2.578316\n",
      "iteration=500, train_loss=2.404959, test_loss=2.582495\n",
      "iteration=600, train_loss=2.398973, test_loss=2.577387\n",
      "iteration=700, train_loss=2.397110, test_loss=2.577130\n",
      "iteration=800, train_loss=2.381254, test_loss=2.560930\n",
      "iteration=900, train_loss=2.380134, test_loss=2.561228\n",
      "Running 32\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.846027, test_loss=10.938219\n",
      "iteration=7, train_loss=9.372172, test_loss=9.417966\n",
      "iteration=8, train_loss=7.617720, test_loss=7.655823\n",
      "iteration=9, train_loss=5.969040, test_loss=6.010517\n",
      "iteration=10, train_loss=4.502914, test_loss=4.566888\n",
      "iteration=11, train_loss=3.396223, test_loss=3.497519\n",
      "iteration=12, train_loss=2.906813, test_loss=3.042639\n",
      "iteration=13, train_loss=2.785128, test_loss=2.930779\n",
      "iteration=23, train_loss=2.755279, test_loss=2.893005\n",
      "iteration=26, train_loss=2.507976, test_loss=2.674776\n",
      "iteration=40, train_loss=2.444382, test_loss=2.623229\n",
      "iteration=41, train_loss=2.435062, test_loss=2.614208\n",
      "iteration=66, train_loss=2.411799, test_loss=2.594385\n",
      "iteration=82, train_loss=2.414385, test_loss=2.596148\n",
      "iteration=100, train_loss=2.424922, test_loss=2.602062\n",
      "iteration=200, train_loss=2.410466, test_loss=2.591330\n",
      "iteration=300, train_loss=2.396713, test_loss=2.578125\n",
      "iteration=400, train_loss=2.403944, test_loss=2.585287\n",
      "iteration=500, train_loss=2.391506, test_loss=2.573088\n",
      "iteration=567, train_loss=2.393031, test_loss=2.571950\n",
      "iteration=600, train_loss=2.382529, test_loss=2.563111\n",
      "iteration=700, train_loss=2.378482, test_loss=2.558108\n",
      "iteration=800, train_loss=2.379723, test_loss=2.557263\n",
      "iteration=900, train_loss=2.372635, test_loss=2.550990\n",
      "Running 33\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.522218, test_loss=13.573615\n",
      "iteration=5, train_loss=8.981616, test_loss=9.065579\n",
      "iteration=6, train_loss=6.616511, test_loss=6.645223\n",
      "iteration=7, train_loss=5.313915, test_loss=5.369235\n",
      "iteration=8, train_loss=5.113460, test_loss=5.208152\n",
      "iteration=10, train_loss=5.109728, test_loss=5.180206\n",
      "iteration=11, train_loss=4.623434, test_loss=4.698715\n",
      "iteration=12, train_loss=3.924489, test_loss=4.020120\n",
      "iteration=13, train_loss=3.201683, test_loss=3.328912\n",
      "iteration=14, train_loss=2.679770, test_loss=2.839348\n",
      "iteration=23, train_loss=2.656331, test_loss=2.806230\n",
      "iteration=24, train_loss=2.602286, test_loss=2.760847\n",
      "iteration=26, train_loss=2.529666, test_loss=2.702027\n",
      "iteration=28, train_loss=2.482296, test_loss=2.664842\n",
      "iteration=29, train_loss=2.467932, test_loss=2.653739\n",
      "iteration=57, train_loss=2.454530, test_loss=2.641798\n",
      "iteration=100, train_loss=2.442283, test_loss=2.628480\n",
      "iteration=200, train_loss=2.431604, test_loss=2.616421\n",
      "iteration=220, train_loss=2.451468, test_loss=2.631223\n",
      "iteration=225, train_loss=2.430034, test_loss=2.615349\n",
      "iteration=300, train_loss=2.423615, test_loss=2.608670\n",
      "iteration=400, train_loss=2.433559, test_loss=2.610343\n",
      "iteration=404, train_loss=2.412752, test_loss=2.595356\n",
      "iteration=500, train_loss=2.404604, test_loss=2.585488\n",
      "iteration=527, train_loss=2.402813, test_loss=2.584034\n",
      "iteration=600, train_loss=2.402135, test_loss=2.585537\n",
      "iteration=700, train_loss=2.399993, test_loss=2.579865\n",
      "iteration=800, train_loss=2.384206, test_loss=2.566295\n",
      "iteration=900, train_loss=2.376892, test_loss=2.557196\n",
      "Running 34\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=10.789036, test_loss=10.878398\n",
      "iteration=7, train_loss=8.929119, test_loss=8.975157\n",
      "iteration=8, train_loss=7.203152, test_loss=7.240248\n",
      "iteration=9, train_loss=5.602969, test_loss=5.645774\n",
      "iteration=10, train_loss=4.258024, test_loss=4.327771\n",
      "iteration=11, train_loss=3.383927, test_loss=3.496690\n",
      "iteration=12, train_loss=3.076525, test_loss=3.207353\n",
      "iteration=13, train_loss=3.000465, test_loss=3.129954\n",
      "iteration=15, train_loss=2.940872, test_loss=3.084304\n",
      "iteration=17, train_loss=2.930791, test_loss=3.067389\n",
      "iteration=20, train_loss=2.930162, test_loss=3.050432\n",
      "iteration=22, train_loss=2.804916, test_loss=2.938042\n",
      "iteration=26, train_loss=2.505064, test_loss=2.666301\n",
      "iteration=27, train_loss=2.461805, test_loss=2.630391\n",
      "iteration=29, train_loss=2.423558, test_loss=2.603021\n",
      "iteration=31, train_loss=2.437855, test_loss=2.617258\n",
      "iteration=32, train_loss=2.455005, test_loss=2.631635\n",
      "iteration=43, train_loss=2.417490, test_loss=2.598603\n",
      "iteration=100, train_loss=2.414083, test_loss=2.596330\n",
      "iteration=200, train_loss=2.408991, test_loss=2.591322\n",
      "iteration=300, train_loss=2.402620, test_loss=2.581896\n",
      "iteration=400, train_loss=2.397497, test_loss=2.579427\n",
      "iteration=500, train_loss=2.388842, test_loss=2.567847\n",
      "iteration=600, train_loss=2.384968, test_loss=2.563976\n",
      "iteration=700, train_loss=2.377592, test_loss=2.557363\n",
      "iteration=737, train_loss=2.376345, test_loss=2.555330\n",
      "iteration=773, train_loss=2.397729, test_loss=2.574870\n",
      "iteration=800, train_loss=2.373208, test_loss=2.552757\n",
      "iteration=833, train_loss=2.374332, test_loss=2.554488\n",
      "iteration=900, train_loss=2.395131, test_loss=2.567409\n",
      "Running 35\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.621193, test_loss=13.657485\n",
      "iteration=7, train_loss=13.317945, test_loss=13.315661\n",
      "iteration=8, train_loss=8.306476, test_loss=8.310312\n",
      "iteration=10, train_loss=6.671579, test_loss=6.703053\n",
      "iteration=11, train_loss=5.794798, test_loss=5.853840\n",
      "iteration=12, train_loss=5.266434, test_loss=5.353881\n",
      "iteration=13, train_loss=4.778017, test_loss=4.872427\n",
      "iteration=14, train_loss=4.406117, test_loss=4.494322\n",
      "iteration=15, train_loss=4.076508, test_loss=4.155571\n",
      "iteration=16, train_loss=3.752798, test_loss=3.834749\n",
      "iteration=17, train_loss=3.409408, test_loss=3.505662\n",
      "iteration=18, train_loss=3.083750, test_loss=3.199484\n",
      "iteration=19, train_loss=2.818417, test_loss=2.957653\n",
      "iteration=20, train_loss=2.645777, test_loss=2.802757\n",
      "iteration=21, train_loss=2.542958, test_loss=2.711544\n",
      "iteration=69, train_loss=2.448953, test_loss=2.632519\n",
      "iteration=100, train_loss=2.436033, test_loss=2.618185\n",
      "iteration=120, train_loss=2.428351, test_loss=2.610674\n",
      "iteration=129, train_loss=2.418466, test_loss=2.599952\n",
      "iteration=200, train_loss=2.395873, test_loss=2.576301\n",
      "iteration=300, train_loss=2.438969, test_loss=2.611310\n",
      "iteration=400, train_loss=3.040652, test_loss=3.156209\n",
      "iteration=500, train_loss=2.707444, test_loss=2.845307\n",
      "iteration=600, train_loss=3.867646, test_loss=3.943632\n",
      "iteration=700, train_loss=4.194151, test_loss=4.255296\n",
      "iteration=800, train_loss=4.246700, test_loss=4.302221\n",
      "iteration=900, train_loss=4.344866, test_loss=4.394598\n",
      "Running 36\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.933670, test_loss=12.979320\n",
      "iteration=11, train_loss=12.189251, test_loss=12.188610\n",
      "iteration=12, train_loss=11.173138, test_loss=11.172851\n",
      "iteration=13, train_loss=10.257357, test_loss=10.257570\n",
      "iteration=14, train_loss=9.427458, test_loss=9.428724\n",
      "iteration=15, train_loss=8.671775, test_loss=8.674721\n",
      "iteration=17, train_loss=7.346521, test_loss=7.356934\n",
      "iteration=18, train_loss=6.766823, test_loss=6.782715\n",
      "iteration=19, train_loss=6.235003, test_loss=6.257357\n",
      "iteration=20, train_loss=5.745676, test_loss=5.775884\n",
      "iteration=21, train_loss=5.297215, test_loss=5.336637\n",
      "iteration=22, train_loss=4.886466, test_loss=4.935688\n",
      "iteration=23, train_loss=4.512302, test_loss=4.572138\n",
      "iteration=25, train_loss=3.874057, test_loss=3.955481\n",
      "iteration=26, train_loss=3.612642, test_loss=3.705398\n",
      "iteration=27, train_loss=3.399068, test_loss=3.503969\n",
      "iteration=28, train_loss=3.223341, test_loss=3.340926\n",
      "iteration=30, train_loss=2.965251, test_loss=3.100260\n",
      "iteration=33, train_loss=2.759370, test_loss=2.907653\n",
      "iteration=38, train_loss=2.685508, test_loss=2.850914\n",
      "iteration=72, train_loss=2.586421, test_loss=2.762829\n",
      "iteration=74, train_loss=2.582688, test_loss=2.758607\n",
      "iteration=85, train_loss=2.565696, test_loss=2.741704\n",
      "iteration=100, train_loss=2.540085, test_loss=2.722254\n",
      "iteration=125, train_loss=2.519808, test_loss=2.706043\n",
      "iteration=139, train_loss=2.515111, test_loss=2.701110\n",
      "iteration=165, train_loss=2.508600, test_loss=2.697192\n",
      "iteration=167, train_loss=2.508849, test_loss=2.697706\n",
      "iteration=200, train_loss=2.504557, test_loss=2.692398\n",
      "iteration=300, train_loss=2.497058, test_loss=2.685803\n",
      "iteration=362, train_loss=2.491871, test_loss=2.679573\n",
      "iteration=400, train_loss=2.489418, test_loss=2.676977\n",
      "iteration=401, train_loss=2.489655, test_loss=2.677076\n",
      "iteration=455, train_loss=2.486363, test_loss=2.674996\n",
      "iteration=500, train_loss=2.482857, test_loss=2.668841\n",
      "iteration=600, train_loss=2.471832, test_loss=2.659242\n",
      "iteration=688, train_loss=2.464070, test_loss=2.650663\n",
      "iteration=700, train_loss=2.463837, test_loss=2.651064\n",
      "iteration=800, train_loss=2.460834, test_loss=2.645488\n",
      "iteration=900, train_loss=2.448583, test_loss=2.635066\n",
      "Running 37\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=11.945219, test_loss=12.031481\n",
      "iteration=5, train_loss=10.125840, test_loss=10.217795\n",
      "iteration=6, train_loss=7.548010, test_loss=7.590793\n",
      "iteration=7, train_loss=5.720383, test_loss=5.756074\n",
      "iteration=8, train_loss=4.588804, test_loss=4.660901\n",
      "iteration=9, train_loss=4.404514, test_loss=4.503945\n",
      "iteration=12, train_loss=4.034749, test_loss=4.117952\n",
      "iteration=13, train_loss=3.455954, test_loss=3.559305\n",
      "iteration=14, train_loss=2.924476, test_loss=3.057053\n",
      "iteration=15, train_loss=2.622483, test_loss=2.790664\n",
      "iteration=16, train_loss=2.517457, test_loss=2.698685\n",
      "iteration=17, train_loss=2.510867, test_loss=2.684616\n",
      "iteration=19, train_loss=2.614235, test_loss=2.764095\n",
      "iteration=27, train_loss=2.460045, test_loss=2.641528\n",
      "iteration=29, train_loss=2.445899, test_loss=2.631189\n",
      "iteration=39, train_loss=2.440630, test_loss=2.623027\n",
      "iteration=75, train_loss=2.430677, test_loss=2.612808\n",
      "iteration=82, train_loss=2.422779, test_loss=2.605425\n",
      "iteration=91, train_loss=2.427680, test_loss=2.609259\n",
      "iteration=100, train_loss=2.419255, test_loss=2.602934\n",
      "iteration=200, train_loss=2.415839, test_loss=2.597299\n",
      "iteration=300, train_loss=2.407541, test_loss=2.588729\n",
      "iteration=400, train_loss=2.403222, test_loss=2.584005\n",
      "iteration=489, train_loss=2.404961, test_loss=2.587915\n",
      "iteration=500, train_loss=2.422713, test_loss=2.598298\n",
      "iteration=600, train_loss=2.395773, test_loss=2.577529\n",
      "iteration=700, train_loss=2.409110, test_loss=2.589573\n",
      "iteration=800, train_loss=2.386330, test_loss=2.567348\n",
      "iteration=900, train_loss=2.382310, test_loss=2.562115\n",
      "iteration=983, train_loss=2.371085, test_loss=2.550577\n",
      "Running 38\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.722190, test_loss=12.784703\n",
      "iteration=5, train_loss=10.338559, test_loss=10.429690\n",
      "iteration=6, train_loss=7.776071, test_loss=7.818284\n",
      "iteration=7, train_loss=5.930767, test_loss=5.963370\n",
      "iteration=8, train_loss=4.668149, test_loss=4.736377\n",
      "iteration=9, train_loss=4.441331, test_loss=4.539841\n",
      "iteration=12, train_loss=4.010015, test_loss=4.093378\n",
      "iteration=13, train_loss=3.446181, test_loss=3.548980\n",
      "iteration=14, train_loss=2.865094, test_loss=3.001686\n",
      "iteration=17, train_loss=2.567115, test_loss=2.724286\n",
      "iteration=19, train_loss=2.739385, test_loss=2.875496\n",
      "iteration=20, train_loss=2.767656, test_loss=2.901392\n",
      "iteration=24, train_loss=2.522162, test_loss=2.683239\n",
      "iteration=28, train_loss=2.450844, test_loss=2.634927\n",
      "iteration=31, train_loss=2.492787, test_loss=2.668994\n",
      "iteration=42, train_loss=2.428378, test_loss=2.610429\n",
      "iteration=43, train_loss=2.427622, test_loss=2.609600\n",
      "iteration=74, train_loss=2.426253, test_loss=2.606210\n",
      "iteration=100, train_loss=2.417370, test_loss=2.601052\n",
      "iteration=143, train_loss=2.417484, test_loss=2.599543\n",
      "iteration=200, train_loss=2.410680, test_loss=2.591767\n",
      "iteration=212, train_loss=2.416650, test_loss=2.596000\n",
      "iteration=300, train_loss=2.402256, test_loss=2.583708\n",
      "iteration=400, train_loss=2.404474, test_loss=2.582746\n",
      "iteration=500, train_loss=2.390600, test_loss=2.570621\n",
      "iteration=553, train_loss=2.391802, test_loss=2.570310\n",
      "iteration=600, train_loss=2.386226, test_loss=2.566686\n",
      "iteration=700, train_loss=2.407830, test_loss=2.588260\n",
      "iteration=800, train_loss=2.378857, test_loss=2.557251\n",
      "iteration=900, train_loss=2.378001, test_loss=2.558438\n",
      "Running 39\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=14.477267, test_loss=14.513793\n",
      "iteration=10, train_loss=13.952025, test_loss=14.036554\n",
      "iteration=11, train_loss=11.762279, test_loss=11.816841\n",
      "iteration=12, train_loss=9.959537, test_loss=9.995890\n",
      "iteration=13, train_loss=8.375642, test_loss=8.401210\n",
      "iteration=14, train_loss=6.957730, test_loss=6.980456\n",
      "iteration=15, train_loss=5.711483, test_loss=5.746390\n",
      "iteration=16, train_loss=4.747561, test_loss=4.811937\n",
      "iteration=17, train_loss=4.185872, test_loss=4.283640\n",
      "iteration=18, train_loss=3.784562, test_loss=3.895712\n",
      "iteration=19, train_loss=3.339607, test_loss=3.463682\n",
      "iteration=20, train_loss=2.867390, test_loss=3.009309\n",
      "iteration=21, train_loss=2.542515, test_loss=2.710315\n",
      "iteration=22, train_loss=2.452708, test_loss=2.634945\n",
      "iteration=86, train_loss=2.432413, test_loss=2.615449\n",
      "iteration=100, train_loss=2.428309, test_loss=2.609689\n",
      "iteration=109, train_loss=2.426841, test_loss=2.609274\n",
      "iteration=168, train_loss=2.453253, test_loss=2.630001\n",
      "iteration=200, train_loss=2.415225, test_loss=2.595848\n",
      "iteration=293, train_loss=2.424924, test_loss=2.607324\n",
      "iteration=300, train_loss=2.414661, test_loss=2.592010\n",
      "iteration=301, train_loss=2.413284, test_loss=2.591035\n",
      "iteration=314, train_loss=2.401326, test_loss=2.582165\n",
      "iteration=400, train_loss=2.393475, test_loss=2.573181\n",
      "iteration=498, train_loss=2.407352, test_loss=2.585876\n",
      "iteration=500, train_loss=2.415052, test_loss=2.591867\n",
      "iteration=600, train_loss=2.381333, test_loss=2.560302\n",
      "iteration=700, train_loss=2.382000, test_loss=2.557607\n",
      "iteration=800, train_loss=2.353467, test_loss=2.531883\n",
      "iteration=870, train_loss=2.341919, test_loss=2.520427\n",
      "iteration=900, train_loss=2.341951, test_loss=2.518224\n",
      "Running 40\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.347239, test_loss=12.421281\n",
      "iteration=5, train_loss=9.270153, test_loss=9.360632\n",
      "iteration=6, train_loss=6.572854, test_loss=6.606781\n",
      "iteration=7, train_loss=5.044689, test_loss=5.102762\n",
      "iteration=8, train_loss=4.871705, test_loss=4.967619\n",
      "iteration=11, train_loss=4.628352, test_loss=4.697940\n",
      "iteration=12, train_loss=3.965760, test_loss=4.055306\n",
      "iteration=14, train_loss=2.719813, test_loss=2.874331\n",
      "iteration=15, train_loss=2.485602, test_loss=2.667881\n",
      "iteration=16, train_loss=2.453314, test_loss=2.635845\n",
      "iteration=18, train_loss=2.616603, test_loss=2.768074\n",
      "iteration=76, train_loss=2.436140, test_loss=2.621549\n",
      "iteration=100, train_loss=2.433196, test_loss=2.618000\n",
      "iteration=200, train_loss=2.431635, test_loss=2.613136\n",
      "iteration=300, train_loss=2.435297, test_loss=2.617625\n",
      "iteration=400, train_loss=2.423054, test_loss=2.606847\n",
      "iteration=500, train_loss=2.398654, test_loss=2.580220\n",
      "iteration=600, train_loss=2.400823, test_loss=2.579077\n",
      "iteration=700, train_loss=2.393540, test_loss=2.572047\n",
      "iteration=800, train_loss=2.390181, test_loss=2.567945\n",
      "iteration=850, train_loss=2.397796, test_loss=2.579914\n",
      "iteration=900, train_loss=2.375454, test_loss=2.555853\n",
      "iteration=981, train_loss=2.388532, test_loss=2.563270\n",
      "Running 41\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.479004, test_loss=13.509208\n",
      "iteration=7, train_loss=11.849583, test_loss=11.858483\n",
      "iteration=8, train_loss=9.969860, test_loss=9.989698\n",
      "iteration=9, train_loss=8.332891, test_loss=8.366443\n",
      "iteration=10, train_loss=6.935854, test_loss=6.988172\n",
      "iteration=11, train_loss=5.783844, test_loss=5.862759\n",
      "iteration=12, train_loss=4.848553, test_loss=4.950057\n",
      "iteration=13, train_loss=4.108952, test_loss=4.225407\n",
      "iteration=14, train_loss=3.542595, test_loss=3.666814\n",
      "iteration=15, train_loss=3.172049, test_loss=3.301635\n",
      "iteration=28, train_loss=3.151227, test_loss=3.288846\n",
      "iteration=29, train_loss=3.010368, test_loss=3.157191\n",
      "iteration=30, train_loss=2.881346, test_loss=3.037891\n",
      "iteration=32, train_loss=2.687228, test_loss=2.860347\n",
      "iteration=33, train_loss=2.627138, test_loss=2.806490\n",
      "iteration=34, train_loss=2.591209, test_loss=2.775322\n",
      "iteration=36, train_loss=2.574909, test_loss=2.763472\n",
      "iteration=37, train_loss=2.589790, test_loss=2.777094\n",
      "iteration=40, train_loss=2.640123, test_loss=2.821184\n",
      "iteration=41, train_loss=2.651975, test_loss=2.831363\n",
      "iteration=58, train_loss=2.572040, test_loss=2.762291\n",
      "iteration=73, train_loss=2.550586, test_loss=2.740157\n",
      "iteration=74, train_loss=2.550438, test_loss=2.739464\n",
      "iteration=100, train_loss=2.553103, test_loss=2.741116\n",
      "iteration=112, train_loss=2.548958, test_loss=2.738438\n",
      "iteration=200, train_loss=2.549624, test_loss=2.736682\n",
      "iteration=287, train_loss=2.553182, test_loss=2.738733\n",
      "iteration=300, train_loss=2.535261, test_loss=2.723422\n",
      "iteration=400, train_loss=2.527341, test_loss=2.715771\n",
      "iteration=498, train_loss=2.520429, test_loss=2.706564\n",
      "iteration=500, train_loss=2.519887, test_loss=2.706121\n",
      "iteration=600, train_loss=2.507969, test_loss=2.696110\n",
      "iteration=700, train_loss=2.497735, test_loss=2.683843\n",
      "iteration=800, train_loss=2.486489, test_loss=2.672745\n",
      "iteration=821, train_loss=2.484179, test_loss=2.670768\n",
      "iteration=900, train_loss=2.486992, test_loss=2.673894\n",
      "Running 42\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.734521, test_loss=13.772891\n",
      "iteration=5, train_loss=8.888653, test_loss=8.975460\n",
      "iteration=6, train_loss=6.403397, test_loss=6.438728\n",
      "iteration=7, train_loss=4.930918, test_loss=4.990703\n",
      "iteration=8, train_loss=4.686709, test_loss=4.782489\n",
      "iteration=11, train_loss=4.047827, test_loss=4.139415\n",
      "iteration=12, train_loss=3.357714, test_loss=3.476654\n",
      "iteration=13, train_loss=2.757192, test_loss=2.907741\n",
      "iteration=14, train_loss=2.483985, test_loss=2.662994\n",
      "iteration=23, train_loss=2.555181, test_loss=2.714179\n",
      "iteration=24, train_loss=2.506760, test_loss=2.676025\n",
      "iteration=26, train_loss=2.464097, test_loss=2.646533\n",
      "iteration=31, train_loss=2.445494, test_loss=2.629806\n",
      "iteration=34, train_loss=2.437094, test_loss=2.620701\n",
      "iteration=43, train_loss=2.429334, test_loss=2.613665\n",
      "iteration=63, train_loss=2.429896, test_loss=2.614465\n",
      "iteration=74, train_loss=2.429837, test_loss=2.612836\n",
      "iteration=100, train_loss=2.426322, test_loss=2.609070\n",
      "iteration=124, train_loss=2.423231, test_loss=2.606187\n",
      "iteration=200, train_loss=2.416803, test_loss=2.600393\n",
      "iteration=300, train_loss=2.409648, test_loss=2.591538\n",
      "iteration=353, train_loss=2.406393, test_loss=2.588488\n",
      "iteration=400, train_loss=2.404305, test_loss=2.584874\n",
      "iteration=500, train_loss=2.393467, test_loss=2.575188\n",
      "iteration=600, train_loss=2.392204, test_loss=2.572486\n",
      "iteration=700, train_loss=2.384182, test_loss=2.562915\n",
      "iteration=800, train_loss=2.373991, test_loss=2.554206\n",
      "iteration=900, train_loss=2.373411, test_loss=2.551819\n",
      "Running 43\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.251521, test_loss=13.301761\n",
      "iteration=6, train_loss=11.863585, test_loss=11.881967\n",
      "iteration=7, train_loss=6.601207, test_loss=6.693150\n",
      "iteration=8, train_loss=4.773978, test_loss=4.860266\n",
      "iteration=9, train_loss=3.972116, test_loss=4.046471\n",
      "iteration=17, train_loss=3.321939, test_loss=3.422669\n",
      "iteration=18, train_loss=3.045658, test_loss=3.162072\n",
      "iteration=20, train_loss=2.662153, test_loss=2.823199\n",
      "iteration=22, train_loss=2.479119, test_loss=2.661200\n",
      "iteration=23, train_loss=2.461574, test_loss=2.644150\n",
      "iteration=29, train_loss=2.545182, test_loss=2.717235\n",
      "iteration=40, train_loss=2.437848, test_loss=2.620210\n",
      "iteration=42, train_loss=2.455312, test_loss=2.637724\n",
      "iteration=52, train_loss=2.446428, test_loss=2.628751\n",
      "iteration=54, train_loss=2.431972, test_loss=2.613844\n",
      "iteration=79, train_loss=2.425354, test_loss=2.606566\n",
      "iteration=87, train_loss=2.423261, test_loss=2.603818\n",
      "iteration=100, train_loss=2.416911, test_loss=2.597729\n",
      "iteration=197, train_loss=2.395750, test_loss=2.572399\n",
      "iteration=200, train_loss=2.408930, test_loss=2.581891\n",
      "iteration=300, train_loss=2.419473, test_loss=2.593728\n",
      "iteration=400, train_loss=2.581834, test_loss=2.734749\n",
      "iteration=500, train_loss=2.451817, test_loss=2.613695\n",
      "iteration=600, train_loss=3.436880, test_loss=3.529501\n",
      "iteration=700, train_loss=3.539519, test_loss=3.628129\n",
      "iteration=800, train_loss=3.797737, test_loss=3.869330\n",
      "iteration=828, train_loss=2.247157, test_loss=2.414004\n",
      "iteration=834, train_loss=2.297617, test_loss=2.455947\n",
      "iteration=900, train_loss=2.227401, test_loss=2.391244\n",
      "Running 44\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=12.251884, test_loss=12.336336\n",
      "iteration=5, train_loss=11.246246, test_loss=11.338840\n",
      "iteration=6, train_loss=8.626667, test_loss=8.684474\n",
      "iteration=7, train_loss=6.607061, test_loss=6.645822\n",
      "iteration=8, train_loss=4.943098, test_loss=4.995292\n",
      "iteration=9, train_loss=3.907301, test_loss=4.002497\n",
      "iteration=10, train_loss=3.683112, test_loss=3.797061\n",
      "iteration=11, train_loss=3.677879, test_loss=3.780005\n",
      "iteration=12, train_loss=3.536602, test_loss=3.635230\n",
      "iteration=14, train_loss=2.889662, test_loss=3.030193\n",
      "iteration=15, train_loss=2.709946, test_loss=2.871440\n",
      "iteration=16, train_loss=2.630431, test_loss=2.794330\n",
      "iteration=17, train_loss=2.628683, test_loss=2.784894\n",
      "iteration=24, train_loss=2.558175, test_loss=2.711575\n",
      "iteration=27, train_loss=2.429284, test_loss=2.608720\n",
      "iteration=30, train_loss=2.428406, test_loss=2.610708\n",
      "iteration=45, train_loss=2.413506, test_loss=2.596522\n",
      "iteration=59, train_loss=2.420065, test_loss=2.600349\n",
      "iteration=88, train_loss=2.411635, test_loss=2.594337\n",
      "iteration=100, train_loss=2.415233, test_loss=2.598902\n",
      "iteration=200, train_loss=2.418872, test_loss=2.599642\n",
      "iteration=300, train_loss=2.403481, test_loss=2.585636\n",
      "iteration=360, train_loss=2.420401, test_loss=2.601471\n",
      "iteration=400, train_loss=2.410271, test_loss=2.589418\n",
      "iteration=480, train_loss=2.388535, test_loss=2.569024\n",
      "iteration=492, train_loss=2.387678, test_loss=2.568460\n",
      "iteration=500, train_loss=2.403925, test_loss=2.582931\n",
      "iteration=600, train_loss=2.389239, test_loss=2.570933\n",
      "iteration=700, train_loss=2.396749, test_loss=2.571221\n",
      "iteration=718, train_loss=2.383215, test_loss=2.564497\n",
      "iteration=744, train_loss=2.377973, test_loss=2.556927\n",
      "iteration=800, train_loss=2.377799, test_loss=2.554924\n",
      "iteration=900, train_loss=2.361491, test_loss=2.540336\n",
      "Running 45\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.302641, test_loss=13.350721\n",
      "iteration=5, train_loss=10.360404, test_loss=10.451707\n",
      "iteration=6, train_loss=7.732523, test_loss=7.777016\n",
      "iteration=7, train_loss=5.815856, test_loss=5.851063\n",
      "iteration=8, train_loss=4.521336, test_loss=4.593024\n",
      "iteration=9, train_loss=4.286848, test_loss=4.387851\n",
      "iteration=13, train_loss=3.199417, test_loss=3.313275\n",
      "iteration=14, train_loss=2.770175, test_loss=2.918496\n",
      "iteration=15, train_loss=2.544024, test_loss=2.722195\n",
      "iteration=18, train_loss=2.679416, test_loss=2.819870\n",
      "iteration=19, train_loss=2.742721, test_loss=2.877797\n",
      "iteration=24, train_loss=2.470966, test_loss=2.641298\n",
      "iteration=28, train_loss=2.438313, test_loss=2.622074\n",
      "iteration=29, train_loss=2.442964, test_loss=2.625592\n",
      "iteration=46, train_loss=2.422367, test_loss=2.606442\n",
      "iteration=83, train_loss=2.425761, test_loss=2.605956\n",
      "iteration=100, train_loss=2.415895, test_loss=2.599094\n",
      "iteration=200, train_loss=2.407381, test_loss=2.589325\n",
      "iteration=220, train_loss=2.412390, test_loss=2.594838\n",
      "iteration=300, train_loss=2.405464, test_loss=2.588354\n",
      "iteration=400, train_loss=2.426918, test_loss=2.602271\n",
      "iteration=427, train_loss=2.393589, test_loss=2.575400\n",
      "iteration=429, train_loss=2.396272, test_loss=2.578253\n",
      "iteration=500, train_loss=2.394604, test_loss=2.573296\n",
      "iteration=537, train_loss=2.388146, test_loss=2.569474\n",
      "iteration=600, train_loss=2.384050, test_loss=2.564762\n",
      "iteration=700, train_loss=2.386596, test_loss=2.563579\n",
      "iteration=800, train_loss=2.374609, test_loss=2.554275\n",
      "iteration=900, train_loss=2.370416, test_loss=2.550355\n",
      "Running 46\n",
      "Cur best 2.2351925\n",
      "iteration=1, train_loss=13.835967, test_loss=13.882109\n",
      "iteration=5, train_loss=8.828205, test_loss=8.910121\n",
      "iteration=6, train_loss=6.634015, test_loss=6.660502\n",
      "iteration=7, train_loss=5.436890, test_loss=5.492984\n",
      "iteration=8, train_loss=5.305871, test_loss=5.400288\n",
      "iteration=9, train_loss=5.459489, test_loss=5.544447\n",
      "iteration=10, train_loss=5.256843, test_loss=5.329391\n",
      "iteration=11, train_loss=4.742447, test_loss=4.819751\n",
      "iteration=12, train_loss=4.013268, test_loss=4.113456\n",
      "iteration=13, train_loss=3.288735, test_loss=3.416460\n",
      "iteration=14, train_loss=2.727704, test_loss=2.881701\n",
      "iteration=17, train_loss=2.588458, test_loss=2.751420\n",
      "iteration=20, train_loss=2.796645, test_loss=2.933897\n",
      "iteration=29, train_loss=2.559433, test_loss=2.741196\n",
      "iteration=35, train_loss=2.457124, test_loss=2.644051\n",
      "iteration=37, train_loss=2.459323, test_loss=2.645145\n",
      "iteration=38, train_loss=2.463636, test_loss=2.649054\n",
      "iteration=50, train_loss=2.458816, test_loss=2.643940\n",
      "iteration=97, train_loss=2.450470, test_loss=2.637378\n",
      "iteration=100, train_loss=2.447832, test_loss=2.633565\n",
      "iteration=106, train_loss=2.447130, test_loss=2.632692\n",
      "iteration=200, train_loss=2.444205, test_loss=2.626733\n",
      "iteration=211, train_loss=2.481617, test_loss=2.665098\n",
      "iteration=251, train_loss=2.430583, test_loss=2.616021\n",
      "iteration=300, train_loss=2.431479, test_loss=2.617016\n"
     ]
    }
   ],
   "source": [
    "for _i in range(10000):\n",
    "    print('Running', _i)\n",
    "    print('Cur best', str(best_loss))\n",
    "    \n",
    "    best_small_loss = np.inf\n",
    "    iteration = 0\n",
    "    train_losses, test_losses = [], []\n",
    "    \n",
    "    lr = args.lr\n",
    "    _, init_params = init_random_params(rng+1, (-1, 4))\n",
    "    rng += 1\n",
    "    opt_init, opt_update, get_params = opti(lr)\n",
    "    init_params = make_new_params(init_params)\n",
    "    opt_state = opt_init(init_params)\n",
    "    bad_iterations = 0\n",
    "    offset = 0\n",
    "    \n",
    "    while iteration < 20000:\n",
    "        iteration += 1\n",
    "        rand_idx = jax.random.randint(rng, (args.batch_size,), 0, len(data['x']))\n",
    "        rng += 1\n",
    "\n",
    "        batch = (data['x'][rand_idx], data['dx'][rand_idx])\n",
    "        \n",
    "        # Compute derivative at halfway point:\n",
    "        half_state, params = update_derivative(iteration+offset, opt_state, batch, args.l2reg, get_params(opt_state))\n",
    "        half_params = get_params(half_state)\n",
    "        opt_state, _ = update_derivative(iteration+offset, opt_state, batch, args.l2reg, half_params)\n",
    "        params = get_params(opt_state)\n",
    "        \n",
    "        del half_params\n",
    "        del half_state\n",
    "        \n",
    "        small_loss = loss(params, batch, 0.0)\n",
    "\n",
    "        new_small_loss = False\n",
    "        if small_loss < best_small_loss:\n",
    "\n",
    "            best_small_loss = small_loss\n",
    "            new_small_loss = True\n",
    "        \n",
    "        if jnp.isnan(small_loss).sum() or new_small_loss or (iteration % 500 == 0) or (iteration < 1000 and iteration % 100 == 0):\n",
    "            params = get_params(opt_state)\n",
    "            train_loss = loss(params, (data['x'], data['dx']), 0.0)/len(data['x'])\n",
    "            train_losses.append(train_loss)\n",
    "            test_loss = loss(params, (data['test_x'], data['test_dx']), 0.0)/len(data['test_x'])\n",
    "            test_losses.append(test_loss)\n",
    "            \n",
    "            if iteration >= 1000 and test_loss > 2.1:\n",
    "                #Only good seeds allowed!\n",
    "                break\n",
    "\n",
    "            if test_loss < best_loss:\n",
    "                best_loss = test_loss\n",
    "                best_params = copy(params)\n",
    "                bad_iterations = 0\n",
    "                offset += iteration\n",
    "                iteration = 0 #Keep going since this one is so good!\n",
    "\n",
    "            if jnp.isnan(test_loss).sum():\n",
    "                break\n",
    "\n",
    "            print(f\"iteration={iteration}, train_loss={train_loss:.6f}, test_loss={test_loss:.6f}\")\n",
    "\n",
    "        bad_iterations += 1\n",
    "    \n",
    "    import pickle as pkl\n",
    "    if best_loss < np.inf:\n",
    "        pkl.dump({'params': best_params, 'args': args},\n",
    "             open('params_for_loss_{}_nupdates=1.pkl'.format(best_loss), 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(lnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lnn import lagrangian_eom_rk4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss(best_params, (data['test_x'], data['test_dx']), 0.0)/len(data['test_x'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "main2",
   "language": "python",
   "name": "main2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}