data_inspect.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import logging\n",
    "import os\n",
    "import random\n",
    "import warnings\n",
    "from multiprocessing import Pool, cpu_count\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from utils.tools import StandardScaler\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "logger = logging.getLogger(\"__main__\")\n",
    "\n",
    "FEATURE_COLS = [\n",
    "    \"bookdata::book=book_UR::data_name=bid_0\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_1\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_2\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_3\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_4\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_size_0\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_size_1\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_size_2\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_size_3\",\n",
    "    \"bookdata::book=book_UR::data_name=bid_size_4\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_0\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_1\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_2\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_3\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_4\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_size_0\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_size_1\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_size_2\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_size_3\",\n",
    "    \"bookdata::book=book_UR::data_name=ask_size_4\",\n",
    "    \"bookdata::book=book_UR::data_name=buy_size\",\n",
    "    \"bookdata::book=book_UR::data_name=buy_price\",\n",
    "    \"bookdata::book=book_UR::data_name=sell_size\",\n",
    "    \"bookdata::book=book_UR::data_name=sell_price\",\n",
    "]\n",
    "\n",
    "VALIDATION_COL = \"book_valid_field::book=book_UR\"\n",
    "\n",
    "LABEL_COLS = [\n",
    "    \"extdata::book=book_UR::data_name=forward_return_vwap_10s\",\n",
    "    \"extdata::book=book_UR::data_name=forward_return_vwap_60s\",\n",
    "    \"extdata::book=book_UR::data_name=forward_return_vwap_600s\",\n",
    "    \"extdata::book=book_UR::data_name=forward_return_vwap_1800s\",\n",
    "]\n",
    "\n",
    "\n",
    "class BaseData(object):\n",
    "    def set_num_processes(self, n_proc):\n",
    "        if (n_proc is None) or (n_proc <= 0):\n",
    "            self.n_proc = cpu_count()  # max(1, cpu_count() - 1)\n",
    "        else:\n",
    "            self.n_proc = min(n_proc, cpu_count())\n",
    "\n",
    "\n",
    "class FutsData(BaseData):\n",
    "    \"\"\"\n",
    "    Dataset class for Machine dataset.\n",
    "    Attributes:\n",
    "        all_df: dataframe indexed by ID, with multiple rows corresponding to the same index (sample).\n",
    "            Each row is a time step; Each column contains either metadata (e.g. timestamp) or a feature.\n",
    "        feature_df: contains the subset of columns of `all_df` which correspond to selected features\n",
    "        feature_names: names of columns contained in `feature_df` (same as feature_df.columns)\n",
    "        all_IDs: IDs contained in `all_df`/`feature_df` (same as all_df.index.unique() )\n",
    "        max_seq_len: maximum sequence (time series) length. If None, script argument `max_seq_len` will be used.\n",
    "            (Moreover, script argument overrides this attribute)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        root_dir,\n",
    "        pattern,\n",
    "        file_list=None,\n",
    "        n_proc=1,\n",
    "        limit_size=None,\n",
    "        config=None,\n",
    "    ):\n",
    "        self.max_seq_len = 1024\n",
    "        self.lookahead = 40\n",
    "        # process features\n",
    "        data_df = self.read_data(os.path.join(root_dir, pattern))\n",
    "        # for large size of data that don't fit into memory\n",
    "        #data_df = self.read_data_xl(os.path.join(root_dir, pattern))\n",
    "        num_rows = data_df.shape[0]\n",
    "        # process labels\n",
    "        feature_df = data_df[FEATURE_COLS]\n",
    "        labels_df = data_df[LABEL_COLS]\n",
    "        # all_IDs uses a compressed representation: i-th position in all_ID maps to (start, end) of the feature_df and start of the label_df.\n",
    "        self.all_IDs = [\n",
    "            [i-self.max_seq_len+1, i]\n",
    "            for i in range(self.max_seq_len-1, num_rows)\n",
    "        ]\n",
    "        self.all_df = feature_df\n",
    "        self.labels_df = labels_df\n",
    "        if limit_size is not None:\n",
    "            if limit_size > 1:\n",
    "                limit_size = int(limit_size)\n",
    "            else:  # interpret as proportion if in (0, 1]\n",
    "                limit_size = int(limit_size * len(self.all_IDs))\n",
    "            self.all_IDs = random.sample(self.all_IDs, k=limit_size)\n",
    "\n",
    "        self.feature_names = list(self.all_df.columns)\n",
    "        self.feature_df = self.all_df[self.feature_names]\n",
    "\n",
    "    def _validate(self, df: pd.DataFrame):\n",
    "        assert VALIDATION_COL in df.columns.to_list()\n",
    "        df = df[df[VALIDATION_COL] > 0]\n",
    "        return df\n",
    "\n",
    "    def read_data(self, pattern: str):\n",
    "        datas = []\n",
    "        logger.info(f\"loading data from {pattern}\")\n",
    "        for file in sorted(glob.glob(pattern)):\n",
    "            if \"xy\" in file:\n",
    "                continue\n",
    "            df = pd.read_parquet(file)\n",
    "            df = self._validate(df)\n",
    "            data = df[FEATURE_COLS + LABEL_COLS]\n",
    "            datas.append(data)\n",
    "        logger.info(f\"number of files loaded: {len(datas)}\")\n",
    "        if len(datas) != 0:\n",
    "            data = pd.concat(datas)\n",
    "            data = data.reset_index(drop=True)\n",
    "        else:\n",
    "            data = None\n",
    "        return data\n",
    "\n",
    "    def read_data_xl(self, pattern: str):\n",
    "        \"\"\"\n",
    "        Same result as get_feature_data, but data is stored in np.memmap so can load large dataset.\n",
    "        \"\"\"\n",
    "        logger.info(f\"preprocssing data from {pattern}\")\n",
    "        num_rows = 0\n",
    "        num_cols = len(FEATURE_COLS + LABEL_COLS)\n",
    "        for file in sorted(glob.glob(pattern)):\n",
    "            if \"xy\" in file:\n",
    "                continue\n",
    "            df = pd.read_parquet(file)\n",
    "            df = self._validate(df)\n",
    "            #data = df[FEATURE_COLS]\n",
    "            num_rows += data.shape[0]\n",
    "        logger.info(f\"Total data size needs to load: {num_rows} x {num_cols}\")\n",
    "\n",
    "        split = \"train\" if \"train\" in pattern else \"val\"\n",
    "        path = f\"/workspace/futs/data/{split}.bin\"\n",
    "\n",
    "        # If file already exists and matches, directly return without importing again.\n",
    "        if os.path.isfile(path):\n",
    "            arr = np.memmap(path, dtype=float, mode=\"r\")\n",
    "            if arr.shape == (num_rows, num_cols):\n",
    "                logger.info(f\"loaded data from preprocessed file {path}\")\n",
    "                return data.pd.DataFrame(arr, copy=False)\n",
    "\n",
    "        # If files don't exist, create and import data.\n",
    "        arr = np.memmap(path, dtype=float, mode=\"w+\", shape=(num_rows, num_cols))\n",
    "        i = 0\n",
    "        for file in sorted(glob.glob(pattern)):\n",
    "            if \"xy\" in file:\n",
    "                continue\n",
    "            # date = file.split(\".\")[-2]\n",
    "            df = pd.read_parquet(file)\n",
    "            df = self._validate(df)\n",
    "            data = df[FEATURE_COLS + LABEL_COLS]\n",
    "            arr[i : i + df.shape[0], :] = data.values\n",
    "            i += data.shape[0]\n",
    "        logger.info(f\"loaded data from {pattern}\")\n",
    "        data = pd.DataFrame(arr, copy=False)\n",
    "        arr.flush()  # save to disk\n",
    "        return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = FutsData(root_dir=\"/Users/tonywy/Desktop/Xode/crossformer\", pattern=\"futs_data/ZCE_CH_UR/daily_frame.*.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_0</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_1</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_2</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_3</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_4</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_size_0</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_size_1</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_size_2</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_size_3</th>\n",
       "      <th>bookdata::book=book_UR::data_name=bid_size_4</th>\n",
       "      <th>...</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_4</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_size_0</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_size_1</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_size_2</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_size_3</th>\n",
       "      <th>bookdata::book=book_UR::data_name=ask_size_4</th>\n",
       "      <th>bookdata::book=book_UR::data_name=buy_size</th>\n",
       "      <th>bookdata::book=book_UR::data_name=buy_price</th>\n",
       "      <th>bookdata::book=book_UR::data_name=sell_size</th>\n",
       "      <th>bookdata::book=book_UR::data_name=sell_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>-0.577350</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>-0.577350</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.154701</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.577350</td>\n",
       "      <td>-0.577350</td>\n",
       "      <td>-0.582021</td>\n",
       "      <td>-1.154701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-1.390759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.489151</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.788854</td>\n",
       "      <td>-1.788854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.447214</td>\n",
       "      <td>0.447214</td>\n",
       "      <td>-1.761410</td>\n",
       "      <td>1.788854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.447214</td>\n",
       "      <td>1.787841</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.788854</td>\n",
       "      <td>1.008505</td>\n",
       "      <td>-1.788854</td>\n",
       "      <td>1.788854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.788854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.095445</td>\n",
       "      <td>-0.213553</td>\n",
       "      <td>0.447214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872032</th>\n",
       "      <td>-0.871900</td>\n",
       "      <td>-0.871900</td>\n",
       "      <td>-0.871900</td>\n",
       "      <td>-0.871900</td>\n",
       "      <td>-0.871900</td>\n",
       "      <td>-1.170703</td>\n",
       "      <td>-0.761341</td>\n",
       "      <td>0.181960</td>\n",
       "      <td>-0.646024</td>\n",
       "      <td>-0.111309</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.892681</td>\n",
       "      <td>-0.374445</td>\n",
       "      <td>-0.537942</td>\n",
       "      <td>-0.811926</td>\n",
       "      <td>-1.526044</td>\n",
       "      <td>-0.813867</td>\n",
       "      <td>0.040438</td>\n",
       "      <td>1.323330</td>\n",
       "      <td>-0.370267</td>\n",
       "      <td>-0.779789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872033</th>\n",
       "      <td>-0.874156</td>\n",
       "      <td>-0.874156</td>\n",
       "      <td>-0.874156</td>\n",
       "      <td>-0.874156</td>\n",
       "      <td>-0.874156</td>\n",
       "      <td>-1.168845</td>\n",
       "      <td>-0.760066</td>\n",
       "      <td>0.183996</td>\n",
       "      <td>-0.644810</td>\n",
       "      <td>-0.108897</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.895006</td>\n",
       "      <td>-0.337508</td>\n",
       "      <td>-0.537306</td>\n",
       "      <td>-0.812818</td>\n",
       "      <td>-1.524757</td>\n",
       "      <td>-0.828303</td>\n",
       "      <td>-0.021192</td>\n",
       "      <td>1.320949</td>\n",
       "      <td>-0.370267</td>\n",
       "      <td>-0.779789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872034</th>\n",
       "      <td>-0.876422</td>\n",
       "      <td>-0.876422</td>\n",
       "      <td>-0.876422</td>\n",
       "      <td>-0.876422</td>\n",
       "      <td>-0.876422</td>\n",
       "      <td>-1.180262</td>\n",
       "      <td>-0.758793</td>\n",
       "      <td>0.195267</td>\n",
       "      <td>-0.643596</td>\n",
       "      <td>-0.106464</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.897341</td>\n",
       "      <td>-0.333637</td>\n",
       "      <td>-0.536683</td>\n",
       "      <td>-0.761048</td>\n",
       "      <td>-1.523471</td>\n",
       "      <td>-0.829672</td>\n",
       "      <td>-0.266902</td>\n",
       "      <td>-0.755118</td>\n",
       "      <td>-0.273969</td>\n",
       "      <td>1.278989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872035</th>\n",
       "      <td>-0.878696</td>\n",
       "      <td>-0.878696</td>\n",
       "      <td>-0.878696</td>\n",
       "      <td>-0.878696</td>\n",
       "      <td>-0.878696</td>\n",
       "      <td>-1.178440</td>\n",
       "      <td>-0.716456</td>\n",
       "      <td>0.197332</td>\n",
       "      <td>-0.642382</td>\n",
       "      <td>-0.104011</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.899685</td>\n",
       "      <td>-0.561011</td>\n",
       "      <td>-0.536075</td>\n",
       "      <td>-0.761986</td>\n",
       "      <td>-1.479252</td>\n",
       "      <td>-0.831044</td>\n",
       "      <td>3.096415</td>\n",
       "      <td>1.318575</td>\n",
       "      <td>-0.369674</td>\n",
       "      <td>-0.778408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872036</th>\n",
       "      <td>-0.880980</td>\n",
       "      <td>-0.880980</td>\n",
       "      <td>-0.880980</td>\n",
       "      <td>-0.880980</td>\n",
       "      <td>-0.880980</td>\n",
       "      <td>-1.242748</td>\n",
       "      <td>-0.715236</td>\n",
       "      <td>0.199407</td>\n",
       "      <td>-0.641170</td>\n",
       "      <td>-0.101536</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.902039</td>\n",
       "      <td>-0.548632</td>\n",
       "      <td>-0.535483</td>\n",
       "      <td>-0.762926</td>\n",
       "      <td>-1.478111</td>\n",
       "      <td>-0.832419</td>\n",
       "      <td>-0.207514</td>\n",
       "      <td>1.316208</td>\n",
       "      <td>0.110131</td>\n",
       "      <td>1.278982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>872037 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        bookdata::book=book_UR::data_name=bid_0  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.000000   \n",
       "2                                      0.000000   \n",
       "3                                      0.000000   \n",
       "4                                     -1.788854   \n",
       "...                                         ...   \n",
       "872032                                -0.871900   \n",
       "872033                                -0.874156   \n",
       "872034                                -0.876422   \n",
       "872035                                -0.878696   \n",
       "872036                                -0.880980   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_1  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.000000   \n",
       "2                                      0.000000   \n",
       "3                                      0.000000   \n",
       "4                                     -1.788854   \n",
       "...                                         ...   \n",
       "872032                                -0.871900   \n",
       "872033                                -0.874156   \n",
       "872034                                -0.876422   \n",
       "872035                                -0.878696   \n",
       "872036                                -0.880980   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_2  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.000000   \n",
       "2                                      0.000000   \n",
       "3                                      0.000000   \n",
       "4                                      0.000000   \n",
       "...                                         ...   \n",
       "872032                                -0.871900   \n",
       "872033                                -0.874156   \n",
       "872034                                -0.876422   \n",
       "872035                                -0.878696   \n",
       "872036                                -0.880980   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_3  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.707107   \n",
       "2                                      0.577350   \n",
       "3                                      0.500000   \n",
       "4                                      0.447214   \n",
       "...                                         ...   \n",
       "872032                                -0.871900   \n",
       "872033                                -0.874156   \n",
       "872034                                -0.876422   \n",
       "872035                                -0.878696   \n",
       "872036                                -0.880980   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_4  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.707107   \n",
       "2                                      0.577350   \n",
       "3                                      0.500000   \n",
       "4                                      0.447214   \n",
       "...                                         ...   \n",
       "872032                                -0.871900   \n",
       "872033                                -0.874156   \n",
       "872034                                -0.876422   \n",
       "872035                                -0.878696   \n",
       "872036                                -0.880980   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_size_0  \\\n",
       "0                                           0.000000   \n",
       "1                                          -0.707107   \n",
       "2                                          -0.577350   \n",
       "3                                          -1.390759   \n",
       "4                                          -1.761410   \n",
       "...                                              ...   \n",
       "872032                                     -1.170703   \n",
       "872033                                     -1.168845   \n",
       "872034                                     -1.180262   \n",
       "872035                                     -1.178440   \n",
       "872036                                     -1.242748   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_size_1  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                           1.788854   \n",
       "...                                              ...   \n",
       "872032                                     -0.761341   \n",
       "872033                                     -0.760066   \n",
       "872034                                     -0.758793   \n",
       "872035                                     -0.716456   \n",
       "872036                                     -0.715236   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_size_2  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                           0.000000   \n",
       "...                                              ...   \n",
       "872032                                      0.181960   \n",
       "872033                                      0.183996   \n",
       "872034                                      0.195267   \n",
       "872035                                      0.197332   \n",
       "872036                                      0.199407   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_size_3  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.707107   \n",
       "2                                           0.577350   \n",
       "3                                           0.500000   \n",
       "4                                           0.447214   \n",
       "...                                              ...   \n",
       "872032                                     -0.646024   \n",
       "872033                                     -0.644810   \n",
       "872034                                     -0.643596   \n",
       "872035                                     -0.642382   \n",
       "872036                                     -0.641170   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=bid_size_4  ...  \\\n",
       "0                                           0.000000  ...   \n",
       "1                                          -0.707107  ...   \n",
       "2                                          -0.577350  ...   \n",
       "3                                          -0.500000  ...   \n",
       "4                                           1.787841  ...   \n",
       "...                                              ...  ...   \n",
       "872032                                     -0.111309  ...   \n",
       "872033                                     -0.108897  ...   \n",
       "872034                                     -0.106464  ...   \n",
       "872035                                     -0.104011  ...   \n",
       "872036                                     -0.101536  ...   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_4  \\\n",
       "0                                      0.000000   \n",
       "1                                      0.000000   \n",
       "2                                      0.000000   \n",
       "3                                      0.000000   \n",
       "4                                     -1.788854   \n",
       "...                                         ...   \n",
       "872032                                -0.892681   \n",
       "872033                                -0.895006   \n",
       "872034                                -0.897341   \n",
       "872035                                -0.899685   \n",
       "872036                                -0.902039   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_size_0  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           1.154701   \n",
       "3                                           0.866025   \n",
       "4                                           1.008505   \n",
       "...                                              ...   \n",
       "872032                                     -0.374445   \n",
       "872033                                     -0.337508   \n",
       "872034                                     -0.333637   \n",
       "872035                                     -0.561011   \n",
       "872036                                     -0.548632   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_size_1  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                          -1.788854   \n",
       "...                                              ...   \n",
       "872032                                     -0.537942   \n",
       "872033                                     -0.537306   \n",
       "872034                                     -0.536683   \n",
       "872035                                     -0.536075   \n",
       "872036                                     -0.535483   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_size_2  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                           1.788854   \n",
       "...                                              ...   \n",
       "872032                                     -0.811926   \n",
       "872033                                     -0.812818   \n",
       "872034                                     -0.761048   \n",
       "872035                                     -0.761986   \n",
       "872036                                     -0.762926   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_size_3  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                           0.000000   \n",
       "...                                              ...   \n",
       "872032                                     -1.526044   \n",
       "872033                                     -1.524757   \n",
       "872034                                     -1.523471   \n",
       "872035                                     -1.479252   \n",
       "872036                                     -1.478111   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=ask_size_4  \\\n",
       "0                                           0.000000   \n",
       "1                                           0.000000   \n",
       "2                                           0.000000   \n",
       "3                                           0.000000   \n",
       "4                                          -1.788854   \n",
       "...                                              ...   \n",
       "872032                                     -0.813867   \n",
       "872033                                     -0.828303   \n",
       "872034                                     -0.829672   \n",
       "872035                                     -0.831044   \n",
       "872036                                     -0.832419   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=buy_size  \\\n",
       "0                                         0.000000   \n",
       "1                                        -0.707107   \n",
       "2                                        -0.577350   \n",
       "3                                        -0.500000   \n",
       "4                                         0.000000   \n",
       "...                                            ...   \n",
       "872032                                    0.040438   \n",
       "872033                                   -0.021192   \n",
       "872034                                   -0.266902   \n",
       "872035                                    3.096415   \n",
       "872036                                   -0.207514   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=buy_price  \\\n",
       "0                                          0.000000   \n",
       "1                                         -0.707107   \n",
       "2                                         -0.577350   \n",
       "3                                         -0.500000   \n",
       "4                                          1.095445   \n",
       "...                                             ...   \n",
       "872032                                     1.323330   \n",
       "872033                                     1.320949   \n",
       "872034                                    -0.755118   \n",
       "872035                                     1.318575   \n",
       "872036                                     1.316208   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=sell_size  \\\n",
       "0                                          0.000000   \n",
       "1                                         -0.707107   \n",
       "2                                         -0.582021   \n",
       "3                                         -0.489151   \n",
       "4                                         -0.213553   \n",
       "...                                             ...   \n",
       "872032                                    -0.370267   \n",
       "872033                                    -0.370267   \n",
       "872034                                    -0.273969   \n",
       "872035                                    -0.369674   \n",
       "872036                                     0.110131   \n",
       "\n",
       "        bookdata::book=book_UR::data_name=sell_price  \n",
       "0                                           0.000000  \n",
       "1                                           0.000000  \n",
       "2                                          -1.154701  \n",
       "3                                           0.500000  \n",
       "4                                           0.447214  \n",
       "...                                              ...  \n",
       "872032                                     -0.779789  \n",
       "872033                                     -0.779789  \n",
       "872034                                      1.278989  \n",
       "872035                                     -0.778408  \n",
       "872036                                      1.278982  \n",
       "\n",
       "[872037 rows x 24 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.all_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0, 1023],\n",
       " [1, 1024],\n",
       " [2, 1025],\n",
       " [3, 1026],\n",
       " [4, 1027],\n",
       " [5, 1028],\n",
       " [6, 1029],\n",
       " [7, 1030],\n",
       " [8, 1031],\n",
       " [9, 1032],\n",
       " [10, 1033],\n",
       " [11, 1034],\n",
       " [12, 1035],\n",
       " [13, 1036],\n",
       " [14, 1037],\n",
       " [15, 1038],\n",
       " [16, 1039],\n",
       " [17, 1040],\n",
       " [18, 1041],\n",
       " [19, 1042],\n",
       " [20, 1043],\n",
       " [21, 1044],\n",
       " [22, 1045],\n",
       " [23, 1046],\n",
       " [24, 1047],\n",
       " [25, 1048],\n",
       " [26, 1049],\n",
       " [27, 1050],\n",
       " [28, 1051],\n",
       " [29, 1052],\n",
       " [30, 1053],\n",
       " [31, 1054],\n",
       " [32, 1055],\n",
       " [33, 1056],\n",
       " [34, 1057],\n",
       " [35, 1058],\n",
       " [36, 1059],\n",
       " [37, 1060],\n",
       " [38, 1061],\n",
       " [39, 1062],\n",
       " [40, 1063],\n",
       " [41, 1064],\n",
       " [42, 1065],\n",
       " [43, 1066],\n",
       " [44, 1067],\n",
       " [45, 1068],\n",
       " [46, 1069],\n",
       " [47, 1070],\n",
       " [48, 1071],\n",
       " [49, 1072],\n",
       " [50, 1073],\n",
       " [51, 1074],\n",
       " [52, 1075],\n",
       " [53, 1076],\n",
       " [54, 1077],\n",
       " [55, 1078],\n",
       " [56, 1079],\n",
       " [57, 1080],\n",
       " [58, 1081],\n",
       " [59, 1082],\n",
       " [60, 1083],\n",
       " [61, 1084],\n",
       " [62, 1085],\n",
       " [63, 1086],\n",
       " [64, 1087],\n",
       " [65, 1088],\n",
       " [66, 1089],\n",
       " [67, 1090],\n",
       " [68, 1091],\n",
       " [69, 1092],\n",
       " [70, 1093],\n",
       " [71, 1094],\n",
       " [72, 1095],\n",
       " [73, 1096],\n",
       " [74, 1097],\n",
       " [75, 1098],\n",
       " [76, 1099],\n",
       " [77, 1100],\n",
       " [78, 1101],\n",
       " [79, 1102],\n",
       " [80, 1103],\n",
       " [81, 1104],\n",
       " [82, 1105],\n",
       " [83, 1106],\n",
       " [84, 1107],\n",
       " [85, 1108],\n",
       " [86, 1109],\n",
       " [87, 1110],\n",
       " [88, 1111],\n",
       " [89, 1112],\n",
       " [90, 1113],\n",
       " [91, 1114],\n",
       " [92, 1115],\n",
       " [93, 1116],\n",
       " [94, 1117],\n",
       " [95, 1118],\n",
       " [96, 1119],\n",
       " [97, 1120],\n",
       " [98, 1121],\n",
       " [99, 1122],\n",
       " [100, 1123],\n",
       " [101, 1124],\n",
       " [102, 1125],\n",
       " [103, 1126],\n",
       " [104, 1127],\n",
       " [105, 1128],\n",
       " [106, 1129],\n",
       " [107, 1130],\n",
       " [108, 1131],\n",
       " [109, 1132],\n",
       " [110, 1133],\n",
       " [111, 1134],\n",
       " [112, 1135],\n",
       " [113, 1136],\n",
       " [114, 1137],\n",
       " [115, 1138],\n",
       " [116, 1139],\n",
       " [117, 1140],\n",
       " [118, 1141],\n",
       " [119, 1142],\n",
       " [120, 1143],\n",
       " [121, 1144],\n",
       " [122, 1145],\n",
       " [123, 1146],\n",
       " [124, 1147],\n",
       " [125, 1148],\n",
       " [126, 1149],\n",
       " [127, 1150],\n",
       " [128, 1151],\n",
       " [129, 1152],\n",
       " [130, 1153],\n",
       " [131, 1154],\n",
       " [132, 1155],\n",
       " [133, 1156],\n",
       " [134, 1157],\n",
       " [135, 1158],\n",
       " [136, 1159],\n",
       " [137, 1160],\n",
       " [138, 1161],\n",
       " [139, 1162],\n",
       " [140, 1163],\n",
       " [141, 1164],\n",
       " [142, 1165],\n",
       " [143, 1166],\n",
       " [144, 1167],\n",
       " [145, 1168],\n",
       " [146, 1169],\n",
       " [147, 1170],\n",
       " [148, 1171],\n",
       " [149, 1172],\n",
       " [150, 1173],\n",
       " [151, 1174],\n",
       " [152, 1175],\n",
       " [153, 1176],\n",
       " [154, 1177],\n",
       " [155, 1178],\n",
       " [156, 1179],\n",
       " [157, 1180],\n",
       " [158, 1181],\n",
       " [159, 1182],\n",
       " [160, 1183],\n",
       " [161, 1184],\n",
       " [162, 1185],\n",
       " [163, 1186],\n",
       " [164, 1187],\n",
       " [165, 1188],\n",
       " [166, 1189],\n",
       " [167, 1190],\n",
       " [168, 1191],\n",
       " [169, 1192],\n",
       " [170, 1193],\n",
       " [171, 1194],\n",
       " [172, 1195],\n",
       " [173, 1196],\n",
       " [174, 1197],\n",
       " [175, 1198],\n",
       " [176, 1199],\n",
       " [177, 1200],\n",
       " [178, 1201],\n",
       " [179, 1202],\n",
       " [180, 1203],\n",
       " [181, 1204],\n",
       " [182, 1205],\n",
       " [183, 1206],\n",
       " [184, 1207],\n",
       " [185, 1208],\n",
       " [186, 1209],\n",
       " [187, 1210],\n",
       " [188, 1211],\n",
       " [189, 1212],\n",
       " [190, 1213],\n",
       " [191, 1214],\n",
       " [192, 1215],\n",
       " [193, 1216],\n",
       " [194, 1217],\n",
       " [195, 1218],\n",
       " [196, 1219],\n",
       " [197, 1220],\n",
       " [198, 1221],\n",
       " [199, 1222],\n",
       " [200, 1223],\n",
       " [201, 1224],\n",
       " [202, 1225],\n",
       " [203, 1226],\n",
       " [204, 1227],\n",
       " [205, 1228],\n",
       " [206, 1229],\n",
       " [207, 1230],\n",
       " [208, 1231],\n",
       " [209, 1232],\n",
       " [210, 1233],\n",
       " [211, 1234],\n",
       " [212, 1235],\n",
       " [213, 1236],\n",
       " [214, 1237],\n",
       " [215, 1238],\n",
       " [216, 1239],\n",
       " [217, 1240],\n",
       " [218, 1241],\n",
       " [219, 1242],\n",
       " [220, 1243],\n",
       " [221, 1244],\n",
       " [222, 1245],\n",
       " [223, 1246],\n",
       " [224, 1247],\n",
       " [225, 1248],\n",
       " [226, 1249],\n",
       " [227, 1250],\n",
       " [228, 1251],\n",
       " [229, 1252],\n",
       " [230, 1253],\n",
       " [231, 1254],\n",
       " [232, 1255],\n",
       " [233, 1256],\n",
       " [234, 1257],\n",
       " [235, 1258],\n",
       " [236, 1259],\n",
       " [237, 1260],\n",
       " [238, 1261],\n",
       " [239, 1262],\n",
       " [240, 1263],\n",
       " [241, 1264],\n",
       " [242, 1265],\n",
       " [243, 1266],\n",
       " [244, 1267],\n",
       " [245, 1268],\n",
       " [246, 1269],\n",
       " [247, 1270],\n",
       " [248, 1271],\n",
       " [249, 1272],\n",
       " [250, 1273],\n",
       " [251, 1274],\n",
       " [252, 1275],\n",
       " [253, 1276],\n",
       " [254, 1277],\n",
       " [255, 1278],\n",
       " [256, 1279],\n",
       " [257, 1280],\n",
       " [258, 1281],\n",
       " [259, 1282],\n",
       " [260, 1283],\n",
       " [261, 1284],\n",
       " [262, 1285],\n",
       " [263, 1286],\n",
       " [264, 1287],\n",
       " [265, 1288],\n",
       " [266, 1289],\n",
       " [267, 1290],\n",
       " [268, 1291],\n",
       " [269, 1292],\n",
       " [270, 1293],\n",
       " [271, 1294],\n",
       " [272, 1295],\n",
       " [273, 1296],\n",
       " [274, 1297],\n",
       " [275, 1298],\n",
       " [276, 1299],\n",
       " [277, 1300],\n",
       " [278, 1301],\n",
       " [279, 1302],\n",
       " [280, 1303],\n",
       " [281, 1304],\n",
       " [282, 1305],\n",
       " [283, 1306],\n",
       " [284, 1307],\n",
       " [285, 1308],\n",
       " [286, 1309],\n",
       " [287, 1310],\n",
       " [288, 1311],\n",
       " [289, 1312],\n",
       " [290, 1313],\n",
       " [291, 1314],\n",
       " [292, 1315],\n",
       " [293, 1316],\n",
       " [294, 1317],\n",
       " [295, 1318],\n",
       " [296, 1319],\n",
       " [297, 1320],\n",
       " [298, 1321],\n",
       " [299, 1322],\n",
       " [300, 1323],\n",
       " [301, 1324],\n",
       " [302, 1325],\n",
       " [303, 1326],\n",
       " [304, 1327],\n",
       " [305, 1328],\n",
       " [306, 1329],\n",
       " [307, 1330],\n",
       " [308, 1331],\n",
       " [309, 1332],\n",
       " [310, 1333],\n",
       " [311, 1334],\n",
       " [312, 1335],\n",
       " [313, 1336],\n",
       " [314, 1337],\n",
       " [315, 1338],\n",
       " [316, 1339],\n",
       " [317, 1340],\n",
       " [318, 1341],\n",
       " [319, 1342],\n",
       " [320, 1343],\n",
       " [321, 1344],\n",
       " [322, 1345],\n",
       " [323, 1346],\n",
       " [324, 1347],\n",
       " [325, 1348],\n",
       " [326, 1349],\n",
       " [327, 1350],\n",
       " [328, 1351],\n",
       " [329, 1352],\n",
       " [330, 1353],\n",
       " [331, 1354],\n",
       " [332, 1355],\n",
       " [333, 1356],\n",
       " [334, 1357],\n",
       " [335, 1358],\n",
       " [336, 1359],\n",
       " [337, 1360],\n",
       " [338, 1361],\n",
       " [339, 1362],\n",
       " [340, 1363],\n",
       " [341, 1364],\n",
       " [342, 1365],\n",
       " [343, 1366],\n",
       " [344, 1367],\n",
       " [345, 1368],\n",
       " [346, 1369],\n",
       " [347, 1370],\n",
       " [348, 1371],\n",
       " [349, 1372],\n",
       " [350, 1373],\n",
       " [351, 1374],\n",
       " [352, 1375],\n",
       " [353, 1376],\n",
       " [354, 1377],\n",
       " [355, 1378],\n",
       " [356, 1379],\n",
       " [357, 1380],\n",
       " [358, 1381],\n",
       " [359, 1382],\n",
       " [360, 1383],\n",
       " [361, 1384],\n",
       " [362, 1385],\n",
       " [363, 1386],\n",
       " [364, 1387],\n",
       " [365, 1388],\n",
       " [366, 1389],\n",
       " [367, 1390],\n",
       " [368, 1391],\n",
       " [369, 1392],\n",
       " [370, 1393],\n",
       " [371, 1394],\n",
       " [372, 1395],\n",
       " [373, 1396],\n",
       " [374, 1397],\n",
       " [375, 1398],\n",
       " [376, 1399],\n",
       " [377, 1400],\n",
       " [378, 1401],\n",
       " [379, 1402],\n",
       " [380, 1403],\n",
       " [381, 1404],\n",
       " [382, 1405],\n",
       " [383, 1406],\n",
       " [384, 1407],\n",
       " [385, 1408],\n",
       " [386, 1409],\n",
       " [387, 1410],\n",
       " [388, 1411],\n",
       " [389, 1412],\n",
       " [390, 1413],\n",
       " [391, 1414],\n",
       " [392, 1415],\n",
       " [393, 1416],\n",
       " [394, 1417],\n",
       " [395, 1418],\n",
       " [396, 1419],\n",
       " [397, 1420],\n",
       " [398, 1421],\n",
       " [399, 1422],\n",
       " [400, 1423],\n",
       " [401, 1424],\n",
       " [402, 1425],\n",
       " [403, 1426],\n",
       " [404, 1427],\n",
       " [405, 1428],\n",
       " [406, 1429],\n",
       " [407, 1430],\n",
       " [408, 1431],\n",
       " [409, 1432],\n",
       " [410, 1433],\n",
       " [411, 1434],\n",
       " [412, 1435],\n",
       " [413, 1436],\n",
       " [414, 1437],\n",
       " [415, 1438],\n",
       " [416, 1439],\n",
       " [417, 1440],\n",
       " [418, 1441],\n",
       " [419, 1442],\n",
       " [420, 1443],\n",
       " [421, 1444],\n",
       " [422, 1445],\n",
       " [423, 1446],\n",
       " [424, 1447],\n",
       " [425, 1448],\n",
       " [426, 1449],\n",
       " [427, 1450],\n",
       " [428, 1451],\n",
       " [429, 1452],\n",
       " [430, 1453],\n",
       " [431, 1454],\n",
       " [432, 1455],\n",
       " [433, 1456],\n",
       " [434, 1457],\n",
       " [435, 1458],\n",
       " [436, 1459],\n",
       " [437, 1460],\n",
       " [438, 1461],\n",
       " [439, 1462],\n",
       " [440, 1463],\n",
       " [441, 1464],\n",
       " [442, 1465],\n",
       " [443, 1466],\n",
       " [444, 1467],\n",
       " [445, 1468],\n",
       " [446, 1469],\n",
       " [447, 1470],\n",
       " [448, 1471],\n",
       " [449, 1472],\n",
       " [450, 1473],\n",
       " [451, 1474],\n",
       " [452, 1475],\n",
       " [453, 1476],\n",
       " [454, 1477],\n",
       " [455, 1478],\n",
       " [456, 1479],\n",
       " [457, 1480],\n",
       " [458, 1481],\n",
       " [459, 1482],\n",
       " [460, 1483],\n",
       " [461, 1484],\n",
       " [462, 1485],\n",
       " [463, 1486],\n",
       " [464, 1487],\n",
       " [465, 1488],\n",
       " [466, 1489],\n",
       " [467, 1490],\n",
       " [468, 1491],\n",
       " [469, 1492],\n",
       " [470, 1493],\n",
       " [471, 1494],\n",
       " [472, 1495],\n",
       " [473, 1496],\n",
       " [474, 1497],\n",
       " [475, 1498],\n",
       " [476, 1499],\n",
       " [477, 1500],\n",
       " [478, 1501],\n",
       " [479, 1502],\n",
       " [480, 1503],\n",
       " [481, 1504],\n",
       " [482, 1505],\n",
       " [483, 1506],\n",
       " [484, 1507],\n",
       " [485, 1508],\n",
       " [486, 1509],\n",
       " [487, 1510],\n",
       " [488, 1511],\n",
       " [489, 1512],\n",
       " [490, 1513],\n",
       " [491, 1514],\n",
       " [492, 1515],\n",
       " [493, 1516],\n",
       " [494, 1517],\n",
       " [495, 1518],\n",
       " [496, 1519],\n",
       " [497, 1520],\n",
       " [498, 1521],\n",
       " [499, 1522],\n",
       " [500, 1523],\n",
       " [501, 1524],\n",
       " [502, 1525],\n",
       " [503, 1526],\n",
       " [504, 1527],\n",
       " [505, 1528],\n",
       " [506, 1529],\n",
       " [507, 1530],\n",
       " [508, 1531],\n",
       " [509, 1532],\n",
       " [510, 1533],\n",
       " [511, 1534],\n",
       " [512, 1535],\n",
       " [513, 1536],\n",
       " [514, 1537],\n",
       " [515, 1538],\n",
       " [516, 1539],\n",
       " [517, 1540],\n",
       " [518, 1541],\n",
       " [519, 1542],\n",
       " [520, 1543],\n",
       " [521, 1544],\n",
       " [522, 1545],\n",
       " [523, 1546],\n",
       " [524, 1547],\n",
       " [525, 1548],\n",
       " [526, 1549],\n",
       " [527, 1550],\n",
       " [528, 1551],\n",
       " [529, 1552],\n",
       " [530, 1553],\n",
       " [531, 1554],\n",
       " [532, 1555],\n",
       " [533, 1556],\n",
       " [534, 1557],\n",
       " [535, 1558],\n",
       " [536, 1559],\n",
       " [537, 1560],\n",
       " [538, 1561],\n",
       " [539, 1562],\n",
       " [540, 1563],\n",
       " [541, 1564],\n",
       " [542, 1565],\n",
       " [543, 1566],\n",
       " [544, 1567],\n",
       " [545, 1568],\n",
       " [546, 1569],\n",
       " [547, 1570],\n",
       " [548, 1571],\n",
       " [549, 1572],\n",
       " [550, 1573],\n",
       " [551, 1574],\n",
       " [552, 1575],\n",
       " [553, 1576],\n",
       " [554, 1577],\n",
       " [555, 1578],\n",
       " [556, 1579],\n",
       " [557, 1580],\n",
       " [558, 1581],\n",
       " [559, 1582],\n",
       " [560, 1583],\n",
       " [561, 1584],\n",
       " [562, 1585],\n",
       " [563, 1586],\n",
       " [564, 1587],\n",
       " [565, 1588],\n",
       " [566, 1589],\n",
       " [567, 1590],\n",
       " [568, 1591],\n",
       " [569, 1592],\n",
       " [570, 1593],\n",
       " [571, 1594],\n",
       " [572, 1595],\n",
       " [573, 1596],\n",
       " [574, 1597],\n",
       " [575, 1598],\n",
       " [576, 1599],\n",
       " [577, 1600],\n",
       " [578, 1601],\n",
       " [579, 1602],\n",
       " [580, 1603],\n",
       " [581, 1604],\n",
       " [582, 1605],\n",
       " [583, 1606],\n",
       " [584, 1607],\n",
       " [585, 1608],\n",
       " [586, 1609],\n",
       " [587, 1610],\n",
       " [588, 1611],\n",
       " [589, 1612],\n",
       " [590, 1613],\n",
       " [591, 1614],\n",
       " [592, 1615],\n",
       " [593, 1616],\n",
       " [594, 1617],\n",
       " [595, 1618],\n",
       " [596, 1619],\n",
       " [597, 1620],\n",
       " [598, 1621],\n",
       " [599, 1622],\n",
       " [600, 1623],\n",
       " [601, 1624],\n",
       " [602, 1625],\n",
       " [603, 1626],\n",
       " [604, 1627],\n",
       " [605, 1628],\n",
       " [606, 1629],\n",
       " [607, 1630],\n",
       " [608, 1631],\n",
       " [609, 1632],\n",
       " [610, 1633],\n",
       " [611, 1634],\n",
       " [612, 1635],\n",
       " [613, 1636],\n",
       " [614, 1637],\n",
       " [615, 1638],\n",
       " [616, 1639],\n",
       " [617, 1640],\n",
       " [618, 1641],\n",
       " [619, 1642],\n",
       " [620, 1643],\n",
       " [621, 1644],\n",
       " [622, 1645],\n",
       " [623, 1646],\n",
       " [624, 1647],\n",
       " [625, 1648],\n",
       " [626, 1649],\n",
       " [627, 1650],\n",
       " [628, 1651],\n",
       " [629, 1652],\n",
       " [630, 1653],\n",
       " [631, 1654],\n",
       " [632, 1655],\n",
       " [633, 1656],\n",
       " [634, 1657],\n",
       " [635, 1658],\n",
       " [636, 1659],\n",
       " [637, 1660],\n",
       " [638, 1661],\n",
       " [639, 1662],\n",
       " [640, 1663],\n",
       " [641, 1664],\n",
       " [642, 1665],\n",
       " [643, 1666],\n",
       " [644, 1667],\n",
       " [645, 1668],\n",
       " [646, 1669],\n",
       " [647, 1670],\n",
       " [648, 1671],\n",
       " [649, 1672],\n",
       " [650, 1673],\n",
       " [651, 1674],\n",
       " [652, 1675],\n",
       " [653, 1676],\n",
       " [654, 1677],\n",
       " [655, 1678],\n",
       " [656, 1679],\n",
       " [657, 1680],\n",
       " [658, 1681],\n",
       " [659, 1682],\n",
       " [660, 1683],\n",
       " [661, 1684],\n",
       " [662, 1685],\n",
       " [663, 1686],\n",
       " [664, 1687],\n",
       " [665, 1688],\n",
       " [666, 1689],\n",
       " [667, 1690],\n",
       " [668, 1691],\n",
       " [669, 1692],\n",
       " [670, 1693],\n",
       " [671, 1694],\n",
       " [672, 1695],\n",
       " [673, 1696],\n",
       " [674, 1697],\n",
       " [675, 1698],\n",
       " [676, 1699],\n",
       " [677, 1700],\n",
       " [678, 1701],\n",
       " [679, 1702],\n",
       " [680, 1703],\n",
       " [681, 1704],\n",
       " [682, 1705],\n",
       " [683, 1706],\n",
       " [684, 1707],\n",
       " [685, 1708],\n",
       " [686, 1709],\n",
       " [687, 1710],\n",
       " [688, 1711],\n",
       " [689, 1712],\n",
       " [690, 1713],\n",
       " [691, 1714],\n",
       " [692, 1715],\n",
       " [693, 1716],\n",
       " [694, 1717],\n",
       " [695, 1718],\n",
       " [696, 1719],\n",
       " [697, 1720],\n",
       " [698, 1721],\n",
       " [699, 1722],\n",
       " [700, 1723],\n",
       " [701, 1724],\n",
       " [702, 1725],\n",
       " [703, 1726],\n",
       " [704, 1727],\n",
       " [705, 1728],\n",
       " [706, 1729],\n",
       " [707, 1730],\n",
       " [708, 1731],\n",
       " [709, 1732],\n",
       " [710, 1733],\n",
       " [711, 1734],\n",
       " [712, 1735],\n",
       " [713, 1736],\n",
       " [714, 1737],\n",
       " [715, 1738],\n",
       " [716, 1739],\n",
       " [717, 1740],\n",
       " [718, 1741],\n",
       " [719, 1742],\n",
       " [720, 1743],\n",
       " [721, 1744],\n",
       " [722, 1745],\n",
       " [723, 1746],\n",
       " [724, 1747],\n",
       " [725, 1748],\n",
       " [726, 1749],\n",
       " [727, 1750],\n",
       " [728, 1751],\n",
       " [729, 1752],\n",
       " [730, 1753],\n",
       " [731, 1754],\n",
       " [732, 1755],\n",
       " [733, 1756],\n",
       " [734, 1757],\n",
       " [735, 1758],\n",
       " [736, 1759],\n",
       " [737, 1760],\n",
       " [738, 1761],\n",
       " [739, 1762],\n",
       " [740, 1763],\n",
       " [741, 1764],\n",
       " [742, 1765],\n",
       " [743, 1766],\n",
       " [744, 1767],\n",
       " [745, 1768],\n",
       " [746, 1769],\n",
       " [747, 1770],\n",
       " [748, 1771],\n",
       " [749, 1772],\n",
       " [750, 1773],\n",
       " [751, 1774],\n",
       " [752, 1775],\n",
       " [753, 1776],\n",
       " [754, 1777],\n",
       " [755, 1778],\n",
       " [756, 1779],\n",
       " [757, 1780],\n",
       " [758, 1781],\n",
       " [759, 1782],\n",
       " [760, 1783],\n",
       " [761, 1784],\n",
       " [762, 1785],\n",
       " [763, 1786],\n",
       " [764, 1787],\n",
       " [765, 1788],\n",
       " [766, 1789],\n",
       " [767, 1790],\n",
       " [768, 1791],\n",
       " [769, 1792],\n",
       " [770, 1793],\n",
       " [771, 1794],\n",
       " [772, 1795],\n",
       " [773, 1796],\n",
       " [774, 1797],\n",
       " [775, 1798],\n",
       " [776, 1799],\n",
       " [777, 1800],\n",
       " [778, 1801],\n",
       " [779, 1802],\n",
       " [780, 1803],\n",
       " [781, 1804],\n",
       " [782, 1805],\n",
       " [783, 1806],\n",
       " [784, 1807],\n",
       " [785, 1808],\n",
       " [786, 1809],\n",
       " [787, 1810],\n",
       " [788, 1811],\n",
       " [789, 1812],\n",
       " [790, 1813],\n",
       " [791, 1814],\n",
       " [792, 1815],\n",
       " [793, 1816],\n",
       " [794, 1817],\n",
       " [795, 1818],\n",
       " [796, 1819],\n",
       " [797, 1820],\n",
       " [798, 1821],\n",
       " [799, 1822],\n",
       " [800, 1823],\n",
       " [801, 1824],\n",
       " [802, 1825],\n",
       " [803, 1826],\n",
       " [804, 1827],\n",
       " [805, 1828],\n",
       " [806, 1829],\n",
       " [807, 1830],\n",
       " [808, 1831],\n",
       " [809, 1832],\n",
       " [810, 1833],\n",
       " [811, 1834],\n",
       " [812, 1835],\n",
       " [813, 1836],\n",
       " [814, 1837],\n",
       " [815, 1838],\n",
       " [816, 1839],\n",
       " [817, 1840],\n",
       " [818, 1841],\n",
       " [819, 1842],\n",
       " [820, 1843],\n",
       " [821, 1844],\n",
       " [822, 1845],\n",
       " [823, 1846],\n",
       " [824, 1847],\n",
       " [825, 1848],\n",
       " [826, 1849],\n",
       " [827, 1850],\n",
       " [828, 1851],\n",
       " [829, 1852],\n",
       " [830, 1853],\n",
       " [831, 1854],\n",
       " [832, 1855],\n",
       " [833, 1856],\n",
       " [834, 1857],\n",
       " [835, 1858],\n",
       " [836, 1859],\n",
       " [837, 1860],\n",
       " [838, 1861],\n",
       " [839, 1862],\n",
       " [840, 1863],\n",
       " [841, 1864],\n",
       " [842, 1865],\n",
       " [843, 1866],\n",
       " [844, 1867],\n",
       " [845, 1868],\n",
       " [846, 1869],\n",
       " [847, 1870],\n",
       " [848, 1871],\n",
       " [849, 1872],\n",
       " [850, 1873],\n",
       " [851, 1874],\n",
       " [852, 1875],\n",
       " [853, 1876],\n",
       " [854, 1877],\n",
       " [855, 1878],\n",
       " [856, 1879],\n",
       " [857, 1880],\n",
       " [858, 1881],\n",
       " [859, 1882],\n",
       " [860, 1883],\n",
       " [861, 1884],\n",
       " [862, 1885],\n",
       " [863, 1886],\n",
       " [864, 1887],\n",
       " [865, 1888],\n",
       " [866, 1889],\n",
       " [867, 1890],\n",
       " [868, 1891],\n",
       " [869, 1892],\n",
       " [870, 1893],\n",
       " [871, 1894],\n",
       " [872, 1895],\n",
       " [873, 1896],\n",
       " [874, 1897],\n",
       " [875, 1898],\n",
       " [876, 1899],\n",
       " [877, 1900],\n",
       " [878, 1901],\n",
       " [879, 1902],\n",
       " [880, 1903],\n",
       " [881, 1904],\n",
       " [882, 1905],\n",
       " [883, 1906],\n",
       " [884, 1907],\n",
       " [885, 1908],\n",
       " [886, 1909],\n",
       " [887, 1910],\n",
       " [888, 1911],\n",
       " [889, 1912],\n",
       " [890, 1913],\n",
       " [891, 1914],\n",
       " [892, 1915],\n",
       " [893, 1916],\n",
       " [894, 1917],\n",
       " [895, 1918],\n",
       " [896, 1919],\n",
       " [897, 1920],\n",
       " [898, 1921],\n",
       " [899, 1922],\n",
       " [900, 1923],\n",
       " [901, 1924],\n",
       " [902, 1925],\n",
       " [903, 1926],\n",
       " [904, 1927],\n",
       " [905, 1928],\n",
       " [906, 1929],\n",
       " [907, 1930],\n",
       " [908, 1931],\n",
       " [909, 1932],\n",
       " [910, 1933],\n",
       " [911, 1934],\n",
       " [912, 1935],\n",
       " [913, 1936],\n",
       " [914, 1937],\n",
       " [915, 1938],\n",
       " [916, 1939],\n",
       " [917, 1940],\n",
       " [918, 1941],\n",
       " [919, 1942],\n",
       " [920, 1943],\n",
       " [921, 1944],\n",
       " [922, 1945],\n",
       " [923, 1946],\n",
       " [924, 1947],\n",
       " [925, 1948],\n",
       " [926, 1949],\n",
       " [927, 1950],\n",
       " [928, 1951],\n",
       " [929, 1952],\n",
       " [930, 1953],\n",
       " [931, 1954],\n",
       " [932, 1955],\n",
       " [933, 1956],\n",
       " [934, 1957],\n",
       " [935, 1958],\n",
       " [936, 1959],\n",
       " [937, 1960],\n",
       " [938, 1961],\n",
       " [939, 1962],\n",
       " [940, 1963],\n",
       " [941, 1964],\n",
       " [942, 1965],\n",
       " [943, 1966],\n",
       " [944, 1967],\n",
       " [945, 1968],\n",
       " [946, 1969],\n",
       " [947, 1970],\n",
       " [948, 1971],\n",
       " [949, 1972],\n",
       " [950, 1973],\n",
       " [951, 1974],\n",
       " [952, 1975],\n",
       " [953, 1976],\n",
       " [954, 1977],\n",
       " [955, 1978],\n",
       " [956, 1979],\n",
       " [957, 1980],\n",
       " [958, 1981],\n",
       " [959, 1982],\n",
       " [960, 1983],\n",
       " [961, 1984],\n",
       " [962, 1985],\n",
       " [963, 1986],\n",
       " [964, 1987],\n",
       " [965, 1988],\n",
       " [966, 1989],\n",
       " [967, 1990],\n",
       " [968, 1991],\n",
       " [969, 1992],\n",
       " [970, 1993],\n",
       " [971, 1994],\n",
       " [972, 1995],\n",
       " [973, 1996],\n",
       " [974, 1997],\n",
       " [975, 1998],\n",
       " [976, 1999],\n",
       " [977, 2000],\n",
       " [978, 2001],\n",
       " [979, 2002],\n",
       " [980, 2003],\n",
       " [981, 2004],\n",
       " [982, 2005],\n",
       " [983, 2006],\n",
       " [984, 2007],\n",
       " [985, 2008],\n",
       " [986, 2009],\n",
       " [987, 2010],\n",
       " [988, 2011],\n",
       " [989, 2012],\n",
       " [990, 2013],\n",
       " [991, 2014],\n",
       " [992, 2015],\n",
       " [993, 2016],\n",
       " [994, 2017],\n",
       " [995, 2018],\n",
       " [996, 2019],\n",
       " [997, 2020],\n",
       " [998, 2021],\n",
       " [999, 2022],\n",
       " ...]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.all_IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>extdata::book=book_UR::data_name=forward_return_vwap_10s</th>\n",
       "      <th>extdata::book=book_UR::data_name=forward_return_vwap_60s</th>\n",
       "      <th>extdata::book=book_UR::data_name=forward_return_vwap_600s</th>\n",
       "      <th>extdata::book=book_UR::data_name=forward_return_vwap_1800s</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.707107</td>\n",
       "      <td>-0.707107</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>0.707107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.581058</td>\n",
       "      <td>-0.577748</td>\n",
       "      <td>0.577836</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.505638</td>\n",
       "      <td>-0.501576</td>\n",
       "      <td>0.507367</td>\n",
       "      <td>0.500129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.004430</td>\n",
       "      <td>-0.250535</td>\n",
       "      <td>1.140760</td>\n",
       "      <td>0.721484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.225485</td>\n",
       "      <td>-0.110365</td>\n",
       "      <td>1.294252</td>\n",
       "      <td>0.831957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.346451</td>\n",
       "      <td>0.540089</td>\n",
       "      <td>1.811106</td>\n",
       "      <td>1.315948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.119430</td>\n",
       "      <td>0.471485</td>\n",
       "      <td>1.414323</td>\n",
       "      <td>1.113687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.975620</td>\n",
       "      <td>0.422953</td>\n",
       "      <td>1.202803</td>\n",
       "      <td>0.985486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.698799</td>\n",
       "      <td>0.263549</td>\n",
       "      <td>0.949845</td>\n",
       "      <td>0.805501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.224626</td>\n",
       "      <td>-0.039403</td>\n",
       "      <td>0.615598</td>\n",
       "      <td>0.555318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.036301</td>\n",
       "      <td>-0.219838</td>\n",
       "      <td>0.456312</td>\n",
       "      <td>0.426951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.085646</td>\n",
       "      <td>-0.233301</td>\n",
       "      <td>0.446148</td>\n",
       "      <td>0.412236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.129631</td>\n",
       "      <td>-0.256435</td>\n",
       "      <td>0.441235</td>\n",
       "      <td>0.400887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-1.116664</td>\n",
       "      <td>-0.884446</td>\n",
       "      <td>-0.224598</td>\n",
       "      <td>-0.096601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.301946</td>\n",
       "      <td>0.052667</td>\n",
       "      <td>0.813118</td>\n",
       "      <td>0.663609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.233448</td>\n",
       "      <td>-0.294624</td>\n",
       "      <td>0.442490</td>\n",
       "      <td>0.386024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.229505</td>\n",
       "      <td>0.689906</td>\n",
       "      <td>1.422011</td>\n",
       "      <td>1.135160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.118472</td>\n",
       "      <td>0.633006</td>\n",
       "      <td>1.319991</td>\n",
       "      <td>1.072516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.014277</td>\n",
       "      <td>0.588357</td>\n",
       "      <td>1.239069</td>\n",
       "      <td>1.020485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.175604</td>\n",
       "      <td>0.719098</td>\n",
       "      <td>1.323769</td>\n",
       "      <td>1.097007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.375328</td>\n",
       "      <td>-0.425604</td>\n",
       "      <td>0.328647</td>\n",
       "      <td>0.297911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.593799</td>\n",
       "      <td>0.299345</td>\n",
       "      <td>0.982667</td>\n",
       "      <td>0.818400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>-0.631259</td>\n",
       "      <td>-0.536343</td>\n",
       "      <td>0.307080</td>\n",
       "      <td>0.271651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.674564</td>\n",
       "      <td>0.418759</td>\n",
       "      <td>1.128313</td>\n",
       "      <td>0.939929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2.066958</td>\n",
       "      <td>1.480276</td>\n",
       "      <td>1.961268</td>\n",
       "      <td>1.656219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.871046</td>\n",
       "      <td>1.392374</td>\n",
       "      <td>1.799404</td>\n",
       "      <td>1.547944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.099815</td>\n",
       "      <td>0.790662</td>\n",
       "      <td>1.275579</td>\n",
       "      <td>1.109448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-0.105032</td>\n",
       "      <td>-0.169924</td>\n",
       "      <td>0.514529</td>\n",
       "      <td>0.449381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.561640</td>\n",
       "      <td>0.381820</td>\n",
       "      <td>0.954041</td>\n",
       "      <td>0.826496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.541153</td>\n",
       "      <td>0.367601</td>\n",
       "      <td>0.928956</td>\n",
       "      <td>0.805978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.736687</td>\n",
       "      <td>0.538088</td>\n",
       "      <td>1.055467</td>\n",
       "      <td>0.916659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.228012</td>\n",
       "      <td>0.101536</td>\n",
       "      <td>0.740587</td>\n",
       "      <td>0.641446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>-0.060649</td>\n",
       "      <td>-0.117845</td>\n",
       "      <td>0.583408</td>\n",
       "      <td>0.499268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>-0.014261</td>\n",
       "      <td>0.077941</td>\n",
       "      <td>0.728079</td>\n",
       "      <td>0.625714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.947397</td>\n",
       "      <td>0.901743</td>\n",
       "      <td>1.311930</td>\n",
       "      <td>1.140354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.905178</td>\n",
       "      <td>0.872129</td>\n",
       "      <td>1.268656</td>\n",
       "      <td>1.107958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1.372629</td>\n",
       "      <td>1.228626</td>\n",
       "      <td>1.518458</td>\n",
       "      <td>1.336857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>-0.492336</td>\n",
       "      <td>-0.270265</td>\n",
       "      <td>0.455839</td>\n",
       "      <td>0.387896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.262532</td>\n",
       "      <td>0.363407</td>\n",
       "      <td>0.900362</td>\n",
       "      <td>0.783516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    extdata::book=book_UR::data_name=forward_return_vwap_10s  \\\n",
       "0                                            0.000000          \n",
       "1                                           -0.707107          \n",
       "2                                           -0.581058          \n",
       "3                                           -0.505638          \n",
       "4                                            0.004430          \n",
       "5                                            0.225485          \n",
       "6                                            1.346451          \n",
       "7                                            1.119430          \n",
       "8                                            0.975620          \n",
       "9                                            0.698799          \n",
       "10                                           0.224626          \n",
       "11                                          -0.036301          \n",
       "12                                          -0.085646          \n",
       "13                                          -0.129631          \n",
       "14                                          -1.116664          \n",
       "15                                           0.301946          \n",
       "16                                          -0.233448          \n",
       "17                                           1.229505          \n",
       "18                                           1.118472          \n",
       "19                                           1.014277          \n",
       "20                                           1.175604          \n",
       "21                                          -0.375328          \n",
       "22                                           0.593799          \n",
       "23                                          -0.631259          \n",
       "24                                           0.674564          \n",
       "25                                           2.066958          \n",
       "26                                           1.871046          \n",
       "27                                           1.099815          \n",
       "28                                          -0.105032          \n",
       "29                                           0.561640          \n",
       "30                                           0.541153          \n",
       "31                                           0.736687          \n",
       "32                                           0.228012          \n",
       "33                                          -0.060649          \n",
       "34                                          -0.014261          \n",
       "35                                           0.947397          \n",
       "36                                           0.905178          \n",
       "37                                           1.372629          \n",
       "38                                          -0.492336          \n",
       "39                                           0.262532          \n",
       "\n",
       "    extdata::book=book_UR::data_name=forward_return_vwap_60s  \\\n",
       "0                                            0.000000          \n",
       "1                                           -0.707107          \n",
       "2                                           -0.577748          \n",
       "3                                           -0.501576          \n",
       "4                                           -0.250535          \n",
       "5                                           -0.110365          \n",
       "6                                            0.540089          \n",
       "7                                            0.471485          \n",
       "8                                            0.422953          \n",
       "9                                            0.263549          \n",
       "10                                          -0.039403          \n",
       "11                                          -0.219838          \n",
       "12                                          -0.233301          \n",
       "13                                          -0.256435          \n",
       "14                                          -0.884446          \n",
       "15                                           0.052667          \n",
       "16                                          -0.294624          \n",
       "17                                           0.689906          \n",
       "18                                           0.633006          \n",
       "19                                           0.588357          \n",
       "20                                           0.719098          \n",
       "21                                          -0.425604          \n",
       "22                                           0.299345          \n",
       "23                                          -0.536343          \n",
       "24                                           0.418759          \n",
       "25                                           1.480276          \n",
       "26                                           1.392374          \n",
       "27                                           0.790662          \n",
       "28                                          -0.169924          \n",
       "29                                           0.381820          \n",
       "30                                           0.367601          \n",
       "31                                           0.538088          \n",
       "32                                           0.101536          \n",
       "33                                          -0.117845          \n",
       "34                                           0.077941          \n",
       "35                                           0.901743          \n",
       "36                                           0.872129          \n",
       "37                                           1.228626          \n",
       "38                                          -0.270265          \n",
       "39                                           0.363407          \n",
       "\n",
       "    extdata::book=book_UR::data_name=forward_return_vwap_600s  \\\n",
       "0                                            0.000000           \n",
       "1                                            0.707107           \n",
       "2                                            0.577836           \n",
       "3                                            0.507367           \n",
       "4                                            1.140760           \n",
       "5                                            1.294252           \n",
       "6                                            1.811106           \n",
       "7                                            1.414323           \n",
       "8                                            1.202803           \n",
       "9                                            0.949845           \n",
       "10                                           0.615598           \n",
       "11                                           0.456312           \n",
       "12                                           0.446148           \n",
       "13                                           0.441235           \n",
       "14                                          -0.224598           \n",
       "15                                           0.813118           \n",
       "16                                           0.442490           \n",
       "17                                           1.422011           \n",
       "18                                           1.319991           \n",
       "19                                           1.239069           \n",
       "20                                           1.323769           \n",
       "21                                           0.328647           \n",
       "22                                           0.982667           \n",
       "23                                           0.307080           \n",
       "24                                           1.128313           \n",
       "25                                           1.961268           \n",
       "26                                           1.799404           \n",
       "27                                           1.275579           \n",
       "28                                           0.514529           \n",
       "29                                           0.954041           \n",
       "30                                           0.928956           \n",
       "31                                           1.055467           \n",
       "32                                           0.740587           \n",
       "33                                           0.583408           \n",
       "34                                           0.728079           \n",
       "35                                           1.311930           \n",
       "36                                           1.268656           \n",
       "37                                           1.518458           \n",
       "38                                           0.455839           \n",
       "39                                           0.900362           \n",
       "\n",
       "    extdata::book=book_UR::data_name=forward_return_vwap_1800s  \n",
       "0                                            0.000000           \n",
       "1                                            0.707107           \n",
       "2                                            0.577350           \n",
       "3                                            0.500129           \n",
       "4                                            0.721484           \n",
       "5                                            0.831957           \n",
       "6                                            1.315948           \n",
       "7                                            1.113687           \n",
       "8                                            0.985486           \n",
       "9                                            0.805501           \n",
       "10                                           0.555318           \n",
       "11                                           0.426951           \n",
       "12                                           0.412236           \n",
       "13                                           0.400887           \n",
       "14                                          -0.096601           \n",
       "15                                           0.663609           \n",
       "16                                           0.386024           \n",
       "17                                           1.135160           \n",
       "18                                           1.072516           \n",
       "19                                           1.020485           \n",
       "20                                           1.097007           \n",
       "21                                           0.297911           \n",
       "22                                           0.818400           \n",
       "23                                           0.271651           \n",
       "24                                           0.939929           \n",
       "25                                           1.656219           \n",
       "26                                           1.547944           \n",
       "27                                           1.109448           \n",
       "28                                           0.449381           \n",
       "29                                           0.826496           \n",
       "30                                           0.805978           \n",
       "31                                           0.916659           \n",
       "32                                           0.641446           \n",
       "33                                           0.499268           \n",
       "34                                           0.625714           \n",
       "35                                           1.140354           \n",
       "36                                           1.107958           \n",
       "37                                           1.336857           \n",
       "38                                           0.387896           \n",
       "39                                           0.783516           "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.labels_df[:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in sorted(glob.glob(pattern)):\n",
    "            df = pd.read_parquet(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_parquet(\"/Users/tonywy/Desktop/Xode/crossformer/futs_data/ZCE_CH_UR/daily_frame.20231226.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['preds', 'trues', 'ts']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "npz = np.load('./checkpoints/futs_01_31_20_57_40_il2048_ol1_sl32_win2_fa10_dm256_nh4_el3_itr0/output.npz')\n",
    "print(npz.files)\n",
    "\n",
    "pred = npz['preds'].flatten()\n",
    "target = npz['trues'].flatten()\n",
    "ts = npz['ts'].flatten()\n",
    "\n",
    "ts_sorted_id = np.argsort(ts)\n",
    "pred = pred[ts_sorted_id]\n",
    "target = target[ts_sorted_id]\n",
    "ts = ts[ts_sorted_id]\n",
    "\n",
    "print(\"Corr: \", np.corrcoef(pred, target))\n",
    "from sklearn.metrics import r2_score\n",
    "print(\"R2: \", r2_score(target, pred))\n",
    "\n",
    "import pandas as pd\n",
    "data = pd.DataFrame({'ts':ts, 'pred': pred, 'target': target})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.66954276],\n",
       "       [0.66954276, 1.        ]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18088789262429916"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts</th>\n",
       "      <th>pred</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1703034568500</td>\n",
       "      <td>2.165842</td>\n",
       "      <td>0.616356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1703034568750</td>\n",
       "      <td>2.115012</td>\n",
       "      <td>0.352395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1703034569250</td>\n",
       "      <td>2.089970</td>\n",
       "      <td>0.610446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1703034569500</td>\n",
       "      <td>2.137356</td>\n",
       "      <td>0.350515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1703034569750</td>\n",
       "      <td>2.195066</td>\n",
       "      <td>0.608521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356347</th>\n",
       "      <td>1704264628750</td>\n",
       "      <td>0.815943</td>\n",
       "      <td>0.258331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356348</th>\n",
       "      <td>1704264629000</td>\n",
       "      <td>0.696816</td>\n",
       "      <td>0.257893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356349</th>\n",
       "      <td>1704264629250</td>\n",
       "      <td>0.627157</td>\n",
       "      <td>0.257455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356350</th>\n",
       "      <td>1704264629500</td>\n",
       "      <td>0.660114</td>\n",
       "      <td>-0.010955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>356351</th>\n",
       "      <td>1704264629750</td>\n",
       "      <td>0.441956</td>\n",
       "      <td>-0.011860</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>356352 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   ts      pred    target\n",
       "0       1703034568500  2.165842  0.616356\n",
       "1       1703034568750  2.115012  0.352395\n",
       "2       1703034569250  2.089970  0.610446\n",
       "3       1703034569500  2.137356  0.350515\n",
       "4       1703034569750  2.195066  0.608521\n",
       "...               ...       ...       ...\n",
       "356347  1704264628750  0.815943  0.258331\n",
       "356348  1704264629000  0.696816  0.257893\n",
       "356349  1704264629250  0.627157  0.257455\n",
       "356350  1704264629500  0.660114 -0.010955\n",
       "356351  1704264629750  0.441956 -0.011860\n",
       "\n",
       "[356352 rows x 3 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}