DANET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Environment_name = Dl_DANET\n",
    "#Python = 3.6\n",
    "#PyTorch version 1.8.2+cu111\n",
    "#einops\n",
    "#seaborn\n",
    "#sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: /home/ma00048/miniconda3/envs/DL/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n",
      "Name: torch\n",
      "Version: 1.8.2\n",
      "Summary: Tensors and Dynamic neural networks in Python with strong GPU acceleration\n",
      "Home-page: https://pytorch.org/\n",
      "Author: PyTorch Team\n",
      "Author-email: packages@pytorch.org\n",
      "License: BSD-3\n",
      "Location: /home/ma00048/miniconda3/envs/Dl_DANET/lib/python3.6/site-packages\n",
      "Requires: typing_extensions, numpy, dataclasses\n",
      "Required-by: torchvision, torchaudio\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip show torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "import argparse\n",
    "from torch.autograd import Variable\n",
    "from DANet.DA_Net import DA_Net\n",
    "from DANet.util import compute_F1_score, exponential_decay, save_result, plot_roc, random_seed\n",
    "from DANet.read_data import load_UEA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix\n",
    "from sklearn.model_selection import cross_val_predict, KFold\n",
    "import sklearn.preprocessing\n",
    "\n",
    "import time\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "from joblib import parallel_backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "length = 1536 * 2\n",
    "parser = argparse.ArgumentParser(description='DA-Net for MTSC')\n",
    "\n",
    "#parser.add_argument('--model', type=str, default='DA-Net')\n",
    "parser.add_argument('--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu')\n",
    "parser.add_argument('--length', type=int, default=1000, help='Embedding length')\n",
    "#parser.add_argument('--writer_path', type=str, default='runs/exp', help='TensorBoard path')\n",
    "parser.add_argument('--data_path', type=str, default=\"../datasets\")\n",
    "#parser.add_argument('--seed', type=int, default=1, help='random seed')\n",
    "parser.add_argument('--dropout', type=float, default=0.05, help='attention dropout rate')\n",
    "parser.add_argument('--batch_size', type=int, default=16)\n",
    "parser.add_argument('--n_epochs', type=int, default=2)\n",
    "parser.add_argument('--cache_path', type=str, default='./DANet/cache')\n",
    "parser.add_argument('--window', type=int, default=64)  # [32,48,64,80,96]\n",
    "parser.add_argument('--M_name', type=str, default='DA-Net')\n",
    "\n",
    "args = parser.parse_args(\"\")\n",
    "M_name=args.M_name\n",
    "\n",
    "length = 1536 * 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetDataAndNet(archive_path, archive_name, wa, prob, mask=1):\n",
    "    train_loader, test_loader, num_class = load_UEA(X_train, X_test,y_train,y_test, args)\n",
    "\n",
    "    # get the length and channel of time series\n",
    "    #time_stmp = next(train_loader.__iter__())[0].shape[2]\n",
    "    time_stmp = train_loader.__iter__().next()[0].shape[2]\n",
    "    #in_channel = next(train_loader.__iter__())[0].shape[1]\n",
    "    in_channel = train_loader.__iter__().next()[0].shape[1]\n",
    "    # num_class = DealDataset(train_path).num_class()\n",
    "\n",
    "    net = DA_Net(\n",
    "        t=time_stmp,\n",
    "        down_dim=length,\n",
    "        hidden_dim=(96, 192, 62),\n",
    "        layers=(2, 2, 6, 2),\n",
    "\n",
    "        heads=(3, 6, 12,24),\n",
    "        channels=in_channel,\n",
    "        num_classes=num_class,\n",
    "        head_dim=32,\n",
    "        window_size=args.window,\n",
    "        downscaling_factors=(4, 2, 2,2),  # 代表多长的时间作为一个特征\n",
    "\n",
    "        relative_pos_embedding=True,\n",
    "        wa=wa,\n",
    "        prob=prob,\n",
    "        mask=mask,).to(device)\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
    "        # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs\n",
    "        net = torch.nn.DataParallel(net)\n",
    "    return train_loader, test_loader, net, num_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch):\n",
    "    total_pred = torch.tensor([], dtype=torch.int64).to(device)\n",
    "    total_true = torch.tensor([], dtype=torch.int64).to(device)\n",
    "    score_list = []\n",
    "    label_list = []\n",
    "    total_test_acc = 0\n",
    "    # for batch_id, (x, y) in tqdm(enumerate(test_loader), total=len(test_loader)):\n",
    "    for batch_id, (x, y) in enumerate(test_loader):\n",
    "\n",
    "        #x = x.float().to(device)\n",
    "        x = Variable(x).float().to(device)\n",
    "        #y = y.to(device)\n",
    "        y = Variable(y).to(device)\n",
    "        net.eval()\n",
    "        start_time = time.time()\n",
    "        embedding, encoder, output, pred_y = net(x)\n",
    "        inference_time = time.time() - start_time\n",
    "\n",
    "        _, y_pred = torch.max(pred_y, -1)\n",
    "        total_test_acc += (y_pred.cpu() == y.cpu()).sum().item()\n",
    "\n",
    "        total_pred = torch.cat([total_pred, y_pred], dim=0)\n",
    "        total_true = torch.cat([total_true, y], dim=0)\n",
    "\n",
    "        test_loss = loss_func(pred_y, y.to(torch.long))\n",
    "\n",
    "        niter = epoch * test_loader.dataset.__len__() + batch_id\n",
    "#         if niter % 10 == 0:\n",
    "#             writer.add_scalar('Test Loss Curve {0}({1})'.format(M_name, length), test_loss.data.item(), niter)\n",
    "\n",
    "        score_list.extend(pred_y.detach().cpu().numpy())\n",
    "        label_list.extend(y.cpu().numpy())\n",
    "\n",
    "    #plot_roc( num_class, label_list, score_list, L=length)\n",
    "\n",
    "    f1_score, precision, recall = compute_F1_score(total_true, total_pred)\n",
    "\n",
    "    return total_test_acc, f1_score, precision, recall, inference_time, test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(optimizer):\n",
    "    train_time = 0\n",
    "    max_accuracy = 0\n",
    "    plot_train_loss = []\n",
    "    plot_test_loss = []\n",
    "    plot_train_acc = []\n",
    "    plot_test_acc = []\n",
    "    for epoch in range(n_epochs):\n",
    "        ls = []\n",
    "        s_time = time.time()\n",
    "        total_train_acc = 0\n",
    "\n",
    "\n",
    "        # for batch_id,(x,y) in tqdm(enumerate(train_loader), total=len(train_loader)):\n",
    "        for batch_id, (x, y) in enumerate(train_loader):\n",
    "            #torch ALEXNET\n",
    "            net.train()\n",
    "            optimizer = exponential_decay(optimizer, LEARNING_RATE, global_epoch, 1, 0.90)\n",
    "\n",
    "            #x = x.float().to(device)\n",
    "            x = Variable(x).float().to(device)\n",
    "            #y = y.to(device)\n",
    "            y = Variable(y).to(device)\n",
    "            # output 我们需要的 all_sample\n",
    "            embedding, encoder, output, pred_y = net(x)\n",
    "            # loss\n",
    "            loss = loss_func(pred_y, y.to(torch.long))\n",
    "\n",
    "            _, y_pred = torch.max(pred_y, -1)\n",
    "            acc_train = (y_pred.cpu() == y.cpu()).sum().item()\n",
    "            total_train_acc += acc_train\n",
    "            niter = epoch * train_loader.dataset.__len__() + batch_id\n",
    "\n",
    "            #if niter % 10 == 0:\n",
    "                #writer.add_scalar('Train Loss Curve {0}({1})'.format(M_name, length), loss.data.item(), niter)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            _, y_pred = torch.max(pred_y, -1)\n",
    "            ls.append(loss)\n",
    "\n",
    "        print('Epoch: {:04d}'.format(epoch + 1),\n",
    "              'loss_train: {:.8f}'.format(loss.item()),\n",
    "              'acc_train: {:.4f}'.format(total_train_acc / train_loader.dataset.__len__()),\n",
    "              'time: {:.4f}s'.format(time.time() - s_time))\n",
    "        plot_train_loss.append(loss.item())\n",
    "        plot_train_acc.append(total_train_acc / train_loader.dataset.__len__())\n",
    "        train_time += time.time() - s_time\n",
    "\n",
    "        # print(\"Total time elapsed: {:.4f}s\".format(train_time))\n",
    "        total_test_acc, f1_score, precision, recall, inference_time, test_loss = test(epoch)\n",
    "        plot_test_loss.append(test_loss.cpu().detach())\n",
    "        plot_test_acc.append(total_test_acc / test_loader.dataset.__len__())\n",
    "\n",
    "        # save model\n",
    "        if os.path.exists(f'DANet/saved_model/{M_name}') == False:\n",
    "            os.makedirs(f'DANet/saved_model/{M_name}')\n",
    "\n",
    "        if total_test_acc > max_accuracy:\n",
    "            print('save best model')\n",
    "            max_accuracy = total_test_acc\n",
    "            torch.save(net,\n",
    "                       f'DANet/saved_model/{M_name}/{Dataset_name} batch={args.batch_size} length={length} window={args.window}.pkl')\n",
    "\n",
    "        print('Epoch: {:04d}'.format(epoch + 1),\n",
    "              'loss_test: {:.8f}'.format(test_loss.item()),\n",
    "              'acc_test: {:.4f}'.format(total_test_acc / test_loader.dataset.__len__()),\n",
    "              'time: {:.4f}s'.format(time.time() - s_time))\n",
    "    #plt.plot()\n",
    "\n",
    "    if os.path.exists(f'./DANet/DA_Net_results') == False:\n",
    "        os.makedirs(f'./DANet/DA_Net_results')\n",
    "    save_result(file, ls[-1], total_test_acc / test_loader.dataset.__len__(), f1_score, precision, recall, train_time,\n",
    "                inference_time, args.window, length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1670796275068,
     "user": {
      "displayName": "Mojtaba Askarzadeh Farahani",
      "userId": "02005001605152753874"
     },
     "user_tz": 300
    },
    "id": "nYPPgDj2e7oy",
    "outputId": "c71b2a21-7bf6-46ac-c958-794ccd9e5531"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(53785, 749, 3)\n",
      "(53785, 749, 1)\n"
     ]
    }
   ],
   "source": [
    "# define a list of datasets\n",
    "datasets = [\n",
    "\"BEARING_Univar\",\n",
    "\"PHM2022_Multivar\",\n",
    "\"PHM2022_Univar_PIN\",\n",
    "\"PHM2022_Univar_PO\",\n",
    "\"PHM2022_Univar_PDIN\",\n",
    "\"ETCHING_Multivar\",\n",
    "\"MFPT_48KHZ_Univar\",\n",
    "\"MFPT_96KHZ_Univar\",\n",
    "\"PADERBORN_6KHZ_Univar\",\n",
    "\"PADERBORN_4KHZ_Univar\",\n",
    "\"PADERBORN_64KHZ_Multivar\",\n",
    "\"PADERBORN_4KHZ_Multivar\",\n",
    "\"Hydraulic_systems_10HZ_Multivar\",\n",
    "\"Hydraulic_systems_100HZ_Multivar\",\n",
    "\"Gas_sensors_home_activity\",\n",
    "\"Control_charts\",\n",
    "\"CWRU_12k_DE_univar\",\n",
    "\"CWRU_12k_DE_multivar\",\n",
    "\"CWRU_12k_FE_univar\",\n",
    "\"CWRU_12k_FE_multivar\",\n",
    "\"CWRU_48k_DE_univar\",\n",
    "\"CWRU_48k_DE_multivar\"\n",
    "]\n",
    "datasets = [\"PHM2022_Multivar\", \"PHM2022_Univar_PDIN\"]\n",
    "datasets_path = \"../datasets\"\n",
    "\n",
    "for dataset in datasets:\n",
    "    Dataset_name = dataset + \"_Dataset\"\n",
    "    Dataset = np.load(datasets_path + \"/\" + Dataset_name + \".npy\")\n",
    "    print(Dataset.shape)\n",
    "    \n",
    "\n",
    "    Labels_name = dataset + \"_Labels\"\n",
    "    Labels = np.load(datasets_path + \"/\"  + Labels_name + \".npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " The dataset shape is:(53785, 749, 3)\n",
      "\n",
      " The number of data samples (N) is:53785\n",
      "\n",
      " The number of TS length (T) is:749\n",
      "\n",
      " The number of TS dimention (M) is:3\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.50085771 acc_train: 0.8358 time: 274.3857s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.27797702 acc_test: 0.8950 time: 300.2248s\n",
      "Epoch: 0002 loss_train: 1.59540093 acc_train: 0.9488 time: 270.9376s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.09731893 acc_test: 0.9555 time: 296.9882s\n",
      "accuracy: 0.9554708561866692\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Multivar_Dataset_fold_1.csv\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.31282175 acc_train: 0.8307 time: 267.6500s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.02904009 acc_test: 0.9381 time: 293.6216s\n",
      "Epoch: 0002 loss_train: 0.00928279 acc_train: 0.9490 time: 269.5931s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.02023693 acc_test: 0.9654 time: 295.8775s\n",
      "accuracy: 0.9654178674351584\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Multivar_Dataset_fold_2.csv\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.02710780 acc_train: 0.8309 time: 265.8772s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.36029434 acc_test: 0.9310 time: 291.7767s\n",
      "Epoch: 0002 loss_train: 0.01264915 acc_train: 0.9427 time: 272.2730s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.00690839 acc_test: 0.9569 time: 299.0822s\n",
      "accuracy: 0.9568652970158966\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Multivar_Dataset_fold_3.csv\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.00933049 acc_train: 0.8302 time: 267.5311s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.00491627 acc_test: 0.9388 time: 293.9908s\n",
      "Epoch: 0002 loss_train: 0.02563783 acc_train: 0.9492 time: 270.9345s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.00092137 acc_test: 0.9673 time: 297.5058s\n",
      "accuracy: 0.9672771218741285\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Multivar_Dataset_fold_4.csv\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.05899946 acc_train: 0.8281 time: 270.7037s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.01650382 acc_test: 0.9330 time: 297.4692s\n",
      "Epoch: 0002 loss_train: 0.04907861 acc_train: 0.9396 time: 272.5836s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.07323061 acc_test: 0.9542 time: 298.7314s\n",
      "accuracy: 0.9541693780793902\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Multivar_Dataset_fold_5.csv\n",
      " Finished!\n",
      "Total time elapsed: 2983.9079s\n",
      "\n",
      " The dataset shape is:(53785, 749, 1)\n",
      "\n",
      " The number of data samples (N) is:53785\n",
      "\n",
      " The number of TS length (T) is:749\n",
      "\n",
      " The number of TS dimention (M) is:1\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.01002010 acc_train: 0.7612 time: 273.0503s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.35904676 acc_test: 0.8537 time: 299.0867s\n",
      "Epoch: 0002 loss_train: 0.08248701 acc_train: 0.9026 time: 277.1580s\n",
      "save best model\n",
      "Epoch: 0002 loss_test: 0.16234262 acc_test: 0.9224 time: 303.7506s\n",
      "accuracy: 0.9223761271730037\n",
      "parameters have been saved to ./DA_Net_results/result_PHM2022_Univar_PDIN_Dataset_fold_1.csv\n",
      "Let's use 2 GPUs!\n",
      "Epoch: 0001 loss_train: 0.13396069 acc_train: 0.7657 time: 272.8087s\n",
      "save best model\n",
      "Epoch: 0001 loss_test: 0.11272772 acc_test: 0.8889 time: 298.8415s\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-995f24daabd8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0;31m## Create Classification module\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-5a90637b4551>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(optimizer)\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0;31m# output 我们需要的 all_sample\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m             \u001b[0membedding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m             \u001b[0;31m# loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m         \u001b[0mreplicas\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py\u001b[0m in \u001b[0;36mparallel_apply\u001b[0;34m(self, replicas, inputs, kwargs)\u001b[0m\n\u001b[1;32m    175\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mparallel_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplicas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py\u001b[0m in \u001b[0;36mparallel_apply\u001b[0;34m(modules, inputs, kwargs_tup, devices)\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mthread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mthread\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mthreads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m             \u001b[0mthread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0m_worker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs_tup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1056\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1057\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1058\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/Dl_DANET/lib/python3.6/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1072\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1073\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1074\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# change this directory for your machine\n",
    "root_dir = './'\n",
    "\n",
    "# define the number of folds\n",
    "n_folds = 5\n",
    "\n",
    "# perform cross-validation for each dataset and algorithm combination\n",
    "for dataset in datasets:\n",
    "    Dataset_name = dataset + \"_Dataset\"\n",
    "    Dataset = np.load(datasets_path + \"/\" + Dataset_name + \".npy\")\n",
    "    \n",
    "\n",
    "    Labels_name = dataset + \"_Labels\"\n",
    "    Labels = np.load(datasets_path + \"/\"  + Labels_name + \".npy\")\n",
    "    \n",
    "\n",
    "\n",
    "    # Create a folder for results\n",
    "    results_path = root_dir + \"Results/\" + Dataset_name\n",
    "    if os.path.exists(results_path):\n",
    "        pass\n",
    "    else:\n",
    "        try:\n",
    "            os.makedirs(results_path)\n",
    "        except:\n",
    "            # in case another machine created the path meanwhile !:(\n",
    "            pass\n",
    "\n",
    "        \n",
    "    t_total = time.time() ##Start timing\n",
    "    \n",
    "\n",
    "\n",
    "    print(f\"\\n The dataset shape is:{Dataset.shape}\")\n",
    "    print(f\"\\n The number of data samples (N) is:{Dataset.shape[0]}\")\n",
    "    print(f\"\\n The number of TS length (T) is:{Dataset.shape[1]}\")\n",
    "    print(f\"\\n The number of TS dimention (M) is:{Dataset.shape[2]}\")\n",
    "\n",
    "\n",
    "\n",
    "    kf = KFold(n_splits=n_folds, shuffle=True)\n",
    "    accuracy_scores = []\n",
    "    f1_scores = []\n",
    "    confusion_matrices = []\n",
    "    report_list = []\n",
    "    for fold, (train_idx, test_idx) in enumerate(kf.split(Dataset)):\n",
    "        # split the data into training and testing sets\n",
    "        X_train, X_test = Dataset[train_idx], Dataset[test_idx]\n",
    "        y_train, y_test = Labels[train_idx], Labels[test_idx]\n",
    "        \n",
    "        wa=1\n",
    "        prob=1\n",
    "        file = r'./DANet/DA_Net_results/result_{0}_fold_{1}.csv'.format(Dataset_name, fold+1)\n",
    "\n",
    "        train_loader, test_loader, net, num_class = GetDataAndNet(0, Dataset_name, wa, prob)\n",
    "\n",
    "        LEARNING_RATE = 0.001\n",
    "        optimizer = torch.optim.Adam(\n",
    "            net.parameters(),\n",
    "            lr=10,\n",
    "            betas=(0.9, 0.999),\n",
    "            eps=1e-08)\n",
    "        global_epoch = 0\n",
    "        global_step = 0\n",
    "        best_tst_accuracy = 0.0\n",
    "        COMPUTE_TRN_METRICS = True\n",
    "        n_epochs = args.n_epochs\n",
    "\n",
    "        loss_func = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "        ## Create Classification module\n",
    "        train(optimizer)\n",
    "        \n",
    "\n",
    "            \n",
    "        # calculate the evaluation metrics\n",
    "#         accuracy = total_test_acc\n",
    "#         print(accuracy)\n",
    "\n",
    "#         f1 = f1_score\n",
    "#         print(f1)\n",
    "\n",
    "#         confusion = confusion_matrix(y_test, y_pred)\n",
    "#         print(confusion)\n",
    "\n",
    "#         accuracy_scores.append(accuracy)\n",
    "#         f1_scores.append(f1)\n",
    "#         confusion_matrices.append(confusion)\n",
    "\n",
    "#         report = classification_report(y_test, y_pred, zero_division=1)\n",
    "#         report_list.append(report)\n",
    "#         print(report)\n",
    "        \n",
    "#         print(f\" fold {fold+1} is Finished!\")\n",
    "        \n",
    "#         # save the output to a text file\n",
    "#         with open(f'{results_path}/dataset_{dataset_name}_DANet_fold_{fold+1}.txt', 'w') as f:\n",
    "#             f.write(f'Accuracy: {accuracy}\\n')\n",
    "#             f.write(f'F1 Score: {f1}\\n')\n",
    "#             f.write(f'Confusion Matrix:\\n{confusion}\\n\\n')\n",
    "#             f.write(f'Classification report:\\n{report}\\n\\n')\n",
    "        \n",
    "#     with open(f'{results_path}/dataset_{dataset_name}_DANet.txt', 'w') as f:\n",
    "#         f.write(\"Mean accuracy: {:.3f} (std={:.3f})\\n\".format(np.mean(accuracy_scores), np.std(accuracy_scores)))\n",
    "#         f.write(\"Mean F1 score: {:.3f} (std={:.3f})\\n\".format(np.mean(f1_scores), np.std(f1_scores)))\n",
    "#         f.write(\"Mean confusion matrix:\\n{}\\n\".format(np.array2string(np.mean(confusion_matrices, axis=0))))\n",
    "#         f.write(\"Total time elapsed: {:.4f}s\".format(time.time() - t_total))\n",
    "\n",
    "    print(\" Finished!\")\n",
    "    print(\"Total time elapsed: {:.4f}s\".format(time.time() - t_total))\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1670797391439,
     "user": {
      "displayName": "Mojtaba Askarzadeh Farahani",
      "userId": "02005001605152753874"
     },
     "user_tz": 300
    },
    "id": "o0hfsmobe7o6"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "import pandas as pd\n",
    "\n",
    "def calculate_metrics(y_true, y_pred, duration, y_true_val=None, y_pred_val=None):\n",
    "    res = pd.DataFrame(data=np.zeros((1, 4), dtype=np.float), index=[0],\n",
    "                       columns=['precision', 'accuracy', 'recall', 'duration'])\n",
    "    res['precision'] = precision_score(y_true, y_pred, average='macro')\n",
    "    res['accuracy'] = accuracy_score(y_true, y_pred)\n",
    "\n",
    "    if not y_true_val is None:\n",
    "        # this is useful when transfer learning is used with cross validation\n",
    "        res['accuracy_val'] = accuracy_score(y_true_val, y_pred_val)\n",
    "\n",
    "    res['recall'] = recall_score(y_true, y_pred, average='macro')\n",
    "    res['duration'] = duration\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_metrics = calculate_metrics(y_true, y_pred, 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 99
    },
    "executionInfo": {
     "elapsed": 1221,
     "status": "ok",
     "timestamp": 1670797392656,
     "user": {
      "displayName": "Mojtaba Askarzadeh Farahani",
      "userId": "02005001605152753874"
     },
     "user_tz": 300
    },
    "id": "BIeCbOs6e7o6",
    "outputId": "973650ee-67f0-43ae-cf19-3b4bf69641fc"
   },
   "outputs": [],
   "source": [
    "classifier.predict(X_test, y_true,X_train,y_train,y_test,return_df_metrics = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "f = h5py.File('./Results/PHM2022_Multivar_Datasetbest_model.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(f.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}