graphVAE_hotel_detect_batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b91df9ab-79c4-46e0-a477-ac62672be489",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 119390 entries, 0 to 119389\n",
      "Data columns (total 20 columns):\n",
      " #   Column  Non-Null Count   Dtype  \n",
      "---  ------  --------------   -----  \n",
      " 0   0       119390 non-null  float64\n",
      " 1   1       119390 non-null  float64\n",
      " 2   2       119390 non-null  float64\n",
      " 3   3       119390 non-null  float64\n",
      " 4   4       119390 non-null  float64\n",
      " 5   5       119390 non-null  float64\n",
      " 6   6       119390 non-null  float64\n",
      " 7   7       119390 non-null  float64\n",
      " 8   8       119390 non-null  float64\n",
      " 9   9       119390 non-null  float64\n",
      " 10  10      119390 non-null  float64\n",
      " 11  11      119390 non-null  float64\n",
      " 12  12      119390 non-null  float64\n",
      " 13  13      119390 non-null  float64\n",
      " 14  14      119390 non-null  float64\n",
      " 15  15      119390 non-null  float64\n",
      " 16  16      119390 non-null  float64\n",
      " 17  17      119390 non-null  float64\n",
      " 18  18      119390 non-null  float64\n",
      " 19  19      119390 non-null  float64\n",
      "dtypes: float64(20)\n",
      "memory usage: 18.2 MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "# 加载数据\n",
    "file_path_clean = '/home/sdong/data/hotel_booking/hotel_booking_process_embeddings.csv'\n",
    "file_path_origi = '/home/sdong/data/hotel_booking/hotel_booking_string_embeddings.csv'\n",
    "data = pd.read_csv(file_path_clean)\n",
    "data_dirty = pd.read_csv(file_path_origi)\n",
    "# 设置显示所有列和部分行\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 10)  \n",
    "# 显示数据集的前几行和数据结构\n",
    "\n",
    "print(data_dirty.info())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "27dc078d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clean data values are within [0, 1]: False\n",
      "Dirty data values are within [0, 1]: False\n",
      "Clean data contains values out of range [0, 1].\n",
      "Dirty data contains values out of range [0, 1].\n",
      "Out of range values in clean data:\n",
      "        0   1   2   3   4   5   6   7   8   9  10  11   12  13  14  15  16  \\\n",
      "75389 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN  1.0 NaN NaN NaN NaN   \n",
      "\n",
      "       17  18  19  \n",
      "75389 NaN NaN NaN  \n",
      "Out of range values in dirty data:\n",
      "        0   1   2   3   4   5   6   7   8   9  10  11   12  13  14  15  16  \\\n",
      "62176 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN  1.0 NaN NaN NaN NaN   \n",
      "\n",
      "       17  18  19  \n",
      "62176 NaN NaN NaN  \n"
     ]
    }
   ],
   "source": [
    "# 检查两个数据集中是否所有值都在0到1之间\n",
    "def check_values_in_range(df, lower=0, upper=1):\n",
    "    return ((df >= lower) & (df <= upper)).all().all()\n",
    "\n",
    "is_data_in_range = check_values_in_range(data)\n",
    "is_data_dirty_in_range = check_values_in_range(data_dirty)\n",
    "\n",
    "print(f\"Clean data values are within [0, 1]: {is_data_in_range}\")\n",
    "print(f\"Dirty data values are within [0, 1]: {is_data_dirty_in_range}\")\n",
    "\n",
    "if not is_data_in_range:\n",
    "    print(\"Clean data contains values out of range [0, 1].\")\n",
    "\n",
    "if not is_data_dirty_in_range:\n",
    "    print(\"Dirty data contains values out of range [0, 1].\")\n",
    "\n",
    "# 如果需要，打印出不在范围内的值和对应的索引\n",
    "def find_out_of_range_values(df, lower=0, upper=1):\n",
    "    out_of_range = df[(df < lower) | (df > upper)]\n",
    "    return out_of_range.dropna(how='all')\n",
    "\n",
    "if not is_data_in_range:\n",
    "    out_of_range_clean = find_out_of_range_values(data)\n",
    "    print(\"Out of range values in clean data:\")\n",
    "    print(out_of_range_clean)\n",
    "    out_of_range_clean_indices = out_of_range_clean.index\n",
    "    data = data.drop(out_of_range_clean_indices)\n",
    "\n",
    "if not is_data_dirty_in_range:\n",
    "    out_of_range_dirty = find_out_of_range_values(data_dirty)\n",
    "    print(\"Out of range values in dirty data:\")\n",
    "    print(out_of_range_dirty)\n",
    "    out_of_range_dirty_indices = out_of_range_dirty.index\n",
    "    data_dirty = data_dirty.drop(out_of_range_dirty_indices)\n",
    "    \n",
    "    # 获取不在范围内的值的索引\n",
    "\n",
    "\n",
    "\n",
    "# 删除不在范围内的行\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14966a57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 119389 entries, 0 to 119389\n",
      "Data columns (total 20 columns):\n",
      " #   Column  Non-Null Count   Dtype  \n",
      "---  ------  --------------   -----  \n",
      " 0   0       119389 non-null  float64\n",
      " 1   1       119389 non-null  float64\n",
      " 2   2       119389 non-null  float64\n",
      " 3   3       119389 non-null  float64\n",
      " 4   4       119389 non-null  float64\n",
      " 5   5       119389 non-null  float64\n",
      " 6   6       119389 non-null  float64\n",
      " 7   7       119389 non-null  float64\n",
      " 8   8       119389 non-null  float64\n",
      " 9   9       119389 non-null  float64\n",
      " 10  10      119389 non-null  float64\n",
      " 11  11      119389 non-null  float64\n",
      " 12  12      119389 non-null  float64\n",
      " 13  13      119389 non-null  float64\n",
      " 14  14      119389 non-null  float64\n",
      " 15  15      119389 non-null  float64\n",
      " 16  16      119389 non-null  float64\n",
      " 17  17      119389 non-null  float64\n",
      " 18  18      119389 non-null  float64\n",
      " 19  19      119389 non-null  float64\n",
      "dtypes: float64(20)\n",
      "memory usage: 19.1 MB\n"
     ]
    }
   ],
   "source": [
    "data_dirty.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5887b650",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 假设 data 已经是一个经过预处理的 DataFrame\n",
    "data_array = data.values.astype(np.float32)  # 转换为浮点数类型的 NumPy 数组\n",
    "\n",
    "# 分割数据为训练集和临时测试集（包括真正的测试集和验证集）\n",
    "train_data, val_test_data = train_test_split(data_array, test_size=0.5, random_state=42)\n",
    "\n",
    "# 将训练验证集进一步分割为训练集和验证集\n",
    "val_data, test_data = train_test_split(val_test_data, test_size=0.5, random_state=42)  # 0.25 x 0.8 = 0.2\n",
    "\n",
    "\n",
    "# 转换为PyTorch张量\n",
    "\n",
    "\n",
    "# 创建数据加载器\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "batch_size = 1280  # 或者任何适合你GPU的大小\n",
    "\n",
    "train_tensor = torch.tensor(train_data) #0.6\n",
    "train_dataset = TensorDataset(train_tensor, train_tensor)\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "val_tensor = torch.tensor(val_data) #0.2\n",
    "val_dataset = TensorDataset(val_tensor, val_tensor)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False)\n",
    "\n",
    "test_tensor = torch.tensor(test_data)  #20%\n",
    "test_dataset = TensorDataset(test_tensor, test_tensor)\n",
    "test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)  # 50个批次\n",
    "#len(test_dataset)// 50\n",
    "\n",
    "data_dirty_array = data_dirty.values.astype(np.float32)  # 转换为浮点数类型的 NumPy 数组\n",
    "test_dirty_tensor = torch.tensor(data_dirty_array)  #20%\n",
    "test_dirty_dataset = TensorDataset(test_dirty_tensor, test_dirty_tensor)\n",
    "test_dirty_loader = DataLoader(test_dirty_dataset, batch_size=1, shuffle=False)  # 50个批次\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e3d31c1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "92722c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 检查数据中是否有NaN或无穷大的值\n",
    "# if torch.isnan(train_tensor).any() or torch.isinf(train_tensor).any():\n",
    "#     print(\"Data contains NaNs or Infs.\")\n",
    "# # 检查数据中是否有NaN或无穷大的值\n",
    "# if torch.isnan(test_dirty_tensor).any() or torch.isinf(test_dirty_tensor).any():\n",
    "#     print(\"Data contains NaNs or Infs.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "63a8f2af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VAE(\n",
      "  (fc1): Linear(in_features=20, out_features=128, bias=True)\n",
      "  (fc2): Linear(in_features=128, out_features=64, bias=True)\n",
      "  (fc31): Linear(in_features=64, out_features=20, bias=True)\n",
      "  (fc32): Linear(in_features=64, out_features=20, bias=True)\n",
      "  (fc4): Linear(in_features=20, out_features=64, bias=True)\n",
      "  (fc5): Linear(in_features=64, out_features=128, bias=True)\n",
      "  (fc6): Linear(in_features=128, out_features=20, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "        # Encoder\n",
    "        self.fc1 = nn.Linear(20, 128)  # Input layer\n",
    "        self.fc2 = nn.Linear(128, 64)  # Hidden layer\n",
    "        self.fc31 = nn.Linear(64, 20)  # Output layer for mu\n",
    "        self.fc32 = nn.Linear(64, 20)  # Output layer for logvar\n",
    "\n",
    "        # Decoder\n",
    "        self.fc4 = nn.Linear(20, 64)   # Input layer\n",
    "        self.fc5 = nn.Linear(64, 128)  # Hidden layer\n",
    "        self.fc6 = nn.Linear(128, 20)  # Output layer\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = F.relu(self.fc1(x))\n",
    "        h2 = F.relu(self.fc2(h1))\n",
    "        return self.fc31(h2), self.fc32(h2)\n",
    "\n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = torch.exp(0.5 * logvar) + 1e-8  # Adding a small constant for numerical stability\n",
    "        eps = torch.randn_like(std)\n",
    "        return mu + eps * std\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = F.relu(self.fc4(z))\n",
    "        h4 = F.relu(self.fc5(h3))\n",
    "        return torch.sigmoid(self.fc6(h4))  # Use sigmoid to ensure output is between 0 and 1\n",
    "\n",
    "    def forward(self, x):\n",
    "        mu, logvar = self.encode(x)\n",
    "        z = self.reparameterize(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "\n",
    "# Instantiate the model\n",
    "model = VAE()\n",
    "print(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c1d03c39",
   "metadata": {},
   "outputs": [],
   "source": [
    "#device = torch.device(\"cpu\")\n",
    "model = VAE().to(device)\n",
    "\n",
    "import torch.optim as optim\n",
    "\n",
    "# 设置优化器\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n",
    "\n",
    "# 定义损失函数\n",
    "def loss_function(recon_x, x, mu, logvar):\n",
    "    # 确保目标张量也是浮点类型且维度匹配\n",
    "    recon_x = torch.clamp(recon_x, 0, 1)  # 确保输出值在[0, 1]范围内\n",
    "    BCE = F.binary_cross_entropy(recon_x, x.view(-1, 20).float(), reduction='sum')\n",
    "    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n",
    "    return BCE, KLD\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c4a9e966",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/59694 (0%)]\tLoss: 13.957637\n",
      "Epoch: 1 Average BCE: 11.759166017522698 Average KLD: 0.025956303900712522 Total Loss: 11.785122287729923\n",
      "Train Epoch: 2 [0/59694 (0%)]\tLoss: 10.950433\n",
      "Epoch: 2 Average BCE: 10.897103204860413 Average KLD: 0.004804554291377296 Total Loss: 10.901907753632065\n",
      "Train Epoch: 3 [0/59694 (0%)]\tLoss: 10.890803\n",
      "Epoch: 3 Average BCE: 10.878972816703731 Average KLD: 0.006134265335128507 Total Loss: 10.885107063602497\n",
      "Train Epoch: 4 [0/59694 (0%)]\tLoss: 10.916163\n",
      "Epoch: 4 Average BCE: 10.87053028060295 Average KLD: 0.006141685371515232 Total Loss: 10.876671938343888\n",
      "Train Epoch: 5 [0/59694 (0%)]\tLoss: 10.882729\n",
      "Epoch: 5 Average BCE: 10.866333568124519 Average KLD: 0.005485647626628819 Total Loss: 10.871819259996817\n",
      "Train Epoch: 6 [0/59694 (0%)]\tLoss: 10.878522\n",
      "Epoch: 6 Average BCE: 10.86218339835243 Average KLD: 0.005061950292819572 Total Loss: 10.867245396181987\n",
      "Train Epoch: 7 [0/59694 (0%)]\tLoss: 10.867618\n",
      "Epoch: 7 Average BCE: 10.859981233064671 Average KLD: 0.004525168791094285 Total Loss: 10.864506443800884\n",
      "Train Epoch: 8 [0/59694 (0%)]\tLoss: 10.869209\n",
      "Epoch: 8 Average BCE: 10.856949805989714 Average KLD: 0.0040963802257127195 Total Loss: 10.861046218526988\n",
      "Train Epoch: 9 [0/59694 (0%)]\tLoss: 10.845842\n",
      "Epoch: 9 Average BCE: 10.855335861984663 Average KLD: 0.0037073127371975616 Total Loss: 10.859043213487746\n",
      "Train Epoch: 10 [0/59694 (0%)]\tLoss: 10.878058\n",
      "Epoch: 10 Average BCE: 10.852549352609769 Average KLD: 0.003337850162330878 Total Loss: 10.855887225370221\n",
      "Model saved to vae_model_hotel_graph.pth\n",
      "CUDA cache cleared.\n"
     ]
    }
   ],
   "source": [
    "# 定义训练函数\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    #train_loss = 0\n",
    "    total_BCE = 0\n",
    "    total_KLD = 0\n",
    "    total_loss = 0\n",
    "    for batch_idx, (data, _) in enumerate(train_loader):  # 由于使用TensorDataset，数据被重复用作输入和标签\n",
    "        data = data.to(device)\n",
    "        # 检查输入数据的范围\n",
    "        if (data < 0).any() or (data > 1).any():\n",
    "            raise ValueError(\"Input data contains values out of range [0, 1]\")\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        recon_batch, mu, logvar = model(data)\n",
    "        if (recon_batch < 0).any() or (recon_batch > 1).any():\n",
    "            raise ValueError(\"Warning: recon_batch contains values out of range [0, 1]\")\n",
    "        BCE, KLD = loss_function(recon_batch, data, mu, logvar)\n",
    "        loss = BCE + KLD\n",
    "        loss.backward()\n",
    "        total_loss += loss.item()\n",
    "        total_BCE += BCE.item()\n",
    "        total_KLD += KLD.item()\n",
    "        optimizer.step()\n",
    "        # if epoch == 1:  # 只在第一个epoch检查\n",
    "        #     print(\"Sample recon_x:\", recon_batch[0].data)\n",
    "        #     print(\"Sample x:\", data[0].data)\n",
    "\n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item() / len(data)))\n",
    "    #print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))\n",
    "    print(f'Epoch: {epoch} Average BCE: {total_BCE / len(train_loader.dataset)} Average KLD: {total_KLD / len(train_loader.dataset)} Total Loss: {total_loss / len(train_loader.dataset)}')\n",
    "\n",
    "# 训练模型\n",
    "num_epochs = 10  # 可根据需要调整\n",
    "for epoch in range(1, num_epochs + 1):\n",
    "    train(epoch)\n",
    "\n",
    "# 保存模型的状态字典\n",
    "torch.save(model.state_dict(), 'vae_model_hotel_graph.pth')\n",
    "\n",
    "print(\"Model saved to vae_model_hotel_graph.pth\")\n",
    "\n",
    "# 添加调试信息\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.synchronize()\n",
    "    torch.cuda.empty_cache()\n",
    "    print(\"CUDA cache cleared.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5b6b9ddf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29847\n",
      "29848\n",
      "119389\n",
      "Average loss on validation data: 10.859621657396028\n",
      "Average loss on test data: 10.86804012717398\n",
      "Average loss on test dirty data: 11.461222172722032\n"
     ]
    }
   ],
   "source": [
    "def evaluate_model(model, data_loader):\n",
    "    model.eval()  # 切换到评估模式\n",
    "    total_loss = 0\n",
    "    with torch.no_grad():  # 关闭梯度计算\n",
    "        for inputs, _ in data_loader:  # 假设 data_loader 返回 inputs 和 targets，这里我们不需要 targets\n",
    "            inputs = inputs.to(device)  # 确保将 inputs 转移到正确的设备\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            loss = BCE + KLD  # 将损失元组中的元素相加\n",
    "            total_loss += loss.item()  # 现在这是一个单一的数值\n",
    "    print(len(data_loader.dataset))\n",
    "    return total_loss / len(data_loader.dataset)\n",
    "\n",
    "\n",
    "model = VAE().to(device)\n",
    "model.load_state_dict(torch.load('vae_model_hotel_graph.pth'))\n",
    "# 计算测试集上的平均损失\n",
    "# 计算测试集和验证集上的平均损失\n",
    "val_loss = evaluate_model(model, val_loader)\n",
    "test_loss = evaluate_model(model, test_loader)\n",
    "test_dirty_loss = evaluate_model(model, test_dirty_loader)\n",
    "print(f\"Average loss on validation data: {val_loss}\")\n",
    "print(f\"Average loss on test data: {test_loss}\")\n",
    "print(f\"Average loss on test dirty data: {test_dirty_loss}\")\n",
    "# 简单的基于阈值的数据质量问题判断\n",
    "# 这里我们需要设置一个阈值来决定什么样的重构误差被认为是“异常”的，此阈值可以基于训练集或验证集的性能来确定\n",
    "# 假设我们根据验证集确定阈值\n",
    "# threshold = np.quantile([loss_function(model(recon, data.to(device), mu, logvar).item() for data, _ in val_loader], 0.95)\n",
    "# print(f\"Loss threshold for detecting data quality issues: {threshold}\")\n",
    "\n",
    "# # 判断测试集\n",
    "# quality_issues = test_loss > threshold\n",
    "# print(f\"Data quality issues detected: {quality_issues}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8950bc94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss threshold for detecting data quality issues: 12.639586925506586\n",
      "Min validation error: 8.914874076843262\n",
      "Max validation error: 15.892179489135742\n",
      "Mean validation error: 10.860653977779606\n",
      "95th percentile of validation errors: 12.639586925506586\n",
      "Maximum validation error (100th percentile): 15.892179489135742\n"
     ]
    }
   ],
   "source": [
    "def collect_reconstruction_errors(model, data_loader):\n",
    "    model.eval()\n",
    "    reconstruction_errors = []\n",
    "    with torch.no_grad():\n",
    "        for inputs, _ in data_loader:  # 假设 data_loader 返回的是 inputs 和 labels，这里我们忽略 labels\n",
    "            inputs = inputs.to(device)  # 将输入数据移动到正确的设备\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            total_loss = BCE + KLD  # 计算总损失\n",
    "            average_loss = total_loss.item() / inputs.size(0)  # 计算平均损失\n",
    "            reconstruction_errors.append(average_loss)  # 添加单个损失值到列表中\n",
    "    return reconstruction_errors\n",
    "\n",
    "# 收集验证集的重构误差\n",
    "val_errors = collect_reconstruction_errors(model, val_loader)\n",
    "threshold = np.quantile(val_errors, 0.95)  # 计算95%分位数作为阈值\n",
    "threshold = threshold * 1\n",
    "print(f\"Loss threshold for detecting data quality issues: {threshold}\")\n",
    "\n",
    "min_val_error = min(val_errors)\n",
    "max_val_error = max(val_errors)\n",
    "mean_val_error = sum(val_errors) / len(val_errors)\n",
    "print(f\"Min validation error: {min_val_error}\")\n",
    "print(f\"Max validation error: {max_val_error}\")\n",
    "print(f\"Mean validation error: {mean_val_error}\")\n",
    "print(f\"95th percentile of validation errors: {np.quantile(val_errors, 0.95)}\")\n",
    "print(f\"Maximum validation error (100th percentile): {np.quantile(val_errors, 1)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "b7b267a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 0 is problematic: 26 out of 596 samples are faulty (4.36%).\n",
      "Batch 1 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 2 is problematic: 33 out of 596 samples are faulty (5.54%).\n",
      "Batch 3 is problematic: 21 out of 596 samples are faulty (3.52%).\n",
      "Batch 4 is problematic: 24 out of 596 samples are faulty (4.03%).\n",
      "Batch 5 is problematic: 35 out of 596 samples are faulty (5.87%).\n",
      "Batch 6 is problematic: 31 out of 596 samples are faulty (5.20%).\n",
      "Batch 7 is problematic: 17 out of 596 samples are faulty (2.85%).\n",
      "Batch 8 is problematic: 37 out of 596 samples are faulty (6.21%).\n",
      "Batch 9 is problematic: 26 out of 596 samples are faulty (4.36%).\n",
      "Batch 10 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 11 is problematic: 51 out of 596 samples are faulty (8.56%).\n",
      "Batch 12 is problematic: 23 out of 596 samples are faulty (3.86%).\n",
      "Batch 13 is problematic: 34 out of 596 samples are faulty (5.70%).\n",
      "Batch 14 is problematic: 32 out of 596 samples are faulty (5.37%).\n",
      "Batch 15 is problematic: 31 out of 596 samples are faulty (5.20%).\n",
      "Batch 16 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 17 is problematic: 32 out of 596 samples are faulty (5.37%).\n",
      "Batch 18 is problematic: 34 out of 596 samples are faulty (5.70%).\n",
      "Batch 19 is problematic: 37 out of 596 samples are faulty (6.21%).\n",
      "Batch 20 is problematic: 29 out of 596 samples are faulty (4.87%).\n",
      "Batch 21 is problematic: 28 out of 596 samples are faulty (4.70%).\n",
      "Batch 22 is problematic: 22 out of 596 samples are faulty (3.69%).\n",
      "Batch 23 is problematic: 23 out of 596 samples are faulty (3.86%).\n",
      "Batch 24 is problematic: 37 out of 596 samples are faulty (6.21%).\n",
      "Batch 25 is problematic: 28 out of 596 samples are faulty (4.70%).\n",
      "Batch 26 is problematic: 31 out of 596 samples are faulty (5.20%).\n",
      "Batch 27 is problematic: 30 out of 596 samples are faulty (5.03%).\n",
      "Batch 28 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 29 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 30 is problematic: 23 out of 596 samples are faulty (3.86%).\n",
      "Batch 31 is problematic: 23 out of 596 samples are faulty (3.86%).\n",
      "Batch 32 is problematic: 27 out of 596 samples are faulty (4.53%).\n",
      "Batch 33 is problematic: 23 out of 596 samples are faulty (3.86%).\n",
      "Batch 34 is problematic: 33 out of 596 samples are faulty (5.54%).\n",
      "Batch 35 is problematic: 37 out of 596 samples are faulty (6.21%).\n",
      "Batch 36 is problematic: 46 out of 596 samples are faulty (7.72%).\n",
      "Batch 37 is problematic: 34 out of 596 samples are faulty (5.70%).\n",
      "Batch 38 is problematic: 28 out of 596 samples are faulty (4.70%).\n",
      "Batch 39 is problematic: 30 out of 596 samples are faulty (5.03%).\n",
      "Batch 40 is problematic: 36 out of 596 samples are faulty (6.04%).\n",
      "Batch 41 is problematic: 21 out of 596 samples are faulty (3.52%).\n",
      "Batch 42 is problematic: 36 out of 596 samples are faulty (6.04%).\n",
      "Batch 43 is problematic: 34 out of 596 samples are faulty (5.70%).\n",
      "Batch 44 is problematic: 32 out of 596 samples are faulty (5.37%).\n",
      "Batch 45 is problematic: 24 out of 596 samples are faulty (4.03%).\n",
      "Batch 46 is problematic: 31 out of 596 samples are faulty (5.20%).\n",
      "Batch 47 is problematic: 31 out of 596 samples are faulty (5.20%).\n",
      "Batch 48 is problematic: 32 out of 596 samples are faulty (5.37%).\n",
      "Batch 49 is problematic: 29 out of 596 samples are faulty (4.87%).\n",
      "Total batches with issues: 50 out of 50\n",
      "Total problematic samples: 1497 out of 29848 (5.02%)\n",
      "Percentage of data quality issues detected in the test set: 5.02%\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "def detect_quality_issues(model, data_loader, threshold):\n",
    "    model.eval()\n",
    "    total_issue_count = 0\n",
    "    total_batches_with_issues = 0\n",
    "    total_samples = 0\n",
    "    current_batch_issues = 0\n",
    "    batch_count = 0\n",
    "    batch_size = len(data_loader.dataset) // 50  # 你希望的批次大小\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, _ in data_loader:\n",
    "            inputs = inputs.to(device)  # 将输入数据移动到正确的设备\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            total_loss = BCE + KLD  # 计算当前样本的总损失\n",
    "            total_samples += 1\n",
    "\n",
    "            # 判断当前样本是否有问题\n",
    "            if total_loss.item() > threshold:\n",
    "                current_batch_issues += 1\n",
    "\n",
    "            # 当累积样本数达到你设定的批次大小时，评估这个批次\n",
    "            if total_samples % batch_size == 0:\n",
    "                if current_batch_issues >= batch_size * 0.02:  # 判断这个批次是否有超过5%的样本有问题\n",
    "                    print(f\"Batch {batch_count} is problematic: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "                    total_batches_with_issues += 1\n",
    "                    total_issue_count = total_issue_count + current_batch_issues\n",
    "                else:\n",
    "                    total_issue_count = total_issue_count + current_batch_issues\n",
    "                    print(f\"Batch {batch_count} is ok: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "                current_batch_issues = 0\n",
    "                batch_count += 1\n",
    "\n",
    "    total_issue_rate = total_issue_count / total_samples\n",
    "    print(f\"Total batches with issues: {total_batches_with_issues} out of {batch_count}\")\n",
    "    print(f\"Total problematic samples: {total_issue_count} out of {total_samples} ({(total_issue_rate * 100):.2f}%)\")\n",
    "    return total_issue_rate\n",
    "\n",
    "# Example usage\n",
    "# test_dataset = TensorDataset(test_tensor, test_tensor)\n",
    "# test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)\n",
    "issue_rate = detect_quality_issues(model, test_loader, threshold)\n",
    "print(f\"Percentage of data quality issues detected in the test set: {issue_rate * 100:.2f}%\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "c139b017",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 0 is problematic: 127 out of 2387 samples are faulty (5.32%).\n",
      "Batch 1 is problematic: 146 out of 2387 samples are faulty (6.12%).\n",
      "Batch 2 is problematic: 139 out of 2387 samples are faulty (5.82%).\n",
      "Batch 3 is problematic: 122 out of 2387 samples are faulty (5.11%).\n",
      "Batch 4 is problematic: 142 out of 2387 samples are faulty (5.95%).\n",
      "Batch 5 is problematic: 145 out of 2387 samples are faulty (6.07%).\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[168], line 52\u001b[0m\n\u001b[1;32m     49\u001b[0m test_dirty_dataset \u001b[38;5;241m=\u001b[39m TensorDataset(test_dirty_tensor, test_dirty_tensor)\n\u001b[1;32m     50\u001b[0m test_dirty_loader \u001b[38;5;241m=\u001b[39m DataLoader(test_dirty_dataset, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m) \n\u001b[0;32m---> 52\u001b[0m issue_rate \u001b[38;5;241m=\u001b[39m \u001b[43mdetect_quality_issues\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest_dirty_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mthreshold\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     53\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPercentage of data quality issues detected in the test set: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00missue_rate\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m100\u001b[39m\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "Cell \u001b[0;32mIn[168], line 16\u001b[0m, in \u001b[0;36mdetect_quality_issues\u001b[0;34m(model, data_loader, threshold)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m inputs, _ \u001b[38;5;129;01min\u001b[39;00m data_loader:\n\u001b[1;32m     15\u001b[0m     inputs \u001b[38;5;241m=\u001b[39m inputs\u001b[38;5;241m.\u001b[39mto(device)  \u001b[38;5;66;03m# 将输入数据移动到正确的设备\u001b[39;00m\n\u001b[0;32m---> 16\u001b[0m     recon, mu, logvar \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     17\u001b[0m     BCE, KLD \u001b[38;5;241m=\u001b[39m loss_function(recon, inputs, mu, logvar)\n\u001b[1;32m     18\u001b[0m     total_loss \u001b[38;5;241m=\u001b[39m BCE \u001b[38;5;241m+\u001b[39m KLD  \u001b[38;5;66;03m# 计算当前样本的总损失\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/mainenv/lib/python3.11/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/mainenv/lib/python3.11/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "Cell \u001b[0;32mIn[162], line 37\u001b[0m, in \u001b[0;36mVAE.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[1;32m     36\u001b[0m     mu, logvar \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencode(x)\n\u001b[0;32m---> 37\u001b[0m     z \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreparameterize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmu\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogvar\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode(z), mu, logvar\n",
      "Cell \u001b[0;32mIn[162], line 25\u001b[0m, in \u001b[0;36mVAE.reparameterize\u001b[0;34m(self, mu, logvar)\u001b[0m\n\u001b[1;32m     22\u001b[0m     h2 \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc2(h1))\n\u001b[1;32m     23\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc31(h2), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc32(h2)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mreparameterize\u001b[39m(\u001b[38;5;28mself\u001b[39m, mu, logvar):\n\u001b[1;32m     26\u001b[0m     std \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mexp(\u001b[38;5;241m0.5\u001b[39m \u001b[38;5;241m*\u001b[39m logvar) \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1e-8\u001b[39m  \u001b[38;5;66;03m# Adding a small constant for numerical stability\u001b[39;00m\n\u001b[1;32m     27\u001b[0m     eps \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrandn_like(std)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "def detect_quality_issues(model, data_loader, threshold):\n",
    "    model.eval()\n",
    "    total_issue_count = 0\n",
    "    total_batches_with_issues = 0\n",
    "    total_samples = 0\n",
    "    current_batch_issues = 0\n",
    "    batch_count = 0\n",
    "    batch_size = len(data_loader.dataset) // 50  # 你希望的批次大小\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, _ in data_loader:\n",
    "            inputs = inputs.to(device)  # 将输入数据移动到正确的设备\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            total_loss = BCE + KLD  # 计算当前样本的总损失\n",
    "            total_samples += 1\n",
    "\n",
    "            # 判断当前样本是否有问题\n",
    "            if total_loss.item() > threshold:\n",
    "                current_batch_issues += 1\n",
    "\n",
    "            # 当累积样本数达到你设定的批次大小时，评估这个批次\n",
    "            if total_samples % batch_size == 0:\n",
    "                if current_batch_issues >= batch_size * 0.02:  # 判断这个批次是否有超过5%的样本有问题\n",
    "                    print(f\"Batch {batch_count} is problematic: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "                    total_batches_with_issues += 1\n",
    "                    total_issue_count = total_issue_count + current_batch_issues\n",
    "                else:\n",
    "                    total_issue_count = total_issue_count + current_batch_issues\n",
    "                    print(f\"Batch {batch_count} is ok: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "                current_batch_issues = 0\n",
    "                batch_count += 1\n",
    "\n",
    "    total_issue_rate = total_issue_count / total_samples\n",
    "    print(f\"Total batches with issues: {total_batches_with_issues} out of {batch_count}\")\n",
    "    print(f\"Total problematic samples: {total_issue_count} out of {total_samples} ({(total_issue_rate * 100):.2f}%)\")\n",
    "    return total_issue_rate\n",
    "\n",
    "# Example usage\n",
    "\n",
    "\n",
    "\n",
    "# 假设 data_dirty 已经是一个经过预处理的 DataFrame\n",
    "data_dirty_array = data_dirty.values.astype(np.float32)  # 转换为浮点数类型的 NumPy 数组\n",
    "test_dirty_tensor = torch.tensor(data_dirty_array)  \n",
    "test_dirty_dataset = TensorDataset(test_dirty_tensor, test_dirty_tensor)\n",
    "test_dirty_loader = DataLoader(test_dirty_dataset, batch_size=1, shuffle=False) \n",
    "\n",
    "issue_rate = detect_quality_issues(model, test_dirty_loader, threshold)\n",
    "print(f\"Percentage of data quality issues detected in the test set: {issue_rate * 100:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b422d31",
   "metadata": {},
   "source": [
    "## loop test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "73c55af4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random sample 0 is problematic: 934 out of 11939 samples are faulty (7.82%).\n",
      "Random sample 1 is problematic: 939 out of 11939 samples are faulty (7.86%).\n",
      "Random sample 2 is problematic: 933 out of 11939 samples are faulty (7.81%).\n",
      "Random sample 3 is problematic: 970 out of 11939 samples are faulty (8.12%).\n",
      "Random sample 4 is problematic: 915 out of 11939 samples are faulty (7.66%).\n",
      "Random sample 5 is problematic: 958 out of 11939 samples are faulty (8.02%).\n",
      "Random sample 6 is problematic: 944 out of 11939 samples are faulty (7.91%).\n",
      "Random sample 7 is problematic: 919 out of 11939 samples are faulty (7.70%).\n",
      "Random sample 8 is problematic: 963 out of 11939 samples are faulty (8.07%).\n",
      "Random sample 9 is problematic: 889 out of 11939 samples are faulty (7.45%).\n",
      "Random sample 10 is problematic: 947 out of 11939 samples are faulty (7.93%).\n",
      "Random sample 11 is problematic: 944 out of 11939 samples are faulty (7.91%).\n",
      "Random sample 12 is problematic: 921 out of 11939 samples are faulty (7.71%).\n",
      "Random sample 13 is problematic: 866 out of 11939 samples are faulty (7.25%).\n",
      "Random sample 14 is problematic: 916 out of 11939 samples are faulty (7.67%).\n",
      "Random sample 15 is problematic: 972 out of 11939 samples are faulty (8.14%).\n",
      "Random sample 16 is problematic: 956 out of 11939 samples are faulty (8.01%).\n",
      "Random sample 17 is problematic: 878 out of 11939 samples are faulty (7.35%).\n",
      "Random sample 18 is problematic: 914 out of 11939 samples are faulty (7.66%).\n",
      "Random sample 19 is problematic: 930 out of 11939 samples are faulty (7.79%).\n",
      "Random sample 20 is problematic: 919 out of 11939 samples are faulty (7.70%).\n",
      "Random sample 21 is problematic: 923 out of 11939 samples are faulty (7.73%).\n",
      "Random sample 22 is problematic: 916 out of 11939 samples are faulty (7.67%).\n",
      "Random sample 23 is problematic: 932 out of 11939 samples are faulty (7.81%).\n",
      "Random sample 24 is problematic: 901 out of 11939 samples are faulty (7.55%).\n",
      "Random sample 25 is problematic: 903 out of 11939 samples are faulty (7.56%).\n",
      "Random sample 26 is problematic: 939 out of 11939 samples are faulty (7.86%).\n",
      "Random sample 27 is problematic: 902 out of 11939 samples are faulty (7.56%).\n",
      "Random sample 28 is problematic: 922 out of 11939 samples are faulty (7.72%).\n",
      "Random sample 29 is problematic: 911 out of 11939 samples are faulty (7.63%).\n",
      "Random sample 30 is problematic: 893 out of 11939 samples are faulty (7.48%).\n",
      "Random sample 31 is problematic: 931 out of 11939 samples are faulty (7.80%).\n",
      "Random sample 32 is problematic: 970 out of 11939 samples are faulty (8.12%).\n",
      "Random sample 33 is problematic: 957 out of 11939 samples are faulty (8.02%).\n",
      "Random sample 34 is problematic: 888 out of 11939 samples are faulty (7.44%).\n",
      "Random sample 35 is problematic: 938 out of 11939 samples are faulty (7.86%).\n",
      "Random sample 36 is problematic: 933 out of 11939 samples are faulty (7.81%).\n",
      "Random sample 37 is problematic: 962 out of 11939 samples are faulty (8.06%).\n",
      "Random sample 38 is problematic: 926 out of 11939 samples are faulty (7.76%).\n",
      "Random sample 39 is problematic: 901 out of 11939 samples are faulty (7.55%).\n",
      "Random sample 40 is problematic: 935 out of 11939 samples are faulty (7.83%).\n",
      "Random sample 41 is problematic: 925 out of 11939 samples are faulty (7.75%).\n",
      "Random sample 42 is problematic: 931 out of 11939 samples are faulty (7.80%).\n",
      "Random sample 43 is problematic: 888 out of 11939 samples are faulty (7.44%).\n",
      "Random sample 44 is problematic: 932 out of 11939 samples are faulty (7.81%).\n",
      "Random sample 45 is problematic: 860 out of 11939 samples are faulty (7.20%).\n",
      "Random sample 46 is problematic: 988 out of 11939 samples are faulty (8.28%).\n",
      "Random sample 47 is problematic: 904 out of 11939 samples are faulty (7.57%).\n",
      "Random sample 48 is problematic: 906 out of 11939 samples are faulty (7.59%).\n",
      "Random sample 49 is problematic: 889 out of 11939 samples are faulty (7.45%).\n",
      "Total problematic batches across all tests: 50\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def detect_quality_issues(model, data_loader, threshold, seed):\n",
    "    model.eval()\n",
    "    current_batch_issues = 0\n",
    "    batch_size = len(data_loader.dataset)  # 这里的批次大小是整个数据集的大小\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, _ in data_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            total_loss = BCE + KLD\n",
    "            # 判断当前样本是否有问题\n",
    "            if total_loss.item() > threshold:\n",
    "                current_batch_issues += 1\n",
    "\n",
    "    # 评估是否有超过5%的样本有问题\n",
    "    if current_batch_issues > batch_size * 0.06:\n",
    "        print(f\"Random sample {seed} is problematic: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "        return True\n",
    "    else:\n",
    "        print(f\"Random sample {seed} is ok: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "        return False\n",
    "\n",
    "# 主函数：执行50次随机采样测试\n",
    "def test_random_samples(model, data_dirty, threshold):\n",
    "    problematic_batches = 0\n",
    "    for seed in range(50):\n",
    "        _, sample_data = train_test_split(data_dirty, test_size=0.1, random_state=seed)  # 随机采样20%\n",
    "        sample_data_array = sample_data.values.astype(np.float32)\n",
    "        sample_tensor = torch.tensor(sample_data_array)\n",
    "        sample_dataset = TensorDataset(sample_tensor, sample_tensor)\n",
    "        test_dirty_loader = DataLoader(sample_dataset, batch_size=1, shuffle=False) \n",
    "        if detect_quality_issues(model, test_dirty_loader, threshold, seed):\n",
    "            problematic_batches += 1\n",
    "\n",
    "    print(f\"Total problematic batches across all tests: {problematic_batches}\")\n",
    "\n",
    "# 假设 model, data_dirty, threshold, device 已经被正确定义和设置\n",
    "test_random_samples(model, data_dirty, threshold)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "74364557",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random sample 0 is ok: 296 out of 5970 samples are faulty (4.96%).\n",
      "Random sample 1 is ok: 301 out of 5970 samples are faulty (5.04%).\n",
      "Random sample 2 is ok: 291 out of 5970 samples are faulty (4.87%).\n",
      "Random sample 3 is ok: 294 out of 5970 samples are faulty (4.92%).\n",
      "Random sample 4 is ok: 314 out of 5970 samples are faulty (5.26%).\n",
      "Random sample 5 is ok: 268 out of 5970 samples are faulty (4.49%).\n",
      "Random sample 6 is ok: 286 out of 5970 samples are faulty (4.79%).\n",
      "Random sample 7 is ok: 314 out of 5970 samples are faulty (5.26%).\n",
      "Random sample 8 is ok: 284 out of 5970 samples are faulty (4.76%).\n",
      "Random sample 9 is ok: 297 out of 5970 samples are faulty (4.97%).\n",
      "Random sample 10 is ok: 272 out of 5970 samples are faulty (4.56%).\n",
      "Random sample 11 is ok: 272 out of 5970 samples are faulty (4.56%).\n",
      "Random sample 12 is ok: 304 out of 5970 samples are faulty (5.09%).\n",
      "Random sample 13 is ok: 285 out of 5970 samples are faulty (4.77%).\n",
      "Random sample 14 is ok: 290 out of 5970 samples are faulty (4.86%).\n",
      "Random sample 15 is ok: 306 out of 5970 samples are faulty (5.13%).\n",
      "Random sample 16 is ok: 289 out of 5970 samples are faulty (4.84%).\n",
      "Random sample 17 is ok: 257 out of 5970 samples are faulty (4.30%).\n",
      "Random sample 18 is ok: 304 out of 5970 samples are faulty (5.09%).\n",
      "Random sample 19 is ok: 274 out of 5970 samples are faulty (4.59%).\n",
      "Random sample 20 is ok: 287 out of 5970 samples are faulty (4.81%).\n",
      "Random sample 21 is ok: 297 out of 5970 samples are faulty (4.97%).\n",
      "Random sample 22 is ok: 295 out of 5970 samples are faulty (4.94%).\n",
      "Random sample 23 is ok: 296 out of 5970 samples are faulty (4.96%).\n",
      "Random sample 24 is ok: 292 out of 5970 samples are faulty (4.89%).\n",
      "Random sample 25 is ok: 303 out of 5970 samples are faulty (5.08%).\n",
      "Random sample 26 is ok: 277 out of 5970 samples are faulty (4.64%).\n",
      "Random sample 27 is ok: 288 out of 5970 samples are faulty (4.82%).\n",
      "Random sample 28 is ok: 290 out of 5970 samples are faulty (4.86%).\n",
      "Random sample 29 is ok: 313 out of 5970 samples are faulty (5.24%).\n",
      "Random sample 30 is ok: 281 out of 5970 samples are faulty (4.71%).\n",
      "Random sample 31 is ok: 318 out of 5970 samples are faulty (5.33%).\n",
      "Random sample 32 is ok: 296 out of 5970 samples are faulty (4.96%).\n",
      "Random sample 33 is ok: 267 out of 5970 samples are faulty (4.47%).\n",
      "Random sample 34 is ok: 297 out of 5970 samples are faulty (4.97%).\n",
      "Random sample 35 is ok: 289 out of 5970 samples are faulty (4.84%).\n",
      "Random sample 36 is ok: 269 out of 5970 samples are faulty (4.51%).\n",
      "Random sample 37 is ok: 308 out of 5970 samples are faulty (5.16%).\n",
      "Random sample 38 is ok: 292 out of 5970 samples are faulty (4.89%).\n",
      "Random sample 39 is ok: 296 out of 5970 samples are faulty (4.96%).\n",
      "Random sample 40 is ok: 276 out of 5970 samples are faulty (4.62%).\n",
      "Random sample 41 is ok: 283 out of 5970 samples are faulty (4.74%).\n",
      "Random sample 42 is ok: 284 out of 5970 samples are faulty (4.76%).\n",
      "Random sample 43 is ok: 290 out of 5970 samples are faulty (4.86%).\n",
      "Random sample 44 is ok: 299 out of 5970 samples are faulty (5.01%).\n",
      "Random sample 45 is ok: 297 out of 5970 samples are faulty (4.97%).\n",
      "Random sample 46 is ok: 273 out of 5970 samples are faulty (4.57%).\n",
      "Random sample 47 is ok: 298 out of 5970 samples are faulty (4.99%).\n",
      "Random sample 48 is ok: 256 out of 5970 samples are faulty (4.29%).\n",
      "Random sample 49 is ok: 315 out of 5970 samples are faulty (5.28%).\n",
      "Total problematic batches across all tests: 0\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def detect_quality_issues(model, data_loader, threshold, seed):\n",
    "    model.eval()\n",
    "    current_batch_issues = 0\n",
    "    batch_size = len(data_loader.dataset)  # 这里的批次大小是整个数据集的大小\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, _ in data_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            recon, mu, logvar = model(inputs)\n",
    "            BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "            total_loss = BCE + KLD\n",
    "            # 判断当前样本是否有问题\n",
    "            if total_loss.item() > threshold:\n",
    "                current_batch_issues += 1\n",
    "\n",
    "    # 评估是否有超过5%的样本有问题\n",
    "    if current_batch_issues > batch_size * 0.05*1.2:\n",
    "        print(f\"Random sample {seed} is problematic: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "        return True\n",
    "    else:\n",
    "        print(f\"Random sample {seed} is ok: {current_batch_issues} out of {batch_size} samples are faulty ({(current_batch_issues/batch_size * 100):.2f}%).\")\n",
    "        return False\n",
    "\n",
    "# 主函数：执行50次随机采样测试\n",
    "def test_random_samples(model, data, threshold):\n",
    "    problematic_batches = 0\n",
    "    for seed in range(50):\n",
    "        _, sample_data = train_test_split(data, test_size=0.2, random_state=seed)  # 随机采样20%\n",
    "        ##sample_data_array = sample_data.values.astype(np.float32)\n",
    "        sample_tensor = torch.tensor(sample_data)\n",
    "        sample_dataset = TensorDataset(sample_tensor, sample_tensor)\n",
    "        test_dirty_loader = DataLoader(sample_dataset, batch_size=1, shuffle=False) \n",
    "        if detect_quality_issues(model, test_dirty_loader, threshold, seed):\n",
    "            problematic_batches += 1\n",
    "\n",
    "    print(f\"Total problematic batches across all tests: {problematic_batches}\")\n",
    "\n",
    "\n",
    "# 假设 model, data_dirty, threshold, device 已经被正确定义和设置\n",
    "test_random_samples(model, test_data, threshold)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b8bc294",
   "metadata": {},
   "source": [
    "## 保存loss 到文件中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "c690c9ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import torch\n",
    "# import numpy as np\n",
    "# from torch.utils.data import DataLoader, TensorDataset\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# import pandas as pd\n",
    "\n",
    "# def record_losses(model, data_loader, seed):\n",
    "#     model.eval()\n",
    "#     losses = []\n",
    "    \n",
    "#     with torch.no_grad():\n",
    "#         for inputs, _ in data_loader:\n",
    "#             inputs = inputs.to(device)\n",
    "#             recon, mu, logvar = model(inputs)\n",
    "#             BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "#             total_loss = BCE + KLD\n",
    "#             losses.append(total_loss.item())\n",
    "    \n",
    "#     # Save the losses to a CSV file\n",
    "#     losses_df = pd.DataFrame(losses, columns=['Loss_dirty_graph'])\n",
    "#     losses_df.to_csv(f'loss_data_dirty_graph.csv', index=False)\n",
    "#     print(f\"Loss data for random sample data_dirty_graph saved.\")\n",
    "\n",
    "# # 主函数：测试2000个随机样本\n",
    "# def test_random_sample(model, data_dirty):\n",
    "#     seed = 42  # Use a fixed seed for reproducibility\n",
    "#     _, sample_data = train_test_split(data_dirty, test_size=2000, train_size=None, random_state=seed)\n",
    "#     sample_data_array = sample_data.values.astype(np.float32)\n",
    "#     sample_tensor = torch.tensor(sample_data_array)\n",
    "#     sample_dataset = TensorDataset(sample_tensor, sample_tensor)\n",
    "#     test_loader = DataLoader(sample_dataset, batch_size=1, shuffle=False) \n",
    "#     record_losses(model, test_loader, seed)\n",
    "\n",
    "# # 假设 model, data_dirty, device, loss_function 已经被正确定义和设置\n",
    "# test_random_sample(model, data_dirty)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "652dc382",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import torch\n",
    "# import numpy as np\n",
    "# from torch.utils.data import DataLoader, TensorDataset\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# import pandas as pd\n",
    "# import os\n",
    "\n",
    "# def record_losses(model, data_loader, seed, device, loss_function):\n",
    "#     model.eval()\n",
    "#     losses = []\n",
    "    \n",
    "#     with torch.no_grad():\n",
    "#         for inputs, _ in data_loader:\n",
    "#             inputs = inputs.to(device)\n",
    "#             recon, mu, logvar = model(inputs)\n",
    "#             BCE, KLD = loss_function(recon, inputs, mu, logvar)\n",
    "#             total_loss = BCE + KLD\n",
    "#             losses.append(total_loss.item())\n",
    "    \n",
    "#     return losses\n",
    "\n",
    "# # 主函数：测试两个数据集并将结果保存到同一个CSV文件\n",
    "# def test_and_save_combined_losses(model, data_clean, data_dirty, file_name='/home/sdong/experiments/VAE_method/results/combined_loss_data_gragh.csv'):\n",
    "#     seed = 42  # Use a fixed seed for reproducibility\n",
    "#     # Process clean data\n",
    "#     _, sample_data_clean = train_test_split(data_clean, test_size=2000, train_size=None, random_state=seed)\n",
    "#     #sample_clean_array = sample_data_clean.values.astype(np.float32)\n",
    "#     sample_clean_tensor = torch.tensor(sample_data_clean)\n",
    "#     sample_clean_dataset = TensorDataset(sample_clean_tensor, sample_clean_tensor)\n",
    "#     clean_loader = DataLoader(sample_clean_dataset, batch_size=1, shuffle=False)\n",
    "#     clean_losses = record_losses(model, clean_loader, seed, device, loss_function)\n",
    "    \n",
    "#     # Process dirty data\n",
    "#     _, sample_data_dirty = train_test_split(data_dirty, test_size=2000, train_size=None, random_state=seed)\n",
    "#     sample_dirty_array = sample_data_dirty.values.astype(np.float32)\n",
    "#     sample_dirty_tensor = torch.tensor(sample_dirty_array)\n",
    "#     sample_dirty_dataset = TensorDataset(sample_dirty_tensor, sample_dirty_tensor)\n",
    "#     dirty_loader = DataLoader(sample_dirty_dataset, batch_size=1, shuffle=False)\n",
    "#     dirty_losses = record_losses(model, dirty_loader, seed, device, loss_function)\n",
    "    \n",
    "#     # Combine and save to CSV\n",
    "#     combined_df = pd.DataFrame({\n",
    "#         'Loss_clean_graph': clean_losses,\n",
    "#         'Loss_dirty_graph': dirty_losses\n",
    "#     })\n",
    "#     combined_df.to_csv(file_name, index=False)\n",
    "#     print(f\"Combined loss data saved to {file_name}.\")\n",
    "\n",
    "# # 假设 model, data_clean, data_dirty, device, loss_function 已经被正确定义和设置\n",
    "# test_and_save_combined_losses(model, test_data, data_dirty)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95383854",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c828937",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface_learn",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}