Add reference annotations to code

TabBench/model/lib/danets/AcceleratedModule.py

@@ -2,6 +2,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 
+# Source: https://github.com/WhatAShot/DANet/blob/main/model/AcceleratedModule.py
 class AcceleratedCreator(object):
     def __init__(self, input_dim, base_out_dim, k):
         super(AcceleratedCreator, self).__init__()

TabBench/model/lib/danets/sparsemax.py

@@ -4,15 +4,7 @@
 
 import torch
 
-"""
-Other possible implementations:
-https://github.com/KrisKorrel/sparsemax-pytorch/blob/master/sparsemax.py
-https://github.com/msobroza/SparsemaxPytorch/blob/master/mnist/sparsemax.py
-https://github.com/vene/sparse-structured-attention/blob/master/pytorch/torchsparseattn/sparsemax.py
-"""
-
-
-# credits to Yandex https://github.com/Qwicen/node/blob/master/lib/nn_utils.py
+# Source: https://github.com/WhatAShot/DANet/blob/main/model/sparsemax.py
 def _make_ix_like(input, dim=0):
     d = input.size(dim)
     rho = torch.arange(1, d + 1, device=input.device, dtype=input.dtype)

TabBench/model/lib/dnnr/__init__.py

@@ -6,3 +6,5 @@
 __author__ = """Youssef Nadar, Leon Sixt"""
 __email__ = 'Your email'
 __version__ = '0.1.0'
+
+# Source: https://github.com/younader/dnnr

TabBench/model/lib/dnnr/dnnr.py

@@ -7,7 +7,7 @@
 import random as random_mod
 import warnings
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
-
+# Source: https://github.com/younader/dnnr
 import numpy as np
 import scipy.optimize
 import scipy.spatial.distance

TabBench/model/lib/dnnr/nn_index.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-
+# Source: https://github.com/younader/dnnr
 import abc
 import dataclasses
 import tempfile

TabBench/model/lib/dnnr/solver.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-
+# Source: https://github.com/younader/dnnr
 import numpy as np
 from scipy.sparse.linalg import lsqr
 from sklearn.linear_model import Lasso, LinearRegression, Ridge

TabBench/model/lib/num_embeddings.py

@@ -57,6 +57,7 @@ def compute_bins(
     regression: Optional[bool] = None,
     verbose: bool = False,
 ) -> List[Tensor]:
+    # Source: https://github.com/yandex-research/rtdl-num-embeddings/blob/main/package/rtdl_num_embeddings.py
     """Compute bin edges for `PiecewiseLinearEmbeddings`.
 
     **Usage**
@@ -233,7 +234,7 @@ class _PiecewiseLinearEncodingImpl(nn.Module):
     edges: Tensor
     width: Tensor
     mask: Tensor
-
+    # Source: https://github.com/yandex-research/rtdl-num-embeddings/blob/main/package/rtdl_num_embeddings.py
     def __init__(self, bins: List[Tensor]) -> None:
         _check_bins(bins)
 
@@ -335,7 +336,7 @@ class PiecewiseLinearEncoding(nn.Module):
       where ``total_n_bins`` is the total number of bins for all features:
       ``total_n_bins = sum(len(b) - 1 for b in bins)``.
     """
-
+    # Source: https://github.com/yandex-research/rtdl-num-embeddings/blob/main/package/rtdl_num_embeddings.py
     def __init__(self, bins: List[Tensor]) -> None:
         """
         Args:

TabBench/model/lib/ptarl/utils.py

@@ -2,7 +2,7 @@
 import numpy as np
 import os.path as osp
 from tqdm import tqdm
-
+# Source: https://github.com/HangtingYe/PTaRL
 def run_one_epoch(model, data_loader, loss_func, model_type, config, regularize, ot_weight, diversity_weight, r_weight, diversity, optimizer=None):
     import torch.nn.functional as F
     running_loss = 0.0

TabBench/model/lib/tabr/utils.py

@@ -9,7 +9,7 @@
 from torch.nn.parameter import Parameter
 import math
 
-
+# adapted from https://github.com/yandex-research/tabular-dl-tabr
 def _initialize_embeddings(weight: Tensor, d: Optional[int]) -> None:
     if d is None:
         d = weight.shape[-1]

TabBench/model/models/autoint.py

@@ -6,8 +6,8 @@
 import torch.nn.functional as F
 import torch.nn.init as nn_init
 from torch import Tensor
-
-
+# Implementation of "AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks"
+# Source: https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/autoint.py
 def reglu(x: Tensor) -> Tensor:
     a, b = x.chunk(2, dim=-1)
     return a * F.relu(b)

TabBench/model/models/danets.py

@@ -3,7 +3,7 @@
 import numpy as np
 import torch.nn.functional as F
 import model.lib.danets.sparsemax as sparsemax
-
+# Source: https://github.com/WhatAShot/DANet/blob/main/model/DANet.py
 def initialize_glu(module, input_dim, output_dim):
     gain_value = np.sqrt((input_dim + output_dim) / np.sqrt(input_dim))
     torch.nn.init.xavier_normal_(module.weight, gain=gain_value)

TabBench/model/models/dcn2.py

@@ -2,7 +2,7 @@
 import typing as ty
 import torch
 import torch.nn as nn
-
+# Source: https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/dcn2.py
 # %%
 class CrossLayer(nn.Module):
     def __init__(self, d, dropout):

TabBench/model/models/ftt.py

@@ -6,7 +6,7 @@
 import torch.nn.functional as F
 import torch.nn.init as nn_init
 from torch import Tensor
-
+# Source: https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/ft_transformer.py
 def reglu(x):
     a, b = x.chunk(2, dim=-1)
     return a * F.relu(b)

TabBench/model/models/grownet.py

@@ -3,7 +3,7 @@
 from enum import Enum
 import torch
 import torch.nn as nn
-
+# Source: https://github.com/sbadirli/GrowNet/
 # %%
 class ForwardType(Enum):
     SIMPLE = 0

TabBench/model/models/node.py

@@ -6,7 +6,7 @@
 import torch.nn.functional as F
 from torch import Tensor
 import model.lib.node as node
-
+# adapted from https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/node.py
 # %%
 class NODE(nn.Module):
     def __init__(

TabBench/model/models/ptarl.py

@@ -5,7 +5,7 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-
+# adapted from https://github.com/HangtingYe/PTaRL
 class MLP(nn.Module):
     def __init__(
         self,

TabBench/model/models/resnet.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torch import Tensor
-
+# adapted from https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/resnet.py
 # %%
 def reglu(x):
     a, b = x.chunk(2, dim=-1)

TabBench/model/models/saint.py

@@ -3,7 +3,7 @@
 from torch import nn, einsum
 import numpy as np
 from einops import rearrange
-
+# Source: https://github.com/kodikarthik21/SAINT/blob/main/models/model.py
 #classes
 class Residual(nn.Module):
     def __init__(self, fn):

TabBench/model/models/snn.py

@@ -5,7 +5,7 @@
 import torch.nn.functional as F
 from torch import Tensor
 import math
-
+# Source: https://github.com/yandex-research/rtdl-revisiting-models/blob/main/bin/snn.py
 # %%
 class SNN(nn.Module):
     def __init__(

TabBench/model/models/switchtab.py

@@ -1,7 +1,7 @@
 import torch
 from torch import nn
 
-
+# Source: https://github.com/avivnur/SwitchTab/blob/main/model.py
 # Feature corruption function
 def feature_corruption(x, corruption_ratio=0.3):
     # We sample a mask of the features to be zeroed out

TabBench/model/models/tabcaps.py

@@ -8,6 +8,7 @@
 from torch.nn.functional import cross_entropy, mse_loss
 from model.utils import Timer, set_seeds
 
+# adapted from https://github.com/WhatAShot/TabCaps
 class TabCapsClassifier(TabCapsModel):
     def __post_init__(self):
         super(TabCapsClassifier, self).__post_init__()

TabBench/model/models/tabnet.py

@@ -6,7 +6,7 @@
 from model.lib.tabnet.multiclass_utils import infer_output_dim, check_output_dim
 from torch.utils.data import DataLoader
 import scipy
-
+# adapted from https://github.com/dreamquark-ai/tabnet/blob/develop/pytorch_tabnet/tab_model.py
 
 class TabNetClassifier(TabModel):
     def __post_init__(self):

TabBench/model/models/tabpfn.py

@@ -20,6 +20,8 @@
 
 from model.lib.tabpfn.utils import CustomUnpickler
 from model.lib.tabpfn.utils import load_model_workflow, get_params_from_config, transformer_predict
+# Source: https://github.com/automl/TabPFN
+
 
 class TabPFNClassifier(BaseEstimator, ClassifierMixin):
 

TabBench/model/models/tabr.py

@@ -11,7 +11,7 @@
 import faiss.contrib.torch_utils
 
 from model.lib.tabr.utils import make_module
-
+# adapted from https://github.com/yandex-research/tabular-dl-tabr/blob/main/bin/tabr.py
 
 class TabR(nn.Module):
     def __init__(

TabBench/model/models/tabtransformer.py

@@ -3,7 +3,7 @@
 from torch import nn, einsum
 
 from einops import rearrange, repeat
-
+# adapted from https://github.com/lucidrains/tab-transformer-pytorch
 # helpers
 
 def exists(val):

TabBench/model/models/tangos.py

@@ -4,7 +4,7 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-
+# adapted from https://github.com/alanjeffares/TANGOS
 # %%
 class Tangos(nn.Module):
     def __init__(