fix bugs in TCN networks;

.gitignore

@@ -4,3 +4,6 @@ build/
 .idea
 **/__pycache__
 docs_output
+docs/generated
+.vscode
+sphinx-build

.readthedocs.yaml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"
+    python: "3.8"
     # You can also specify other tool versions:
     # nodejs: "20"
     # rust: "1.70"

README.rst

@@ -5,6 +5,14 @@ Python Deep Outlier/Anomaly Detection (DeepOD)
    :target: https://github.com/xuhongzuo/DeepOD/actions/workflows/testing.yml
    :alt: testing2
 
+.. image:: https://readthedocs.org/projects/deepod/badge/?version=latest
+    :target: https://deepod.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+.. image:: https://app.codacy.com/project/badge/Grade/2c587126aac2441abb917c032189fbe8
+    :target: https://app.codacy.com/gh/xuhongzuo/DeepOD/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade
+    :alt: codacy
+
 .. image:: https://coveralls.io/repos/github/xuhongzuo/DeepOD/badge.svg?branch=main
     :target: https://coveralls.io/github/xuhongzuo/DeepOD?branch=main
     :alt: coveralls
@@ -18,7 +26,7 @@ Python Deep Outlier/Anomaly Detection (DeepOD)
 and `Anomaly Detection <https://en.wikipedia.org/wiki/Anomaly_detection>`_. ``DeepOD`` supports tabular anomaly detection and time-series anomaly detection.
 
 
-DeepOD includes **26** deep outlier detection / anomaly detection algorithms (in unsupervised/weakly-supervised paradigm).
+DeepOD includes **27** deep outlier detection / anomaly detection algorithms (in unsupervised/weakly-supervised paradigm).
 More baseline algorithms will be included later.
 
 
@@ -169,14 +177,14 @@ Implemented Models
  RCA, IJCAI, 2021, unsupervised, RCA: A Deep Collaborative Autoencoder Approach for Anomaly Detection [#Liu2021RCA]_
  GOAD, ICLR, 2020, unsupervised, Classification-Based Anomaly Detection for General Data [#Bergman2020GOAD]_
  NeuTraL, ICML, 2021, unsupervised, Neural Transformation Learning for Deep Anomaly Detection Beyond Images [#Qiu2021Neutral]_
- ICL, ICLR, 2022, unsupervised, Anomaly Detection for Tabular Data with Internal Contrastive Learning
- DIF, TKDE, 2023, unsupervised, Deep Isolation Forest for Anomaly Detection
- SLAD, ICML, 2023, unsupervised, Fascinating Supervisory Signals and Where to Find Them: Deep Anomaly Detection with Scale Learning
- DevNet, KDD, 2019, weakly-supervised, Deep Anomaly Detection with Deviation Networks
- PReNet, KDD, 2023, weakly-supervised, Deep Weakly-supervised Anomaly Detection
- Deep SAD, ICLR, 2020, weakly-supervised, Deep Semi-Supervised Anomaly Detection
- FeaWAD, TNNLS, 2021, weakly-supervised, Feature Encoding with AutoEncoders for Weakly-supervised Anomaly Detection
- RoSAS, IP&M, 2023, weakly-supervised, RoSAS: Deep semi-supervised anomaly detection with contamination-resilient continuous supervision
+ ICL, ICLR, 2022, unsupervised, Anomaly Detection for Tabular Data with Internal Contrastive Learning [#Shenkar2022ICL]_
+ DIF, TKDE, 2023, unsupervised, Deep Isolation Forest for Anomaly Detection [#Xu2023DIF]_
+ SLAD, ICML, 2023, unsupervised, Fascinating Supervisory Signals and Where to Find Them: Deep Anomaly Detection with Scale Learning [#Xu2023SLAD]_
+ DevNet, KDD, 2019, weakly-supervised, Deep Anomaly Detection with Deviation Networks [#Pang2019DevNet]_
+ PReNet, KDD, 2023, weakly-supervised, Deep Weakly-supervised Anomaly Detection [#Pang2023PreNet]_
+ Deep SAD, ICLR, 2020, weakly-supervised, Deep Semi-Supervised Anomaly Detection [#Ruff2020DSAD]_
+ FeaWAD, TNNLS, 2021, weakly-supervised, Feature Encoding with AutoEncoders for Weakly-supervised Anomaly Detection [#Zhou2021FeaWAD]_
+ RoSAS, IP&M, 2023, weakly-supervised, RoSAS: Deep semi-supervised anomaly detection with contamination-resilient continuous supervision [#Xu2023RoSAS]_
 
 **Time-series Anomaly Detection models:**
 
@@ -187,15 +195,16 @@ Implemented Models
  DCdetector, KDD, 2023, unsupervised, DCdetector: Dual Attention Contrastive Representation Learning for Time Series Anomaly Detection [#Yang2023dcdetector]_
  TimesNet, ICLR, 2023, unsupervised, TIMESNET: Temporal 2D-Variation Modeling for General Time Series Analysis [#Wu2023timesnet]_
  AnomalyTransformer, ICLR, 2022, unsupervised, Anomaly Transformer: Time Series Anomaly Detection with Association Discrepancy [#Xu2022transformer]_
- TranAD, VLDB, 2022, unsupervised, TranAD: Deep Transformer Networks for Anomaly Detection in Multivariate Time Series Data  
- COUTA, arXiv, 2022, unsupervised, Calibrated One-class Classification for Unsupervised Time Series Anomaly Detection
+ NCAD, IJCAI, 2022, unsupervised, Neural Contextual Anomaly Detection for Time Series [#Carmona2022NCAD]_
+ TranAD, VLDB, 2022, unsupervised, TranAD: Deep Transformer Networks for Anomaly Detection in Multivariate Time Series Data [#Tuli2022TranAD]_
+ COUTA, arXiv, 2022, unsupervised, Calibrated One-class Classification for Unsupervised Time Series Anomaly Detection [#Xu2022COUTA]_
  USAD, KDD, 2020, unsupervised, USAD: UnSupervised Anomaly Detection on Multivariate Time Series  
- DIF, TKDE, 2023, unsupervised, Deep Isolation Forest for Anomaly Detection
- TcnED, TNNLS, 2021, unsupervised, An Evaluation of Anomaly Detection and Diagnosis in Multivariate Time Series
- Deep SVDD (TS), ICML, 2018, unsupervised, Deep One-Class Classification  
- DevNet (TS), KDD, 2019, weakly-supervised, Deep Anomaly Detection with Deviation Networks
- PReNet (TS), KDD, 2023, weakly-supervised, Deep Weakly-supervised Anomaly Detection
- Deep SAD (TS), ICLR, 2020, weakly-supervised, Deep Semi-Supervised Anomaly Detection
+ DIF, TKDE, 2023, unsupervised, Deep Isolation Forest for Anomaly Detection [#Xu2023DIF]_
+ TcnED, TNNLS, 2021, unsupervised, An Evaluation of Anomaly Detection and Diagnosis in Multivariate Time Series [#Garg2021Evaluation]_
+ Deep SVDD (TS), ICML, 2018, unsupervised, Deep One-Class Classification [#Ruff2018Deep]_
+ DevNet (TS), KDD, 2019, weakly-supervised, Deep Anomaly Detection with Deviation Networks [#Pang2019DevNet]_
+ PReNet (TS), KDD, 2023, weakly-supervised, Deep Weakly-supervised Anomaly Detection [#Pang2023PreNet]_
+ Deep SAD (TS), ICLR, 2020, weakly-supervised, Deep Semi-Supervised Anomaly Detection [#Ruff2020DSAD]_
 
 NOTE:
 
@@ -252,8 +261,34 @@ Reference
 
 .. [#Qiu2021Neutral] Qiu, Chen, et al. "Neural Transformation Learning for Deep Anomaly Detection Beyond Images". ICML. 2021.
 
-.. [#Xu2022transformer] Xu Jiehui, et al. "Anomaly Transformer: Time Series Anomaly Detection with Association Discrepancy". ICLR, 2022.
+.. [#Shenkar2022ICL] Shenkar, Tom, et al. "Anomaly Detection for Tabular Data with Internal Contrastive Learning". ICLR. 2022.
+
+.. [#Pang2019DevNet] Pang, Guansong, et al. "Deep Anomaly Detection with Deviation Networks". KDD. 2019.
+
+.. [#Pang2023PreNet] Pang, Guansong, et al. "Deep Weakly-supervised Anomaly Detection". KDD. 2023. 
+
+.. [#Ruff2020DSAD] Ruff, Lukas, et al. "Deep Semi-Supervised Anomaly Detection". ICLR. 2020. 
+
+.. [#Zhou2021FeaWAD] Zhou, Yingjie, et al. "Feature Encoding with AutoEncoders for Weakly-supervised Anomaly Detection". TNNLS. 2021. 
+
+.. [#Xu2022transformer] Xu, Jiehui, et al. "Anomaly Transformer: Time Series Anomaly Detection with Association Discrepancy". ICLR, 2022.
+
+.. [#Wu2023timesnet] Wu, Haixu, et al. "TimesNet: Temporal 2D-Variation Modeling for General Time Series Analysis". ICLR. 2023.
+
+.. [#Yang2023dcdetector] Yang, Yiyuan, et al. "DCdetector: Dual Attention Contrastive Representation Learning for Time Series Anomaly Detection". KDD. 2023
+
+.. [#Tuli2022TranAD] Tuli, Shreshth, et al. "TranAD: Deep Transformer Networks for Anomaly Detection in Multivariate Time Series Data". VLDB. 2022.
+
+.. [#Carmona2022NCAD] Carmona, Chris U., et al. "Neural Contextual Anomaly Detection for Time Series". IJCAI. 2022. 
+
+.. [#Garg2021Evaluation] Garg, Astha, et al. "An Evaluation of Anomaly Detection and Diagnosis in Multivariate Time Series". TNNLS. 2021. 
+
+.. [#Xu2022COUTA] Xu, Hongzuo et al. "Calibrated One-class Classification for Unsupervised Time Series Anomaly Detection". arXiv:2207.12201. 2022.
+
+.. [#Xu2023DIF] Xu, Hongzuo et al. "Deep Isolation Forest for Anomaly Detection". TKDE. 2023.
+
+.. [#Xu2023SLAD] Xu, Hongzuo et al. "Fascinating supervisory signals and where to find them: deep anomaly detection with scale learning". ICML. 2023. 
+
+.. [#Xu2023RoSAS] Xu, Hongzuo et al. "RoSAS: Deep semi-supervised anomaly detection with contamination-resilient continuous supervision". IP&M. 2023. 
 
-.. [#Wu2023timesnet] Wu Haixu, et al. "TimesNet: Temporal 2D-Variation Modeling for General Time Series Analysis". ICLR. 2023.
 
-.. [#Yang2023dcdetector] Yang Yiyuan et al. "DCdetector: Dual Attention Contrastive Representation Learning for Time Series Anomaly Detection". KDD. 2023

deepod/core/base_model.py

@@ -21,6 +21,7 @@
 from deepod.utils.utility import get_sub_seqs, get_sub_seqs_label
 import pickle
 
+
 class BaseDeepAD(metaclass=ABCMeta):
     """
     Abstract class for deep outlier detection models

deepod/core/networks/__init__.py

@@ -0,0 +1,12 @@
+from .base_networks import MLPnet
+from .base_networks import MlpAE
+from .base_networks import GRUNet
+from .base_networks import LSTMNet
+from .base_networks import ConvSeqEncoder
+from .base_networks import ConvNet
+from .ts_network_transformer import TSTransformerEncoder
+from .ts_network_tcn import TCNnet
+from .ts_network_tcn import TcnAE
+
+__all__ = ['MLPnet', 'MlpAE', 'GRUNet', 'LSTMNet', 'ConvSeqEncoder',
+           'ConvNet', 'TSTransformerEncoder', 'TCNnet', 'TcnAE']

deepod/core/networks/base_networks.py

@@ -6,7 +6,7 @@
 from deepod.core.networks.ts_network_tcn import TCNnet, TcnAE
 # from deepod.core.base_transformer_network_dev import TSTransformerEncoder
 from deepod.core.networks.network_utility import _instantiate_class, _handle_n_hidden
-
+import torch.nn.modules.activation
 
 sequential_net_name = ['TCN', 'GRU', 'LSTM', 'Transformer', 'ConvSeq', 'DilatedConv']
 
@@ -32,6 +32,26 @@ def get_network(network_name):
 
 
 class ConvNet(torch.nn.Module):
+    """Convolutional Network
+
+    Args:
+        n_features (int):
+            number of input data features
+        kernel_size (int):
+            kernel size (Default=1)
+        n_hidden (int):
+            number of hidden units in hidden layers (Default=8)
+        n_layers (int):
+            number of layers (Default=5)
+        activation (str):
+            name of activation layer,
+            activation should be implemented in torch.nn.module.activation
+            (Default='ReLU')
+        bias (bool):
+            use bias or not
+            (Default=False)
+
+    """
     def __init__(self, n_features, kernel_size=1, n_hidden=8, n_layers=5,
                  activation='ReLU', bias=False):
         super(ConvNet, self).__init__()
@@ -49,7 +69,7 @@ def __init__(self, n_features, kernel_size=1, n_hidden=8, n_layers=5,
                 self.layers += [
                     # torch.nn.LeakyReLU(inplace=True)
                     _instantiate_class(module_name="torch.nn.modules.activation",
-                                      class_name=activation)
+                                       class_name=activation)
                 ]
             in_channels = n_hidden
 
@@ -62,6 +82,7 @@ def forward(self, x):
 
 
 class MlpAE(torch.nn.Module):
+    """MLP-based AutoEncoder"""
     def __init__(self, n_features, n_hidden='500,100', n_emb=20, activation='ReLU',
                  bias=False, batch_norm=False,
                  skip_connection=None, dropout=None
@@ -105,6 +126,7 @@ def forward(self, x):
 
 
 class MLPnet(torch.nn.Module):
+    """MLP-based Representation Network"""
     def __init__(self, n_features, n_hidden='500,100', n_output=20, mid_channels=None,
                  activation='ReLU', bias=False, batch_norm=False,
                  skip_connection=None, dropout=None):
@@ -140,7 +162,6 @@ def __init__(self, n_features, n_hidden='500,100', n_output=20, mid_channels=Non
             ]
         self.network = torch.nn.Sequential(*self.layers)
 
-
     def forward(self, x):
         x = self.network(x)
         return x
@@ -158,6 +179,7 @@ def get_in_out_channels(self, i, num_layers, n_features, n_hidden, n_output, ski
 
 
 class LinearBlock(torch.nn.Module):
+    """Linear Block"""
     def __init__(self, in_channels, out_channels, mid_channels=None,
                  activation='Tanh', bias=False, batch_norm=False,
                  skip_connection=None, dropout=None):
@@ -214,6 +236,7 @@ def forward(self, x):
 
 
 class GRUNet(torch.nn.Module):
+    """GRU Network"""
     def __init__(self, n_features, n_hidden='20', n_output=20,
                  bias=False, dropout=None, activation='ReLU'):
         super(GRUNet, self).__init__()
@@ -235,8 +258,8 @@ def forward(self, x):
         return out
 
 
-
 class LSTMNet(torch.nn.Module):
+    """LSTM Network"""
     def __init__(self, n_features, n_hidden='20', n_output=20,
                  bias=False, dropout=None, activation='ReLU'):
         super(LSTMNet, self).__init__()
@@ -309,6 +332,7 @@ def forward(self, x):
 
 
 class ConvResBlock(torch.nn.Module):
+    """Convolutional Residual Block"""
     def __init__(self, in_dim, out_dim, conv_param=None, down_sample=None,
                  batch_norm=False, bias=False, activation='ReLU'):
         super(ConvResBlock, self).__init__()

deepod/core/networks/ts_network_tcn.py

@@ -1,9 +1,12 @@
+# TCN is partially adapted from https://github.com/locuslab/TCN
+
 import torch
 from torch.nn.utils import weight_norm
 from deepod.core.networks.network_utility import _instantiate_class, _handle_n_hidden
 
 
 class TcnAE(torch.nn.Module):
+    """Temporal Convolutional Network-based AutoEncoder"""
     def __init__(self, n_features, n_hidden='500,100', n_emb=20, activation='ReLU', bias=False,
                  kernel_size=2, dropout=0.2):
         super(TcnAE, self).__init__()
@@ -60,7 +63,7 @@ def forward(self, x):
 
 
 class TCNnet(torch.nn.Module):
-    """TCN is adapted from https://github.com/locuslab/TCN"""
+    """Temporal Convolutional Network (TCN) for encoding/representing input time series sequences"""
     def __init__(self, n_features, n_hidden='8', n_output=20,
                  kernel_size=2, bias=False,
                  dropout=0.2, activation='ReLU'):

deepod/core/networks/ts_network_transformer.py

@@ -271,16 +271,13 @@ def forward(self, src, src_mask=None, src_key_padding_mask=None):
         return src
 
 
-
-
 class TSTransformerEncoder(torch.nn.Module):
     """
-    Simplest classifier/regressor. Can be either regressor or classifier because the output does not include
-    softmax. Concatenates final layer embeddings and uses 0s to ignore padding embeddings in final output layer.
+    Transformer for encoding/representing input time series sequences
     """
 
     def __init__(self, n_features, n_output=20, seq_len=100, d_model=128,
-                 n_heads=8, n_hidden='128', dropout=0.1,
+                 n_heads=8, n_hidden='512', dropout=0.1,
                  token_encoding='convolutional', pos_encoding='fixed', activation='GELU', bias=False,
                  attn='self_attn', norm='LayerNorm', freeze=False):
         super(TSTransformerEncoder, self).__init__()
@@ -290,11 +287,11 @@ def __init__(self, n_features, n_output=20, seq_len=100, d_model=128,
         n_hidden, n_layers = _handle_n_hidden(n_hidden)
 
         # parameter check
-        assert  token_encoding in ['linear', 'convolutional'], \
+        assert token_encoding in ['linear', 'convolutional'], \
             f"use 'linear' or 'convolutional', {token_encoding} is not supported in token_encoding"
-        assert  pos_encoding in ['learnable', 'fixed'],\
+        assert pos_encoding in ['learnable', 'fixed'],\
             f"use 'learnable' or 'fixed', {pos_encoding} is not supported in pos_encoding"
-        assert  norm in ['LayerNorm', 'BatchNorm'],\
+        assert norm in ['LayerNorm', 'BatchNorm'],\
             f"use 'learnable' or 'fixed', {norm} is not supported in norm"
 
         if token_encoding == 'linear':

deepod/metrics/_tsad_adjustment.py

@@ -13,7 +13,13 @@ def point_adjustment(y_true, y_score):
         data label, 0 indicates normal timestamp, and 1 is anomaly
 
     y_score: np.array, required
-        anomaly score, higher score indicates higher likelihoods to be anomaly
+        predicted anomaly scores, higher score indicates higher likelihoods to be anomaly
+
+    Returns
+    -------
+    score: np.array
+        adjusted anomaly scores
+
     """
     score = y_score.copy()
     assert len(score) == len(y_true)

deepod/models/__init__.py

@@ -23,16 +23,17 @@
 from deepod.models.time_series.dcdetector import DCdetector
 from deepod.models.time_series.timesnet import TimesNet
 from deepod.models.time_series.anomalytransformer import AnomalyTransformer
+from deepod.models.time_series.ncad import NCAD
 from deepod.models.time_series.tranad import TranAD
 from deepod.models.time_series.couta import COUTA
 from deepod.models.time_series.usad import USAD
 from deepod.models.time_series.tcned import TcnED
 
-
 __all__ = [
     'RCA', 'DeepSVDD', 'GOAD', 'NeuTraL', 'RDP', 'ICL', 'SLAD', 'DeepIsolationForest',
     'DeepSAD', 'DevNet', 'PReNet', 'FeaWAD', 'REPEN', 'RoSAS',
-    'DCdetector', 'TimesNet', 'AnomalyTransformer', 'TranAD', 'COUTA', 'USAD', 'TcnED',
+    'DCdetector', 'TimesNet', 'AnomalyTransformer', 'NCAD',
+    'TranAD', 'COUTA', 'USAD', 'TcnED',
     'DeepIsolationForestTS', 'DeepSVDDTS',
     'PReNetTS', 'DeepSADTS', 'DevNetTS'
 ]