Update benchmarks

reczoo · Jun 6, 2024 · 918abde · 918abde
1 parent bed1f4f
commit 918abde
Show file tree

Hide file tree

Showing 168 changed files with 26,860 additions and 1 deletion.
diff --git a/CITATION b/CITATION
@@ -0,0 +1,14 @@
+@incollection{BARS,
+  author    = {Jieming Zhu and
+               Quanyu Dai and
+               Liangcai Su and
+               Rong Ma and
+               Jinyang Liu and
+               Guohao Cai and
+               Xi Xiao and
+               Rui Zhang},
+  title     = {BARS: Towards Open Benchmarking for Recommender Systems},
+  booktitle = {The 45th International ACM SIGIR Conference on Research 
+               and Development in Information Retrieval (SIGIR'22)},
+  year      = {2022}
+}
diff --git a/docs/CTR/leaderboard/amazonelectronics_x1.csv b/docs/CTR/leaderboard/amazonelectronics_x1.csv
@@ -11,3 +11,7 @@ Year,Publication,Model,Paper URL,gAUC,AUC,Logloss,Running Steps,Contributor
 2018,KDD'18,DIN,https://www.kdd.org/kdd2018/accepted-papers/view/deep-interest-network-for-click-through-rate-prediction,0.883526,0.886028,0.43019,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIN/DIN_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,AAAI'19,DIEN,https://arxiv.org/abs/1809.03672,0.885625,0.888777,0.425708,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIEN/DIEN_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,DLP-KDD'19,BST,https://arxiv.org/abs/1905.06874,0.884108,0.886424,0.430077,https://github.com/reczoo/BARS/tree/main/ranking/ctr/BST/BST_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,CIKM'20,DMIN,https://dl.acm.org/doi/10.1145/3340531.3412092,0.885636,0.887665,0.424567,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMIN/DMIN_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,AAAI'20,DMR,https://ojs.aaai.org/index.php/AAAI/article/view/5346,0.885142,0.887335,0.427744,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMR/DMR_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2023,KDD'23,PPNet,https://arxiv.org/abs/2302.01115,0.87969,0.881667,0.439829,https://github.com/reczoo/BARS/tree/main/ranking/ctr/PPNet/PPNet_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2022,NeurIPS'22,APG_DCNv2,https://arxiv.org/abs/2203.16218,0.87942,0.882007,0.437314,https://github.com/reczoo/BARS/tree/main/ranking/ctr/APG/APG_DCNv2_amazonelectronics_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
diff --git a/docs/CTR/leaderboard/kuaivideo_x1.csv b/docs/CTR/leaderboard/kuaivideo_x1.csv
@@ -11,3 +11,7 @@ Year,Publication,Model,Paper URL,gAUC,AUC,Logloss,Running Steps,Contributor
 2018,KDD'18,DIN,https://www.kdd.org/kdd2018/accepted-papers/view/deep-interest-network-for-click-through-rate-prediction,0.669568,0.749537,0.43212,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIN/DIN_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,AAAI'19,DIEN,https://arxiv.org/abs/1809.03672,0.671148,0.750372,0.433049,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIEN/DIEN_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,DLP-KDD'19,BST,https://arxiv.org/abs/1905.06874,0.669039,0.748407,0.433819,https://github.com/reczoo/BARS/tree/main/ranking/ctr/BST/BST_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,CIKM'20,DMIN,https://dl.acm.org/doi/10.1145/3340531.3412092,0.672621,0.750777,0.430187,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMIN/DMIN_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,AAAI'20,DMR,https://ojs.aaai.org/index.php/AAAI/article/view/5346,0.66888,0.748489,0.435421,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMR/DMR_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2023,KDD'23,PPNet,https://arxiv.org/abs/2302.01115,0.666808,0.746437,0.437027,https://github.com/reczoo/BARS/tree/main/ranking/ctr/PPNet/PPNet_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2022,NeurIPS'22,APG_DCNv2,https://arxiv.org/abs/2203.16218,0.667191,0.746629,0.439467,https://github.com/reczoo/BARS/tree/main/ranking/ctr/APG/APG_DCNv2_kuaivideo_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
diff --git a/docs/CTR/leaderboard/microvideo1.7m_x1.csv b/docs/CTR/leaderboard/microvideo1.7m_x1.csv
@@ -11,3 +11,7 @@ Year,Publication,Model,Paper URL,gAUC,AUC,Logloss,Running Steps,Contributor
 2018,KDD'18,DIN,https://www.kdd.org/kdd2018/accepted-papers/view/deep-interest-network-for-click-through-rate-prediction,0.688282,0.736006,0.411558,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIN/DIN_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,AAAI'19,DIEN,https://arxiv.org/abs/1809.03672,0.68672,0.732075,0.412213,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIEN/DIEN_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,DLP-KDD'19,BST,https://arxiv.org/abs/1905.06874,0.685436,0.73415,0.411837,https://github.com/reczoo/BARS/tree/main/ranking/ctr/BST/BST_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,CIKM'20,DMIN,https://dl.acm.org/doi/10.1145/3340531.3412092,0.68791,0.733174,0.411456,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMIN/DMIN_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,AAAI'20,DMR,https://ojs.aaai.org/index.php/AAAI/article/view/5346,0.687533,0.735436,0.412249,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMR/DMR_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2023,KDD'23,PPNet,https://arxiv.org/abs/2302.01115,0.686542,0.734938,0.411962,https://github.com/reczoo/BARS/tree/main/ranking/ctr/PPNet/PPNet_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2022,NeurIPS'22,APG_DCNv2,https://arxiv.org/abs/2203.16218,0.682735,0.732541,0.412684,https://github.com/reczoo/BARS/tree/main/ranking/ctr/APG/APG_DCNv2_microvideo1.7m_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
diff --git a/docs/CTR/leaderboard/microvideo1.7m_x1.md b/docs/CTR/leaderboard/microvideo1.7m_x1.md
@@ -14,7 +14,7 @@ jupytext:
 
 ```{note}
 Please use the following evaluation settings for this benchmark:
-+ Dataset split: [MicroVideo1.7M_x1](https://github.com/reczoo/Datasets/tree/main/MicroVideo1.7M/MicroVideo1.7M_x1)
++ Dataset split: [MicroVideo1.7M_x1](https://github.com/reczoo/Datasets/tree/main/MicroVideo/MicroVideo1.7M_x1)
 + Rare features filtering: min_categr_count=1
 + Embedding size: 64
 ```

diff --git a/docs/CTR/leaderboard/taobaoad_x1.csv b/docs/CTR/leaderboard/taobaoad_x1.csv
@@ -11,3 +11,7 @@ Year,Publication,Model,Paper URL,gAUC,AUC,Logloss,Running Steps,Contributor
 2018,KDD'18,DIN,https://www.kdd.org/kdd2018/accepted-papers/view/deep-interest-network-for-click-through-rate-prediction,0.576459,0.652399,0.192445,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIN/DIN_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,AAAI'19,DIEN,https://arxiv.org/abs/1809.03672,0.576916,0.652937,0.19284,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DIEN/DIEN_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
 2019,DLP-KDD'19,BST,https://arxiv.org/abs/1905.06874,0.576304,0.651131,0.192842,https://github.com/reczoo/BARS/tree/main/ranking/ctr/BST/BST_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,CIKM'20,DMIN,https://dl.acm.org/doi/10.1145/3340531.3412092,0.577,0.651103,0.192813,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMIN/DMIN_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2020,AAAI'20,DMR,https://ojs.aaai.org/index.php/AAAI/article/view/5346,0.57677,0.652411,0.193055,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DMR/DMR_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2023,KDD'23,PPNet,https://arxiv.org/abs/2302.01115,0.572597,0.647442,0.194488,https://github.com/reczoo/BARS/tree/main/ranking/ctr/PPNet/PPNet_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
+2022,NeurIPS'22,APG_DCNv2,https://arxiv.org/abs/2203.16218,0.575258,0.649595,0.19288,https://github.com/reczoo/BARS/tree/main/ranking/ctr/APG/APG_DCNv2_taobaoad_x1,"<a href=""https://arxiv.org/abs/2205.09626"">Zhu et al.</a>"
diff --git a/docs/Pretraining/papers.md b/docs/Pretraining/papers.md
@@ -0,0 +1,39 @@
+# Pretraining
+
+A curated list of pretraining models
+
+
+## User/Item Embedding Pretraining
+
+|   |   |   |    |    | |
+| :---------:|:------:|:------:|:------:|:------:|:------:|
+| **2023**   | [MAP](https://arxiv.org/abs/2308.01737) {cite}`MAP`<br>KDD'23<br>Huawei |   [BERT4CTR](https://arxiv.org/abs/2308.11527) {cite}`BERT4CTR`<br>KDD'23<br>Microsoft |  [SUM](https://arxiv.org/abs/2311.09544) {cite}`SUM`<br>Arxiv'23<br>Meta | [UniM^2Rec](https://arxiv.org/abs/2311.01831) {cite}`UniM2Rec`<br>Arxiv'23<br>Tencent | 
+| **2022**   | [GUIM](https://arxiv.org/abs/2207.00750) {cite}`GUIM`<br>Arxiv'22<br>Alibaba |  
+
+
+## User Model Pretraining
+
+|   |   |   |    |    | |
+| :---------:|:------:|:------:|:------:|:------:|:------:|
+| **2023**   | [MAP](https://arxiv.org/abs/2308.01737) {cite}`MAP`<br>KDD'23<br>Huawei |   [BERT4CTR](https://arxiv.org/abs/2308.11527) {cite}`BERT4CTR`<br>KDD'23<br>Microsoft |  [SUM](https://arxiv.org/abs/2311.09544) {cite}`SUM`<br>Arxiv'23<br>Meta | [UniM^2Rec](https://arxiv.org/abs/2311.01831) {cite}`UniM2Rec`<br>Arxiv'23<br>Tencent | 
+| **2022**   | [GUIM](https://arxiv.org/abs/2207.00750) {cite}`GUIM`<br>Arxiv'22<br>Alibaba |  
+
+
+## Network Pretraining
+
+|   |   |   |    |    | |
+| :---------:|:------:|:------:|:------:|:------:|:------:|
+| **2023**   | [MAP](https://arxiv.org/abs/2308.01737) {cite}`MAP`<br>KDD'23<br>Huawei |   [BERT4CTR](https://arxiv.org/abs/2308.11527) {cite}`BERT4CTR`<br>KDD'23<br>Microsoft |  [SUM](https://arxiv.org/abs/2311.09544) {cite}`SUM`<br>Arxiv'23<br>Meta | [UniM^2Rec](https://arxiv.org/abs/2311.01831) {cite}`UniM2Rec`<br>Arxiv'23<br>Tencent | 
+| **2022**   | [GUIM](https://arxiv.org/abs/2207.00750) {cite}`GUIM`<br>Arxiv'22<br>Alibaba |  
+
+
+## Finetuning
+
+
+
+## References
+
+```{bibliography}
+:style: unsrt
+:filter: docname in docnames
+```
diff --git a/...ng/ctr/APG/APG_DCNv2_amazonelectronics_x1/APG_DCNv2_amazonelectronics_x1_015_a0cca3e4.log b/...ng/ctr/APG/APG_DCNv2_amazonelectronics_x1/APG_DCNv2_amazonelectronics_x1_015_a0cca3e4.log
@@ -0,0 +1,145 @@
+2023-05-31 09:46:57,297 P12611 INFO Params: {
+    "batch_norm": "True",
+    "batch_size": "1024",
+    "condition_features": "None",
+    "condition_mode": "self-wise",
+    "data_format": "csv",
+    "data_root": "../data/Amazon/",
+    "dataset_id": "amazonelectronics_x1_b7a43f49",
+    "debug_mode": "False",
+    "dnn_activations": "relu",
+    "early_stop_patience": "2",
+    "embedding_dim": "64",
+    "embedding_regularizer": "0.005",
+    "epochs": "100",
+    "eval_steps": "None",
+    "feature_cols": "[{'active': True, 'dtype': 'int', 'name': 'user_id', 'remap': False, 'type': 'meta'}, {'active': True, 'dtype': 'str', 'name': 'item_id', 'type': 'categorical'}, {'active': True, 'dtype': 'str', 'name': 'cate_id', 'type': 'categorical'}, {'active': True, 'dtype': 'str', 'feature_encoder': 'layers.MaskedAveragePooling()', 'max_len': 100, 'name': 'item_history', 'share_embedding': 'item_id', 'splitter': '^', 'type': 'sequence'}, {'active': True, 'dtype': 'str', 'feature_encoder': 'layers.MaskedAveragePooling()', 'max_len': 100, 'name': 'cate_history', 'share_embedding': 'cate_id', 'splitter': '^', 'type': 'sequence'}]",
+    "feature_config": "None",
+    "feature_specs": "None",
+    "generate_bias": "True",
+    "gpu": "6",
+    "group_id": "user_id",
+    "hypernet_config": "{'dropout_rates': 0.1, 'hidden_activations': 'relu', 'hidden_units': []}",
+    "label_col": "{'dtype': 'float', 'name': 'label'}",
+    "learning_rate": "0.0005",
+    "loss": "binary_crossentropy",
+    "low_rank": "32",
+    "metrics": "['gAUC', 'AUC', 'logloss']",
+    "min_categr_count": "1",
+    "model": "APG_DCNv2",
+    "model_id": "APG_DCNv2_amazonelectronics_x1_015_a0cca3e4",
+    "model_root": "./checkpoints/APG_DCNv2_amazonelectronics_x1/",
+    "model_structure": "parallel",
+    "monitor": "{'AUC': 1, 'gAUC': 1}",
+    "monitor_mode": "max",
+    "net_dropout": "0.1",
+    "net_regularizer": "0",
+    "new_condition_emb": "False",
+    "num_cross_layers": "2",
+    "num_experts": "4",
+    "num_workers": "3",
+    "optimizer": "adam",
+    "overparam_p": "[32, 16, 8]",
+    "parallel_dnn_hidden_units": "[1024, 512, 256]",
+    "pickle_feature_encoder": "True",
+    "rank_k": "[32, 16, 8]",
+    "save_best_only": "True",
+    "seed": "2022",
+    "shuffle": "True",
+    "stacked_dnn_hidden_units": "[500, 500, 500]",
+    "task": "binary_classification",
+    "test_data": "../data/Amazon/AmazonElectronics_x1/test.csv",
+    "train_data": "../data/Amazon/AmazonElectronics_x1/train.csv",
+    "use_features": "None",
+    "use_low_rank_mixture": "False",
+    "valid_data": "../data/Amazon/AmazonElectronics_x1/test.csv",
+    "verbose": "1"
+}
+2023-05-31 09:46:57,297 P12611 INFO Set up feature processor...
+2023-05-31 09:46:57,297 P12611 WARNING Skip rebuilding ../data/Amazon/amazonelectronics_x1_b7a43f49/feature_map.json. Please delete it manually if rebuilding is required.
+2023-05-31 09:46:57,298 P12611 INFO Load feature_map from json: ../data/Amazon/amazonelectronics_x1_b7a43f49/feature_map.json
+2023-05-31 09:46:57,298 P12611 INFO Set column index...
+2023-05-31 09:46:57,298 P12611 INFO Feature specs: {
+    "cate_history": "{'source': '', 'type': 'sequence', 'feature_encoder': 'layers.MaskedAveragePooling()', 'share_embedding': 'cate_id', 'padding_idx': 0, 'oov_idx': 802, 'vocab_size': 803, 'max_len': 100}",
+    "cate_id": "{'source': '', 'type': 'categorical', 'padding_idx': 0, 'oov_idx': 802, 'vocab_size': 803}",
+    "item_history": "{'source': '', 'type': 'sequence', 'feature_encoder': 'layers.MaskedAveragePooling()', 'share_embedding': 'item_id', 'padding_idx': 0, 'oov_idx': 63002, 'vocab_size': 63003, 'max_len': 100}",
+    "item_id": "{'source': '', 'type': 'categorical', 'padding_idx': 0, 'oov_idx': 63002, 'vocab_size': 63003}",
+    "user_id": "{'type': 'meta'}"
+}
+2023-05-31 09:47:03,303 P12611 INFO Total number of parameters: 5771329.
+2023-05-31 09:47:03,304 P12611 INFO Loading data...
+2023-05-31 09:47:03,304 P12611 INFO Loading data from h5: ../data/Amazon/amazonelectronics_x1_b7a43f49/train.h5
+2023-05-31 09:47:06,616 P12611 INFO Train samples: total/2608764, blocks/1
+2023-05-31 09:47:06,617 P12611 INFO Loading data from h5: ../data/Amazon/amazonelectronics_x1_b7a43f49/valid.h5
+2023-05-31 09:47:07,029 P12611 INFO Validation samples: total/384806, blocks/1
+2023-05-31 09:47:07,029 P12611 INFO Loading train and validation data done.
+2023-05-31 09:47:07,029 P12611 INFO Start training: 2548 batches/epoch
+2023-05-31 09:47:07,029 P12611 INFO ************ Epoch=1 start ************
+2023-05-31 09:52:05,224 P12611 INFO Train loss: 0.637748
+2023-05-31 09:52:05,224 P12611 INFO Evaluation @epoch 1 - batch 2548: 
+2023-05-31 09:53:38,497 P12611 INFO [Metrics] AUC: 0.834987 - gAUC: 0.833734
+2023-05-31 09:53:38,498 P12611 INFO Save best model: monitor(max)=1.668721
+2023-05-31 09:53:38,600 P12611 INFO ************ Epoch=1 end ************
+2023-05-31 09:58:32,438 P12611 INFO Train loss: 0.595188
+2023-05-31 09:58:32,439 P12611 INFO Evaluation @epoch 2 - batch 2548: 
+2023-05-31 10:00:06,624 P12611 INFO [Metrics] AUC: 0.847464 - gAUC: 0.844956
+2023-05-31 10:00:06,627 P12611 INFO Save best model: monitor(max)=1.692420
+2023-05-31 10:00:06,797 P12611 INFO ************ Epoch=2 end ************
+2023-05-31 10:05:02,904 P12611 INFO Train loss: 0.578905
+2023-05-31 10:05:02,905 P12611 INFO Evaluation @epoch 3 - batch 2548: 
+2023-05-31 10:06:36,308 P12611 INFO [Metrics] AUC: 0.852364 - gAUC: 0.850231
+2023-05-31 10:06:36,309 P12611 INFO Save best model: monitor(max)=1.702595
+2023-05-31 10:06:36,509 P12611 INFO ************ Epoch=3 end ************
+2023-05-31 10:11:31,521 P12611 INFO Train loss: 0.573118
+2023-05-31 10:11:31,525 P12611 INFO Evaluation @epoch 4 - batch 2548: 
+2023-05-31 10:13:05,624 P12611 INFO [Metrics] AUC: 0.854765 - gAUC: 0.851972
+2023-05-31 10:13:05,625 P12611 INFO Save best model: monitor(max)=1.706737
+2023-05-31 10:13:05,741 P12611 INFO ************ Epoch=4 end ************
+2023-05-31 10:18:04,625 P12611 INFO Train loss: 0.569993
+2023-05-31 10:18:04,625 P12611 INFO Evaluation @epoch 5 - batch 2548: 
+2023-05-31 10:19:39,244 P12611 INFO [Metrics] AUC: 0.856304 - gAUC: 0.854181
+2023-05-31 10:19:39,245 P12611 INFO Save best model: monitor(max)=1.710485
+2023-05-31 10:19:39,372 P12611 INFO ************ Epoch=5 end ************
+2023-05-31 10:24:36,877 P12611 INFO Train loss: 0.568825
+2023-05-31 10:24:36,878 P12611 INFO Evaluation @epoch 6 - batch 2548: 
+2023-05-31 10:26:09,168 P12611 INFO [Metrics] AUC: 0.855471 - gAUC: 0.853173
+2023-05-31 10:26:09,248 P12611 INFO Monitor(max)=1.708644 STOP!
+2023-05-31 10:26:09,248 P12611 INFO Reduce learning rate on plateau: 0.000050
+2023-05-31 10:26:09,353 P12611 INFO ************ Epoch=6 end ************
+2023-05-31 10:30:59,981 P12611 INFO Train loss: 0.472181
+2023-05-31 10:30:59,981 P12611 INFO Evaluation @epoch 7 - batch 2548: 
+2023-05-31 10:32:35,254 P12611 INFO [Metrics] AUC: 0.876567 - gAUC: 0.873755
+2023-05-31 10:32:35,256 P12611 INFO Save best model: monitor(max)=1.750322
+2023-05-31 10:32:35,391 P12611 INFO ************ Epoch=7 end ************
+2023-05-31 10:37:11,148 P12611 INFO Train loss: 0.427074
+2023-05-31 10:37:11,149 P12611 INFO Evaluation @epoch 8 - batch 2548: 
+2023-05-31 10:38:38,473 P12611 INFO [Metrics] AUC: 0.880211 - gAUC: 0.877897
+2023-05-31 10:38:38,475 P12611 INFO Save best model: monitor(max)=1.758108
+2023-05-31 10:38:38,601 P12611 INFO ************ Epoch=8 end ************
+2023-05-31 10:43:10,243 P12611 INFO Train loss: 0.407486
+2023-05-31 10:43:10,243 P12611 INFO Evaluation @epoch 9 - batch 2548: 
+2023-05-31 10:44:33,860 P12611 INFO [Metrics] AUC: 0.882007 - gAUC: 0.879420
+2023-05-31 10:44:33,860 P12611 INFO Save best model: monitor(max)=1.761427
+2023-05-31 10:44:33,990 P12611 INFO ************ Epoch=9 end ************
+2023-05-31 10:48:39,854 P12611 INFO Train loss: 0.394435
+2023-05-31 10:48:39,855 P12611 INFO Evaluation @epoch 10 - batch 2548: 
+2023-05-31 10:49:55,253 P12611 INFO [Metrics] AUC: 0.880280 - gAUC: 0.877923
+2023-05-31 10:49:55,254 P12611 INFO Monitor(max)=1.758203 STOP!
+2023-05-31 10:49:55,254 P12611 INFO Reduce learning rate on plateau: 0.000005
+2023-05-31 10:49:55,331 P12611 INFO ************ Epoch=10 end ************
+2023-05-31 10:53:22,343 P12611 INFO Train loss: 0.332896
+2023-05-31 10:53:22,344 P12611 INFO Evaluation @epoch 11 - batch 2548: 
+2023-05-31 10:54:28,346 P12611 INFO [Metrics] AUC: 0.874604 - gAUC: 0.872393
+2023-05-31 10:54:28,347 P12611 INFO Monitor(max)=1.746997 STOP!
+2023-05-31 10:54:28,347 P12611 INFO Reduce learning rate on plateau: 0.000001
+2023-05-31 10:54:28,347 P12611 INFO ********* Epoch==11 early stop *********
+2023-05-31 10:54:28,415 P12611 INFO Training finished.
+2023-05-31 10:54:28,415 P12611 INFO Load best model: /cache/FuxiCTR/benchmark/checkpoints/APG_DCNv2_amazonelectronics_x1/amazonelectronics_x1_b7a43f49/APG_DCNv2_amazonelectronics_x1_015_a0cca3e4.model
+2023-05-31 10:54:28,452 P12611 INFO ****** Validation evaluation ******
+2023-05-31 10:55:32,159 P12611 INFO [Metrics] gAUC: 0.879420 - AUC: 0.882007 - logloss: 0.437314
+2023-05-31 10:55:32,252 P12611 INFO ******** Test evaluation ********
+2023-05-31 10:55:32,253 P12611 INFO Loading data...
+2023-05-31 10:55:32,253 P12611 INFO Loading data from h5: ../data/Amazon/amazonelectronics_x1_b7a43f49/test.h5
+2023-05-31 10:55:32,694 P12611 INFO Test samples: total/384806, blocks/1
+2023-05-31 10:55:32,694 P12611 INFO Loading test data done.
+2023-05-31 10:56:41,092 P12611 INFO [Metrics] gAUC: 0.879420 - AUC: 0.882007 - logloss: 0.437314