Add pre-trained version of NeuMF model (PreferredAI#218)

README.md

@@ -140,11 +140,11 @@ The recommender models supported by Cornac are listed below. Why don't you join
 |      | [Probabilistic Collaborative Representation Learning (PCRL)](cornac/models/pcrl), [paper](http://www.hadylauw.com/publications/uai18.pdf) | [requirements.txt](cornac/models/pcrl/requirements.txt) | [pcrl_exp.py](examples/pcrl_example.py)
 |      | [Variational Autoencoder for Collaborative Filtering (VAECF)](cornac/models/vaecf), [paper](https://arxiv.org/pdf/1802.05814.pdf) | [requirements.txt](cornac/models/vaecf/requirements.txt) |
 | 2017 | [Collaborative Variational Autoencoder (CVAE)](cornac/models/cvae), [paper](http://eelxpeng.github.io/assets/paper/Collaborative_Variational_Autoencoder.pdf) | [requirements.txt](cornac/models/cvae/requirements.txt) | [cvae_exp.py](examples/cvae_example.py)
-|      | [Generalized Matrix Factorization (GMF)](cornac/models/neu_cf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/neu_cf/requirements.txt) | [neucf_exp.py](examples/neucf_example.py)
+|      | [Generalized Matrix Factorization (GMF)](cornac/models/ncf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/ncf/requirements.txt) | [ncf_exp.py](examples/ncf_example.py)
 |      | [Indexable Bayesian Personalized Ranking (IBPR)](cornac/models/ibpr), [paper](http://www.hadylauw.com/publications/cikm17a.pdf) | [requirements.txt](cornac/models/ibpr/requirements.txt) | [ibpr_exp.py](examples/ibpr_example.py)
 |      | [Matrix Co-Factorization (MCF)](cornac/models/mcf), [paper](http://papers.www2017.com.au.s3-website-ap-southeast-2.amazonaws.com/proceedings/p1113.pdf) | N/A | [mcf_office.py](examples/mcf_office.py)
-|      | [Multi-Layer Perceptron (MLP)](cornac/models/neu_cf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/neu_cf/requirements.txt) | [neucf_exp.py](examples/neucf_example.py)
-|      | [Neural Matrix Factorization (NeuMF)](cornac/models/neu_cf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/neu_cf/requirements.txt) | [neucf_exp.py](examples/neucf_example.py)
+|      | [Multi-Layer Perceptron (MLP)](cornac/models/ncf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/ncf/requirements.txt) | [ncf_exp.py](examples/ncf_example.py)
+|      | [Neural Matrix Factorization (NeuMF) / Neural Collaborative Filtering (NCF)](cornac/models/ncf), [paper](https://arxiv.org/pdf/1708.05031.pdf) | [requirements.txt](cornac/models/ncf/requirements.txt) | [ncf_exp.py](examples/ncf_example.py)
 |      | [Online Indexable Bayesian Personalized Ranking (Online IBPR)](cornac/models/online_ibpr), [paper](http://www.hadylauw.com/publications/cikm17a.pdf) | [requirements.txt](cornac/models/online_ibpr/requirements.txt) |
 |      | [Visual Matrix Factorization (VMF)](cornac/models/vmf), [paper](http://papers.www2017.com.au.s3-website-ap-southeast-2.amazonaws.com/proceedings/p1113.pdf) | [requirements.txt](cornac/models/vmf/requirements.txt) |
 | 2016 | [Collaborative Deep Ranking (CDR)](cornac/models/cdr), [paper](http://inpluslab.com/chenliang/homepagefiles/paper/hao-pakdd2016.pdf) | [requirements.txt](cornac/models/cdr/requirements.txt) | [cdr_exp.py](examples/cdr_example.py)

cornac/models/__init__.py

@@ -39,6 +39,6 @@
 from .vaecf import VAECF
 from .ctr import CTR
 from .hft import HFT
-from .neu_cf import GMF
-from .neu_cf import MLP
-from .neu_cf import NeuMF
+from .ncf import GMF
+from .ncf import MLP
+from .ncf import NeuMF

cornac/models/neu_cf/ops.py → cornac/models/ncf/ops.py

@@ -28,7 +28,7 @@
 
 
 def loss_fn(labels, logits):
-    cross_entropy = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits))
+    cross_entropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits))
     reg_loss = tf.losses.get_regularization_loss()
     return cross_entropy + reg_loss
 
@@ -74,7 +74,7 @@ def mlp(uid, iid, num_users, num_items, layers, reg_layers, act_fn, seed=None):
                                  seed=seed, scope=scope)
         interaction = tf.concat([user_emb, item_emb], axis=-1)
         for i, layer in enumerate(layers[1:]):
-            interaction = tf.layers.dense(interaction, units=layer, name='layer{}'.format(i),
+            interaction = tf.layers.dense(interaction, units=layer, name='layer{}'.format(i + 1),
                                           activation=act_functions.get(act_fn, tf.nn.relu),
                                           kernel_initializer=tf.initializers.lecun_uniform(seed),
                                           kernel_regularizer=tf.contrib.layers.l2_regularizer(reg_layers[i + 1]))

cornac/models/neu_cf/recom_gmf.py → cornac/models/ncf/recom_gmf.py

@@ -91,19 +91,20 @@ def fit(self, train_set):
         Recommender.fit(self, train_set)
 
         if self.trainable:
-            self._fit_neumf()
+            self._fit_gmf()
 
-    def _fit_neumf(self):
+        return self
+
+    def _fit_gmf(self):
         import os
         import tensorflow as tf
         from tqdm import trange
+        from .ops import gmf, loss_fn, train_fn
 
         np.random.seed(self.seed)
         os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
         tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 
-        from .ops import gmf, mlp, loss_fn, train_fn
-
         graph = tf.Graph()
         with graph.as_default():
             tf.set_random_seed(self.seed)
@@ -112,11 +113,11 @@ def _fit_neumf(self):
             self.item_id = tf.placeholder(shape=[None, ], dtype=tf.int32, name='item_id')
             self.labels = tf.placeholder(shape=[None, 1], dtype=tf.float32, name='labels')
 
-            interaction = gmf(uid=self.user_id, iid=self.item_id, num_users=self.train_set.num_users,
-                              num_items=self.train_set.num_items, emb_size=self.num_factors,
-                              reg_user=self.regs[0], reg_item=self.regs[1], seed=self.seed)
+            self.interaction = gmf(uid=self.user_id, iid=self.item_id, num_users=self.train_set.num_users,
+                                   num_items=self.train_set.num_items, emb_size=self.num_factors,
+                                   reg_user=self.regs[0], reg_item=self.regs[1], seed=self.seed)
 
-            logits = tf.layers.dense(interaction, units=1, name='logits',
+            logits = tf.layers.dense(self.interaction, units=1, name='logits',
                                      kernel_initializer=tf.initializers.lecun_uniform(self.seed))
             self.prediction = tf.nn.sigmoid(logits)
 
@@ -144,7 +145,7 @@ def _fit_neumf(self):
                                          })
 
                 count += len(batch_ratings)
-                sum_loss += _loss
+                sum_loss += _loss * len(batch_ratings)
                 if i % 10 == 0:
                     loop.set_postfix(loss=(sum_loss / count))
 

cornac/models/neu_cf/recom_mlp.py → cornac/models/ncf/recom_mlp.py

@@ -101,17 +101,18 @@ def fit(self, train_set):
         if self.trainable:
             self._fit_neumf()
 
+        return self
+
     def _fit_neumf(self):
         import os
         import tensorflow as tf
         from tqdm import trange
+        from .ops import mlp, loss_fn, train_fn
 
         np.random.seed(self.seed)
         os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
         tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 
-        from .ops import gmf, mlp, loss_fn, train_fn
-
         graph = tf.Graph()
         with graph.as_default():
             tf.set_random_seed(self.seed)
@@ -120,10 +121,10 @@ def _fit_neumf(self):
             self.item_id = tf.placeholder(shape=[None, ], dtype=tf.int32, name='item_id')
             self.labels = tf.placeholder(shape=[None, 1], dtype=tf.float32, name='labels')
 
-            interaction = mlp(uid=self.user_id, iid=self.item_id, num_users=self.train_set.num_users,
-                              num_items=self.train_set.num_items, layers=self.layers,
-                              reg_layers=self.reg_layers, act_fn=self.act_fn, seed=self.seed)
-            logits = tf.layers.dense(interaction, units=1, name='logits',
+            self.interaction = mlp(uid=self.user_id, iid=self.item_id, num_users=self.train_set.num_users,
+                                   num_items=self.train_set.num_items, layers=self.layers,
+                                   reg_layers=self.reg_layers, act_fn=self.act_fn, seed=self.seed)
+            logits = tf.layers.dense(self.interaction, units=1, name='logits',
                                      kernel_initializer=tf.initializers.lecun_uniform(self.seed))
             self.prediction = tf.nn.sigmoid(logits)
 
@@ -151,7 +152,7 @@ def _fit_neumf(self):
                                          })
 
                 count += len(batch_ratings)
-                sum_loss += _loss
+                sum_loss += _loss * len(batch_ratings)
                 if i % 10 == 0:
                     loop.set_postfix(loss=(sum_loss / count))
 

cornac/models/neu_cf/recom_neumf.py → cornac/models/ncf/recom_neumf.py

@@ -94,6 +94,28 @@ def __init__(self, name='NeuMF',
         self.learning_rate = lr
         self.learner = learner
         self.seed = seed
+        self.pretrained = False
+
+    def pretrain(self, gmf_model, mlp_model, alpha=0.5):
+        """Provide pre-trained GMF and MLP models. Section 3.4.1 of the paper.
+
+        Parameters
+        ----------
+        gmf_model: object of type GMF, required
+            Reference to trained/fitted GMF model.
+
+        gmf_model: object of type GMF, required
+            Reference to trained/fitted GMF model.
+
+        alpha: float, optional, default: 0.5
+            Hyper-parameter determining the trade-off between the two pre-trained models.
+            Details are described in the section 3.4.1 of the paper.
+        """
+        self.pretrained = True
+        self.gmf_model = gmf_model
+        self.mlp_model = mlp_model
+        self.alpha = alpha
+        return self
 
     def fit(self, train_set):
         """Fit the model to observations.
@@ -110,17 +132,18 @@ def fit(self, train_set):
         if self.trainable:
             self._fit_neumf()
 
+        return self
+
     def _fit_neumf(self):
         import os
         import tensorflow as tf
         from tqdm import trange
+        from .ops import gmf, mlp, loss_fn, train_fn
 
         np.random.seed(self.seed)
         os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
         tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 
-        from .ops import gmf, mlp, loss_fn, train_fn
-
         graph = tf.Graph()
         with graph.as_default():
             tf.set_random_seed(self.seed)
@@ -152,22 +175,40 @@ def _fit_neumf(self):
         self.sess = tf.Session(graph=graph, config=config)
         self.sess.run(initializer)
 
+        if self.pretrained:
+            gmf_kernel = self.gmf_model.sess.run(self.gmf_model.sess.graph.get_tensor_by_name('logits/kernel:0'))
+            gmf_bias = self.gmf_model.sess.run(self.gmf_model.sess.graph.get_tensor_by_name('logits/bias:0'))
+            mlp_kernel = self.mlp_model.sess.run(self.mlp_model.sess.graph.get_tensor_by_name('logits/kernel:0'))
+            mlp_bias = self.mlp_model.sess.run(self.mlp_model.sess.graph.get_tensor_by_name('logits/bias:0'))
+            logits_kernel = np.concatenate([self.alpha * gmf_kernel, (1 - self.alpha) * mlp_kernel])
+            logits_bias = self.alpha * gmf_bias + (1 - self.alpha) * mlp_bias
+
+            for v in graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
+                if v.name.startswith('GMF'):
+                    sess = self.gmf_model.sess
+                    self.sess.run(tf.assign(v, sess.run(sess.graph.get_tensor_by_name(v.name))))
+                elif v.name.startswith('MLP'):
+                    sess = self.mlp_model.sess
+                    self.sess.run(tf.assign(v, sess.run(sess.graph.get_tensor_by_name(v.name))))
+                elif v.name.startswith('logits/kernel'):
+                    self.sess.run(tf.assign(v, logits_kernel))
+                elif v.name.startswith('logits/bias'):
+                    self.sess.run(tf.assign(v, logits_bias))
+
         loop = trange(self.num_epochs, disable=not self.verbose)
         for _ in loop:
             count = 0
             sum_loss = 0
             for i, (batch_users, batch_items, batch_ratings) in enumerate(
                     self.train_set.uir_iter(self.batch_size, shuffle=True, num_zeros=self.num_neg)):
-                _, _loss = self.sess.run([train_op, loss],
-                                         feed_dict={
-                                             self.gmf_user_id: batch_users,
-                                             self.mlp_user_id: batch_users,
-                                             self.item_id: batch_items,
-                                             self.labels: batch_ratings.reshape(-1, 1)
-                                         })
-
+                _, _loss = self.sess.run([train_op, loss], feed_dict={
+                    self.gmf_user_id: batch_users,
+                    self.mlp_user_id: batch_users,
+                    self.item_id: batch_items,
+                    self.labels: batch_ratings.reshape(-1, 1)
+                })
                 count += len(batch_ratings)
-                sum_loss += _loss
+                sum_loss += _loss * len(batch_ratings)
                 if i % 10 == 0:
                     loop.set_postfix(loss=(sum_loss / count))
 

docs/source/models.rst

@@ -27,7 +27,7 @@ Collaborative Variational Autoencoder (CVAE)
 
 Generalized Matrix Factorization (NeuMF)
 -----------------------------------------
-.. automodule:: cornac.models.neu_cf.recom_gmf
+.. automodule:: cornac.models.ncf.recom_gmf
    :members:
 
 Indexable Bayesian Personalized Ranking (IBPR)
@@ -42,12 +42,12 @@ Matrix Co-Factorization (MCF)
 
 Multi-Layer Perceptron (MLP)
 -----------------------------------------
-.. automodule:: cornac.models.neu_cf.recom_mlp
+.. automodule:: cornac.models.ncf.recom_mlp
    :members:
 
 Neural Matrix Factorization (NeuMF)
 -----------------------------------------
-.. automodule:: cornac.models.neu_cf.recom_neumf
+.. automodule:: cornac.models.ncf.recom_neumf
    :members:
 
 Online Indexable Bayesian Personalized Ranking (OIBPR)

examples/neucf_example.py → examples/ncf_example.py

@@ -19,20 +19,23 @@
 from cornac.datasets import amazon_clothing
 from cornac.data import Reader
 
-ratio_split = RatioSplit(data=amazon_clothing.load_rating(Reader(bin_threshold=1.0)),
+ratio_split = RatioSplit(data=amazon_clothing.load_rating(reader=Reader(bin_threshold=1.0)),
                          test_size=0.2, rating_threshold=1.0, seed=123,
                          exclude_unknowns=True, verbose=True)
 
-gmf = cornac.models.GMF(num_factors=8, num_epochs=20, learner='adam',
-                        batch_size=256, lr=0.001, num_neg=10, seed=123)
+gmf = cornac.models.GMF(num_factors=8, num_epochs=10, learner='adam',
+                        batch_size=256, lr=0.001, num_neg=50, seed=123)
 mlp = cornac.models.MLP(layers=[64, 32, 16, 8], act_fn='tanh', learner='adam',
-                        num_epochs=20, batch_size=256, lr=0.001, num_neg=10, seed=123)
-neumf = cornac.models.NeuMF(num_factors=8, layers=[64, 32, 16, 8], act_fn='tanh', learner='adam',
-                            num_epochs=20, batch_size=256, lr=0.001, num_neg=10, seed=123)
+                        num_epochs=10, batch_size=256, lr=0.001, num_neg=50, seed=123)
+neumf1 = cornac.models.NeuMF(num_factors=8, layers=[64, 32, 16, 8], act_fn='tanh', learner='adam',
+                             num_epochs=10, batch_size=256, lr=0.001, num_neg=50, seed=123)
+neumf2 = cornac.models.NeuMF(name='NeuMF_pretrained', learner='adam',
+                             num_epochs=10, batch_size=256, lr=0.001, num_neg=50, seed=123,
+                             num_factors=gmf.num_factors, layers=mlp.layers, act_fn=mlp.act_fn).pretrain(gmf, mlp)
 
 ndcg_50 = cornac.metrics.NDCG(k=50)
 rec_50 = cornac.metrics.Recall(k=50)
 
 cornac.Experiment(eval_method=ratio_split,
-                  models=[gmf, mlp, neumf],
+                  models=[gmf, mlp, neumf1, neumf2],
                   metrics=[ndcg_50, rec_50]).run()