Skip to content

Commit

Permalink
Updata docs&minor changes
Browse files Browse the repository at this point in the history 
1. Modified some default parameters
2. Remove the dropout before first dense layer in MLP
  • Loading branch information
@shenweichen
shenweichen committed Dec 4, 2018
1 parent 3da641f commit 34e8fa5
Show file tree
Hide file tree
Showing 14 changed files with 118 additions and 106 deletions.
4 changes: 2 additions & 2 deletions deepctr/layers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ class CrossNet(Layer):
References
- [Deep & Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123)
"""
def __init__(self, layer_num=1,l2_reg=0,seed=1024, **kwargs):
def __init__(self, layer_num=2,l2_reg=0,seed=1024, **kwargs):
self.layer_num = layer_num
self.l2_reg = l2_reg
self.seed = seed
Expand Down Expand Up @@ -287,7 +287,7 @@ def build(self, input_shape):

def call(self, inputs,**kwargs):
deep_input = inputs
deep_input = Dropout(1 - self.keep_prob)(deep_input)
#deep_input = Dropout(1 - self.keep_prob)(deep_input)

for l in range(len(self.hidden_size)):
fc = Dense(self.hidden_size[l], activation=None, \
Expand Down
65 changes: 35 additions & 30 deletions deepctr/models/afm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@
"""

from tensorflow.python.keras.layers import Dense, Embedding, Concatenate, Reshape, add
from tensorflow.python.keras.layers import Dense, Embedding, Concatenate, Reshape, add
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.initializers import RandomNormal
from tensorflow.python.keras.regularizers import l2

from ..utils import get_input
from ..layers import PredictionLayer,AFMLayer,FM
from ..layers import PredictionLayer, AFMLayer, FM


def AFM(feature_dim_dict, embedding_size=8,use_attention=True, attention_factor=4,
l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_att=1e-5,keep_prob=1.0,init_std=0.0001,seed=1024,
final_activation='sigmoid',):
def AFM(feature_dim_dict, embedding_size=8, use_attention=True, attention_factor=8,
l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_att=1e-5, keep_prob=1.0, init_std=0.0001, seed=1024,
final_activation='sigmoid',):
"""Instantiates the Attentonal Factorization Machine architecture.
:param feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
Expand All @@ -42,22 +42,24 @@ def AFM(feature_dim_dict, embedding_size=8,use_attention=True, attention_factor=
dict) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim_dict must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}")
if not isinstance(feature_dim_dict["sparse"],dict):
raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is",type(feature_dim_dict['sparse']))
if not isinstance(feature_dim_dict["dense"],list):
raise ValueError("feature_dim_dict['dense'] must be a list,cur is", type(feature_dim_dict['dense']))




sparse_input, dense_input = get_input(feature_dim_dict,None)
sparse_embedding, linear_embedding, = get_embeddings(feature_dim_dict,embedding_size,init_std,seed,l2_reg_embedding,l2_reg_linear)

embed_list = [sparse_embedding[i](sparse_input[i]) for i in range(len(sparse_input))]
linear_term = [linear_embedding[i](sparse_input[i]) for i in range(len(sparse_input))]
if not isinstance(feature_dim_dict["sparse"], dict):
raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is", type(
feature_dim_dict['sparse']))
if not isinstance(feature_dim_dict["dense"], list):
raise ValueError("feature_dim_dict['dense'] must be a list,cur is", type(
feature_dim_dict['dense']))

sparse_input, dense_input = get_input(feature_dim_dict, None)
sparse_embedding, linear_embedding, = get_embeddings(
feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding, l2_reg_linear)

embed_list = [sparse_embedding[i](sparse_input[i])
for i in range(len(sparse_input))]
linear_term = [linear_embedding[i](sparse_input[i])
for i in range(len(sparse_input))]
if len(linear_term) > 1:
linear_term = add(linear_term)
elif len(linear_term) >0:
elif len(linear_term) > 0:
linear_term = linear_term[0]
else:
linear_term = 0
Expand All @@ -66,37 +68,40 @@ def AFM(feature_dim_dict, embedding_size=8,use_attention=True, attention_factor=
continuous_embedding_list = list(
map(Dense(embedding_size, use_bias=False, kernel_regularizer=l2(l2_reg_embedding), ),
dense_input))
continuous_embedding_list = list(map(Reshape((1, embedding_size)), continuous_embedding_list))
continuous_embedding_list = list(
map(Reshape((1, embedding_size)), continuous_embedding_list))
embed_list += continuous_embedding_list

dense_input_ = dense_input[0] if len(dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(1,activation=None,use_bias=False,kernel_regularizer=l2(l2_reg_linear))(dense_input_)
linear_term = add([linear_dense_logit,linear_term])
dense_input_ = dense_input[0] if len(
dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(
1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg_linear))(dense_input_)
linear_term = add([linear_dense_logit, linear_term])

fm_input = Concatenate(axis=1)(embed_list)
if use_attention:
fm_out = AFMLayer(attention_factor,l2_reg_att,keep_prob,seed)(embed_list)
fm_out = AFMLayer(attention_factor, l2_reg_att,
keep_prob, seed)(embed_list)
else:
fm_out = FM()(fm_input)

final_logit = add([linear_term,fm_out])
final_logit = add([linear_term, fm_out])
output = PredictionLayer(final_activation)(final_logit)
model = Model(inputs=sparse_input + dense_input , outputs=output)
model = Model(inputs=sparse_input + dense_input, outputs=output)
return model


def get_embeddings(feature_dim_dict, embedding_size, init_std, seed, l2_rev_V, l2_reg_w):
sparse_embedding = [Embedding(feature_dim_dict["sparse"][feat], embedding_size,
embeddings_initializer=RandomNormal(mean=0.0, stddev=init_std, seed=seed),
embeddings_initializer=RandomNormal(
mean=0.0, stddev=init_std, seed=seed),
embeddings_regularizer=l2(l2_rev_V),
name='sparse_emb_' + str(i) + '-' + feat) for i, feat in
enumerate(feature_dim_dict["sparse"])]
linear_embedding = [Embedding(feature_dim_dict["sparse"][feat], 1,
embeddings_initializer=RandomNormal(mean=0.0, stddev=init_std,
seed=seed)
, embeddings_regularizer=l2(l2_reg_w),
seed=seed), embeddings_regularizer=l2(l2_reg_w),
name='linear_emb_' + str(i) + '-' + feat) for
i, feat in enumerate(feature_dim_dict["sparse"])]

return sparse_embedding, linear_embedding

8 changes: 4 additions & 4 deletions deepctr/models/dcn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@


def DCN(feature_dim_dict, embedding_size='auto',
cross_num=2, hidden_size=(32,), l2_reg_embedding=1e-5, l2_reg_cross=0, l2_reg_deep=0,
init_std=0.0001, seed=1024, keep_prob=1, use_bn=True, activation='relu', final_activation='sigmoid',
cross_num=2, hidden_size=[128, 128, ], l2_reg_embedding=1e-5, l2_reg_cross=1e-5, l2_reg_deep=0,
init_std=0.0001, seed=1024, keep_prob=1, use_bn=False, activation='relu', final_activation='sigmoid',
):
"""Instantiates the Deep&Cross Network architecture.
Expand Down Expand Up @@ -90,8 +90,8 @@ def get_embeddings(feature_dim_dict, embedding_size, init_std, seed, l2_rev_V):
embeddings_regularizer=l2(l2_rev_V), name='sparse_emb_' + str(i) + '-'+feat) for i, feat in
enumerate(feature_dim_dict["sparse"])]

print("DCN total auto embed size", sum(
[int(pow(feature_dim_dict["sparse"][k], 0.25)) for k, v in feature_dim_dict["sparse"].items()]))
print("Using auto embedding size,the connected vector dimension is", sum(
[6*int(pow(feature_dim_dict["sparse"][k], 0.25)) for k, v in feature_dim_dict["sparse"].items()]))
else:
sparse_embedding = [Embedding(feature_dim_dict["sparse"][feat], embedding_size,
embeddings_initializer=RandomNormal(
Expand Down
60 changes: 32 additions & 28 deletions deepctr/models/deepfm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,19 @@
"""

from tensorflow.python.keras.layers import Dense, Embedding, Concatenate, Reshape, Flatten, add
from tensorflow.python.keras.layers import Dense, Embedding, Concatenate, Reshape, Flatten, add
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.initializers import RandomNormal
from tensorflow.python.keras.regularizers import l2


from ..utils import get_input
from ..layers import PredictionLayer,MLP,FM
from ..layers import PredictionLayer, MLP, FM


def DeepFM(feature_dim_dict, embedding_size=8,
use_fm=True, hidden_size=[256,256], l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_deep=0,
init_std=0.0001, seed=1024, keep_prob=0.5, activation='relu', final_activation='sigmoid', use_bn=False):
use_fm=True, hidden_size=[128, 128], l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_deep=0,
init_std=0.0001, seed=1024, keep_prob=1, activation='relu', final_activation='sigmoid', use_bn=False):
"""Instantiates the DeepFM Network architecture.
:param feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
Expand All @@ -42,22 +42,24 @@ def DeepFM(feature_dim_dict, embedding_size=8,
dict) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim_dict must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}")
if not isinstance(feature_dim_dict["sparse"],dict):
raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is",type(feature_dim_dict['sparse']))
if not isinstance(feature_dim_dict["dense"],list):
raise ValueError("feature_dim_dict['dense'] must be a list,cur is", type(feature_dim_dict['dense']))




sparse_input, dense_input = get_input(feature_dim_dict,None)
sparse_embedding, linear_embedding, = get_embeddings(feature_dim_dict,embedding_size,init_std,seed,l2_reg_deep,l2_reg_linear)

embed_list = [sparse_embedding[i](sparse_input[i]) for i in range(len(sparse_input))]
linear_term = [linear_embedding[i](sparse_input[i]) for i in range(len(sparse_input))]
if not isinstance(feature_dim_dict["sparse"], dict):
raise ValueError("feature_dim_dict['sparse'] must be a dict,cur is", type(
feature_dim_dict['sparse']))
if not isinstance(feature_dim_dict["dense"], list):
raise ValueError("feature_dim_dict['dense'] must be a list,cur is", type(
feature_dim_dict['dense']))

sparse_input, dense_input = get_input(feature_dim_dict, None)
sparse_embedding, linear_embedding, = get_embeddings(
feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding, l2_reg_linear)

embed_list = [sparse_embedding[i](sparse_input[i])
for i in range(len(sparse_input))]
linear_term = [linear_embedding[i](sparse_input[i])
for i in range(len(sparse_input))]
if len(linear_term) > 1:
linear_term = add(linear_term)
elif len(linear_term) >0:
elif len(linear_term) > 0:
linear_term = linear_term[0]
else:
linear_term = 0
Expand All @@ -66,19 +68,22 @@ def DeepFM(feature_dim_dict, embedding_size=8,
continuous_embedding_list = list(
map(Dense(embedding_size, use_bias=False, kernel_regularizer=l2(l2_reg_embedding), ),
dense_input))
continuous_embedding_list = list(map(Reshape((1, embedding_size)), continuous_embedding_list))
continuous_embedding_list = list(
map(Reshape((1, embedding_size)), continuous_embedding_list))
embed_list += continuous_embedding_list

dense_input_ = dense_input[0] if len(dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(1,activation=None,use_bias=False,kernel_regularizer=l2(l2_reg_linear))(dense_input_)
linear_term = add([linear_dense_logit,linear_term])
dense_input_ = dense_input[0] if len(
dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(
1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg_linear))(dense_input_)
linear_term = add([linear_dense_logit, linear_term])

fm_input = Concatenate(axis=1)(embed_list)
deep_input = Flatten()(fm_input)
fm_out = FM()(fm_input)
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob,
use_bn, seed)(deep_input)
deep_logit = Dense(1,use_bias=False, activation=None)(deep_out)
deep_logit = Dense(1, use_bias=False, activation=None)(deep_out)

if len(hidden_size) == 0 and use_fm == False: # only linear
final_logit = linear_term
Expand All @@ -92,22 +97,21 @@ def DeepFM(feature_dim_dict, embedding_size=8,
raise NotImplementedError

output = PredictionLayer(final_activation)(final_logit)
model = Model(inputs=sparse_input + dense_input , outputs=output)
model = Model(inputs=sparse_input + dense_input, outputs=output)
return model


def get_embeddings(feature_dim_dict, embedding_size, init_std, seed, l2_rev_V, l2_reg_w):
sparse_embedding = [Embedding(feature_dim_dict["sparse"][feat], embedding_size,
embeddings_initializer=RandomNormal(mean=0.0, stddev=init_std, seed=seed),
embeddings_initializer=RandomNormal(
mean=0.0, stddev=init_std, seed=seed),
embeddings_regularizer=l2(l2_rev_V),
name='sparse_emb_' + str(i) + '-' + feat) for i, feat in
enumerate(feature_dim_dict["sparse"])]
linear_embedding = [Embedding(feature_dim_dict["sparse"][feat], 1,
embeddings_initializer=RandomNormal(mean=0.0, stddev=init_std,
seed=seed)
, embeddings_regularizer=l2(l2_reg_w),
seed=seed), embeddings_regularizer=l2(l2_reg_w),
name='linear_emb_' + str(i) + '-' + feat) for
i, feat in enumerate(feature_dim_dict["sparse"])]

return sparse_embedding, linear_embedding

6 changes: 3 additions & 3 deletions deepctr/models/din.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ def get_input(feature_dim_dict, seq_feature_list, seq_max_len):
return sparse_input, user_behavior_input, user_behavior_length


def DIN(feature_dim_dict, seq_feature_list, embedding_size=4, hist_len_max=16,
use_din=True, use_bn=False, hidden_size=[200, 80], activation=Dice, att_hidden_size=[80, 40], att_activation='sigmoid', att_weight_normalization=True,
l2_reg_deep=5e-5, l2_reg_embedding=0, final_activation='sigmoid', keep_prob=1, init_std=0.0001, seed=1024, ):
def DIN(feature_dim_dict, seq_feature_list, embedding_size=8, hist_len_max=16,
use_din=True, use_bn=False, hidden_size=[200, 80], activation='relu', att_hidden_size=[80, 40], att_activation='sigmoid', att_weight_normalization=True,
l2_reg_deep=0, l2_reg_embedding=1e-5, final_activation='sigmoid', keep_prob=1, init_std=0.0001, seed=1024, ):
"""Instantiates the Deep Interest Network architecture.
:param feature_dim_dict: dict,to indicate sparse field (**now only support sparse feature**)like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':[]}
Expand Down
Loading

0 comments on commit 34e8fa5

Please sign in to comment.