From 444ce74bad186aa73a94befe3d241068fa509760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=B5=A9=E5=A4=A9?= Date: Mon, 29 Jul 2019 13:29:19 +0800 Subject: [PATCH] New model NFM --- README.md | 1 + core/blocks.py | 11 +++++--- core/features.py | 4 +-- models/DCN.py | 2 +- models/NFM.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 models/NFM.py diff --git a/README.md b/README.md index 7f4cd88..79f12c8 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,4 @@ | DeepFM | [IJCAI 2017][DeepFM: A Factorization-Machine based Neural Network for CTR Prediction](http://www.ijcai.org/proceedings/2017/0239.pdf)| | Deep & Cross Network (DCN) | [ADKDD 2017][Deep & Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123) | | xDeepFM | [KDD 2018][xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems](https://arxiv.org/pdf/1803.05170.pdf) | +| Neural Factorization Machine (NFM) | [SIGIR 2017][Neural Factorization Machines for Sparse Predictive Analytics](https://arxiv.org/pdf/1708.05027.pdf) | diff --git a/core/blocks.py b/core/blocks.py index cf3a5c0..f1e3fb2 100644 --- a/core/blocks.py +++ b/core/blocks.py @@ -120,22 +120,27 @@ def __init__(self, **kwargs): super(FM, self).__init__(**kwargs) - def call(self, inputs, **kwargs): + def call(self, inputs, require_logit=True, **kwargs): """ :param inputs: - list of 2D tensor with shape [batch_size, number_of_features, embedding_size] + list of 2D tensor with shape [batch_size, embedding_size] all the features should be embedded and have the same embedding size :return: 2D tensor with shape [batch_size, 1] sum of all cross features """ + # [b, n, m] inputs_3d = tf.stack(inputs, axis=1) + # [b, m] # (a + b) ^ 2 - (a ^ 2 + b ^ 2) = 2 * ab, we need the cross feature "ab" square_of_sum = tf.square(tf.reduce_sum(inputs_3d, axis=1, keepdims=False)) sum_of_square = tf.reduce_sum(tf.square(inputs_3d), axis=1, keepdims=False) - outputs = 0.5 * tf.reduce_sum(square_of_sum - sum_of_square, axis=1, keepdims=True) + if require_logit: + outputs = 0.5 * tf.reduce_sum(square_of_sum - sum_of_square, axis=1, keepdims=True) + else: + outputs = 0.5 * (square_of_sum - sum_of_square) return outputs diff --git a/core/features.py b/core/features.py index 17ddaea..c90a5e1 100644 --- a/core/features.py +++ b/core/features.py @@ -354,6 +354,8 @@ def get_embedded_dict(self, slots_filter = list(filter(lambda slot_name: slot_name in self.sparse_feats_slots, slots_filter)) + embedded_dict = dict() + if fixed_embedding_dim is not None: group_name = group_name + '_' + str(fixed_embedding_dim) @@ -362,8 +364,6 @@ def get_embedded_dict(self, self.embedded_groups[group_name] = dict() group = self.embedded_groups[group_name] - embedded_dict = dict() - # if the slot is not in this group, make a new embedding for it. for slot_name in slots_filter: if slot_name not in group: diff --git a/models/DCN.py b/models/DCN.py index 9a052f0..5cd909f 100644 --- a/models/DCN.py +++ b/models/DCN.py @@ -23,7 +23,7 @@ def DCN( dnn_bias_initializers='zeros', dnn_kernel_regularizers=tf.keras.regularizers.l2(1e-5), dnn_bias_regularizers=None, - name='Deep&Cross Network'): + name='Deep&CrossNetwork'): assert isinstance(feature_metas, FeatureMetas) diff --git a/models/NFM.py b/models/NFM.py new file mode 100644 index 0000000..c9f98d2 --- /dev/null +++ b/models/NFM.py @@ -0,0 +1,70 @@ +import tensorflow as tf + +from core.features import FeatureMetas, Features, group_embedded_by_dim +from core.blocks import DNN, FM + + +def NFM( + feature_metas, + linear_slots, + fm_slots, + embedding_initializer='glorot_uniform', + embedding_regularizer=tf.keras.regularizers.l2(1e-5), + fm_fixed_embedding_dim=None, + linear_use_bias=True, + linear_kernel_initializer='glorot_uniform', + linear_kernel_regularizer=None, + dnn_hidden_units=(128, 64, 1), + dnn_activations=('relu', 'relu', None), + dnn_use_bias=True, + dnn_use_bn=False, + dnn_dropout=0, + dnn_kernel_initializers='glorot_uniform', + dnn_bias_initializers='zeros', + dnn_kernel_regularizers=tf.keras.regularizers.l2(1e-5), + dnn_bias_regularizers=None, + name='NFM'): + + assert isinstance(feature_metas, FeatureMetas) + + with tf.name_scope(name): + + features = Features(metas=feature_metas) + + # Linear Part + with tf.name_scope('Linear'): + linear_output = features.get_linear_logit(use_bias=linear_use_bias, + kernel_initializer=linear_kernel_initializer, + kernel_regularizer=linear_kernel_regularizer, + embedding_group='dot_embedding', + slots_filter=linear_slots) + + # Bi-interaction + with tf.name_scope('BiInteraction'): + fm_embedded_dict = features.get_embedded_dict(group_name='embedding', + fixed_embedding_dim=fm_fixed_embedding_dim, + embedding_initializer=embedding_initializer, + embedding_regularizer=embedding_regularizer, + slots_filter=fm_slots) + fm_dim_groups = group_embedded_by_dim(fm_embedded_dict) + fms = [FM()(group, require_logit=False) for group in fm_dim_groups.values() if len(group) > 1] + dnn_inputs = tf.concat(fms, axis=1) + dnn_output = DNN( + units=dnn_hidden_units, + use_bias=dnn_use_bias, + activations=dnn_activations, + use_bn=dnn_use_bn, + dropout=dnn_dropout, + kernel_initializers=dnn_kernel_initializers, + bias_initializers=dnn_bias_initializers, + kernel_regularizers=dnn_kernel_regularizers, + bias_regularizers=dnn_bias_regularizers + )(dnn_inputs) + + # Output + output = tf.add_n([linear_output, dnn_output]) + output = tf.keras.activations.sigmoid(output) + + model = tf.keras.Model(inputs=features.get_inputs_list(), outputs=output) + + return model