diff --git a/README.md b/README.md index 626f3c4c9..7ecc17ae8 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ The recommender models supported by Cornac are listed below. Why don't you join | | [Hybrid neural recommendation with joint deep representation learning of ratings and reviews (HRDR)](cornac/models/hrdr), [paper](https://www.sciencedirect.com/science/article/abs/pii/S0925231219313207) | [requirements.txt](cornac/models/hrdr/requirements.txt) | [hrdr_example.py](examples/hrdr_example.py) | | [LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation](cornac/models/lightgcn), [paper](https://arxiv.org/pdf/2002.02126.pdf) | [requirements.txt](cornac/models/lightgcn/requirements.txt) | [lightgcn_example.py](examples/lightgcn_example.py) | 2019 | [Embarrassingly Shallow Autoencoders for Sparse Data (EASEá´¿)](cornac/models/ease), [paper](https://arxiv.org/pdf/1905.03375.pdf) | N/A | [ease_movielens.py](examples/ease_movielens.py) +| | [Neural Graph Collaborative Filtering](cornac/models/ngcf), [paper](https://arxiv.org/pdf/1905.08108.pdf) | [requirements.txt](cornac/models/ngcf/requirements.txt) | [ngcf_example.py](examples/ngcf_example.py) | 2018 | [Collaborative Context Poisson Factorization (C2PF)](cornac/models/c2pf), [paper](https://www.ijcai.org/proceedings/2018/0370.pdf) | N/A | [c2pf_exp.py](examples/c2pf_example.py) | | [Graph Convolutional Matrix Completion (GCMC)](cornac/models/gcmc), [paper](https://www.kdd.org/kdd2018/files/deep-learning-day/DLDay18_paper_32.pdf) | [requirements.txt](cornac/models/gcmc/requirements.txt) | [gcmc_example.py](examples/gcmc_example.py) | | [Multi-Task Explainable Recommendation (MTER)](cornac/models/mter), [paper](https://arxiv.org/pdf/1806.03568.pdf) | N/A | [mter_exp.py](examples/mter_example.py) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 4674a36db..82b2a9e0d 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -51,6 +51,7 @@ from .ncf import GMF from .ncf import MLP from .ncf import NeuMF +from .ngcf import NGCF from .nmf import NMF from .online_ibpr import OnlineIBPR from .pcrl import PCRL diff --git a/cornac/models/ngcf/__init__.py b/cornac/models/ngcf/__init__.py new file mode 100644 index 000000000..ab703d890 --- /dev/null +++ b/cornac/models/ngcf/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from .recom_ngcf import NGCF diff --git a/cornac/models/ngcf/ngcf.py b/cornac/models/ngcf/ngcf.py new file mode 100644 index 000000000..64e473bbc --- /dev/null +++ b/cornac/models/ngcf/ngcf.py @@ -0,0 +1,185 @@ +# Reference: https://github.com/dmlc/dgl/blob/master/examples/pytorch/NGCF/NGCF/model.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +import dgl +import dgl.function as fn + + +USER_KEY = "user" +ITEM_KEY = "item" + + +def construct_graph(data_set): + """ + Generates graph given a cornac data set + + Parameters + ---------- + data_set : cornac.data.dataset.Dataset + The data set as provided by cornac + """ + user_indices, item_indices, _ = data_set.uir_tuple + + # construct graph from the train data and add self-loops + user_selfs = [i for i in range(data_set.total_users)] + item_selfs = [i for i in range(data_set.total_items)] + + data_dict = { + (USER_KEY, "user_self", USER_KEY): (user_selfs, user_selfs), + (ITEM_KEY, "item_self", ITEM_KEY): (item_selfs, item_selfs), + (USER_KEY, "user_item", ITEM_KEY): (user_indices, item_indices), + (ITEM_KEY, "item_user", USER_KEY): (item_indices, user_indices), + } + num_dict = {USER_KEY: data_set.total_users, ITEM_KEY: data_set.total_items} + + return dgl.heterograph(data_dict, num_nodes_dict=num_dict) + + +class NGCFLayer(nn.Module): + def __init__(self, in_size, out_size, norm_dict, dropout): + super(NGCFLayer, self).__init__() + self.in_size = in_size + self.out_size = out_size + + # weights for different types of messages + self.W1 = nn.Linear(in_size, out_size, bias=True) + self.W2 = nn.Linear(in_size, out_size, bias=True) + + # leaky relu + self.leaky_relu = nn.LeakyReLU(0.2) + + # dropout layer + self.dropout = nn.Dropout(dropout) + + # initialization + torch.nn.init.xavier_uniform_(self.W1.weight) + torch.nn.init.constant_(self.W1.bias, 0) + torch.nn.init.xavier_uniform_(self.W2.weight) + torch.nn.init.constant_(self.W2.bias, 0) + + # norm + self.norm_dict = norm_dict + + def forward(self, g, feat_dict): + funcs = {} # message and reduce functions dict + # for each type of edges, compute messages and reduce them all + for srctype, etype, dsttype in g.canonical_etypes: + if srctype == dsttype: # for self loops + messages = self.W1(feat_dict[srctype]) + g.nodes[srctype].data[etype] = messages # store in ndata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_u(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + else: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + norm = self.norm_dict[(srctype, etype, dsttype)] + messages = norm * ( + self.W1(feat_dict[srctype][src]) + + self.W2(feat_dict[srctype][src] * feat_dict[dsttype][dst]) + ) # compute messages + g.edges[(srctype, etype, dsttype)].data[ + etype + ] = messages # store in edata + funcs[(srctype, etype, dsttype)] = ( + fn.copy_e(etype, "m"), + fn.sum("m", "h"), + ) # define message and reduce functions + + g.multi_update_all( + funcs, "sum" + ) # update all, reduce by first type-wisely then across different types + feature_dict = {} + for ntype in g.ntypes: + h = self.leaky_relu(g.nodes[ntype].data["h"]) # leaky relu + h = self.dropout(h) # dropout + h = F.normalize(h, dim=1, p=2) # l2 normalize + feature_dict[ntype] = h + return feature_dict + + +class Model(nn.Module): + def __init__(self, g, in_size, layer_sizes, dropout_rates, lambda_reg, device=None): + super(Model, self).__init__() + self.norm_dict = dict() + self.lambda_reg = lambda_reg + self.device = device + + for srctype, etype, dsttype in g.canonical_etypes: + src, dst = g.edges(etype=(srctype, etype, dsttype)) + dst_degree = g.in_degrees( + dst, etype=(srctype, etype, dsttype) + ).float() # obtain degrees + src_degree = g.out_degrees(src, etype=(srctype, etype, dsttype)).float() + norm = torch.pow(src_degree * dst_degree, -0.5).unsqueeze(1) # compute norm + self.norm_dict[(srctype, etype, dsttype)] = norm + + self.layers = nn.ModuleList() + + # sanity check, just to ensure layer sizes and dropout_rates have the same size + assert len(layer_sizes) == len(dropout_rates), "'layer_sizes' and " \ + "'dropout_rates' must be of the same size" + + self.layers.append( + NGCFLayer(in_size, layer_sizes[0], self.norm_dict, dropout_rates[0]) + ) + self.num_layers = len(layer_sizes) + for i in range(self.num_layers - 1): + self.layers.append( + NGCFLayer( + layer_sizes[i], + layer_sizes[i + 1], + self.norm_dict, + dropout_rates[i + 1], + ) + ) + self.initializer = nn.init.xavier_uniform_ + + # embeddings for different types of nodes + self.feature_dict = nn.ParameterDict( + { + ntype: nn.Parameter( + self.initializer(torch.empty(g.num_nodes(ntype), in_size)) + ) + for ntype in g.ntypes + } + ) + + def forward(self, g, users=None, pos_items=None, neg_items=None): + h_dict = {ntype: self.feature_dict[ntype] for ntype in g.ntypes} + # obtain features of each layer and concatenate them all + user_embeds = [] + item_embeds = [] + user_embeds.append(h_dict[USER_KEY]) + item_embeds.append(h_dict[ITEM_KEY]) + for layer in self.layers: + h_dict = layer(g, h_dict) + user_embeds.append(h_dict[USER_KEY]) + item_embeds.append(h_dict[ITEM_KEY]) + user_embd = torch.cat(user_embeds, 1) + item_embd = torch.cat(item_embeds, 1) + + u_g_embeddings = user_embd if users is None else user_embd[users, :] + pos_i_g_embeddings = item_embd if pos_items is None else item_embd[pos_items, :] + neg_i_g_embeddings = item_embd if neg_items is None else item_embd[neg_items, :] + + return u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + + def loss_fn(self, users, pos_items, neg_items): + pos_scores = (users * pos_items).sum(1) + neg_scores = (users * neg_items).sum(1) + + bpr_loss = F.softplus(neg_scores - pos_scores).mean() + reg_loss = ( + (1 / 2) + * ( + torch.norm(users) ** 2 + + torch.norm(pos_items) ** 2 + + torch.norm(neg_items) ** 2 + ) + / len(users) + ) + + return bpr_loss + self.lambda_reg * reg_loss, bpr_loss, reg_loss diff --git a/cornac/models/ngcf/recom_ngcf.py b/cornac/models/ngcf/recom_ngcf.py new file mode 100644 index 000000000..7284b8af7 --- /dev/null +++ b/cornac/models/ngcf/recom_ngcf.py @@ -0,0 +1,261 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from ..recommender import Recommender +from ...exception import ScoreException + +from tqdm.auto import tqdm, trange + + +class NGCF(Recommender): + """ + Neural Graph Collaborative Filtering + + Parameters + ---------- + name: string, default: 'NGCF' + The name of the recommender model. + + emb_size: int, default: 64 + Size of the node embeddings. + + layer_sizes: list, default: [64, 64, 64] + Size of the output of convolution layers. + + dropout_rates: list, default: [0.1, 0.1, 0.1] + Dropout rate for each of the convolution layers. + - Number of values should be the same as 'layer_sizes' + + num_epochs: int, default: 1000 + Maximum number of iterations or the number of epochs. + + learning_rate: float, default: 0.001 + The learning rate that determines the step size at each iteration + + train_batch_size: int, default: 1024 + Mini-batch size used for train set + + test_batch_size: int, default: 100 + Mini-batch size used for test set + + early_stopping: {min_delta: float, patience: int}, optional, default: None + If `None`, no early stopping. Meaning of the arguments: + + - `min_delta`: the minimum increase in monitored value on validation + set to be considered as improvement, + i.e. an increment of less than min_delta will count as + no improvement. + + - `patience`: number of epochs with no improvement after which + training should be stopped. + + lambda_reg: float, default: 1e-4 + Weight decay for the L2 normalization + + trainable: boolean, optional, default: True + When False, the model is not trained and Cornac assumes that the model + is already pre-trained. + + verbose: boolean, optional, default: False + When True, some running logs are displayed. + + seed: int, optional, default: 2020 + Random seed for parameters initialization. + + References + ---------- + * Wang, Xiang, et al. "Neural graph collaborative filtering." Proceedings of the 42nd international ACM SIGIR conference on Research and development in Information Retrieval. 2019. + """ + + def __init__( + self, + name="NGCF", + emb_size=64, + layer_sizes=[64, 64, 64], + dropout_rates=[0.1, 0.1, 0.1], + num_epochs=1000, + learning_rate=0.001, + train_batch_size=1024, + test_batch_size=100, + early_stopping=None, + lambda_reg=1e-4, + trainable=True, + verbose=False, + seed=2020, + ): + super().__init__(name=name, trainable=trainable, verbose=verbose) + self.emb_size = emb_size + self.layer_sizes = layer_sizes + self.dropout_rates = dropout_rates + self.num_epochs = num_epochs + self.learning_rate = learning_rate + self.train_batch_size = train_batch_size + self.test_batch_size = test_batch_size + self.early_stopping = early_stopping + self.lambda_reg = lambda_reg + self.seed = seed + + def fit(self, train_set, val_set=None): + """Fit the model to observations. + + Parameters + ---------- + train_set: :obj:`cornac.data.Dataset`, required + User-Item preference data as well as additional modalities. + + val_set: :obj:`cornac.data.Dataset`, optional, default: None + User-Item preference data for model selection purposes (e.g., early stopping). + + Returns + ------- + self : object + """ + Recommender.fit(self, train_set, val_set) + + if not self.trainable: + return self + + # model setup + import torch + from .ngcf import Model + from .ngcf import construct_graph + + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if self.seed is not None: + torch.manual_seed(self.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(self.seed) + + graph = construct_graph(train_set).to(self.device) + model = Model( + graph, + self.emb_size, + self.layer_sizes, + self.dropout_rates, + self.lambda_reg, + ).to(self.device) + + optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate) + + # model training + pbar = trange( + self.num_epochs, + desc="Training", + unit="iter", + position=0, + leave=False, + disable=not self.verbose, + ) + for _ in pbar: + model.train() + accum_loss = 0.0 + for batch_u, batch_i, batch_j in tqdm( + train_set.uij_iter( + batch_size=self.train_batch_size, + shuffle=True, + ), + desc="Epoch", + total=train_set.num_batches(self.train_batch_size), + leave=False, + position=1, + disable=not self.verbose, + ): + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings = model( + graph, batch_u, batch_i, batch_j + ) + + batch_loss, batch_bpr_loss, batch_reg_loss = model.loss_fn( + u_g_embeddings, pos_i_g_embeddings, neg_i_g_embeddings + ) + accum_loss += batch_loss.cpu().item() * len(batch_u) + + optimizer.zero_grad() + batch_loss.backward() + optimizer.step() + + accum_loss /= len(train_set.uir_tuple[0]) # normalize over all observations + pbar.set_postfix(loss=accum_loss) + + # store user and item embedding matrices for prediction + model.eval() + u_embs, i_embs, _ = model(graph) + # we will use numpy for faster prediction in the score function, no need torch + self.U = u_embs.cpu().detach().numpy() + self.V = i_embs.cpu().detach().numpy() + + if self.early_stopping is not None and self.early_stop( + **self.early_stopping + ): + break + + def monitor_value(self): + """Calculating monitored value used for early stopping on validation set (`val_set`). + This function will be called by `early_stop()` function. + + Returns + ------- + res : float + Monitored value on validation set. + Return `None` if `val_set` is `None`. + """ + if self.val_set is None: + return None + + from ...metrics import Recall + from ...eval_methods import ranking_eval + + recall_20 = ranking_eval( + model=self, + metrics=[Recall(k=20)], + train_set=self.train_set, + test_set=self.val_set, + )[0][0] + + return recall_20 # Section 4.2.3 in the paper + + def score(self, user_idx, item_idx=None): + """Predict the scores/ratings of a user for an item. + + Parameters + ---------- + user_idx: int, required + The index of the user for whom to perform score prediction. + + item_idx: int, optional, default: None + The index of the item for which to perform score prediction. + If None, scores for all known items will be returned. + + Returns + ------- + res : A scalar or a Numpy array + Relative scores that the user gives to the item or to all known items + + """ + if item_idx is None: + if self.train_set.is_unk_user(user_idx): + raise ScoreException( + "Can't make score prediction for (user_id=%d)" % user_idx + ) + known_item_scores = self.V.dot(self.U[user_idx, :]) + return known_item_scores + else: + if self.train_set.is_unk_user(user_idx) or self.train_set.is_unk_item( + item_idx + ): + raise ScoreException( + "Can't make score prediction for (user_id=%d, item_id=%d)" + % (user_idx, item_idx) + ) + return self.V[item_idx, :].dot(self.U[user_idx, :]) diff --git a/cornac/models/ngcf/requirements.txt b/cornac/models/ngcf/requirements.txt new file mode 100644 index 000000000..32f294fbc --- /dev/null +++ b/cornac/models/ngcf/requirements.txt @@ -0,0 +1,2 @@ +torch>=2.0.0 +dgl>=1.1.0 \ No newline at end of file diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index f36d616a0..ba603a80b 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -68,6 +68,11 @@ Neural Attention Rating Regression with Review-level Explanations (NARRE) .. automodule:: cornac.models.narre.recom_narre :members: +Neural Graph Collaborative Filtering (NGCF) +---------------------------------------------------- +.. automodule:: cornac.models.ngcf.recom_ngcf + :members: + Probabilistic Collaborative Representation Learning (PCRL) ------------------------------------------------------------ .. automodule:: cornac.models.pcrl.recom_pcrl diff --git a/examples/README.md b/examples/README.md index 966f6c6ba..5ae6d09d5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,6 +34,8 @@ [mcf_office.py](mcf_office.py) - Fit Matrix Co-Factorization (MCF) to the Amazon Office dataset. +[ngcf_example.py](ngcf_example.py) - NGCF example with CiteULike dataset. + [pcrl_example.py](pcrl_example.py) - Probabilistic Collaborative Representation Learning (PCRL) Amazon Office dataset. [sbpr_epinions.py](sbpr_epinions.py) - Social Bayesian Personalized Ranking (SBPR) with Epinions dataset. diff --git a/examples/ngcf_example.py b/examples/ngcf_example.py new file mode 100644 index 000000000..89abba836 --- /dev/null +++ b/examples/ngcf_example.py @@ -0,0 +1,60 @@ +# Copyright 2018 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Example for NGCF, using the CiteULike dataset +""" +import cornac +from cornac.datasets import citeulike +from cornac.eval_methods import RatioSplit + +# Load user-item feedback +data = citeulike.load_feedback() + +# Instantiate an evaluation method to split data into train and test sets. +ratio_split = RatioSplit( + data=data, + val_size=0.1, + test_size=0.1, + exclude_unknowns=True, + verbose=True, + seed=123, + rating_threshold=0.5, +) + +# Instantiate the NGCF model +ngcf = cornac.models.NGCF( + seed=123, + num_epochs=1000, + emb_size=64, + layer_sizes=[64, 64, 64], + dropout_rates=[0.1, 0.1, 0.1], + early_stopping={"min_delta": 1e-4, "patience": 50}, + train_batch_size=1024, + learning_rate=0.001, + lambda_reg=1e-5, + verbose=True, +) + +# Instantiate evaluation measures +rec_20 = cornac.metrics.Recall(k=20) +ndcg_20 = cornac.metrics.NDCG(k=20) + +# Put everything together into an experiment and run it +cornac.Experiment( + eval_method=ratio_split, + models=[ngcf], + metrics=[rec_20, ndcg_20], + user_based=True, +).run()