diff --git a/README.md b/README.md index f0d8a32f..fb5ff221 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ CogDL is a graph deep learning toolkit that allows researchers and developers to We summarize the contributions of CogDL as follows: -- **High Efficiency**: CogDL utilizes well-optimized operators to speed up training and save GPU memory of GNN models. -- **Easy-to-Use**: CogDL provides easy-to-use APIs for running experiments with the given models and datasets using hyper-parameter search. +- **Efficiency**: CogDL utilizes well-optimized operators to speed up training and save GPU memory of GNN models. +- **Ease of Use**: CogDL provides easy-to-use APIs for running experiments with the given models and datasets using hyper-parameter search. - **Extensibility**: The design of CogDL makes it easy to apply GNN models to new scenarios based on our framework. -- **Reproducibility**: CogDL provides reproducible leaderboards for state-of-the-art models on most of important tasks in the graph domain. ## ❗ News @@ -223,7 +222,7 @@ Please cite [our paper](https://arxiv.org/abs/2103.00959) if you find our code o ``` @article{cen2021cogdl, - title={CogDL: An Extensive Toolkit for Deep Learning on Graphs}, + title={CogDL: Toolkit for Deep Learning on Graphs}, author={Yukuo Cen and Zhenyu Hou and Yan Wang and Qibin Chen and Yizhen Luo and Xingcheng Yao and Aohan Zeng and Shiguang Guo and Peng Zhang and Guohao Dai and Yu Wang and Chang Zhou and Hongxia Yang and Jie Tang}, journal={arXiv preprint arXiv:2103.00959}, year={2021} diff --git a/README_CN.md b/README_CN.md index 6a6aa14a..2e7f2fa0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -14,10 +14,9 @@ CogDL是一款图深度学习工具包,基于[PyTorch](https://github.com/pyto CogDL的特性包括: -- 高效:CogDL支持使用优化好的算子来加速GNN模型的训练。 +- 高效性:CogDL支持使用优化好的算子来加速GNN模型的训练。 - 易用性:CogDL提供了非常易用的API来在给定的模型和数据集上运行实验。 -- 可扩展性:用户可以基于CogDL已有的框架来实现和提交新的数据集、模型和任务。 -- 可复现性:CogDL对图领域大多数重要的任务都提供了可复现的排行榜。 +- 扩展性:用户可以基于CogDL已有的框架来扩展新的数据集、模型。 ## ❗ 最新 @@ -200,7 +199,7 @@ CogDL核心开发团队可以通过[cogdlteam@gmail.com](mailto:cogdlteam@gmail. ``` @article{cen2021cogdl, - title={CogDL: An Extensive Toolkit for Deep Learning on Graphs}, + title={CogDL: Toolkit for Deep Learning on Graphs}, author={Yukuo Cen and Zhenyu Hou and Yan Wang and Qibin Chen and Yizhen Luo and Xingcheng Yao and Aohan Zeng and Shiguang Guo and Peng Zhang and Guohao Dai and Yu Wang and Chang Zhou and Hongxia Yang and Jie Tang}, journal={arXiv preprint arXiv:2103.00959}, year={2021} diff --git a/cogdl/__init__.py b/cogdl/__init__.py index 9fa0f5dd..a9e4d6af 100644 --- a/cogdl/__init__.py +++ b/cogdl/__init__.py @@ -1,5 +1,4 @@ __version__ = "0.5.1.post1" from .experiments import experiment -from .oag import oagbert from .pipelines import pipeline diff --git a/cogdl/data/data.py b/cogdl/data/data.py index 9fedba14..71f8d2be 100644 --- a/cogdl/data/data.py +++ b/cogdl/data/data.py @@ -17,6 +17,8 @@ from cogdl.utils import RandomWalker from cogdl.operators.sample import sample_adj_c, subgraph_c +subgraph_c = None # noqa: F811 + class BaseGraph(object): def __init__(self): @@ -628,7 +630,11 @@ def edge_index(self, edge_index): self._adj.row_ptr = None self._adj.row = row self._adj.col = col - self.__num_nodes__ = None + if self.x is not None: + self._adj.__num_nodes__ = self.x.shape[0] + self.__num_nodes__ = self.x.shape[0] + else: + self.__num_nodes__ = None @edge_weight.setter def edge_weight(self, edge_weight): @@ -857,6 +863,7 @@ def csr_subgraph(self, node_idx, keep_order=False): continue data._adj[key] = self._adj[key][edges] data.num_nodes = node_idx.shape[0] + data.edge_weight = None return data def subgraph(self, node_idx, keep_order=False): diff --git a/cogdl/layers/sage_layer.py b/cogdl/layers/sage_layer.py index 59883cdd..d910f3dc 100644 --- a/cogdl/layers/sage_layer.py +++ b/cogdl/layers/sage_layer.py @@ -51,11 +51,6 @@ def __init__( else: raise NotImplementedError - if dropout > 0: - self.dropout = nn.Dropout(dropout) - else: - self.dropout = None - if activation is not None: self.act = get_activation(activation, inplace=True) else: diff --git a/cogdl/models/nn/daegc.py b/cogdl/models/nn/daegc.py index 6222963b..03257c8b 100644 --- a/cogdl/models/nn/daegc.py +++ b/cogdl/models/nn/daegc.py @@ -81,4 +81,4 @@ def get_features(self, data): def recon_loss(self, z, adj): # print(torch.mm(z, z.t()), adj) - return F.binary_cross_entropy(F.softmax(torch.mm(z, z.t())), adj, reduction="sum") + return F.binary_cross_entropy(F.sigmoid(torch.mm(z, z.t())), adj, reduction="sum") diff --git a/cogdl/models/nn/gae.py b/cogdl/models/nn/gae.py index a62b61e9..0d5b5ebd 100644 --- a/cogdl/models/nn/gae.py +++ b/cogdl/models/nn/gae.py @@ -47,6 +47,7 @@ def __init__(self, num_features, hidden_size): self.conv2_var = GCNLayer(self.hidden_size, self.hidden_size) def reparameterize(self, mean, log_var): + log_var = log_var.clamp(max=10) sigma = torch.exp(log_var) z = mean + torch.randn_like(log_var) * sigma return z diff --git a/cogdl/models/nn/infograph.py b/cogdl/models/nn/infograph.py index f31c074e..1b8f19f8 100644 --- a/cogdl/models/nn/infograph.py +++ b/cogdl/models/nn/infograph.py @@ -88,7 +88,7 @@ def forward(self, x): class InfoGraph(BaseModel): - r"""Implimentation of Infograph in paper `"InfoGraph: Unsupervised and Semi-supervised Graph-Level Representation + r"""Implementation of Infograph in paper `"InfoGraph: Unsupervised and Semi-supervised Graph-Level Representation Learning via Mutual Information Maximization" __. ` Parameters diff --git a/cogdl/oag/README.md b/cogdl/oag/README.md index f68a3ee6..e408b1bb 100644 --- a/cogdl/oag/README.md +++ b/cogdl/oag/README.md @@ -8,7 +8,7 @@ A basic version OAG-BERT. Similar to [SciBERT](https://github.com/allenai/sciber The usage of OAG-BERT is the same of ordinary SciBERT or BERT. For example, you can use the following code to encode two text sequences and retrieve their outputs ```python -from cogdl import oagbert +from cogdl.oag import oagbert tokenizer, bert_model = oagbert() @@ -20,7 +20,7 @@ outputs = bert_model(**tokens) ## V2: The entity augmented version An extension to the vanilla OAG-BERT. We incorporate rich entity information in Open Academic Graph such as **authors** and **field-of-study**. Thus, you can encode various type of entities in OAG-BERT v2. For example, to encode the paper of BERT, you can use the following code ```python -from cogdl import oagbert +from cogdl.oag import oagbert import torch tokenizer, model = oagbert("oagbert-v2") @@ -47,7 +47,7 @@ sequence_output, pooled_output = model.bert.forward( ``` If you want to encode various type of entities separately, you can use the following code instead ```python -from cogdl import oagbert +from cogdl.oag import oagbert tokenizer, model = oagbert("oagbert-v2") title = 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding' @@ -87,7 +87,7 @@ We also release another two V2 version for users. One is a generation based version which can be used for generating texts based on other information. For example, use the following code to automatically generate paper titles with abstracts. ```python -from cogdl import oagbert +from cogdl.oag import oagbert tokenizer, model = oagbert('oagbert-v2-lm') model.eval() @@ -101,7 +101,7 @@ for seq, prob in model.generate_title(abstract="To enrich language models with d In addition to that, we fine-tune the OAG-BERT for calculating paper similarity based on name disambiguation tasks, which is named as Sentence-OAGBERT following [Sentence-BERT](https://github.com/UKPLab/sentence-transformers). The following codes demonstrate an example of using Sentence-OAGBERT to calculate paper similarity. ```python import os -from cogdl import oagbert +from cogdl.oag import oagbert import torch import torch.nn.functional as F import numpy as np diff --git a/cogdl/operators/spmm/spmm_cpu.cpp b/cogdl/operators/spmm/spmm_cpu.cpp index bd8683f8..477c9b92 100644 --- a/cogdl/operators/spmm/spmm_cpu.cpp +++ b/cogdl/operators/spmm/spmm_cpu.cpp @@ -13,7 +13,7 @@ torch::Tensor spmm_cpu( const auto k = dense.size(1); auto devid = dense.device().index(); auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCPU, devid); - auto out = torch::empty({m,k}, options); + auto out = torch::zeros({m,k}, options); int *rowptr_ptr = rowptr.data_ptr(); int *colind_ptr = colind.data_ptr(); diff --git a/cogdl/pipelines.py b/cogdl/pipelines.py index 557e1500..7a3819da 100644 --- a/cogdl/pipelines.py +++ b/cogdl/pipelines.py @@ -9,7 +9,6 @@ from grave import plot_network, use_attributes from tabulate import tabulate -from cogdl import oagbert from cogdl.data import Graph from cogdl.datasets import build_dataset_from_name, NodeDataset from cogdl.models import build_model @@ -126,6 +125,9 @@ def __init__(self, app: str, model: str, **kwargs): super(OAGBertInferencePipepline, self).__init__(app, model=model, **kwargs) load_weights = kwargs["load_weights"] if "load_weights" in kwargs else True + + from cogdl.oag import oagbert + self.tokenizer, self.bert_model = oagbert(model, load_weights=load_weights) def __call__(self, sequence, **kwargs): diff --git a/cogdl/trainer/trainer.py b/cogdl/trainer/trainer.py index 18f26dcf..186c3685 100644 --- a/cogdl/trainer/trainer.py +++ b/cogdl/trainer/trainer.py @@ -275,7 +275,7 @@ def initialize(self, model_w, rank=0, master_addr: str = "localhost", master_por else: return model_w.to(rank), None - def train(self, rank, model_w, dataset_w): + def train(self, rank, model_w, dataset_w): # noqa: C901 model_w, _ = self.initialize(model_w, rank=rank, master_addr=self.master_addr, master_port=self.master_port) self.data_controller.prepare_data_wrapper(dataset_w, rank) self.eval_data_back_to_cpu = dataset_w.data_back_to_cpu @@ -306,10 +306,10 @@ def train(self, rank, model_w, dataset_w): self.data_controller.training_proc_per_stage(dataset_w, rank) if self.progress_bar == "epoch": - epoch_iter = tqdm(range(self.epochs)) + epoch_iter = tqdm(range(1, self.epochs + 1)) epoch_printer = Printer(epoch_iter.set_description, rank=rank, world_size=self.world_size) else: - epoch_iter = range(self.epochs) + epoch_iter = range(1, self.epochs + 1) epoch_printer = Printer(print, rank=rank, world_size=self.world_size) self.logger.start() @@ -321,7 +321,10 @@ def train(self, rank, model_w, dataset_w): # inductive setting .. dataset_w.train() train_loader = dataset_w.on_train_wrapper() - training_loss = self.training_step(model_w, train_loader, optimizers, lr_schedulers, rank) + train_dataset = train_loader.get_dataset_from_loader() + if hasattr(train_dataset, "shuffle"): + train_dataset.shuffle() + training_loss = self.train_step(model_w, train_loader, optimizers, lr_schedulers, rank) print_str_dict["Epoch"] = epoch print_str_dict["train_loss"] = training_loss @@ -386,12 +389,13 @@ def validate(self, model_w: ModelWrapper, dataset_w: DataWrapper, device): dataset_w.eval() if self.cpu_inference: model_w.to("cpu") - _device = device + _device = "cpu" else: _device = device val_loader = dataset_w.on_val_wrapper() - result = self.val_step(model_w, val_loader, _device) + with torch.no_grad(): + result = self.val_step(model_w, val_loader, _device) model_w.to(device) return result @@ -406,12 +410,16 @@ def test(self, model_w: ModelWrapper, dataset_w: DataWrapper, device): dataset_w.eval() if self.cpu_inference: model_w.to("cpu") - _device = device + _device = "cpu" else: _device = device test_loader = dataset_w.on_test_wrapper() - result = self.test_step(model_w, test_loader, _device) + if model_w.training_type == "unsupervised": + result = self.test_step(model_w, test_loader, _device) + else: + with torch.no_grad(): + result = self.test_step(model_w, test_loader, _device) model_w.to(device) return result @@ -425,7 +433,8 @@ def distributed_test(self, model_w: ModelWrapper, loader, rank, fn): _device = "cpu" else: _device = rank - result = fn(model_w, loader, _device) + with torch.no_grad(): + result = fn(model_w, loader, _device) model_w.to(rank) object_list = [result] @@ -434,7 +443,7 @@ def distributed_test(self, model_w: ModelWrapper, loader, rank, fn): dist.broadcast_object_list(object_list, src=0) return object_list[0] - def training_step(self, model_w, train_loader, optimizers, lr_schedulers, device): + def train_step(self, model_w, train_loader, optimizers, lr_schedulers, device): model_w.train() losses = [] @@ -443,6 +452,8 @@ def training_step(self, model_w, train_loader, optimizers, lr_schedulers, device for batch in train_loader: batch = move_to_device(batch, device) + if hasattr(batch, "train_mask") and batch.train_mask.sum().item() == 0: + continue loss = model_w.on_train_step(batch) for optimizer in optimizers: @@ -458,9 +469,9 @@ def training_step(self, model_w, train_loader, optimizers, lr_schedulers, device if lr_schedulers is not None: for lr_schedular in lr_schedulers: lr_schedular.step() + return np.mean(losses) - @torch.no_grad() def val_step(self, model_w, val_loader, device): model_w.eval() if val_loader is None: @@ -472,7 +483,6 @@ def val_step(self, model_w, val_loader, device): move_to_device(batch, "cpu") return model_w.collect_notes() - # @torch.no_grad() def test_step(self, model_w, test_loader, device): model_w.eval() if test_loader is None: diff --git a/cogdl/utils/spmm_utils.py b/cogdl/utils/spmm_utils.py index cfdc2acb..ac3dad09 100644 --- a/cogdl/utils/spmm_utils.py +++ b/cogdl/utils/spmm_utils.py @@ -82,10 +82,13 @@ def forward(self, graph, x): return spmm_cpu(graph, x, self.fast_spmm_cpu) -def spmm(graph, x, actnn=False, fast_spmm=None): +def spmm(graph, x, actnn=False, fast_spmm=None, fast_spmm_cpu=None): if fast_spmm is None: initialize_spmm() fast_spmm = CONFIGS["fast_spmm"] + if fast_spmm_cpu is None: + initialize_spmm_cpu() + fast_spmm_cpu = CONFIGS["fast_spmm_cpu"] if fast_spmm is not None and str(x.device) != "cpu": if graph.out_norm is not None: x = graph.out_norm * x @@ -94,6 +97,16 @@ def spmm(graph, x, actnn=False, fast_spmm=None): csr_data = graph.raw_edge_weight x = fast_spmm(row_ptr.int(), col_indices.int(), x, csr_data, graph.is_symmetric(), actnn=actnn) + if graph.in_norm is not None: + x = graph.in_norm * x + elif fast_spmm_cpu is not None and str(x.device) == "cpu" and x.requires_grad is False: + if graph.out_norm is not None: + x = graph.out_norm * x + + row_ptr, col_indices = graph.row_indptr, graph.col_indices + csr_data = graph.raw_edge_weight + x = fast_spmm_cpu(row_ptr.int(), col_indices.int(), csr_data, x) + if graph.in_norm is not None: x = graph.in_norm * x else: diff --git a/cogdl/wrappers/model_wrapper/__init__.py b/cogdl/wrappers/model_wrapper/__init__.py index 782579d5..ff3d7446 100644 --- a/cogdl/wrappers/model_wrapper/__init__.py +++ b/cogdl/wrappers/model_wrapper/__init__.py @@ -1,6 +1,6 @@ import importlib -from .base_model_wrapper import ModelWrapper, EmbeddingModelWrapper +from .base_model_wrapper import ModelWrapper, EmbeddingModelWrapper, UnsupervisedModelWrapper def register_model_wrapper(name): diff --git a/cogdl/wrappers/model_wrapper/base_model_wrapper.py b/cogdl/wrappers/model_wrapper/base_model_wrapper.py index fc53f832..c9490204 100644 --- a/cogdl/wrappers/model_wrapper/base_model_wrapper.py +++ b/cogdl/wrappers/model_wrapper/base_model_wrapper.py @@ -17,6 +17,7 @@ def __init__(self): self._evaluator = None self._evaluator_metric = None self.__record__ = dict() + self.training_type = "" def forward(self): pass @@ -196,3 +197,9 @@ def _model_key_(self): class EmbeddingModelWrapper(ModelWrapper): def setup_optimizer(self): pass + + +class UnsupervisedModelWrapper(ModelWrapper): + def __init__(self): + super(UnsupervisedModelWrapper, self).__init__() + self.training_type = "unsupervised" diff --git a/cogdl/wrappers/model_wrapper/node_classification/dgi_mw.py b/cogdl/wrappers/model_wrapper/node_classification/dgi_mw.py index 3347d1fe..3a7377c6 100644 --- a/cogdl/wrappers/model_wrapper/node_classification/dgi_mw.py +++ b/cogdl/wrappers/model_wrapper/node_classification/dgi_mw.py @@ -3,11 +3,11 @@ import torch import torch.nn as nn -from .. import ModelWrapper +from .. import UnsupervisedModelWrapper from cogdl.wrappers.tools.wrapper_utils import evaluate_node_embeddings_using_logreg -class DGIModelWrapper(ModelWrapper): +class DGIModelWrapper(UnsupervisedModelWrapper): @staticmethod def add_args(parser): # fmt: off diff --git a/cogdl/wrappers/model_wrapper/node_classification/grace_mw.py b/cogdl/wrappers/model_wrapper/node_classification/grace_mw.py index 22a81c80..e6c6ec4c 100644 --- a/cogdl/wrappers/model_wrapper/node_classification/grace_mw.py +++ b/cogdl/wrappers/model_wrapper/node_classification/grace_mw.py @@ -3,12 +3,12 @@ import torch.nn.functional as F from cogdl.data import Graph -from .. import ModelWrapper +from .. import UnsupervisedModelWrapper from cogdl.wrappers.tools.wrapper_utils import evaluate_node_embeddings_using_logreg from cogdl.utils import dropout_adj, dropout_features -class GRACEModelWrapper(ModelWrapper): +class GRACEModelWrapper(UnsupervisedModelWrapper): @staticmethod def add_args(parser): # fmt: off diff --git a/cogdl/wrappers/model_wrapper/node_classification/mvgrl_mw.py b/cogdl/wrappers/model_wrapper/node_classification/mvgrl_mw.py index eee8d583..5f77e574 100644 --- a/cogdl/wrappers/model_wrapper/node_classification/mvgrl_mw.py +++ b/cogdl/wrappers/model_wrapper/node_classification/mvgrl_mw.py @@ -1,11 +1,11 @@ import torch import torch.nn as nn -from .. import ModelWrapper +from .. import UnsupervisedModelWrapper from cogdl.wrappers.tools.wrapper_utils import evaluate_node_embeddings_using_logreg -class MVGRLModelWrapper(ModelWrapper): +class MVGRLModelWrapper(UnsupervisedModelWrapper): def __init__(self, model, optimizer_cfg): super(MVGRLModelWrapper, self).__init__() self.model = model diff --git a/cogdl/wrappers/model_wrapper/node_classification/self_auxiliary_mw.py b/cogdl/wrappers/model_wrapper/node_classification/self_auxiliary_mw.py index bc5f5ce2..bc66c1ee 100644 --- a/cogdl/wrappers/model_wrapper/node_classification/self_auxiliary_mw.py +++ b/cogdl/wrappers/model_wrapper/node_classification/self_auxiliary_mw.py @@ -10,10 +10,10 @@ from cogdl.wrappers.tools.wrapper_utils import evaluate_node_embeddings_using_logreg from tqdm import tqdm -from .. import ModelWrapper +from .. import UnsupervisedModelWrapper -class SelfAuxiliaryModelWrapper(ModelWrapper): +class SelfAuxiliaryModelWrapper(UnsupervisedModelWrapper): @staticmethod def add_args(parser): # fmt: off diff --git a/cogdl/wrappers/model_wrapper/node_classification/unsup_graphsage_mw.py b/cogdl/wrappers/model_wrapper/node_classification/unsup_graphsage_mw.py index 5b1447fc..91313681 100644 --- a/cogdl/wrappers/model_wrapper/node_classification/unsup_graphsage_mw.py +++ b/cogdl/wrappers/model_wrapper/node_classification/unsup_graphsage_mw.py @@ -3,10 +3,10 @@ import numpy as np from cogdl.utils import RandomWalker from cogdl.wrappers.tools.wrapper_utils import evaluate_node_embeddings_using_logreg -from .. import ModelWrapper +from .. import UnsupervisedModelWrapper -class UnsupGraphSAGEModelWrapper(ModelWrapper): +class UnsupGraphSAGEModelWrapper(UnsupervisedModelWrapper): @staticmethod def add_args(parser): # fmt: off diff --git a/docs/requirements.txt b/docs/requirements.txt index ab8126d6..2762c681 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,7 +10,7 @@ numpy<1.21,>=1.17 scipy gensim<4.0 grave -scikit_learn +scikit_learn==0.24.2 tabulate optuna==2.4.0 ogb diff --git a/examples/oagbert/calculate_paper_similarity.py b/examples/oagbert/calculate_paper_similarity.py index 09164ab3..5f95d3e4 100644 --- a/examples/oagbert/calculate_paper_similarity.py +++ b/examples/oagbert/calculate_paper_similarity.py @@ -1,5 +1,5 @@ import os -from cogdl import oagbert +from cogdl.oag import oagbert import torch import torch.nn.functional as F import numpy as np @@ -10,15 +10,24 @@ model.eval() # Paper 1 -title = 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding' -abstract = 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation...' -authors = ['Jacob Devlin', 'Ming-Wei Chang', 'Kenton Lee', 'Kristina Toutanova'] -venue = 'north american chapter of the association for computational linguistics' -affiliations = ['Google'] -concepts = ['language model', 'natural language inference', 'question answering'] +title = "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding" +abstract = "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation..." +authors = ["Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova"] +venue = "north american chapter of the association for computational linguistics" +affiliations = ["Google"] +concepts = ["language model", "natural language inference", "question answering"] # encode first paper -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs( title=title, abstract=abstract, venue=venue, authors=authors, concepts=concepts, affiliations=affiliations ) _, paper_embed_1 = model.bert.forward( @@ -28,18 +37,27 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # Positive Paper 2 -title = 'Attention Is All You Need' -abstract = 'We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely...' -authors = ['Ashish Vaswani', 'Noam Shazeer', 'Niki Parmar', 'Jakob Uszkoreit'] -venue = 'neural information processing systems' -affiliations = ['Google'] -concepts = ['machine translation', 'computation and language', 'language model'] +title = "Attention Is All You Need" +abstract = "We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely..." +authors = ["Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit"] +venue = "neural information processing systems" +affiliations = ["Google"] +concepts = ["machine translation", "computation and language", "language model"] -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs( title=title, abstract=abstract, venue=venue, authors=authors, concepts=concepts, affiliations=affiliations ) # encode second paper @@ -50,18 +68,27 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # Negative Paper 3 title = "Traceability and international comparison of ultraviolet irradiance" abstract = "NIM took part in the CIPM Key Comparison of ″Spectral Irradiance 250 to 2500 nm″. In UV and NIR wavelength, the international comparison results showed that the consistency between Chinese value and the international reference one" -authors = ['Jing Yu', 'Bo Huang', 'Jia-Lin Yu', 'Yan-Dong Lin', 'Cai-Hong Dai'] -veune = 'Jiliang Xuebao/Acta Metrologica Sinica' -affiliations = ['Department of Electronic Engineering'] -concept = ['Optical Division'] +authors = ["Jing Yu", "Bo Huang", "Jia-Lin Yu", "Yan-Dong Lin", "Cai-Hong Dai"] +veune = "Jiliang Xuebao/Acta Metrologica Sinica" +affiliations = ["Department of Electronic Engineering"] +concept = ["Optical Division"] -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs( title=title, abstract=abstract, venue=venue, authors=authors, concepts=concepts, affiliations=affiliations ) # encode thrid paper @@ -72,7 +99,7 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # calulate text similarity diff --git a/examples/oagbert/generate_title.py b/examples/oagbert/generate_title.py index ab4ca140..2a594cff 100644 --- a/examples/oagbert/generate_title.py +++ b/examples/oagbert/generate_title.py @@ -1,8 +1,10 @@ -from cogdl import oagbert +from cogdl.oag import oagbert -tokenizer, model = oagbert('oagbert-v2-lm') +tokenizer, model = oagbert("oagbert-v2-lm") model.eval() -for seq, prob in model.generate_title(abstract="To enrich language models with domain knowledge is crucial but difficult. Based on the world's largest public academic graph Open Academic Graph (OAG), we pre-train an academic language model, namely OAG-BERT, which integrates massive heterogeneous entities including paper, author, concept, venue, and affiliation. To better endow OAG-BERT with the ability to capture entity information, we develop novel pre-training strategies including heterogeneous entity type embedding, entity-aware 2D positional encoding, and span-aware entity masking. For zero-shot inference, we design a special decoding strategy to allow OAG-BERT to generate entity names from scratch. We evaluate the OAG-BERT on various downstream academic tasks, including NLP benchmarks, zero-shot entity inference, heterogeneous graph link prediction, and author name disambiguation. Results demonstrate the effectiveness of the proposed pre-training approach to both comprehending academic texts and modeling knowledge from heterogeneous entities. OAG-BERT has been deployed to multiple real-world applications, such as reviewer recommendations for NSFC (National Nature Science Foundation of China) and paper tagging in the AMiner system. It is also available to the public through the CogDL package."): - print('Title: %s' % seq) - print('Perplexity: %.4f' % prob) +for seq, prob in model.generate_title( + abstract="To enrich language models with domain knowledge is crucial but difficult. Based on the world's largest public academic graph Open Academic Graph (OAG), we pre-train an academic language model, namely OAG-BERT, which integrates massive heterogeneous entities including paper, author, concept, venue, and affiliation. To better endow OAG-BERT with the ability to capture entity information, we develop novel pre-training strategies including heterogeneous entity type embedding, entity-aware 2D positional encoding, and span-aware entity masking. For zero-shot inference, we design a special decoding strategy to allow OAG-BERT to generate entity names from scratch. We evaluate the OAG-BERT on various downstream academic tasks, including NLP benchmarks, zero-shot entity inference, heterogeneous graph link prediction, and author name disambiguation. Results demonstrate the effectiveness of the proposed pre-training approach to both comprehending academic texts and modeling knowledge from heterogeneous entities. OAG-BERT has been deployed to multiple real-world applications, such as reviewer recommendations for NSFC (National Nature Science Foundation of China) and paper tagging in the AMiner system. It is also available to the public through the CogDL package." +): + print("Title: %s" % seq) + print("Perplexity: %.4f" % prob) diff --git a/examples/oagbert/oagbert.py b/examples/oagbert/oagbert.py index fa1abf54..c1135943 100644 --- a/examples/oagbert/oagbert.py +++ b/examples/oagbert/oagbert.py @@ -1,5 +1,5 @@ import torch -from cogdl import oagbert +from cogdl.oag import oagbert tokenizer, bert_model = oagbert() bert_model.eval() diff --git a/examples/oagbert/oagbert_encode_paper.py b/examples/oagbert/oagbert_encode_paper.py index 2a43c893..12e4f11d 100644 --- a/examples/oagbert/oagbert_encode_paper.py +++ b/examples/oagbert/oagbert_encode_paper.py @@ -1,18 +1,23 @@ -from cogdl import oagbert +from cogdl.oag import oagbert tokenizer, model = oagbert("oagbert-v2") -title = 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding' -abstract = 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation...' -authors = ['Jacob Devlin', 'Ming-Wei Chang', 'Kenton Lee', 'Kristina Toutanova'] -venue = 'north american chapter of the association for computational linguistics' -affiliations = ['Google'] -concepts = ['language model', 'natural language inference', 'question answering'] +title = "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding" +abstract = "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation..." +authors = ["Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova"] +venue = "north american chapter of the association for computational linguistics" +affiliations = ["Google"] +concepts = ["language model", "natural language inference", "question answering"] # encode paper paper_info = model.encode_paper( - title=title, abstract=abstract, venue=venue, authors=authors, concepts=concepts, affiliations=affiliations, reduction="max" + title=title, + abstract=abstract, + venue=venue, + authors=authors, + concepts=concepts, + affiliations=affiliations, + reduction="max", ) for name, content in paper_info.items(): print(name) print(content) - diff --git a/examples/oagbert/oagbert_metainfo.py b/examples/oagbert/oagbert_metainfo.py index d9c13442..bc31d73e 100644 --- a/examples/oagbert/oagbert_metainfo.py +++ b/examples/oagbert/oagbert_metainfo.py @@ -1,4 +1,4 @@ -from cogdl import oagbert +from cogdl.oag import oagbert from cogdl.oag.utils import colored import math diff --git a/examples/oagbert/oagbert_metainfo_zh.py b/examples/oagbert/oagbert_metainfo_zh.py index a942b2cd..960ccf50 100644 --- a/examples/oagbert/oagbert_metainfo_zh.py +++ b/examples/oagbert/oagbert_metainfo_zh.py @@ -1,32 +1,47 @@ -from cogdl import oagbert +from cogdl.oag import oagbert from cogdl.oag.utils import colored import math tokenizer, model = oagbert("oagbert-v2-zh") model.eval() -title = '基于随机化矩阵分解的网络嵌入方法' -abstract = '''随着互联网的普及,越来越多的问题以社交网络这样的网络形式出现.网络通常用图数据表示,由于图数据处理的挑战性,如何从图中学习到重要的信息是当前被广泛关注的问题.网络嵌入就是通过分析图数据得到反映网络结构的特征向量,利用它们进而实现各种数据挖掘任务,例如边预测、节点分类、网络重构、标签推荐和异常检测.最近,基于矩阵分解的网络嵌入方法NetMF被提出,它在理论上统一了多种网络嵌入方法,并且在处理实际数据时表现出很好的效果.然而,在处理大规模网络时,NetMF需要极大的时间和空间开销.本文使用快速随机化特征值分解和单遍历奇异值分解技术对NetMF进行改进,提出一种高效率、且内存用量小的矩阵分解网络嵌入算法eNetMF.首先,我们提出了适合于对称稀疏矩阵的随机化特征值分解算法freigs,它在处理实际的归一化网络矩阵时比传统的截断特征值分解算法快近10倍,且几乎不损失准确度.其次,我们提出使用单遍历奇异值分解处理NetMF方法中高次近似矩阵从而避免稠密矩阵存储的技术,它大大减少了网络嵌入所需的内存用量.最后,我们提出一种简洁的、且保证分解结果对称的随机化单遍历奇异值分解算法,将它与上述技术结合得到eNetMF算法.基于5个实际的网络数据集,我们评估了eNetMF学习到的网络低维表示在多标签节点分类和边预测上的有效性.实验结果表明,使用eNetMF替代NetMF后在后续得到的多标签分类性能指标上几乎没有损失,但在处理大规模数据时有超过40倍的加速与内存用量节省.在一台32核的机器上,eNetMF仅需约1.3 h即可对含一百多万节点的YouTube数据学习到网络嵌入,内存用量仅为120GB,并得到较高质量的分类结果.此外,最近被提出的网络嵌入算法NetSMF由于图稀疏化过程的内存需求太大,无法在256 GB内存的机器上处理两个较大的网络数据,而ProNE算法则在多标签分类的结果上表现不稳定,得到的Macro-F1值都比较差.因此,eNetMF算法在结果质量上明显优于NetSMF和ProNE算法.在边预测任务上,eNetMF算法也表现出与其它方法差不多甚至更好的性能.''' +title = "基于随机化矩阵分解的网络嵌入方法" +abstract = """随着互联网的普及,越来越多的问题以社交网络这样的网络形式出现.网络通常用图数据表示,由于图数据处理的挑战性,如何从图中学习到重要的信息是当前被广泛关注的问题.网络嵌入就是通过分析图数据得到反映网络结构的特征向量,利用它们进而实现各种数据挖掘任务,例如边预测、节点分类、网络重构、标签推荐和异常检测.最近,基于矩阵分解的网络嵌入方法NetMF被提出,它在理论上统一了多种网络嵌入方法,并且在处理实际数据时表现出很好的效果.然而,在处理大规模网络时,NetMF需要极大的时间和空间开销.本文使用快速随机化特征值分解和单遍历奇异值分解技术对NetMF进行改进,提出一种高效率、且内存用量小的矩阵分解网络嵌入算法eNetMF.首先,我们提出了适合于对称稀疏矩阵的随机化特征值分解算法freigs,它在处理实际的归一化网络矩阵时比传统的截断特征值分解算法快近10倍,且几乎不损失准确度.其次,我们提出使用单遍历奇异值分解处理NetMF方法中高次近似矩阵从而避免稠密矩阵存储的技术,它大大减少了网络嵌入所需的内存用量.最后,我们提出一种简洁的、且保证分解结果对称的随机化单遍历奇异值分解算法,将它与上述技术结合得到eNetMF算法.基于5个实际的网络数据集,我们评估了eNetMF学习到的网络低维表示在多标签节点分类和边预测上的有效性.实验结果表明,使用eNetMF替代NetMF后在后续得到的多标签分类性能指标上几乎没有损失,但在处理大规模数据时有超过40倍的加速与内存用量节省.在一台32核的机器上,eNetMF仅需约1.3 h即可对含一百多万节点的YouTube数据学习到网络嵌入,内存用量仅为120GB,并得到较高质量的分类结果.此外,最近被提出的网络嵌入算法NetSMF由于图稀疏化过程的内存需求太大,无法在256 GB内存的机器上处理两个较大的网络数据,而ProNE算法则在多标签分类的结果上表现不稳定,得到的Macro-F1值都比较差.因此,eNetMF算法在结果质量上明显优于NetSMF和ProNE算法.在边预测任务上,eNetMF算法也表现出与其它方法差不多甚至更好的性能.""" # calculate the probability of `machine learning`, `artificial intelligence`, `language model` for GPT-3 paper -print('=== Span Probability ===') -for span in ['机器学习', '网络嵌入', '随机化特征值分解']: +print("=== Span Probability ===") +for span in ["机器学习", "网络嵌入", "随机化特征值分解"]: span_prob, token_probs = model.calculate_span_prob( - title=title, abstract=abstract, decode_span_type='FOS', decode_span=span, mask_propmt_text='Field of Study:', debug=False) - print('%s probability: %.4f' % (span.ljust(30), span_prob)) + title=title, + abstract=abstract, + decode_span_type="FOS", + decode_span=span, + mask_propmt_text="Field of Study:", + debug=False, + ) + print("%s probability: %.4f" % (span.ljust(30), span_prob)) print() # decode a list of Field-Of-Study using beam search concepts = [] -print('=== Generated FOS ===') +print("=== Generated FOS ===") for i in range(4): candidates = [] for span_length in range(1, 5): - results = model.decode_beamsearch(title=title, abstract=abstract, authors=[ - ], concepts=concepts, decode_span_type='FOS', decode_span_length=span_length, beam_width=8, force_forward=False) + results = model.decode_beamsearch( + title=title, + abstract=abstract, + authors=[], + concepts=concepts, + decode_span_type="FOS", + decode_span_length=span_length, + beam_width=8, + force_forward=False, + ) candidates.append(results[0]) candidates.sort(key=lambda x: -x[1]) span, prob = candidates[0] - print("%2d. %s %s" % (i + 1, span, colored('[%s]' % - (','.join(['%s(%.4f)' % (k, v) for k, v in candidates])), 'blue'))) + print( + "%2d. %s %s" % (i + 1, span, colored("[%s]" % (",".join(["%s(%.4f)" % (k, v) for k, v in candidates])), "blue")) + ) concepts.append(span) diff --git a/examples/oagbert/oagbert_metainfo_zh_similarity.py b/examples/oagbert/oagbert_metainfo_zh_similarity.py index 76b31476..fdcea27e 100644 --- a/examples/oagbert/oagbert_metainfo_zh_similarity.py +++ b/examples/oagbert/oagbert_metainfo_zh_similarity.py @@ -1,5 +1,5 @@ import os -from cogdl import oagbert +from cogdl.oag import oagbert import torch import torch.nn.functional as F import numpy as np @@ -14,9 +14,16 @@ abstract = "通过搜集已有尾矿坝事故资料,分析了国内外尾矿坝事故与坝高、筑坝工艺及致灾因素的关系。对147起尾矿坝事故的分析研究表明, 引起尾矿坝事故的主要因素为降雨,其次为地震、管理等;" # Encode first paper -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( - title=title, abstract=abstract -) +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs(title=title, abstract=abstract) _, paper_embed_1 = model.bert.forward( input_ids=torch.LongTensor(input_ids).unsqueeze(0), token_type_ids=torch.LongTensor(token_type_ids).unsqueeze(0), @@ -24,16 +31,23 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # Positive Paper 2 title = "尾矿库工程特性及其安全监控系统研究" abstract = "总结了尾矿坝工程的特殊性和复杂性.为了保证尾矿坝在全生命周期(包括运行期及其闭库后)的安全,发展尾矿库安全监控系统具有重要意义.提出了尾矿库安全监控的基础框架,分析了尾矿库安全监测的主要内容及关键问题,为保证尾矿库的安全提供强有力的科学和技术依据." # Encode second paper -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( - title=title, abstract=abstract -) +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs(title=title, abstract=abstract) _, paper_embed_2 = model.bert.forward( input_ids=torch.LongTensor(input_ids).unsqueeze(0), token_type_ids=torch.LongTensor(token_type_ids).unsqueeze(0), @@ -41,16 +55,23 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # Negative Paper 3 title = "Windows下EPA技术的研究与改进" abstract = "该文对Windows下rookit的几种检测技术进行了比较和研究,并着重分析了基于可执行路径分析(EPA)技术.同时还讨论了其在Win2k下的代码实现,并提出改进方案。" # encode third paper -input_ids, input_masks, token_type_ids, masked_lm_labels, position_ids, position_ids_second, masked_positions, num_spans = model.build_inputs( - title=title, abstract=abstract -) +( + input_ids, + input_masks, + token_type_ids, + masked_lm_labels, + position_ids, + position_ids_second, + masked_positions, + num_spans, +) = model.build_inputs(title=title, abstract=abstract) _, paper_embed_3 = model.bert.forward( input_ids=torch.LongTensor(input_ids).unsqueeze(0), token_type_ids=torch.LongTensor(token_type_ids).unsqueeze(0), @@ -58,7 +79,7 @@ output_all_encoded_layers=False, checkpoint_activations=False, position_ids=torch.LongTensor(position_ids).unsqueeze(0), - position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0) + position_ids_second=torch.LongTensor(position_ids_second).unsqueeze(0), ) # calulate text similarity diff --git a/examples/ogb/products/README.md b/examples/ogb/products/README.md new file mode 100644 index 00000000..4d4da8c1 --- /dev/null +++ b/examples/ogb/products/README.md @@ -0,0 +1,25 @@ +# CogDL examples for ogbn-products + +CogDL implementation of ClusterGCN (SAGE aggr) for [ogbn-products](https://ogb.stanford.edu/docs/nodeprop/#ogbn-products). + +Requires CogDL 0.5.1 or later versions. + + +## Training & Evaluation + +``` +# Run with sage model with default config +python gnn.py + +# Run with sage model with custom config +python gnn.py --hidden-size 128 +``` +For more hyper-parameters, please find them in the `gnn.py`. + +## Results + +Here are the results over 10 runs which are comparable with OGB official results reported in the leaderboard. + +| Method | Test Accuracy | Validation Accuracy | #Parameters | +|:-------------------------------:|:---------------:|:-------------------:|:-----------:| +| ClusterGCN (SAGE aggr) | 0.7906 ± 0.0032 | 0.9168 ± 0.0006 | 207,919 | diff --git a/examples/ogb/products/gnn.py b/examples/ogb/products/gnn.py new file mode 100644 index 00000000..d856b676 --- /dev/null +++ b/examples/ogb/products/gnn.py @@ -0,0 +1,86 @@ +import argparse +import torch.nn as nn +from cogdl import experiment +from cogdl.models import BaseModel +from cogdl.layers import SAGELayer +from cogdl.datasets.ogb import OGBProductsDataset + + +class SAGE(BaseModel): + def __init__( + self, + in_feats, + out_feats, + hidden_size, + num_layers, + aggr="mean", + dropout=0.5, + norm="batchnorm", + activation="relu", + normalize=False, + ): + super(SAGE, self).__init__() + shapes = [in_feats] + [hidden_size] * (num_layers - 1) + [out_feats] + self.num_layers = num_layers + self.layers = nn.ModuleList( + [ + SAGELayer( + shapes[i], + shapes[i + 1], + aggr=aggr, + normalize=normalize if i != num_layers - 1 else False, + dropout=dropout if i != num_layers - 1 else False, + norm=norm if i != num_layers - 1 else None, + activation=activation if i != num_layers - 1 else None, + ) + for i in range(num_layers) + ] + ) + + def forward(self, graph): + x = graph.x + for layer in self.layers: + x = layer(graph, x) + return x + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="OGBN-Products (CogDL GNNs)") + parser.add_argument("--num-layers", type=int, default=3) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--dropout", type=float, default=0.5) + parser.add_argument("--lr", type=float, default=0.001) + parser.add_argument("--weight-decay", type=float, default=0.0) + parser.add_argument("--epochs", type=int, default=100) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--n-cluster", type=int, default=15000) + parser.add_argument("--eval-step", type=int, default=10) + parser.add_argument("--patience", type=int, default=10) + parser.add_argument("--logger", type=str, default=None) + parser.add_argument("--runs", type=int, default=10) + args = parser.parse_args() + + dataset = OGBProductsDataset() + gnn = SAGE( + in_feats=dataset.num_features, + hidden_size=args.hidden_size, + out_feats=dataset.num_classes, + num_layers=args.num_layers, + dropout=args.dropout, + ) + + experiment( + model=gnn, + dataset=dataset, + lr=args.lr, + weight_decay=args.weight_decay, + epochs=args.epochs, + seed=list(range(args.runs)), + dw="cluster_dw", + batch_size=args.batch_size, + n_cluster=args.n_cluster, + cpu_inference=True, + eval_step=args.eval_step, + logger=args.logger, + patience=args.patience, + ) diff --git a/tests/tasks/test_attributed_graph_clustering.py b/tests/tasks/test_attributed_graph_clustering.py index a2e51f6c..c51b071d 100644 --- a/tests/tasks/test_attributed_graph_clustering.py +++ b/tests/tasks/test_attributed_graph_clustering.py @@ -49,7 +49,7 @@ def test_kmeans_cora(): args.model_type = "content" args.cluster_method = "kmeans" ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_spectral_cora(): @@ -57,7 +57,7 @@ def test_spectral_cora(): args.model_type = "content" args.cluster_method = "spectral" ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_prone_cora(): @@ -65,16 +65,16 @@ def test_prone_cora(): args.model_type = "spectral" args.cluster_method = "kmeans" ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_agc_cora(): args = get_default_args_agc(dataset="cora", model="agc", mw="agc_mw", dw="node_classification_dw") args.model_type = "both" args.cluster_method = "spectral" - args.max_iter = 2 + args.max_iter = 1 ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_daegc_cora(): @@ -82,7 +82,7 @@ def test_daegc_cora(): args.model_type = "both" args.cluster_method = "kmeans" ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_gae_cora(): @@ -91,15 +91,16 @@ def test_gae_cora(): args.model_type = "both" args.cluster_method = "kmeans" ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 def test_vgae_cora(): args = get_default_args_agc(dataset="cora", model="vgae", mw="gae_mw", dw="node_classification_dw") args.model_type = "both" args.cluster_method = "kmeans" + args.epochs = 1 ret = train(args) - assert ret["nmi"] > 0 + assert ret["nmi"] >= 0 if __name__ == "__main__": diff --git a/tests/tasks/test_encode_paper.py b/tests/tasks/test_encode_paper.py index f6a101c9..d6fa16ee 100644 --- a/tests/tasks/test_encode_paper.py +++ b/tests/tasks/test_encode_paper.py @@ -1,4 +1,4 @@ -from cogdl import oagbert +from cogdl.oag import oagbert def test_encode_paper(): diff --git a/tests/tasks/test_graph_classification.py b/tests/tasks/test_graph_classification.py index 94b126db..65c0cf47 100644 --- a/tests/tasks/test_graph_classification.py +++ b/tests/tasks/test_graph_classification.py @@ -105,10 +105,10 @@ def test_gin_proteins(): assert ret["test_acc"] > 0 -def test_diffpool_mutag(): - args = get_default_args_graph_clf(dataset="mutag", model="diffpool") +def test_diffpool_imdb_binary(): + args = get_default_args_graph_clf(dataset="imdb-b", model="diffpool") args = add_diffpool_args(args) - args.batch_size = 5 + args.batch_size = 100 args.train_ratio = 0.6 args.test_ratio = 0.2 ret = train(args) @@ -138,5 +138,5 @@ def test_patchy_san_mutag(): test_gin_proteins() test_sortpool_mutag() - test_diffpool_mutag() + test_diffpool_imdb_binary() test_patchy_san_mutag() diff --git a/tests/tasks/test_unsupervised_graph_classification.py b/tests/tasks/test_unsupervised_graph_classification.py index f4eb97bb..c89e2503 100644 --- a/tests/tasks/test_unsupervised_graph_classification.py +++ b/tests/tasks/test_unsupervised_graph_classification.py @@ -38,16 +38,16 @@ def get_default_args_graph_clf(dataset, model, dw="graph_embedding_dw", mw="grap def add_infograp_args(args): args.hidden_size = 16 - args.batch_size = 10 + args.batch_size = 32 args.target = 0 args.train_num = 100 - args.num_layers = 3 + args.num_layers = 2 args.sup = False - args.epochs = 3 + args.epochs = 2 args.nn = True args.lr = 0.0001 args.train_ratio = 0.7 - args.test_ratio = 0.1 + args.test_ratio = 0.2 args.model = "infograph" args.degree_node_features = False return args diff --git a/tests/test_oag.py b/tests/test_oag.py index 6afe0ece..0de561bf 100644 --- a/tests/test_oag.py +++ b/tests/test_oag.py @@ -1,4 +1,4 @@ -from cogdl import oagbert +from cogdl.oag import oagbert def test_oagbert(): @@ -17,11 +17,22 @@ def test_oagbert_v2(): tokenizer, model = oagbert("oagbert-v2-test") sequence = "CogDL is developed by KEG, Tsinghua." span_prob, token_probs = model.calculate_span_prob( - title=sequence, decode_span_type='FOS', decode_span='data mining', mask_propmt_text='Field of Study:', debug=False) + title=sequence, + decode_span_type="FOS", + decode_span="data mining", + mask_propmt_text="Field of Study:", + debug=False, + ) assert span_prob >= 0 and span_prob <= 1 - results = model.decode_beamsearch(title=sequence, decode_span_type='FOS', decode_span_length=2, beam_width=2, force_forward=False) + results = model.decode_beamsearch( + title=sequence, decode_span_type="FOS", decode_span_length=2, beam_width=2, force_forward=False + ) assert len(results) == 2 - model.generate_title(abstract="To enrich language models with domain knowledge is crucial but difficult. Based on the world's largest public academic graph Open Academic Graph (OAG), we pre-train an academic language model, namely OAG-BERT, which integrates massive heterogeneous entities including paper, author, concept, venue, and affiliation. To better endow OAG-BERT with the ability to capture entity information, we develop novel pre-training strategies including heterogeneous entity type embedding, entity-aware 2D positional encoding, and span-aware entity masking. For zero-shot inference, we design a special decoding strategy to allow OAG-BERT to generate entity names from scratch. We evaluate the OAG-BERT on various downstream academic tasks, including NLP benchmarks, zero-shot entity inference, heterogeneous graph link prediction, and author name disambiguation. Results demonstrate the effectiveness of the proposed pre-training approach to both comprehending academic texts and modeling knowledge from heterogeneous entities. OAG-BERT has been deployed to multiple real-world applications, such as reviewer recommendations for NSFC (National Nature Science Foundation of China) and paper tagging in the AMiner system. It is also available to the public through the CogDL package.", max_length=20) + model.generate_title( + abstract="To enrich language models with domain knowledge is crucial but difficult. Based on the world's largest public academic graph Open Academic Graph (OAG), we pre-train an academic language model, namely OAG-BERT, which integrates massive heterogeneous entities including paper, author, concept, venue, and affiliation. To better endow OAG-BERT with the ability to capture entity information, we develop novel pre-training strategies including heterogeneous entity type embedding, entity-aware 2D positional encoding, and span-aware entity masking. For zero-shot inference, we design a special decoding strategy to allow OAG-BERT to generate entity names from scratch. We evaluate the OAG-BERT on various downstream academic tasks, including NLP benchmarks, zero-shot entity inference, heterogeneous graph link prediction, and author name disambiguation. Results demonstrate the effectiveness of the proposed pre-training approach to both comprehending academic texts and modeling knowledge from heterogeneous entities. OAG-BERT has been deployed to multiple real-world applications, such as reviewer recommendations for NSFC (National Nature Science Foundation of China) and paper tagging in the AMiner system. It is also available to the public through the CogDL package.", + max_length=20, + ) + if __name__ == "__main__": test_oagbert()