From d4dca2db5ef21628d2528ccf9d74be9bc12f3464 Mon Sep 17 00:00:00 2001 From: joseph-arulraj Date: Thu, 5 Sep 2024 17:05:11 +0100 Subject: [PATCH 01/22] CSAI Added CSAI implementation to pypots --- csai.py | 76 ++++++++ pypots/classification/__init__.py | 2 + pypots/classification/base.py | 2 + pypots/classification/csai/__init__.py | 5 + pypots/classification/csai/core.py | 119 +++++++++++ pypots/classification/csai/data.py | 32 +++ pypots/classification/csai/model.py | 260 +++++++++++++++++++++++++ pypots/data/utils.py | 224 +++++++++++++++++++++ pypots/imputation/__init__.py | 2 + pypots/imputation/base.py | 2 + pypots/imputation/csai/__init__.py | 5 + pypots/imputation/csai/core.py | 55 ++++++ pypots/imputation/csai/data.py | 176 +++++++++++++++++ pypots/imputation/csai/model.py | 250 ++++++++++++++++++++++++ pypots/nn/modules/csai/__init__.py | 13 ++ pypots/nn/modules/csai/backbone.py | 165 ++++++++++++++++ pypots/nn/modules/csai/layers.py | 128 ++++++++++++ 17 files changed, 1516 insertions(+) create mode 100644 csai.py create mode 100644 pypots/classification/csai/__init__.py create mode 100644 pypots/classification/csai/core.py create mode 100644 pypots/classification/csai/data.py create mode 100644 pypots/classification/csai/model.py create mode 100644 pypots/imputation/csai/__init__.py create mode 100644 pypots/imputation/csai/core.py create mode 100644 pypots/imputation/csai/data.py create mode 100644 pypots/imputation/csai/model.py create mode 100644 pypots/nn/modules/csai/__init__.py create mode 100644 pypots/nn/modules/csai/backbone.py create mode 100644 pypots/nn/modules/csai/layers.py diff --git a/csai.py b/csai.py new file mode 100644 index 00000000..ee77261e --- /dev/null +++ b/csai.py @@ -0,0 +1,76 @@ +from pypots.classification import CSAI +import pickle +from pypots.optim import Adam +from pypots.utils.metrics import calc_mae +import torch +from pypots.utils.metrics import calc_binary_classification_metrics + +kfold_data = pickle.load(open(r"C:\Users\jsphr\Documents\Imputation\data\data_nan.pkl", 'rb')) +kfold_label = pickle.load(open(r"C:\Users\jsphr\Documents\Imputation\data\label.pkl", 'rb')) + + +train_data = kfold_data[0][0] +train_label = kfold_label[0][0] + +valid_data = kfold_data[0][1] +valid_label = kfold_label[0][1] + + +test_data = kfold_data[0][2] +test_label = kfold_label[0][2] + +dataset_for_training = { + "X": train_data, + "y": train_label +} + +dataset_for_validating = { + "X": valid_data, + "y": valid_label +} + +dataset_for_testing = { + "X": test_data, + "y": test_label +} + +csai = CSAI(n_steps = 48, + n_features = 35, + rnn_hidden_size = 108, + imputation_weight = 0.3, + consistency_weight = 0.1, + classification_weight = 1, + n_classes = 1, + removal_percent = 10, + increase_factor = 0.1, + compute_intervals = True, + step_channels = 512, + batch_size = 32, + epochs = 10, + patience = None, + optimizer = Adam(lr=1e-3), + num_workers = 0, + device = torch.device("cuda" if torch.cuda.is_available() else "cpu"), + saving_path = r'C:\Users\jsphr\Documents\Imputation', + model_saving_strategy = "best", + verbose = True) + +csai.fit(dataset_for_training, dataset_for_validating) +csai_results = csai.predict(dataset_for_testing) +print(csai_results.keys()) +classification_predictions = csai_results['classification'] +metrics = calc_binary_classification_metrics(classification_predictions, dataset_for_testing["y"]) + +# or0 = csai_results["X_ori"] +# indi = csai_results["indicating_mask"] + +# mae = calc_mae(imputed_data, or0, indi) +# print(mae) + +print("Testing classification metrics: \n" + f'ROC_AUC: {metrics["roc_auc"]}, \n' + f'PR_AUC: {metrics["pr_auc"]},\n' + f'F1: {metrics["f1"]},\n' + f'Precision: {metrics["precision"]},\n' + f'Recall: {metrics["recall"]},\n' +) \ No newline at end of file diff --git a/pypots/classification/__init__.py b/pypots/classification/__init__.py index 73f6524d..1d044d9a 100644 --- a/pypots/classification/__init__.py +++ b/pypots/classification/__init__.py @@ -8,9 +8,11 @@ from .brits import BRITS from .grud import GRUD from .raindrop import Raindrop +from .csai import CSAI __all__ = [ "BRITS", "GRUD", "Raindrop", + "CSAI", ] diff --git a/pypots/classification/base.py b/pypots/classification/base.py index ca587c29..c86d2993 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -287,7 +287,9 @@ def _train_model( try: training_step = 0 + print("Training model......................................") for epoch in range(1, self.epochs + 1): + print(f"Epoch {epoch}......................................") self.model.train() epoch_train_loss_collector = [] for idx, data in enumerate(training_loader): diff --git a/pypots/classification/csai/__init__.py b/pypots/classification/csai/__init__.py new file mode 100644 index 00000000..d856c4b3 --- /dev/null +++ b/pypots/classification/csai/__init__.py @@ -0,0 +1,5 @@ +from .model import CSAI + +__all__ = [ + "CSAI", +] \ No newline at end of file diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py new file mode 100644 index 00000000..35e24890 --- /dev/null +++ b/pypots/classification/csai/core.py @@ -0,0 +1,119 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ...nn.modules.csai import BackboneBCSAI + + + + +class DiceBCELoss(nn.Module): + def __init__(self, weight=None, size_average=True): + super(DiceBCELoss, self).__init__() + self.bcelogits = nn.BCEWithLogitsLoss() + + def forward(self, y_score, y_out, targets, smooth=1): + + #comment out if your model contains a sigmoid or equivalent activation layer + # inputs = F.sigmoid(inputs) + + #flatten label and prediction tensors + BCE = self.bcelogits(y_out, targets) + + y_score = y_score.view(-1) + targets = targets.view(-1) + intersection = (y_score * targets).sum() + dice_loss = 1 - (2.*intersection + smooth)/(y_score.sum() + targets.sum() + smooth) + + Dice_BCE = BCE + dice_loss + + return BCE, Dice_BCE + + +class _BCSAI(nn.Module): + def __init__( + self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + classification_weight: float, + n_classes: int, + step_channels: int, + dropout: float = 0.5, + intervals: list = None, + device = None, + + ): + super().__init__() + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.classification_weight = classification_weight + self.n_classes = n_classes + self.step_channels = step_channels + self.intervals = intervals + self.device = device + + + # create models + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) + self.classifier = nn.Linear(self.rnn_hidden_size, n_classes) + self.imputer = nn.Linear(self.rnn_hidden_size, n_features) + self.dropout = nn.Dropout(dropout) + + def forward(self, inputs: dict, training: bool = True) -> dict: + + ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) = self.model(inputs) + + results = { + "imputed_data": imputed_data, + } + + f_logits = self.classifier(self.dropout(f_hidden_states)) + b_logits = self.classifier(self.dropout(b_hidden_states)) + + f_prediction = torch.sigmoid(f_logits) + b_prediction = torch.sigmoid(b_logits) + + classification_pred = (f_prediction + b_prediction) / 2 + + results = { + "imputed_data": imputed_data, + "classification_pred": classification_pred, + } + + # if in training mode, return results with losses + if training: + criterion = DiceBCELoss().to(self.device) + results["consistency_loss"] = consistency_loss + results["reconstruction_loss"] = reconstruction_loss + # print(inputs["labels"].unsqueeze(1)) + f_classification_loss, _ = criterion(f_prediction, f_logits, inputs["labels"].unsqueeze(1).float()) + b_classification_loss, _ = criterion(b_prediction, b_logits, inputs["labels"].unsqueeze(1).float()) + classification_loss = (f_classification_loss + b_classification_loss) + + loss = ( + self.consistency_weight * consistency_loss + + self.imputation_weight * reconstruction_loss + + self.classification_weight * classification_loss + ) + + results["loss"] = loss + # results["reconstruction"] = (f_reconstruction + b_reconstruction) / 2 + results["classification_loss"] = classification_loss + results["f_reconstruction"] = f_reconstruction + results["b_reconstruction"] = b_reconstruction + + return results \ No newline at end of file diff --git a/pypots/classification/csai/data.py b/pypots/classification/csai/data.py new file mode 100644 index 00000000..3766b4f2 --- /dev/null +++ b/pypots/classification/csai/data.py @@ -0,0 +1,32 @@ +from typing import Union +from ...imputation.csai.data import DatasetForCSAI as DatasetForCSAI_Imputation + + + +class DatasetForCSAI(DatasetForCSAI_Imputation): + def __init__(self, + data: dict | str, + file_type: str = "hdf5", + return_y: bool = True, + removal_percent: float = 0.0, + increase_factor: float = 0.1, + compute_intervals: bool = False, + replacement_probabilities = None, + normalise_mean : list = [], + normalise_std: list = [], + training: bool = True + ): + super().__init__( + data=data, + return_X_ori=False, + return_y=return_y, + file_type=file_type, + removal_percent=removal_percent, + increase_factor=increase_factor, + compute_intervals=compute_intervals, + replacement_probabilities=replacement_probabilities, + normalise_mean=normalise_mean, + normalise_std=normalise_std, + training=training + ) + \ No newline at end of file diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py new file mode 100644 index 00000000..e3e077f8 --- /dev/null +++ b/pypots/classification/csai/model.py @@ -0,0 +1,260 @@ + +from typing import Optional, Union +import numpy as np +import torch +from torch.utils.data import DataLoader + +from .core import _BCSAI +from .data import DatasetForCSAI +from ..base import BaseNNClassifier +from ...optim.adam import Adam +from ...optim.base import Optimizer + + +class CSAI(BaseNNClassifier): + def __init__(self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + classification_weight: float, + n_classes: int, + removal_percent: int, + increase_factor: float, + compute_intervals: bool, + step_channels:int, + batch_size: int, + epochs: int, + dropout: float = 0.5, + patience: int | None = None, + optimizer: Optimizer = Adam(), + num_workers: int = 0, + device: Optional[Union[str, torch.device, list]] = None, + saving_path: str = None, + model_saving_strategy: str | None = "best", + verbose: bool = True): + super().__init__( + n_classes, + batch_size, + epochs, + patience, + num_workers, + device, + saving_path, + model_saving_strategy, + verbose) + + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.classification_weight = classification_weight + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.step_channels = step_channels + self.compute_intervals = compute_intervals + self.intervals = None + self.dropout = dropout + self.device = device + + # Initialise empty model + self.model = None + self.optimizer = optimizer + + def _assemble_input_for_training(self, data: list, training=True) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + labels + ) = self._send_data_to_given_device(sample) + + inputs = { + "indices": indices, + "labels": labels, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + } + + + return inputs + def _assemble_input_for_validating(self, data: list) -> dict: + return self._assemble_input_for_training(data) + + def _assemble_input_for_testing(self, data: list) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + # X_ori, + # indicating_mask, + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + # "X_ori": X_ori, + # "indicating_mask": indicating_mask, + } + + return inputs + + def fit( + self, + train_set, + val_set=None, + file_type: str = "hdf5", + ): + # Create dataset + self.training_set = DatasetForCSAI( + data=train_set, + file_type=file_type, + return_y=True, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + ) + + train_loader = DataLoader( + self.training_set, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + ) + + val_loader = None + if val_set is not None: + val_set = DatasetForCSAI( + data=val_set, + file_type=file_type, + return_y=True, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + replacement_probabilities=self.training_set.replacement_probabilities, + normalise_mean=self.training_set.normalise_mean, + normalise_std=self.training_set.normalise_std, + training=False + + ) + val_loader = DataLoader( + val_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + # Create model + self.model = _BCSAI( + n_steps=self.n_steps, + n_features=self.n_features, + rnn_hidden_size=self.rnn_hidden_size, + imputation_weight=self.imputation_weight, + consistency_weight=self.consistency_weight, + classification_weight=self.classification_weight, + n_classes=self.n_classes, + step_channels=self.step_channels, + dropout=self.dropout, + intervals=self.training_set.intervals, + device=self.device, + ) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer + self.optimizer.init_optimizer(self.model.parameters()) + + # train the model + self._train_model(train_loader, val_loader) + self.model.load_state_dict(self.best_model_dict) + self.model.eval() + + self._auto_save_model_if_necessary(confirm_saving=True) + + + def predict( + self, + test_set, + file_type: str = "hdf5"): + + self.model.eval() + test_set = DatasetForCSAI( + data=test_set, + file_type=file_type, + return_y=False, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + replacement_probabilities=self.training_set.replacement_probabilities, + normalise_mean=self.training_set.normalise_mean, + normalise_std=self.training_set.normalise_std, + training=False + ) + test_loader = DataLoader( + test_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + + classificaion_results = [] + + with torch.no_grad(): + for idx, data in enumerate(test_loader): + inputs = self._assemble_input_for_testing(data) + results = self.model.forward(inputs, training=False) + classificaion_results.append(results['classification_pred']) + + + classification = torch.cat(classificaion_results).cpu().detach().numpy() + result_dict = { + "classification": classification, + } + return result_dict + + def classify( + self, + test_set, + file_type): + + result_dict = self.predict(test_set, file_type) + return result_dict['classification'] \ No newline at end of file diff --git a/pypots/data/utils.py b/pypots/data/utils.py index 7762ff7f..f89cc18a 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -5,6 +5,7 @@ # Created by Wenjie Du # License: BSD-3-Clause +import copy from typing import Union import benchpots @@ -231,3 +232,226 @@ def inverse_sliding_window(X, sliding_len): X, sliding_len, ) + + +def parse_delta_bidirectional(masks, direction): + if direction == 'backward': + masks = masks[::-1] == 1.0 + + [T, D] = masks.shape + deltas = [] + for t in range(T): + if t == 0: + deltas.append(np.ones(D)) + else: + deltas.append(np.ones(D) + (1 - masks[t]) * deltas[-1]) + + return np.array(deltas) + +def compute_last_obs(data, masks, direction): + """ + Compute the last observed values for each time step. + + Parameters: + - data (np.array): Original data array. + - masks (np.array): Binary masks indicating where data is not NaN. + + Returns: + - last_obs (np.array): Array of the last observed values. + """ + if direction == 'backward': + masks = masks[::-1] == 1.0 + data = data[::-1] + + [T, D] = masks.shape + last_obs = np.full((T, D), np.nan) # Initialize last observed values with NaNs + last_obs_val = np.full(D, np.nan) # Initialize last observed values for first time step with NaNs + + for t in range(1, T): + mask = masks[t-1] + # Update last observed values + # last_obs_val = np.where(masks[t], data[t], last_obs_val) + last_obs_val[mask] = data[t-1, mask] + # Assign last observed values to the current time step + last_obs[t] = last_obs_val + + return last_obs + +def adjust_probability_vectorized(obs_count, avg_count, base_prob, increase_factor=0.5): + if obs_count < avg_count: + return min(base_prob * (avg_count / obs_count) * increase_factor, 1.0) + else: + return max(base_prob * (obs_count / avg_count) / increase_factor, 0) + + + +def normalize_csai(data, mean, std, compute_intervals=False): + n_patients = data.shape[0] + n_hours = data.shape[1] + n_variables = data.shape[2] + + + measure = copy.deepcopy(data).reshape(n_patients * n_hours, n_variables) + + if compute_intervals: + intervals_list = {v: [] for v in range(n_variables)} + + isnew = 0 + if len(mean) == 0 or len(std) == 0: + isnew = 1 + mean_set = np.zeros([n_variables]) + std_set = np.zeros([n_variables]) + else: + mean_set = mean + std_set = std + + for v in range(n_variables): + + if isnew: + mask_v = ~np.isnan(measure[:,v]) * 1 + idx_global = np.where(mask_v == 1)[0] + + if idx_global.sum() == 0: + continue + + measure_mean = np.mean(measure[:, v][idx_global]) + measure_std = np.std(measure[:, v][idx_global]) + + mean_set[v] = measure_mean + std_set[v] = measure_std + else: + measure_mean = mean[v] + measure_std = std[v] + for p in range(n_patients): + mask_p_v = ~np.isnan(data[p, :, v]) * 1 + idx_p = np.where(mask_p_v == 1)[0] + if compute_intervals and len(idx_p) > 1: + intervals_list[v].extend([idx_p[i+1] - idx_p[i] for i in range(len(idx_p)-1)]) + + for ix in idx_p: + if measure_std != 0: + data[p, ix, v] = (data[p, ix, v] - measure_mean) / measure_std + else: + data[p, ix, v] = data[p, ix, v] - measure_mean + + if compute_intervals: + intervals_list = {v: np.median(intervals_list[v]) if intervals_list[v] else np.nan for v in intervals_list} + + if compute_intervals: + return data, mean_set, std_set, intervals_list + else: + return data, mean_set, std_set + + + +def non_uniform_sample_loader_bidirectional(data, removal_percent, pre_replacement_probabilities=None, increase_factor=0.5): + + # Random seed + np.random.seed(1) + torch.manual_seed(1) + + # Get Dimensionality + [N, T, D] = data.shape + + if pre_replacement_probabilities is None: + + + observations_per_feature = np.sum(~np.isnan(data), axis=(0, 1)) + average_observations = np.mean(observations_per_feature) + replacement_probabilities = np.full(D, removal_percent / 100) + + if increase_factor > 0: + print('The increase_factor is {}!'.format(increase_factor)) + for feature_idx in range(D): + replacement_probabilities[feature_idx] = adjust_probability_vectorized( + observations_per_feature[feature_idx], + average_observations, + replacement_probabilities[feature_idx], + increase_factor=increase_factor + ) + + # print('before:\n',replacement_probabilities) + total_observations = np.sum(observations_per_feature) + total_replacement_target = total_observations * removal_percent / 100 + + for _ in range(1000): # Limit iterations to prevent infinite loop + total_replacement = np.sum(replacement_probabilities * observations_per_feature) + if np.isclose(total_replacement, total_replacement_target, rtol=1e-3): + break + adjustment_factor = total_replacement_target / total_replacement + replacement_probabilities *= adjustment_factor + + # print('after:\n',replacement_probabilities) + else: + replacement_probabilities = pre_replacement_probabilities + + recs = [] + number = 0 + masks_sum = np.zeros(D) + eval_masks_sum = np.zeros(D) + values = copy.deepcopy(data) + random_matrix = np.random.rand(N, T, D) + values[(~np.isnan(values)) & (random_matrix < replacement_probabilities)] = np.nan +# mask = (~torch.isnan(values)) & (random_matrix < replacement_probabilities) + +# # Ensure mask is boolean +# mask = mask.to(torch.bool) + +# # Apply mask +# values[mask] = float('nan') + for i in range(N): + masks = ~np.isnan(values[i, :, :]) + eval_masks = (~np.isnan(values[i, :, :])) ^ (~np.isnan(data[i, :, :])) + evals = data[i, :, :] + rec = {} + # rec['label'] = label[i] + deltas_f = parse_delta_bidirectional(masks, direction='forward') + deltas_b = parse_delta_bidirectional(masks, direction='backward') + last_obs_f = compute_last_obs(values[i, :, :], masks, direction='forward') + last_obs_b = compute_last_obs(values[i, :, :], masks, direction='backward') + rec['values'] = np.nan_to_num(values[i, :, :]).tolist() + rec['last_obs_f'] = np.nan_to_num(last_obs_f).tolist() + rec['last_obs_b'] = np.nan_to_num(last_obs_b).tolist() + rec['masks'] = masks.astype('int32').tolist() + rec['evals'] = np.nan_to_num(evals).tolist() + rec['eval_masks'] = eval_masks.astype('int32').tolist() + rec['deltas_f'] = deltas_f.tolist() + rec['deltas_b'] = deltas_b.tolist() + recs.append(rec) + number += 1 + masks_sum += np.sum(masks, axis=0) + eval_masks_sum += np.sum(eval_masks, axis=0) + + + return recs, replacement_probabilities + + +def collate_fn_bidirectional(recs): + + def to_tensor_dict(recs): + + values = torch.FloatTensor(np.array([r['values'] for r in recs])) + last_obs_f = torch.FloatTensor(np.array([r['last_obs_f'] for r in recs])) + last_obs_b = torch.FloatTensor(np.array([r['last_obs_b'] for r in recs])) + masks = torch.FloatTensor(np.array([r['masks'] for r in recs])) + deltas_f = torch.FloatTensor(np.array([r['deltas_f'] for r in recs])) + deltas_b = torch.FloatTensor(np.array([r['deltas_b'] for r in recs])) + + evals = torch.FloatTensor(np.array([r['evals'] for r in recs])) + eval_masks = torch.FloatTensor(np.array([r['eval_masks'] for r in recs])) + + return {'values': values, + 'last_obs_f': last_obs_f, + 'last_obs_b': last_obs_b, + 'masks': masks, + 'deltas_f': deltas_f, + 'deltas_b': deltas_b, + 'evals': evals, + 'eval_masks': eval_masks} + + ret_dict = to_tensor_dict(recs) + + # ret_dict['labels'] = torch.FloatTensor(np.array([r['label'] for r in recs])) + # ret_dict['labels'] = torch.LongTensor(np.array([r['label'] for r in recs])) + + return ret_dict \ No newline at end of file diff --git a/pypots/imputation/__init__.py b/pypots/imputation/__init__.py index 8136b0f8..d48e2f9a 100644 --- a/pypots/imputation/__init__.py +++ b/pypots/imputation/__init__.py @@ -38,6 +38,7 @@ from .imputeformer import ImputeFormer from .timemixer import TimeMixer from .moderntcn import ModernTCN +from .csai import CSAI # naive imputation methods from .locf import LOCF @@ -47,6 +48,7 @@ __all__ = [ # neural network imputation methods + "CSAI", "SAITS", "Transformer", "iTransformer", diff --git a/pypots/imputation/base.py b/pypots/imputation/base.py index 6ca8bcb2..5366e74a 100644 --- a/pypots/imputation/base.py +++ b/pypots/imputation/base.py @@ -275,7 +275,9 @@ def _train_model( try: training_step = 0 + print("Training model......................................") for epoch in range(1, self.epochs + 1): + print(f"Epoch {epoch}......................................") self.model.train() epoch_train_loss_collector = [] for idx, data in enumerate(training_loader): diff --git a/pypots/imputation/csai/__init__.py b/pypots/imputation/csai/__init__.py new file mode 100644 index 00000000..d856c4b3 --- /dev/null +++ b/pypots/imputation/csai/__init__.py @@ -0,0 +1,5 @@ +from .model import CSAI + +__all__ = [ + "CSAI", +] \ No newline at end of file diff --git a/pypots/imputation/csai/core.py b/pypots/imputation/csai/core.py new file mode 100644 index 00000000..efc9d9df --- /dev/null +++ b/pypots/imputation/csai/core.py @@ -0,0 +1,55 @@ +import torch.nn as nn +from ...nn.modules.csai.backbone import BackboneBCSAI + + + +class _BCSAI(nn.Module): + def __init__(self, n_steps, + n_features, + rnn_hidden_size, + step_channels, + intervals, + consistency_weight, + imputation_weight) : + super().__init__() + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.step_channels = step_channels + self.intervals = intervals + self.consistency_weight = consistency_weight + self.imputation_weight = imputation_weight + + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) + + def forward(self, inputs:dict, training:bool = True) -> dict: + ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) = self.model(inputs) + + results = { + "imputed_data": imputed_data, + } + + # if in training mode, return results with losses + if training: + results["consistency_loss"] = consistency_loss + results["reconstruction_loss"] = reconstruction_loss + loss = self.consistency_weight * consistency_loss + self.imputation_weight * reconstruction_loss + + # `loss` is always the item for backward propagating to update the model + results["loss"] = loss + # results["reconstruction"] = (f_reconstruction + b_reconstruction) / 2 + results["f_reconstruction"] = f_reconstruction + results["b_reconstruction"] = b_reconstruction + if not training: + results["X_ori"] = inputs["X_ori"] + results["indicating_mask"] = inputs["indicating_mask"] + + return results diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py new file mode 100644 index 00000000..5d8b9288 --- /dev/null +++ b/pypots/imputation/csai/data.py @@ -0,0 +1,176 @@ +from typing import Iterable +from ...data.dataset import BaseDataset +import numpy as np +import torch +from ...data.utils import collate_fn_bidirectional, non_uniform_sample_loader_bidirectional, normalize_csai + +class DatasetForCSAI(BaseDataset): + def __init__(self, data: dict | str, + return_X_ori: bool, + return_y: bool, + file_type: str = "hdf5", + removal_percent: float = 0.0, + increase_factor: float = 0.1, + compute_intervals: bool = False, + replacement_probabilities = None, + normalise_mean : list = [], + normalise_std: list = [], + training: bool = True + ): + super().__init__(data = data, + return_X_ori = return_X_ori, + return_X_pred = False, + return_y = return_y, + file_type = file_type) + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.compute_intervals = compute_intervals + self.replacement_probabilities = replacement_probabilities + self.normalise_mean = normalise_mean + self.normalise_std = normalise_std + self.training = training + + if not isinstance(self.data, str): + self.normalized_data, self.mean_set, self.std_set, self.intervals = normalize_csai(self.data['X'], normalise_mean, + normalise_std, compute_intervals) + _data, self.replacement_probabilities = non_uniform_sample_loader_bidirectional(self.normalized_data, + removal_percent, + replacement_probabilities, + increase_factor) + self.processed_data = collate_fn_bidirectional(_data) + self.forward_X = self.processed_data['values'] + self.forward_missing_mask = self.processed_data['masks'] + self.backward_X = torch.flip(self.forward_X, dims=[1]) + self.backward_missing_mask = torch.flip(self.forward_missing_mask, dims=[1]) + + self.X_ori = self.processed_data['evals'] + self.indicating_mask = self.processed_data['eval_masks'] + # if self.return_y: + # self.y = self.processed_data['labels'] + + + + def _fetch_data_from_array(self, idx: int) -> Iterable: + """Fetch data from self.X if it is given. + + Parameters + ---------- + idx : + The index of the sample to be return. + + Returns + ------- + sample : + A list contains + + index : int tensor, + The index of the sample. + + X : tensor, + The feature vector for model input. + + missing_mask : tensor, + The mask indicates all missing values in X. + + delta : tensor, + The delta matrix contains time gaps of missing values. + + label (optional) : tensor, + The target label of the time-series sample. + """ + + + sample = [ + torch.tensor(idx), + # for forward + self.forward_X[idx], + self.forward_missing_mask[idx], + self.processed_data["deltas_f"][idx], + self.processed_data["last_obs_f"][idx], + # for backward + self.backward_X[idx], + self.backward_missing_mask[idx], + self.processed_data["deltas_b"][idx], + self.processed_data["last_obs_b"][idx], + ] + + # if not self.training: + # sample.extend([self.X_ori[idx], self.indicating_mask[idx]]) + + if self.return_y: + sample.append(self.y[idx].to(torch.long)) + + return { + 'sample': sample, + 'replacement_probabilities': self.replacement_probabilities, + 'mean_set': self.mean_set, + 'std_set': self.std_set, + 'intervals': self.intervals + } + + def _fetch_data_from_file(self, idx: int) -> Iterable: + """Fetch data with the lazy-loading strategy, i.e. only loading data from the file while requesting for samples. + Here the opened file handle doesn't load the entire dataset into RAM but only load the currently accessed slice. + + Parameters + ---------- + idx : + The index of the sample to be return. + + Returns + ------- + sample : + The collated data sample, a list including all necessary sample info. + """ + + if self.file_handle is None: + self.file_handle = self._open_file_handle() + + X = torch.from_numpy(self.file_handle["X"][idx]) + normalized_data, mean_set, std_set, intervals = normalize_csai(X, self.normalise_mean, + self.normalise_std, + self.compute_intervals) + processed_data, replacement_probabilities = non_uniform_sample_loader_bidirectional(normalized_data, + self.removal_percent, + self.replacement_probabilities, + self.increase_factor) + forward_X = processed_data['values'] + forward_missing_mask = processed_data['masks'] + backward_X = torch.flip(forward_X, dims=[1]) + backward_missing_mask = torch.flip(forward_missing_mask, dims=[1]) + + X_ori = self.processed_data['evals'] + indicating_mask = self.processed_data['eval_masks'] + if self.return_y: + y = self.processed_data['labels'] + + + sample = [ + torch.tensor(idx), + # for forward + forward_X, + forward_missing_mask, + processed_data["deltas_f"], + processed_data["last_obs_f"], + # for backward + backward_X, + backward_missing_mask, + processed_data["deltas_b"], + processed_data["last_obs_b"] + ] + + if self.return_X_ori: + sample.extend([X_ori, indicating_mask]) + + # if the dataset has labels and is for training, then fetch it from the file + if self.return_y: + sample.append(y) + + return { + 'sample': sample, + 'replacement_probabilities': replacement_probabilities, + 'mean_set': mean_set, + 'std_set': std_set, + 'intervals': intervals + } + diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py new file mode 100644 index 00000000..10348fa2 --- /dev/null +++ b/pypots/imputation/csai/model.py @@ -0,0 +1,250 @@ + +from typing import Union, Optional + +import numpy as np +import torch +from torch.utils.data import DataLoader + + +from .core import _BCSAI +from .data import DatasetForCSAI +from ..base import BaseNNImputer +from ...data.checking import key_in_data_set +from ...optim.adam import Adam +from ...optim.base import Optimizer + + +class CSAI(BaseNNImputer): + def __init__(self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + removal_percent: int, + increase_factor: float, + compute_intervals: bool, + step_channels:int, + batch_size: int, + epochs: int, + patience: int | None = None, + optimizer: Optional[Optimizer] = Adam(), + num_workers: int = 0, + device: str | torch.device | list | None = None, + saving_path: str = None, + model_saving_strategy: str | None = "best", + verbose: bool = True): + super().__init__(batch_size, epochs, patience, num_workers, device, saving_path, model_saving_strategy, verbose) + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.step_channels = step_channels + self.compute_intervals = compute_intervals + self.intervals = None + + # Initialise empty model + self.model = None + self.optimizer = optimizer + + def _assemble_input_for_training(self, data: list, training=True) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + } + + + return inputs + + def _assemble_input_for_validating(self, data: list) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + X_ori, + indicating_mask, + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + "X_ori": X_ori, + "indicating_mask": indicating_mask, + } + return inputs + + def _assemble_input_for_testing(self, data: list) -> dict: + return self._assemble_input_for_validating(data) + + def fit(self, train_set, val_set, file_type: str = "hdf5",): + + self.training_set = DatasetForCSAI( + train_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals + ) + + training_loader = DataLoader( + self.training_set, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + + if val_set is not None: + # if not key_in_data_set("X_ori", val_set): + # raise ValueError("val_set must contain 'X_ori' for model validation.") + val_set = DatasetForCSAI( + val_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals, + self.training_set.replacement_probabilities, + self.training_set.mean_set, self.training_set.std_set, False + ) + val_loader = DataLoader( + val_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + + # set up the model + self.model = _BCSAI(self.n_steps, + self.n_features, + self.rnn_hidden_size, + self.step_channels, + self.training_set.intervals, + self.consistency_weight, + self.imputation_weight) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer + self.optimizer = self.optimizer + self.optimizer.init_optimizer(self.model.parameters()) + + # train the model + self._train_model(training_loader, val_loader) + self.model.load_state_dict(self.best_model_dict) + self.model.eval() # set the model as eval status to freeze it. + + # Step 3: save the model if necessary + self._auto_save_model_if_necessary(confirm_saving=True) + + def predict(self, test_set: dict | str, file_type: str = "hdf5") -> dict: + + if self.model == None: + raise ValueError("Training must be run before predict") + + self.model.eval() + test_set = DatasetForCSAI( + test_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals, + self.training_set.replacement_probabilities, + self.training_set.mean_set, self.training_set.std_set, False + ) + test_loader = DataLoader( + test_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + imputation_collector = [] + x_ori_collector = [] + indicating_mask_collector = [] + with torch.no_grad(): + for idx, data in enumerate(test_loader): + inputs = self._assemble_input_for_testing(data) + results = self.model.forward(inputs, training=False) + imputed_data = results["imputed_data"] + imputation_collector.append(imputed_data) + x_ori_collector.append(inputs["X_ori"]) + indicating_mask_collector.append(inputs["indicating_mask"]) + + imputation = torch.cat(imputation_collector).cpu().detach().numpy() + result_dict = { + "imputation": imputation, + "X_ori": torch.cat(x_ori_collector).cpu().detach().numpy(), + "indicating_mask": torch.cat(indicating_mask_collector).cpu().detach().numpy(), + } + return result_dict + + def impute( + self, + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> np.ndarray: + """Impute missing values in the given data with the trained model. + + Parameters + ---------- + test_set : + The data samples for testing, should be array-like of shape [n_samples, sequence length (n_steps), + n_features], or a path string locating a data file, e.g. h5 file. + + file_type : + The type of the given file if X is a path string. + + Returns + ------- + array-like, shape [n_samples, sequence length (n_steps), n_features], + Imputed data. + """ + + result_dict = self.predict(test_set, file_type=file_type) + return result_dict["imputation"] + + \ No newline at end of file diff --git a/pypots/nn/modules/csai/__init__.py b/pypots/nn/modules/csai/__init__.py new file mode 100644 index 00000000..84e04f4f --- /dev/null +++ b/pypots/nn/modules/csai/__init__.py @@ -0,0 +1,13 @@ +from .backbone import BackboneCSAI, BackboneBCSAI +from .layers import FeatureRegression + +__all__ = [ + "BackboneCSAI", + "BackboneBCSAI", + "FeatureRegression", + "Decay", + "Decay_obs", + "PositionalEncoding", + "Conv1dWithInit", + "TorchTransformerEncoder" +] diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py new file mode 100644 index 00000000..d71b8f9c --- /dev/null +++ b/pypots/nn/modules/csai/backbone.py @@ -0,0 +1,165 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from .layers import FeatureRegression, Decay, Decay_obs, PositionalEncoding, Conv1dWithInit, TorchTransformerEncoder + + +class BackboneCSAI(nn.Module): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + super(BackboneCSAI, self).__init__() + + + if medians_df is not None: + self.medians_tensor = torch.tensor(list(medians_df.values())).float() + else: + self.medians_tensor = None + + self.n_steps = n_steps + self.step_channels = step_channels + self.input_size = n_features + self.hidden_size = rnn_hidden_size + self.temp_decay_h = Decay(input_size=self.input_size, output_size=self.hidden_size, diag = False) + self.temp_decay_x = Decay(input_size=self.input_size, output_size=self.input_size, diag = True) + self.hist = nn.Linear(self.hidden_size, self.input_size) + self.feat_reg_v = FeatureRegression(self.input_size) + self.weight_combine = nn.Linear(self.input_size * 2, self.input_size) + self.weighted_obs = Decay_obs(self.input_size, self.input_size) + self.gru = nn.GRUCell(self.input_size * 2, self.hidden_size) + + self.pos_encoder = PositionalEncoding(self.step_channels) + self.input_projection = Conv1dWithInit(self.input_size, self.step_channels, 1) + self.output_projection1 = Conv1dWithInit(self.step_channels, self.hidden_size, 1) + self.output_projection2 = Conv1dWithInit(self.n_steps*2, 1, 1) + self.time_layer = TorchTransformerEncoder(channels=self.step_channels) + self.device = device + + self.reset_parameters() + + def reset_parameters(self): + for weight in self.parameters(): + if len(weight.size()) == 1: + continue + stv = 1. / math.sqrt(weight.size(1)) + nn.init.uniform_(weight, -stv, stv) + + def forward(self, x, mask, deltas, last_obs, h=None): + + # Get dimensionality + [B, _, _] = x.shape + + if self.medians_tensor is not None: + medians_t = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(self.device) + + decay_factor = self.weighted_obs(deltas - medians_t.unsqueeze(1)) + + if h == None: + data_last_obs = self.input_projection(last_obs.permute(0, 2, 1)).permute(0, 2, 1) + data_decay_factor = self.input_projection(decay_factor.permute(0, 2, 1)).permute(0, 2, 1) + + data_last_obs = self.pos_encoder(data_last_obs.permute(1, 0, 2)).permute(1, 0, 2) + data_decay_factor = self.pos_encoder(data_decay_factor.permute(1, 0, 2)).permute(1, 0, 2) + + data = torch.cat([data_last_obs, data_decay_factor], dim=1) + + data = self.time_layer(data) + data = self.output_projection1(data.permute(0, 2, 1)).permute(0, 2, 1) + h = self.output_projection2(data).squeeze() + + x_loss = 0 + x_imp = x.clone() + Hiddens = [] + reconstruction = [] + for t in range(self.n_steps): + x_t = x[:, t, :] + d_t = deltas[:, t, :] + m_t = mask[:, t, :] + + # Decayed Hidden States + gamma_h = self.temp_decay_h(d_t) + h = h * gamma_h + + # history based estimation + x_h = self.hist(h) + + x_r_t = (m_t * x_t) + ((1 - m_t) * x_h) + + # feature based estimation + xu = self.feat_reg_v(x_r_t) + gamma_x = self.temp_decay_x(d_t) + + beta = self.weight_combine(torch.cat([gamma_x, m_t], dim=1)) + x_comb_t = beta * xu + (1 - beta) * x_h + + x_loss += torch.sum(torch.abs(x_t - x_comb_t) * m_t) / (torch.sum(m_t) + 1e-5) + + # Final Imputation Estimates + x_imp[:, t, :] = (m_t * x_t) + ((1 - m_t) * x_comb_t) + + # Set input the RNN + input_t = torch.cat([x_imp[:, t, :], m_t], dim=1) + + h = self.gru(input_t, h) + Hiddens.append(h.unsqueeze(dim=1)) + reconstruction.append(x_comb_t.unsqueeze(dim=1)) + Hiddens = torch.cat(Hiddens, dim=1) + + return x_imp, reconstruction, h, x_loss + + +class BackboneBCSAI(nn.Module): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + super(BackboneBCSAI, self).__init__() + + self.model_f = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) + self.model_b = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) + + + def forward(self, xdata): + + # Fetching forward data from xdata + x = xdata['forward']['X'] + m = xdata['forward']['missing_mask'] + d_f = xdata['forward']['deltas'] + last_obs_f = xdata['forward']['last_obs'] + + # Fetching backward data from xdata + x_b = xdata['backward']['X'] + m_b = xdata['backward']['missing_mask'] + d_b = xdata['backward']['deltas'] + last_obs_b = xdata['backward']['last_obs'] + + # Call forward model + ( + f_imputed_data, + f_reconstruction, + f_hidden_states, + f_reconstruction_loss, + ) = self.model_f(x, m, d_f, last_obs_f) + + # Call backward model + ( + b_imputed_data, + b_reconstruction, + b_hidden_states, + b_reconstruction_loss, + ) = self.model_b(x_b, m_b, d_b, last_obs_b) + + # Averaging the imputations and prediction + x_imp = (f_imputed_data + b_imputed_data.flip(dims=[1])) / 2 + imputed_data = (x * m)+ ((1-m) * x_imp) + + # average consistency loss + consistency_loss = torch.abs(f_imputed_data - b_imputed_data.flip(dims=[1])).mean() * 1e-1 + + # Merge the regression loss + reconstruction_loss = f_reconstruction_loss + b_reconstruction_loss + return ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) diff --git a/pypots/nn/modules/csai/layers.py b/pypots/nn/modules/csai/layers.py new file mode 100644 index 00000000..ecfadd7f --- /dev/null +++ b/pypots/nn/modules/csai/layers.py @@ -0,0 +1,128 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +from torch.nn.parameter import Parameter +import math +import numpy as np +import os +import copy +import pandas as pd +from torch.nn.modules import TransformerEncoderLayer + + +class FeatureRegression(nn.Module): + def __init__(self, input_size): + super(FeatureRegression, self).__init__() + self.build(input_size) + + def build(self, input_size): + self.W = Parameter(torch.Tensor(input_size, input_size)) + self.b = Parameter(torch.Tensor(input_size)) + m = torch.ones(input_size, input_size) - torch.eye(input_size, input_size) + self.register_buffer('m', m) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.W.size(0)) + self.W.data.uniform_(-stdv, stdv) + if self.b is not None: + self.b.data.uniform_(-stdv, stdv) + + def forward(self, x): + z_h = F.linear(x, self.W * Variable(self.m), self.b) + return z_h + +class Decay(nn.Module): + def __init__(self, input_size, output_size, diag=False): + super(Decay, self).__init__() + self.diag = diag + self.build(input_size, output_size) + + def build(self, input_size, output_size): + self.W = Parameter(torch.Tensor(output_size, input_size)) + self.b = Parameter(torch.Tensor(output_size)) + + if self.diag == True: + assert(input_size == output_size) + m = torch.eye(input_size, input_size) + self.register_buffer('m', m) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.W.size(0)) + self.W.data.uniform_(-stdv, stdv) + if self.b is not None: + self.b.data.uniform_(-stdv, stdv) + + def forward(self, d): + if self.diag == True: + gamma = F.relu(F.linear(d, self.W * Variable(self.m), self.b)) + else: + gamma = F.relu(F.linear(d, self.W, self.b)) + gamma = torch.exp(-gamma) + return gamma + +class Decay_obs(nn.Module): + def __init__(self, input_size, output_size): + super(Decay_obs, self).__init__() + self.linear = nn.Linear(input_size, output_size) + + def forward(self, delta_diff): + # When delta_diff is negative, weight tends to 1. + # When delta_diff is positive, weight tends to 0. + sign = torch.sign(delta_diff) + weight_diff = self.linear(delta_diff) + # weight_diff can be either positive or negative for each delta_diff + positive_part = F.relu(weight_diff) + negative_part = F.relu(-weight_diff) + weight_diff = positive_part + negative_part + weight_diff = sign * weight_diff + # Using a tanh activation to squeeze values between -1 and 1 + weight_diff = torch.tanh(weight_diff) + # This will move the weight values towards 1 if delta_diff is negative + # and towards 0 if delta_diff is positive + weight = 0.5 * (1 - weight_diff) + + return weight + +class TorchTransformerEncoder(nn.Module): + def __init__(self, heads=8, layers=1, channels=64): + super(TorchTransformerEncoder, self).__init__() + self.encoder_layer = nn.TransformerEncoderLayer( + d_model=channels, nhead=heads, dim_feedforward=64, activation="gelu" + ) + self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=layers) + + def forward(self, x): + return self.transformer_encoder(x) + +class Conv1dWithInit(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size): + super(Conv1dWithInit, self).__init__() + self.conv = nn.Conv1d(in_channels, out_channels, kernel_size) + nn.init.kaiming_normal_(self.conv.weight) + + def forward(self, x): + return self.conv(x) + +class PositionalEncoding(nn.Module): + + def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): + super().__init__() + self.dropout = nn.Dropout(p=dropout) + + position = torch.arange(max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) + pe = torch.zeros(max_len, 1, d_model) + pe[:, 0, 0::2] = torch.sin(position * div_term) + pe[:, 0, 1::2] = torch.cos(position * div_term) + self.register_buffer('pe', pe) + + def forward(self, x): + """ + Arguments: + x: Tensor, shape ``[seq_len, batch_size, embedding_dim]`` + """ + x = x + self.pe[:x.size(0)] + return self.dropout(x) \ No newline at end of file From 7e0c5f5397f0cf6cf24e6387bbec0e23d880e715 Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Mon, 16 Sep 2024 07:31:47 +0100 Subject: [PATCH 02/22] Minor changes --- csai.py | 247 +++++++---- output_logs/Classification.out | 636 ++++++++++++++++++++++++++++ output_logs/Imputation.out | 605 ++++++++++++++++++++++++++ pypots/classification/base.py | 18 +- pypots/classification/csai/core.py | 2 +- pypots/classification/csai/data.py | 3 +- pypots/classification/csai/model.py | 4 +- pypots/data/utils.py | 2 +- pypots/imputation/base.py | 2 - pypots/imputation/csai/core.py | 6 +- pypots/imputation/csai/data.py | 9 +- pypots/imputation/csai/model.py | 15 +- run_pypots.sh | 25 ++ 13 files changed, 1475 insertions(+), 99 deletions(-) create mode 100644 output_logs/Classification.out create mode 100644 output_logs/Imputation.out create mode 100644 run_pypots.sh diff --git a/csai.py b/csai.py index ee77261e..d987955d 100644 --- a/csai.py +++ b/csai.py @@ -1,76 +1,175 @@ -from pypots.classification import CSAI import pickle -from pypots.optim import Adam -from pypots.utils.metrics import calc_mae +import logging +import numpy as np import torch -from pypots.utils.metrics import calc_binary_classification_metrics - -kfold_data = pickle.load(open(r"C:\Users\jsphr\Documents\Imputation\data\data_nan.pkl", 'rb')) -kfold_label = pickle.load(open(r"C:\Users\jsphr\Documents\Imputation\data\label.pkl", 'rb')) - - -train_data = kfold_data[0][0] -train_label = kfold_label[0][0] - -valid_data = kfold_data[0][1] -valid_label = kfold_label[0][1] - - -test_data = kfold_data[0][2] -test_label = kfold_label[0][2] - -dataset_for_training = { - "X": train_data, - "y": train_label -} - -dataset_for_validating = { - "X": valid_data, - "y": valid_label -} - -dataset_for_testing = { - "X": test_data, - "y": test_label -} - -csai = CSAI(n_steps = 48, - n_features = 35, - rnn_hidden_size = 108, - imputation_weight = 0.3, - consistency_weight = 0.1, - classification_weight = 1, - n_classes = 1, - removal_percent = 10, - increase_factor = 0.1, - compute_intervals = True, - step_channels = 512, - batch_size = 32, - epochs = 10, - patience = None, - optimizer = Adam(lr=1e-3), - num_workers = 0, - device = torch.device("cuda" if torch.cuda.is_available() else "cpu"), - saving_path = r'C:\Users\jsphr\Documents\Imputation', - model_saving_strategy = "best", - verbose = True) - -csai.fit(dataset_for_training, dataset_for_validating) -csai_results = csai.predict(dataset_for_testing) -print(csai_results.keys()) -classification_predictions = csai_results['classification'] -metrics = calc_binary_classification_metrics(classification_predictions, dataset_for_testing["y"]) - -# or0 = csai_results["X_ori"] -# indi = csai_results["indicating_mask"] - -# mae = calc_mae(imputed_data, or0, indi) -# print(mae) - -print("Testing classification metrics: \n" - f'ROC_AUC: {metrics["roc_auc"]}, \n' - f'PR_AUC: {metrics["pr_auc"]},\n' - f'F1: {metrics["f1"]},\n' - f'Precision: {metrics["precision"]},\n' - f'Recall: {metrics["recall"]},\n' -) \ No newline at end of file +from sklearn.metrics import ( + confusion_matrix, roc_auc_score, accuracy_score, precision_recall_curve, + balanced_accuracy_score, recall_score, precision_score, f1_score, + PrecisionRecallDisplay, ConfusionMatrixDisplay, roc_curve, RocCurveDisplay +) +from pypots.classification import CSAI as classify +from pypots.imputation import CSAI as imputer +from pypots.optim import Adam +from pypots.utils.metrics import calc_mae, calc_mre, calc_binary_classification_metrics +import json +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def calculate_performance_brits(y, y_score, y_pred, pr_display=False, cm_display=False, roc_display=False): + + acc = accuracy_score(y, y_pred) + auc = roc_auc_score(y, y_score) + prec_macro = precision_score(y, y_pred, average='macro') + recall_macro = recall_score(y, y_pred, average='macro') + f1_macro = f1_score(y, y_pred, average='macro') + bal_acc = balanced_accuracy_score(y, y_pred) + + if pr_display: + prec, recall, _ = precision_recall_curve(y, y_score) + PrecisionRecallDisplay(precision=prec, recall=recall).plot() + + if cm_display: + cm = confusion_matrix(y, y_pred) + ConfusionMatrixDisplay(confusion_matrix=cm).plot() + + if roc_display: + fpr, tpr, _ = roc_curve(y, y_score) + RocCurveDisplay(fpr=fpr, tpr=tpr).plot() + + return acc, auc, prec_macro, recall_macro, f1_macro, bal_acc + + +def load_data(kfold_data_path, kfold_label_path, fold=2): + + kfold_data = pickle.load(open(kfold_data_path, 'rb')) + kfold_label = pickle.load(open(kfold_label_path, 'rb')) + + return { + "train": (kfold_data[fold][0], kfold_label[fold][0]), + "valid": (kfold_data[fold][1], kfold_label[fold][1]), + "test": (kfold_data[fold][2], kfold_label[fold][2]) + } + + +def get_csai_model(mode, **kwargs): + + if mode == 'classify': + return classify(**kwargs) + else: + return imputer(**kwargs) + + +def run_pypots(mode, data): + + + dataset_for_training = {"X": data["train"][0], "y": data["train"][1]} + dataset_for_validating = {"X": data["valid"][0], "y": data["valid"][1]} + dataset_for_testing = {"X": data["test"][0], "y": data["test"][1]} + + model_params = { + "n_steps": 48, + "n_features": 35, + "rnn_hidden_size": 108, + "imputation_weight": 0.3, + "consistency_weight": 0.1, + "removal_percent": 10, + "increase_factor": 0.1, + "compute_intervals": True, + "step_channels": 512, + "batch_size": 64, + "epochs": 200, + "patience": 5, + "optimizer": Adam(lr=0.0005, weight_decay=0.00001), + "num_workers": 0, + "device": torch.device("cuda" if torch.cuda.is_available() else "cpu"), + "saving_path": '/scratch/users/k23031260/PyPOTS', + "model_saving_strategy": "best", + "verbose": True + } + + if mode == 'classify': + model_params["classification_weight"] = 1 + model_params["n_classes"] = 1 + + csai_model = get_csai_model(mode, **model_params) + + logger.info("\n Training..................................... \n") + csai_model.fit(dataset_for_training, dataset_for_validating) + + logger.info("\n Testing...................................... \n") + csai_results = csai_model.predict(dataset_for_testing) + + if mode == 'classify': + classification_predictions = csai_results['classification'] + metrics = calc_binary_classification_metrics(classification_predictions, dataset_for_testing["y"]) + + y_pred = np.round(classification_predictions) + acc, auc, prec_macro, recall_macro, f1_macro, bal_acc = calculate_performance_brits( + dataset_for_testing["y"], classification_predictions, y_pred + ) + + return { + "PyPots_evaluation": { + "ROC_AUC": metrics['roc_auc'], + "PR_AUC": metrics['pr_auc'], + "F1": metrics['f1'], + "Precision": metrics['precision'], + "Recall": metrics['recall'] + }, + "CSAI_evaluation": { + "ACC": acc, + "AUC": auc, + "Prec_Macro": prec_macro, + "Recall_Macro": recall_macro, + "F1_Macro": f1_macro, + "Bal_ACC": bal_acc + } + } + + else: + + original_data = csai_results["X_ori"] + indicating_mask = csai_results["indicating_mask"] + imputed_data = csai_results['imputation'] + + mae = calc_mae(imputed_data, original_data, indicating_mask) + mre = calc_mre(imputed_data, original_data, indicating_mask) + + return { + "MAE": mae, + "MRE": mre + } + + + +if __name__ == "__main__": + + mode = 'impute' + data_path = "/scratch/users/k23031260/data/physionet/data_nan.pkl" + label_path = "/scratch/users/k23031260/data/physionet/label.pkl" + + all_fold_results = [] + for fold in range(5): + logger.info(f"\n Fold {fold} started ...................................... \n") + data = load_data(data_path, label_path, fold) + results = run_pypots(mode, data) + all_fold_results.append(results) + logger.info(f"\n Fold {fold} ended ........................................ \n") + logger.info(f"\n Fold {fold} Results: \n{json.dumps(results, indent=4)} \n") + + logger.info(f"\n Average performance across all folds.......................... \n") + if mode == 'impute': + avg_result = {metric: np.mean([fold[metric] for fold in all_fold_results]) for metric in all_fold_results[0].keys()} + logger.info(f"\n Imputation performance: \n {avg_result} \n") + else: + PyPots_evaluation = [{k: v for k,v in fold_result['PyPots_evaluation'].items()} for fold_result in all_fold_results] + CSAI_evaluation = [{k: v for k,v in fold_result['CSAI_evaluation'].items()} for fold_result in all_fold_results] + + avg_result = {metric: np.mean([fold[metric] for fold in PyPots_evaluation]) for metric in PyPots_evaluation[0].keys()} + logger.info(f"\n Classification performance using Pypots evaluation: \n {avg_result} \n") + + avg_result = {metric: np.mean([fold[metric] for fold in CSAI_evaluation]) for metric in CSAI_evaluation[0].keys()} + logger.info(f"\n Classification performance using CSAI evaluation: \n {avg_result} \n") + + diff --git a/output_logs/Classification.out b/output_logs/Classification.out new file mode 100644 index 00000000..06fd4f28 --- /dev/null +++ b/output_logs/Classification.out @@ -0,0 +1,636 @@ +Lmod has detected the following error: The following module(s) are unknown: +"anaconda3/2021.05-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "anaconda3/2021.05-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Lmod has detected the following error: The following module(s) are unknown: +"cuda/11.1.1-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "cuda/11.1.1-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Lmod has detected the following error: The following module(s) are unknown: +"cudnn/8.0.5.39-11.1-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "cudnn/8.0.5.39-11.1-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Fri Sep 13 12:02:41 2024 ++---------------------------------------------------------------------------------------+ +| NVIDIA-SMI 535.183.01 Driver Version: 535.183.01 CUDA Version: 12.2 | +|-----------------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+======================+======================| +| 0 NVIDIA A100-SXM4-40GB On | 00000000:17:00.0 Off | 0 | +| N/A 33C P0 53W / 400W | 0MiB / 40960MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+----------------------+----------------------+ + ++---------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=======================================================================================| +| No running processes found | ++---------------------------------------------------------------------------------------+ +/var/lib/slurm/slurmd/job21076806/slurm_script: line 18: activate: No such file or directory +2024-09-13 12:04:04.605532: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2024-09-13 12:04:08.365187: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2024-09-13 12:04:30.558164: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +INFO:__main__: + Fold 0 started ...................................... + +2024-09-13 12:05:06 [INFO]: Using the given device: cuda +2024-09-13 12:05:06 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120506 +2024-09-13 12:05:06 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120506/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:05:16 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 +2024-09-13 12:05:32 [INFO]: Epoch 001 - training loss: 19.8551, validation loss: 17.4215, validation AUC: 0.4966053925763644 +2024-09-13 12:05:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch1_loss17.421496171217697.pypots +2024-09-13 12:05:38 [INFO]: Epoch 002 - training loss: 15.9493, validation loss: 14.9795, validation AUC: 0.6574584374536451 +2024-09-13 12:05:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch2_loss14.979489913353554.pypots +2024-09-13 12:05:44 [INFO]: Epoch 003 - training loss: 14.2548, validation loss: 13.8747, validation AUC: 0.7760471936010224 +2024-09-13 12:05:44 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch3_loss13.874650735121508.pypots +2024-09-13 12:05:50 [INFO]: Epoch 004 - training loss: 13.3428, validation loss: 13.1115, validation AUC: 0.8137358938372186 +2024-09-13 12:05:50 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch4_loss13.111526049100435.pypots +2024-09-13 12:05:56 [INFO]: Epoch 005 - training loss: 12.6785, validation loss: 12.5040, validation AUC: 0.8340807174887893 +2024-09-13 12:05:56 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch5_loss12.50400081047645.pypots +2024-09-13 12:06:02 [INFO]: Epoch 006 - training loss: 12.1627, validation loss: 12.0352, validation AUC: 0.8508198404819771 +2024-09-13 12:06:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch6_loss12.035209068885216.pypots +2024-09-13 12:06:09 [INFO]: Epoch 007 - training loss: 11.6513, validation loss: 11.5828, validation AUC: 0.8498499526466526 +2024-09-13 12:06:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch7_loss11.582774309011606.pypots +2024-09-13 12:06:15 [INFO]: Epoch 008 - training loss: 11.3052, validation loss: 11.2321, validation AUC: 0.8583621447072651 +2024-09-13 12:06:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch8_loss11.232126529400166.pypots +2024-09-13 12:06:21 [INFO]: Epoch 009 - training loss: 10.9663, validation loss: 10.9465, validation AUC: 0.8650828968838074 +2024-09-13 12:06:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch9_loss10.946472608126127.pypots +2024-09-13 12:06:27 [INFO]: Epoch 010 - training loss: 10.7618, validation loss: 10.7389, validation AUC: 0.8638049270302034 +2024-09-13 12:06:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch10_loss10.738909794734074.pypots +2024-09-13 12:06:33 [INFO]: Epoch 011 - training loss: 10.5480, validation loss: 10.6198, validation AUC: 0.8506258629149124 +2024-09-13 12:06:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch11_loss10.619767115666317.pypots +2024-09-13 12:06:39 [INFO]: Epoch 012 - training loss: 10.3870, validation loss: 10.4472, validation AUC: 0.8605643606157076 +2024-09-13 12:06:39 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch12_loss10.44724391056941.pypots +2024-09-13 12:06:45 [INFO]: Epoch 013 - training loss: 10.2086, validation loss: 10.3096, validation AUC: 0.8556350483232352 +2024-09-13 12:06:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch13_loss10.309552999643179.pypots +2024-09-13 12:06:51 [INFO]: Epoch 014 - training loss: 10.0798, validation loss: 10.1739, validation AUC: 0.8625269571765994 +2024-09-13 12:06:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch14_loss10.173876102154072.pypots +2024-09-13 12:06:57 [INFO]: Epoch 015 - training loss: 9.9415, validation loss: 10.0565, validation AUC: 0.8666461278654479 +2024-09-13 12:06:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch15_loss10.05653770153339.pypots +2024-09-13 12:07:03 [INFO]: Epoch 016 - training loss: 9.8283, validation loss: 9.9481, validation AUC: 0.863325688335102 +2024-09-13 12:07:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch16_loss9.94806275000939.pypots +2024-09-13 12:07:09 [INFO]: Epoch 017 - training loss: 9.6922, validation loss: 9.9070, validation AUC: 0.8664635607435046 +2024-09-13 12:07:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch17_loss9.907046978290264.pypots +2024-09-13 12:07:15 [INFO]: Epoch 018 - training loss: 9.6060, validation loss: 9.8097, validation AUC: 0.8598911443535412 +2024-09-13 12:07:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch18_loss9.809694950397198.pypots +2024-09-13 12:07:21 [INFO]: Epoch 019 - training loss: 9.5409, validation loss: 9.7606, validation AUC: 0.869989388286037 +2024-09-13 12:07:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch19_loss9.760592313913198.pypots +2024-09-13 12:07:27 [INFO]: Epoch 020 - training loss: 9.4575, validation loss: 9.7574, validation AUC: 0.8667716427617842 +2024-09-13 12:07:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch20_loss9.757406088022085.pypots +2024-09-13 12:07:34 [INFO]: Epoch 021 - training loss: 9.3948, validation loss: 9.6642, validation AUC: 0.8601535845913348 +2024-09-13 12:07:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch21_loss9.664151485149677.pypots +2024-09-13 12:07:40 [INFO]: Epoch 022 - training loss: 9.3306, validation loss: 9.6316, validation AUC: 0.8560458243476078 +2024-09-13 12:07:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch22_loss9.631555557250977.pypots +2024-09-13 12:07:46 [INFO]: Epoch 023 - training loss: 9.3060, validation loss: 9.6051, validation AUC: 0.854722212713518 +2024-09-13 12:07:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch23_loss9.605094029353214.pypots +2024-09-13 12:07:52 [INFO]: Epoch 024 - training loss: 9.2244, validation loss: 9.6119, validation AUC: 0.854699391823275 +2024-09-13 12:07:57 [INFO]: Epoch 025 - training loss: 9.1648, validation loss: 9.5212, validation AUC: 0.8517554969819372 +2024-09-13 12:07:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch25_loss9.521164967463566.pypots +2024-09-13 12:08:03 [INFO]: Epoch 026 - training loss: 9.1255, validation loss: 9.4918, validation AUC: 0.8550873469574047 +2024-09-13 12:08:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch26_loss9.491754311781664.pypots +2024-09-13 12:08:10 [INFO]: Epoch 027 - training loss: 9.1028, validation loss: 9.4686, validation AUC: 0.8524743550245896 +2024-09-13 12:08:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch27_loss9.468608709482046.pypots +2024-09-13 12:08:16 [INFO]: Epoch 028 - training loss: 9.0499, validation loss: 9.4682, validation AUC: 0.8566277570488026 +2024-09-13 12:08:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch28_loss9.468163930452787.pypots +2024-09-13 12:08:22 [INFO]: Epoch 029 - training loss: 9.0216, validation loss: 9.5157, validation AUC: 0.8501238033295678 +2024-09-13 12:08:28 [INFO]: Epoch 030 - training loss: 8.9307, validation loss: 9.4084, validation AUC: 0.8450119239151519 +2024-09-13 12:08:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch30_loss9.408410145686222.pypots +2024-09-13 12:08:34 [INFO]: Epoch 031 - training loss: 8.8653, validation loss: 9.4911, validation AUC: 0.8337612250253883 +2024-09-13 12:08:40 [INFO]: Epoch 032 - training loss: 8.8738, validation loss: 9.4475, validation AUC: 0.8396604251531853 +2024-09-13 12:08:46 [INFO]: Epoch 033 - training loss: 8.8115, validation loss: 9.4653, validation AUC: 0.8333048072205297 +2024-09-13 12:08:52 [INFO]: Epoch 034 - training loss: 8.7482, validation loss: 9.4411, validation AUC: 0.8224877052453815 +2024-09-13 12:08:58 [INFO]: Epoch 035 - training loss: 8.7117, validation loss: 9.4449, validation AUC: 0.8189504672577277 +2024-09-13 12:08:58 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:08:58 [INFO]: Finished training. The best model is from epoch#30. +2024-09-13 12:08:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 0 ended ........................................ + +INFO:__main__: + Fold 0 Results: +{ + "PyPots_evaluation": { + "ROC_AUC": 0.7999868247694335, + "PR_AUC": 0.40923534439915027, + "F1": 0.326797385620915, + "Precision": 0.5813953488372093, + "Recall": 0.22727272727272727 + }, + "CSAI_evaluation": { + "ACC": 0.87125, + "AUC": 0.7999868247694335, + "Prec_Macro": 0.7345550059905994, + "Recall_Macro": 0.600592885375494, + "F1_Macro": 0.627807815132503, + "Bal_ACC": 0.600592885375494 + } +} + +INFO:__main__: + Fold 1 started ...................................... + +2024-09-13 12:09:02 [INFO]: Using the given device: cuda +2024-09-13 12:09:02 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120902 +2024-09-13 12:09:02 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120902/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:09:11 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 +2024-09-13 12:09:17 [INFO]: Epoch 001 - training loss: 19.8380, validation loss: 17.5980, validation AUC: 0.5338787497377806 +2024-09-13 12:09:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch1_loss17.59797653785119.pypots +2024-09-13 12:09:23 [INFO]: Epoch 002 - training loss: 15.9641, validation loss: 15.1263, validation AUC: 0.6651488789348339 +2024-09-13 12:09:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch2_loss15.12628489274245.pypots +2024-09-13 12:09:29 [INFO]: Epoch 003 - training loss: 14.3313, validation loss: 14.0248, validation AUC: 0.7368674341983489 +2024-09-13 12:09:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch3_loss14.024825316209059.pypots +2024-09-13 12:09:35 [INFO]: Epoch 004 - training loss: 13.4015, validation loss: 13.3206, validation AUC: 0.7668036377546614 +2024-09-13 12:09:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch4_loss13.320613787724422.pypots +2024-09-13 12:09:41 [INFO]: Epoch 005 - training loss: 12.7353, validation loss: 12.7163, validation AUC: 0.7872012241019756 +2024-09-13 12:09:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch5_loss12.716337717496431.pypots +2024-09-13 12:09:47 [INFO]: Epoch 006 - training loss: 12.1400, validation loss: 12.1862, validation AUC: 0.8018608324387023 +2024-09-13 12:09:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch6_loss12.186163535484901.pypots +2024-09-13 12:09:54 [INFO]: Epoch 007 - training loss: 11.6714, validation loss: 11.7651, validation AUC: 0.8000839102160688 +2024-09-13 12:09:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch7_loss11.765064973097582.pypots +2024-09-13 12:10:00 [INFO]: Epoch 008 - training loss: 11.2592, validation loss: 11.4473, validation AUC: 0.7982823085181209 +2024-09-13 12:10:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch8_loss11.447268706101637.pypots +2024-09-13 12:10:06 [INFO]: Epoch 009 - training loss: 10.9614, validation loss: 11.1697, validation AUC: 0.8008366342131565 +2024-09-13 12:10:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch9_loss11.169659467843863.pypots +2024-09-13 12:10:12 [INFO]: Epoch 010 - training loss: 10.7048, validation loss: 10.9466, validation AUC: 0.8111032959439283 +2024-09-13 12:10:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch10_loss10.946574284480167.pypots +2024-09-13 12:10:18 [INFO]: Epoch 011 - training loss: 10.4913, validation loss: 10.7678, validation AUC: 0.8016510568985304 +2024-09-13 12:10:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch11_loss10.767754774827223.pypots +2024-09-13 12:10:24 [INFO]: Epoch 012 - training loss: 10.2789, validation loss: 10.5824, validation AUC: 0.8090795789681512 +2024-09-13 12:10:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch12_loss10.582400542039137.pypots +2024-09-13 12:10:30 [INFO]: Epoch 013 - training loss: 10.1285, validation loss: 10.4989, validation AUC: 0.8066116314367157 +2024-09-13 12:10:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch13_loss10.498880899869478.pypots +2024-09-13 12:10:36 [INFO]: Epoch 014 - training loss: 10.0109, validation loss: 10.3582, validation AUC: 0.806290798257629 +2024-09-13 12:10:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch14_loss10.358169629023624.pypots +2024-09-13 12:10:42 [INFO]: Epoch 015 - training loss: 9.8685, validation loss: 10.2522, validation AUC: 0.8099557003418107 +2024-09-13 12:10:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch15_loss10.252222941471981.pypots +2024-09-13 12:10:48 [INFO]: Epoch 016 - training loss: 9.7759, validation loss: 10.2081, validation AUC: 0.8008119547378423 +2024-09-13 12:10:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch16_loss10.208140519949106.pypots +2024-09-13 12:10:54 [INFO]: Epoch 017 - training loss: 9.6913, validation loss: 10.1220, validation AUC: 0.7995656412344674 +2024-09-13 12:10:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch17_loss10.121957192054161.pypots +2024-09-13 12:11:00 [INFO]: Epoch 018 - training loss: 9.6082, validation loss: 10.0452, validation AUC: 0.7976900011105763 +2024-09-13 12:11:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch18_loss10.04515633216271.pypots +2024-09-13 12:11:07 [INFO]: Epoch 019 - training loss: 9.5294, validation loss: 9.9853, validation AUC: 0.8068707659275163 +2024-09-13 12:11:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch19_loss9.985262870788574.pypots +2024-09-13 12:11:13 [INFO]: Epoch 020 - training loss: 9.4589, validation loss: 10.0217, validation AUC: 0.8058218882266563 +2024-09-13 12:11:19 [INFO]: Epoch 021 - training loss: 9.3960, validation loss: 9.9241, validation AUC: 0.8132750897715915 +2024-09-13 12:11:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch21_loss9.924118335430439.pypots +2024-09-13 12:11:25 [INFO]: Epoch 022 - training loss: 9.3173, validation loss: 9.8431, validation AUC: 0.8125100260368465 +2024-09-13 12:11:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch22_loss9.843071497403658.pypots +2024-09-13 12:11:31 [INFO]: Epoch 023 - training loss: 9.2848, validation loss: 9.8281, validation AUC: 0.8046989720998532 +2024-09-13 12:11:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch23_loss9.828058609595665.pypots +2024-09-13 12:11:37 [INFO]: Epoch 024 - training loss: 9.2376, validation loss: 9.8259, validation AUC: 0.8040943249546514 +2024-09-13 12:11:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch24_loss9.825871027432955.pypots +2024-09-13 12:11:43 [INFO]: Epoch 025 - training loss: 9.1740, validation loss: 9.7791, validation AUC: 0.8060193240291711 +2024-09-13 12:11:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch25_loss9.779062491196852.pypots +2024-09-13 12:11:49 [INFO]: Epoch 026 - training loss: 9.1133, validation loss: 9.7563, validation AUC: 0.8030207677784771 +2024-09-13 12:11:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch26_loss9.756314864525429.pypots +2024-09-13 12:11:55 [INFO]: Epoch 027 - training loss: 9.0885, validation loss: 9.8235, validation AUC: 0.804402818396081 +2024-09-13 12:12:01 [INFO]: Epoch 028 - training loss: 9.0216, validation loss: 9.7160, validation AUC: 0.8045015362973382 +2024-09-13 12:12:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch28_loss9.71602880037748.pypots +2024-09-13 12:12:07 [INFO]: Epoch 029 - training loss: 8.9750, validation loss: 9.7326, validation AUC: 0.8029467293525341 +2024-09-13 12:12:13 [INFO]: Epoch 030 - training loss: 8.9495, validation loss: 9.7374, validation AUC: 0.8016140376855588 +2024-09-13 12:12:19 [INFO]: Epoch 031 - training loss: 8.9167, validation loss: 9.6785, validation AUC: 0.8000839102160688 +2024-09-13 12:12:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch31_loss9.678529225862944.pypots +2024-09-13 12:12:25 [INFO]: Epoch 032 - training loss: 8.8744, validation loss: 9.6473, validation AUC: 0.8043904786584237 +2024-09-13 12:12:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch32_loss9.647251716026894.pypots +2024-09-13 12:12:31 [INFO]: Epoch 033 - training loss: 8.8384, validation loss: 9.6853, validation AUC: 0.7965917644590875 +2024-09-13 12:12:37 [INFO]: Epoch 034 - training loss: 8.8245, validation loss: 9.6900, validation AUC: 0.8031194856797345 +2024-09-13 12:12:43 [INFO]: Epoch 035 - training loss: 8.7635, validation loss: 9.7357, validation AUC: 0.7963079504929726 +2024-09-13 12:12:49 [INFO]: Epoch 036 - training loss: 8.7447, validation loss: 9.7541, validation AUC: 0.7911129209393007 +2024-09-13 12:12:55 [INFO]: Epoch 037 - training loss: 8.6846, validation loss: 9.6365, validation AUC: 0.7853996224040277 +2024-09-13 12:12:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch37_loss9.636484439556416.pypots +2024-09-13 12:13:01 [INFO]: Epoch 038 - training loss: 8.6647, validation loss: 9.7143, validation AUC: 0.7943089129925098 +2024-09-13 12:13:07 [INFO]: Epoch 039 - training loss: 8.7310, validation loss: 9.6946, validation AUC: 0.7955922457088561 +2024-09-13 12:13:14 [INFO]: Epoch 040 - training loss: 8.6248, validation loss: 9.6612, validation AUC: 0.7932600352916497 +2024-09-13 12:13:20 [INFO]: Epoch 041 - training loss: 8.6021, validation loss: 9.7136, validation AUC: 0.7877318328212342 +2024-09-13 12:13:26 [INFO]: Epoch 042 - training loss: 8.5472, validation loss: 9.6806, validation AUC: 0.7863497822036304 +2024-09-13 12:13:26 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:13:26 [INFO]: Finished training. The best model is from epoch#37. +2024-09-13 12:13:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 1 ended ........................................ + +INFO:__main__: + Fold 1 Results: +{ + "PyPots_evaluation": { + "ROC_AUC": 0.7708333333333334, + "PR_AUC": 0.4004018527922971, + "F1": 0.39263803680981596, + "Precision": 0.5423728813559322, + "Recall": 0.3076923076923077 + }, + "CSAI_evaluation": { + "ACC": 0.87625, + "AUC": 0.7708333333333334, + "Prec_Macro": 0.722603444726549, + "Recall_Macro": 0.6344496021220158, + "F1_Macro": 0.6618722543130499, + "Bal_ACC": 0.6344496021220158 + } +} + +INFO:__main__: + Fold 2 started ...................................... + +2024-09-13 12:13:29 [INFO]: Using the given device: cuda +2024-09-13 12:13:29 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121329 +2024-09-13 12:13:29 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121329/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:13:39 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 +2024-09-13 12:13:45 [INFO]: Epoch 001 - training loss: 19.8475, validation loss: 17.6167, validation AUC: 0.5809640640785105 +2024-09-13 12:13:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch1_loss17.61674191401555.pypots +2024-09-13 12:13:51 [INFO]: Epoch 002 - training loss: 15.9218, validation loss: 15.1607, validation AUC: 0.6880502236975032 +2024-09-13 12:13:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch2_loss15.160728748028095.pypots +2024-09-13 12:13:57 [INFO]: Epoch 003 - training loss: 14.2607, validation loss: 13.9902, validation AUC: 0.7676793188050224 +2024-09-13 12:13:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch3_loss13.990155880267803.pypots +2024-09-13 12:14:03 [INFO]: Epoch 004 - training loss: 13.3474, validation loss: 13.2878, validation AUC: 0.7781666426131718 +2024-09-13 12:14:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch4_loss13.287820229163536.pypots +2024-09-13 12:14:09 [INFO]: Epoch 005 - training loss: 12.6907, validation loss: 12.7066, validation AUC: 0.7864650983787944 +2024-09-13 12:14:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch5_loss12.70658830495981.pypots +2024-09-13 12:14:15 [INFO]: Epoch 006 - training loss: 12.1345, validation loss: 12.2049, validation AUC: 0.7893394910280463 +2024-09-13 12:14:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch6_loss12.204926637502817.pypots +2024-09-13 12:14:21 [INFO]: Epoch 007 - training loss: 11.6593, validation loss: 11.7718, validation AUC: 0.7989248087747151 +2024-09-13 12:14:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch7_loss11.771849412184496.pypots +2024-09-13 12:14:28 [INFO]: Epoch 008 - training loss: 11.2531, validation loss: 11.4333, validation AUC: 0.7916125463029777 +2024-09-13 12:14:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch8_loss11.43331028864934.pypots +2024-09-13 12:14:33 [INFO]: Epoch 009 - training loss: 10.9356, validation loss: 11.1864, validation AUC: 0.7990210227546063 +2024-09-13 12:14:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch9_loss11.186383614173302.pypots +2024-09-13 12:14:39 [INFO]: Epoch 010 - training loss: 10.6980, validation loss: 11.0176, validation AUC: 0.8005243661904073 +2024-09-13 12:14:39 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch10_loss11.017565727233887.pypots +2024-09-13 12:14:45 [INFO]: Epoch 011 - training loss: 10.5164, validation loss: 10.8152, validation AUC: 0.8005724731803532 +2024-09-13 12:14:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch11_loss10.81515957758977.pypots +2024-09-13 12:14:52 [INFO]: Epoch 012 - training loss: 10.3187, validation loss: 10.6378, validation AUC: 0.8013421850194833 +2024-09-13 12:14:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch12_loss10.63784591968243.pypots +2024-09-13 12:14:58 [INFO]: Epoch 013 - training loss: 10.1152, validation loss: 10.5081, validation AUC: 0.8024486457882332 +2024-09-13 12:14:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch13_loss10.508104764498198.pypots +2024-09-13 12:15:04 [INFO]: Epoch 014 - training loss: 9.9899, validation loss: 10.3729, validation AUC: 0.796459325540001 +2024-09-13 12:15:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch14_loss10.372918495765099.pypots +2024-09-13 12:15:10 [INFO]: Epoch 015 - training loss: 9.8594, validation loss: 10.3116, validation AUC: 0.8013782652619426 +2024-09-13 12:15:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch15_loss10.311591001657339.pypots +2024-09-13 12:15:16 [INFO]: Epoch 016 - training loss: 9.7446, validation loss: 10.2354, validation AUC: 0.8044811661134363 +2024-09-13 12:15:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch16_loss10.235374890840971.pypots +2024-09-13 12:15:22 [INFO]: Epoch 017 - training loss: 9.6591, validation loss: 10.1343, validation AUC: 0.8100134699571848 +2024-09-13 12:15:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch17_loss10.134286440335787.pypots +2024-09-13 12:15:28 [INFO]: Epoch 018 - training loss: 9.5688, validation loss: 10.0724, validation AUC: 0.8039880694664934 +2024-09-13 12:15:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch18_loss10.072449977581318.pypots +2024-09-13 12:15:34 [INFO]: Epoch 019 - training loss: 9.5057, validation loss: 10.0097, validation AUC: 0.8036272670419011 +2024-09-13 12:15:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch19_loss10.009667323185848.pypots +2024-09-13 12:15:40 [INFO]: Epoch 020 - training loss: 9.4289, validation loss: 9.9999, validation AUC: 0.8061769375090201 +2024-09-13 12:15:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch20_loss9.999937864450308.pypots +2024-09-13 12:15:46 [INFO]: Epoch 021 - training loss: 9.3561, validation loss: 9.9410, validation AUC: 0.8058762688218599 +2024-09-13 12:15:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch21_loss9.940957436194786.pypots +2024-09-13 12:15:52 [INFO]: Epoch 022 - training loss: 9.2925, validation loss: 9.9246, validation AUC: 0.8167604752970608 +2024-09-13 12:15:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch22_loss9.924552550682655.pypots +2024-09-13 12:15:58 [INFO]: Epoch 023 - training loss: 9.2463, validation loss: 9.8549, validation AUC: 0.8096165872901333 +2024-09-13 12:15:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch23_loss9.854878132159893.pypots +2024-09-13 12:16:05 [INFO]: Epoch 024 - training loss: 9.1737, validation loss: 9.8140, validation AUC: 0.808558233511329 +2024-09-13 12:16:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch24_loss9.814040257380558.pypots +2024-09-13 12:16:11 [INFO]: Epoch 025 - training loss: 9.1330, validation loss: 9.7736, validation AUC: 0.8207653822100353 +2024-09-13 12:16:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch25_loss9.773633883549618.pypots +2024-09-13 12:16:17 [INFO]: Epoch 026 - training loss: 9.0987, validation loss: 9.7521, validation AUC: 0.8032784913647953 +2024-09-13 12:16:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch26_loss9.752125373253456.pypots +2024-09-13 12:16:23 [INFO]: Epoch 027 - training loss: 9.0337, validation loss: 9.7578, validation AUC: 0.8066580074084766 +2024-09-13 12:16:29 [INFO]: Epoch 028 - training loss: 8.9899, validation loss: 9.7796, validation AUC: 0.7995622263914947 +2024-09-13 12:16:35 [INFO]: Epoch 029 - training loss: 8.9836, validation loss: 9.7480, validation AUC: 0.795665560205898 +2024-09-13 12:16:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch29_loss9.748024500333345.pypots +2024-09-13 12:16:41 [INFO]: Epoch 030 - training loss: 8.9059, validation loss: 9.7315, validation AUC: 0.7934165584259394 +2024-09-13 12:16:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch30_loss9.731545301584097.pypots +2024-09-13 12:16:47 [INFO]: Epoch 031 - training loss: 8.8881, validation loss: 9.7377, validation AUC: 0.8008370616250542 +2024-09-13 12:16:53 [INFO]: Epoch 032 - training loss: 8.8780, validation loss: 9.7357, validation AUC: 0.8071511040554193 +2024-09-13 12:16:59 [INFO]: Epoch 033 - training loss: 8.8250, validation loss: 9.7180, validation AUC: 0.7788281137249242 +2024-09-13 12:16:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch33_loss9.717986547029936.pypots +2024-09-13 12:17:05 [INFO]: Epoch 034 - training loss: 8.7778, validation loss: 9.6552, validation AUC: 0.8032063308798769 +2024-09-13 12:17:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch34_loss9.655178436866173.pypots +2024-09-13 12:17:11 [INFO]: Epoch 035 - training loss: 8.7297, validation loss: 9.7163, validation AUC: 0.8014744792418339 +2024-09-13 12:17:17 [INFO]: Epoch 036 - training loss: 8.7866, validation loss: 9.6828, validation AUC: 0.7946432866695531 +2024-09-13 12:17:23 [INFO]: Epoch 037 - training loss: 8.6777, validation loss: 9.6914, validation AUC: 0.7854668783374225 +2024-09-13 12:17:29 [INFO]: Epoch 038 - training loss: 8.7008, validation loss: 9.7403, validation AUC: 0.796988502429403 +2024-09-13 12:17:35 [INFO]: Epoch 039 - training loss: 8.6678, validation loss: 9.6835, validation AUC: 0.7959421753980854 +2024-09-13 12:17:35 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:17:35 [INFO]: Finished training. The best model is from epoch#34. +2024-09-13 12:17:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 2 ended ........................................ + +INFO:__main__: + Fold 2 Results: +{ + "PyPots_evaluation": { + "ROC_AUC": 0.8132865385489072, + "PR_AUC": 0.4924055309152301, + "F1": 0.32954545454545453, + "Precision": 0.6590909090909091, + "Recall": 0.2196969696969697 + }, + "CSAI_evaluation": { + "ACC": 0.8523153942428036, + "AUC": 0.8132865385489072, + "Prec_Macro": 0.7613335340156532, + "Recall_Macro": 0.5986041070373904, + "F1_Macro": 0.623281869326173, + "Bal_ACC": 0.5986041070373904 + } +} + +INFO:__main__: + Fold 3 started ...................................... + +2024-09-13 12:17:39 [INFO]: Using the given device: cuda +2024-09-13 12:17:39 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121739 +2024-09-13 12:17:39 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121739/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:17:48 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 +2024-09-13 12:17:54 [INFO]: Epoch 001 - training loss: 19.8825, validation loss: 17.4137, validation AUC: 0.5158057340583293 +2024-09-13 12:17:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch1_loss17.413655647864708.pypots +2024-09-13 12:18:01 [INFO]: Epoch 002 - training loss: 16.0221, validation loss: 14.9347, validation AUC: 0.6432773109243698 +2024-09-13 12:18:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch2_loss14.93472458766057.pypots +2024-09-13 12:18:07 [INFO]: Epoch 003 - training loss: 14.3395, validation loss: 13.8289, validation AUC: 0.7859861591695502 +2024-09-13 12:18:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch3_loss13.82885162646954.pypots +2024-09-13 12:18:13 [INFO]: Epoch 004 - training loss: 13.4141, validation loss: 13.1185, validation AUC: 0.8242956005931786 +2024-09-13 12:18:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch4_loss13.118463369516226.pypots +2024-09-13 12:18:19 [INFO]: Epoch 005 - training loss: 12.7445, validation loss: 12.5076, validation AUC: 0.831957488877904 +2024-09-13 12:18:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch5_loss12.50760790017935.pypots +2024-09-13 12:18:24 [INFO]: Epoch 006 - training loss: 12.1803, validation loss: 12.0181, validation AUC: 0.8430548690064261 +2024-09-13 12:18:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch6_loss12.018106460571289.pypots +2024-09-13 12:18:31 [INFO]: Epoch 007 - training loss: 11.7076, validation loss: 11.6133, validation AUC: 0.8420044488383589 +2024-09-13 12:18:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch7_loss11.613336343031664.pypots +2024-09-13 12:18:37 [INFO]: Epoch 008 - training loss: 11.3546, validation loss: 11.3158, validation AUC: 0.8478868017795355 +2024-09-13 12:18:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch8_loss11.315820840688852.pypots +2024-09-13 12:18:43 [INFO]: Epoch 009 - training loss: 11.0431, validation loss: 11.1116, validation AUC: 0.845304003954523 +2024-09-13 12:18:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch9_loss11.111641370333158.pypots +2024-09-13 12:18:49 [INFO]: Epoch 010 - training loss: 10.8137, validation loss: 10.9210, validation AUC: 0.8418685121107267 +2024-09-13 12:18:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch10_loss10.920994465167706.pypots +2024-09-13 12:18:55 [INFO]: Epoch 011 - training loss: 10.6497, validation loss: 10.7272, validation AUC: 0.8377656945130993 +2024-09-13 12:18:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch11_loss10.727198380690355.pypots +2024-09-13 12:19:01 [INFO]: Epoch 012 - training loss: 10.4457, validation loss: 10.5835, validation AUC: 0.846280276816609 +2024-09-13 12:19:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch12_loss10.583506510807918.pypots +2024-09-13 12:19:07 [INFO]: Epoch 013 - training loss: 10.2859, validation loss: 10.4547, validation AUC: 0.8366658428077114 +2024-09-13 12:19:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch13_loss10.454722331120418.pypots +2024-09-13 12:19:13 [INFO]: Epoch 014 - training loss: 10.1591, validation loss: 10.3023, validation AUC: 0.8379263470093921 +2024-09-13 12:19:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch14_loss10.302282993610088.pypots +2024-09-13 12:19:20 [INFO]: Epoch 015 - training loss: 10.0102, validation loss: 10.2317, validation AUC: 0.8472936233316857 +2024-09-13 12:19:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch15_loss10.23169524853046.pypots +2024-09-13 12:19:26 [INFO]: Epoch 016 - training loss: 9.8710, validation loss: 10.1016, validation AUC: 0.8339841819080575 +2024-09-13 12:19:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch16_loss10.101593971252441.pypots +2024-09-13 12:19:32 [INFO]: Epoch 017 - training loss: 9.7457, validation loss: 10.0257, validation AUC: 0.834268413247652 +2024-09-13 12:19:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch17_loss10.025650537931002.pypots +2024-09-13 12:19:38 [INFO]: Epoch 018 - training loss: 9.6776, validation loss: 9.9535, validation AUC: 0.8422639644092931 +2024-09-13 12:19:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch18_loss9.953543516305777.pypots +2024-09-13 12:19:44 [INFO]: Epoch 019 - training loss: 9.5949, validation loss: 9.9773, validation AUC: 0.8271131982204647 +2024-09-13 12:19:50 [INFO]: Epoch 020 - training loss: 9.5266, validation loss: 9.9032, validation AUC: 0.8192535837864557 +2024-09-13 12:19:50 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch20_loss9.903202497042143.pypots +2024-09-13 12:19:56 [INFO]: Epoch 021 - training loss: 9.4495, validation loss: 9.8674, validation AUC: 0.8209218981710331 +2024-09-13 12:19:56 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch21_loss9.867419976454515.pypots +2024-09-13 12:20:02 [INFO]: Epoch 022 - training loss: 9.3996, validation loss: 9.8199, validation AUC: 0.8238012852199703 +2024-09-13 12:20:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch22_loss9.81988026545598.pypots +2024-09-13 12:20:08 [INFO]: Epoch 023 - training loss: 9.3449, validation loss: 9.8067, validation AUC: 0.8128769154720712 +2024-09-13 12:20:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch23_loss9.806729390070988.pypots +2024-09-13 12:20:14 [INFO]: Epoch 024 - training loss: 9.2911, validation loss: 9.7355, validation AUC: 0.8299678695007415 +2024-09-13 12:20:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch24_loss9.735456540034367.pypots +2024-09-13 12:20:21 [INFO]: Epoch 025 - training loss: 9.2479, validation loss: 9.7462, validation AUC: 0.8284602076124568 +2024-09-13 12:20:27 [INFO]: Epoch 026 - training loss: 9.1890, validation loss: 9.7657, validation AUC: 0.8208601087493821 +2024-09-13 12:20:33 [INFO]: Epoch 027 - training loss: 9.1731, validation loss: 9.6901, validation AUC: 0.82194760257044 +2024-09-13 12:20:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch27_loss9.690123631403996.pypots +2024-09-13 12:20:39 [INFO]: Epoch 028 - training loss: 9.1152, validation loss: 9.6960, validation AUC: 0.8299555116164113 +2024-09-13 12:20:45 [INFO]: Epoch 029 - training loss: 9.0335, validation loss: 9.6730, validation AUC: 0.8254695996045477 +2024-09-13 12:20:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch29_loss9.672953972449669.pypots +2024-09-13 12:20:51 [INFO]: Epoch 030 - training loss: 9.0020, validation loss: 9.6380, validation AUC: 0.827805239742956 +2024-09-13 12:20:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch30_loss9.637961314274715.pypots +2024-09-13 12:20:57 [INFO]: Epoch 031 - training loss: 8.9571, validation loss: 9.6237, validation AUC: 0.8336875926841324 +2024-09-13 12:20:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch31_loss9.62369889479417.pypots +2024-09-13 12:21:03 [INFO]: Epoch 032 - training loss: 8.9231, validation loss: 9.6379, validation AUC: 0.8148912506178941 +2024-09-13 12:21:09 [INFO]: Epoch 033 - training loss: 8.8739, validation loss: 9.6433, validation AUC: 0.8089594661393971 +2024-09-13 12:21:15 [INFO]: Epoch 034 - training loss: 8.8328, validation loss: 9.6885, validation AUC: 0.804695996045477 +2024-09-13 12:21:21 [INFO]: Epoch 035 - training loss: 8.8352, validation loss: 9.5828, validation AUC: 0.8196613939693524 +2024-09-13 12:21:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch35_loss9.58282654102032.pypots +2024-09-13 12:21:27 [INFO]: Epoch 036 - training loss: 8.7549, validation loss: 9.6210, validation AUC: 0.7901631240731587 +2024-09-13 12:21:33 [INFO]: Epoch 037 - training loss: 8.7074, validation loss: 9.5951, validation AUC: 0.7956376668314384 +2024-09-13 12:21:39 [INFO]: Epoch 038 - training loss: 8.6545, validation loss: 9.6687, validation AUC: 0.7702916460701927 +2024-09-13 12:21:45 [INFO]: Epoch 039 - training loss: 8.6706, validation loss: 9.6737, validation AUC: 0.7659540286702917 +2024-09-13 12:21:51 [INFO]: Epoch 040 - training loss: 8.6064, validation loss: 9.5919, validation AUC: 0.7831562036579338 +2024-09-13 12:21:51 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:21:51 [INFO]: Finished training. The best model is from epoch#35. +2024-09-13 12:21:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 3 ended ........................................ + +INFO:__main__: + Fold 3 Results: +{ + "PyPots_evaluation": { + "ROC_AUC": 0.8221998842486825, + "PR_AUC": 0.42611023661860026, + "F1": 0.39999999999999997, + "Precision": 0.4583333333333333, + "Recall": 0.3548387096774194 + }, + "CSAI_evaluation": { + "ACC": 0.8760951188986232, + "AUC": 0.8221998842486825, + "Prec_Macro": 0.6879011921137093, + "Recall_Macro": 0.6497989582381432, + "F1_Macro": 0.665457083042568, + "Bal_ACC": 0.6497989582381432 + } +} + +INFO:__main__: + Fold 4 started ...................................... + +2024-09-13 12:21:55 [INFO]: Using the given device: cuda +2024-09-13 12:21:55 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T122155 +2024-09-13 12:21:55 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T122155/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:22:04 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 +2024-09-13 12:22:10 [INFO]: Epoch 001 - training loss: 19.7550, validation loss: 17.4351, validation AUC: 0.5277519903226714 +2024-09-13 12:22:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch1_loss17.435132100031925.pypots +2024-09-13 12:22:16 [INFO]: Epoch 002 - training loss: 15.7833, validation loss: 15.0071, validation AUC: 0.642318620418038 +2024-09-13 12:22:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch2_loss15.007051687974196.pypots +2024-09-13 12:22:22 [INFO]: Epoch 003 - training loss: 14.2049, validation loss: 13.9272, validation AUC: 0.7292879849213717 +2024-09-13 12:22:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch3_loss13.927179703345665.pypots +2024-09-13 12:22:28 [INFO]: Epoch 004 - training loss: 13.3239, validation loss: 13.2162, validation AUC: 0.7507103271724759 +2024-09-13 12:22:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch4_loss13.21615402515118.pypots +2024-09-13 12:22:34 [INFO]: Epoch 005 - training loss: 12.6727, validation loss: 12.6155, validation AUC: 0.7631445691619545 +2024-09-13 12:22:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch5_loss12.615543512197641.pypots +2024-09-13 12:22:40 [INFO]: Epoch 006 - training loss: 12.0835, validation loss: 12.0658, validation AUC: 0.7773370467268688 +2024-09-13 12:22:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch6_loss12.065805288461538.pypots +2024-09-13 12:22:47 [INFO]: Epoch 007 - training loss: 11.6361, validation loss: 11.6526, validation AUC: 0.7797704447632713 +2024-09-13 12:22:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch7_loss11.652607697706957.pypots +2024-09-13 12:22:53 [INFO]: Epoch 008 - training loss: 11.2179, validation loss: 11.2851, validation AUC: 0.7865923987959602 +2024-09-13 12:22:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch8_loss11.285062423119179.pypots +2024-09-13 12:22:59 [INFO]: Epoch 009 - training loss: 10.9033, validation loss: 11.0406, validation AUC: 0.7826539511069851 +2024-09-13 12:22:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch9_loss11.040606351999136.pypots +2024-09-13 12:23:05 [INFO]: Epoch 010 - training loss: 10.6604, validation loss: 10.8240, validation AUC: 0.7906574394463667 +2024-09-13 12:23:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch10_loss10.823973142183744.pypots +2024-09-13 12:23:12 [INFO]: Epoch 011 - training loss: 10.4568, validation loss: 10.6639, validation AUC: 0.785649984527527 +2024-09-13 12:23:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch11_loss10.663858340336727.pypots +2024-09-13 12:23:18 [INFO]: Epoch 012 - training loss: 10.2834, validation loss: 10.4640, validation AUC: 0.7900666722930204 +2024-09-13 12:23:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch12_loss10.463965269235464.pypots +2024-09-13 12:23:24 [INFO]: Epoch 013 - training loss: 10.1037, validation loss: 10.3295, validation AUC: 0.7903198582158832 +2024-09-13 12:23:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch13_loss10.329508854792667.pypots +2024-09-13 12:23:31 [INFO]: Epoch 014 - training loss: 9.9464, validation loss: 10.2101, validation AUC: 0.7876754719104284 +2024-09-13 12:23:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch14_loss10.210116239694448.pypots +2024-09-13 12:23:37 [INFO]: Epoch 015 - training loss: 9.8494, validation loss: 10.1288, validation AUC: 0.792556333867837 +2024-09-13 12:23:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch15_loss10.128803693331205.pypots +2024-09-13 12:23:43 [INFO]: Epoch 016 - training loss: 9.7545, validation loss: 10.0486, validation AUC: 0.7848060314513179 +2024-09-13 12:23:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch16_loss10.048647660475512.pypots +2024-09-13 12:23:49 [INFO]: Epoch 017 - training loss: 9.6540, validation loss: 10.0037, validation AUC: 0.7942301741356514 +2024-09-13 12:23:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch17_loss10.003671866196852.pypots +2024-09-13 12:23:55 [INFO]: Epoch 018 - training loss: 9.5533, validation loss: 9.9426, validation AUC: 0.781486482684896 +2024-09-13 12:23:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch18_loss9.942629153911884.pypots +2024-09-13 12:24:01 [INFO]: Epoch 019 - training loss: 9.4786, validation loss: 9.8861, validation AUC: 0.7929923762905449 +2024-09-13 12:24:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch19_loss9.886068124037523.pypots +2024-09-13 12:24:08 [INFO]: Epoch 020 - training loss: 9.4157, validation loss: 9.7993, validation AUC: 0.7877317354488423 +2024-09-13 12:24:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch20_loss9.799341935377855.pypots +2024-09-13 12:24:14 [INFO]: Epoch 021 - training loss: 9.3380, validation loss: 9.8065, validation AUC: 0.786789321180409 +2024-09-13 12:24:20 [INFO]: Epoch 022 - training loss: 9.2901, validation loss: 9.7783, validation AUC: 0.7809097814161531 +2024-09-13 12:24:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch22_loss9.778309235206017.pypots +2024-09-13 12:24:26 [INFO]: Epoch 023 - training loss: 9.2254, validation loss: 9.7417, validation AUC: 0.7812051649928263 +2024-09-13 12:24:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch23_loss9.74165703700139.pypots +2024-09-13 12:24:32 [INFO]: Epoch 024 - training loss: 9.1778, validation loss: 9.7390, validation AUC: 0.7845669114130588 +2024-09-13 12:24:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch24_loss9.739037880530724.pypots +2024-09-13 12:24:38 [INFO]: Epoch 025 - training loss: 9.1469, validation loss: 9.7079, validation AUC: 0.7792781388021494 +2024-09-13 12:24:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch25_loss9.707893738379845.pypots +2024-09-13 12:24:44 [INFO]: Epoch 026 - training loss: 9.1043, validation loss: 9.6506, validation AUC: 0.78469350437449 +2024-09-13 12:24:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch26_loss9.650553336510292.pypots +2024-09-13 12:24:50 [INFO]: Epoch 027 - training loss: 9.0213, validation loss: 9.6380, validation AUC: 0.7773511126114723 +2024-09-13 12:24:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch27_loss9.63803085914025.pypots +2024-09-13 12:24:57 [INFO]: Epoch 028 - training loss: 8.9983, validation loss: 9.6047, validation AUC: 0.7793625341097702 +2024-09-13 12:24:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch28_loss9.604698988107534.pypots +2024-09-13 12:25:03 [INFO]: Epoch 029 - training loss: 8.9608, validation loss: 9.6082, validation AUC: 0.785213942104819 +2024-09-13 12:25:09 [INFO]: Epoch 030 - training loss: 8.8889, validation loss: 9.5917, validation AUC: 0.7713590457703885 +2024-09-13 12:25:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch30_loss9.591749264643742.pypots +2024-09-13 12:25:15 [INFO]: Epoch 031 - training loss: 8.8505, validation loss: 9.5793, validation AUC: 0.7748614510366558 +2024-09-13 12:25:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch31_loss9.579335065988394.pypots +2024-09-13 12:25:21 [INFO]: Epoch 032 - training loss: 8.8151, validation loss: 9.5944, validation AUC: 0.7682645511576223 +2024-09-13 12:25:27 [INFO]: Epoch 033 - training loss: 8.7853, validation loss: 9.5422, validation AUC: 0.772624975384702 +2024-09-13 12:25:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch33_loss9.54223764859713.pypots +2024-09-13 12:25:33 [INFO]: Epoch 034 - training loss: 8.7152, validation loss: 9.5936, validation AUC: 0.7766618842659014 +2024-09-13 12:25:40 [INFO]: Epoch 035 - training loss: 8.6958, validation loss: 9.6095, validation AUC: 0.7647340141221483 +2024-09-13 12:25:46 [INFO]: Epoch 036 - training loss: 8.7181, validation loss: 9.5777, validation AUC: 0.7849185585281458 +2024-09-13 12:25:52 [INFO]: Epoch 037 - training loss: 8.6481, validation loss: 9.4928, validation AUC: 0.7674065321968098 +2024-09-13 12:25:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch37_loss9.492766306950497.pypots +2024-09-13 12:25:58 [INFO]: Epoch 038 - training loss: 8.5793, validation loss: 9.4996, validation AUC: 0.7613019382788984 +2024-09-13 12:26:04 [INFO]: Epoch 039 - training loss: 8.5310, validation loss: 9.5502, validation AUC: 0.7683067488114328 +2024-09-13 12:26:10 [INFO]: Epoch 040 - training loss: 8.5103, validation loss: 9.6960, validation AUC: 0.7804596731088417 +2024-09-13 12:26:16 [INFO]: Epoch 041 - training loss: 8.6236, validation loss: 9.4729, validation AUC: 0.7637494021999043 +2024-09-13 12:26:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch41_loss9.472880730262169.pypots +2024-09-13 12:26:22 [INFO]: Epoch 042 - training loss: 8.4435, validation loss: 9.4721, validation AUC: 0.7571665682054746 +2024-09-13 12:26:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch42_loss9.472069373497597.pypots +2024-09-13 12:26:28 [INFO]: Epoch 043 - training loss: 8.4412, validation loss: 9.5390, validation AUC: 0.7747911216136383 +2024-09-13 12:26:34 [INFO]: Epoch 044 - training loss: 8.3844, validation loss: 9.4883, validation AUC: 0.7643261034686472 +2024-09-13 12:26:40 [INFO]: Epoch 045 - training loss: 8.3603, validation loss: 9.4581, validation AUC: 0.7577995330126311 +2024-09-13 12:26:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch45_loss9.458125554598295.pypots +2024-09-13 12:26:46 [INFO]: Epoch 046 - training loss: 8.3310, validation loss: 9.4998, validation AUC: 0.7775902326497314 +2024-09-13 12:26:53 [INFO]: Epoch 047 - training loss: 8.2575, validation loss: 9.5115, validation AUC: 0.7640166540073705 +2024-09-13 12:26:59 [INFO]: Epoch 048 - training loss: 8.2970, validation loss: 9.4353, validation AUC: 0.7679691675809491 +2024-09-13 12:26:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch48_loss9.435322248018705.pypots +2024-09-13 12:27:05 [INFO]: Epoch 049 - training loss: 8.2504, validation loss: 9.4680, validation AUC: 0.7643542352378541 +2024-09-13 12:27:11 [INFO]: Epoch 050 - training loss: 8.2382, validation loss: 9.4753, validation AUC: 0.7467015500604832 +2024-09-13 12:27:17 [INFO]: Epoch 051 - training loss: 8.2924, validation loss: 9.4827, validation AUC: 0.7728640954229612 +2024-09-13 12:27:23 [INFO]: Epoch 052 - training loss: 8.1721, validation loss: 9.4675, validation AUC: 0.7588685402424958 +2024-09-13 12:27:29 [INFO]: Epoch 053 - training loss: 8.1054, validation loss: 9.5912, validation AUC: 0.7507947224800968 +2024-09-13 12:27:29 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:27:29 [INFO]: Finished training. The best model is from epoch#48. +2024-09-13 12:27:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 4 ended ........................................ + +INFO:__main__: + Fold 4 Results: +{ + "PyPots_evaluation": { + "ROC_AUC": 0.839816933638444, + "PR_AUC": 0.4424882640898875, + "F1": 0.3870967741935485, + "Precision": 0.5070422535211268, + "Recall": 0.3130434782608696 + }, + "CSAI_evaluation": { + "ACC": 0.8573216520650814, + "AUC": 0.839816933638444, + "Prec_Macro": 0.6992628850023217, + "Recall_Macro": 0.6309369438087974, + "F1_Macro": 0.6531801151421001, + "Bal_ACC": 0.6309369438087974 + } +} + +INFO:__main__: + Average performance across all folds.......................... + +INFO:__main__: + Classification performance using Pypots evaluation: + {'ROC_AUC': 0.8092247029077602, 'PR_AUC': 0.434128245763033, 'F1': 0.36721553023394676, 'Precision': 0.5496469452277022, 'Recall': 0.28450883852005876} + +INFO:__main__: + Classification performance using CSAI evaluation: + {'ACC': 0.8666464330413017, 'AUC': 0.8092247029077602, 'Prec_Macro': 0.7211312123697666, 'Recall_Macro': 0.6228764993163682, 'F1_Macro': 0.6463198273912789, 'Bal_ACC': 0.6228764993163682} + diff --git a/output_logs/Imputation.out b/output_logs/Imputation.out new file mode 100644 index 00000000..f67a58c1 --- /dev/null +++ b/output_logs/Imputation.out @@ -0,0 +1,605 @@ +Lmod has detected the following error: The following module(s) are unknown: +"anaconda3/2021.05-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "anaconda3/2021.05-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Lmod has detected the following error: The following module(s) are unknown: +"cuda/11.1.1-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "cuda/11.1.1-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Lmod has detected the following error: The following module(s) are unknown: +"cudnn/8.0.5.39-11.1-gcc-9.4.0" + +Please check the spelling or version number. Also try "module spider ..." +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load "cudnn/8.0.5.39-11.1-gcc-9.4.0" + +Also make sure that all modulefiles written in TCL start with the string +#%Module + + + +Fri Sep 13 12:32:45 2024 ++---------------------------------------------------------------------------------------+ +| NVIDIA-SMI 535.183.01 Driver Version: 535.183.01 CUDA Version: 12.2 | +|-----------------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+======================+======================| +| 0 NVIDIA A100-SXM4-40GB On | 00000000:17:00.0 Off | 0 | +| N/A 33C P0 53W / 400W | 0MiB / 40960MiB | 0% Default | +| | | Disabled | ++-----------------------------------------+----------------------+----------------------+ + ++---------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=======================================================================================| +| No running processes found | ++---------------------------------------------------------------------------------------+ +/var/lib/slurm/slurmd/job21076914/slurm_script: line 18: activate: No such file or directory +2024-09-13 12:32:51.319467: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2024-09-13 12:32:51.361282: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2024-09-13 12:32:52.037976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +INFO:__main__: + Fold 0 started ...................................... + +2024-09-13 12:33:09 [INFO]: Using the given device: cuda +2024-09-13 12:33:09 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123309 +2024-09-13 12:33:09 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123309/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:33:19 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 +2024-09-13 12:33:29 [INFO]: Epoch 001 - training loss: 18.5395, validation loss: 0.6407 +2024-09-13 12:33:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch1_loss0.6407435054962451.pypots +2024-09-13 12:33:35 [INFO]: Epoch 002 - training loss: 14.9314, validation loss: 0.5001 +2024-09-13 12:33:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch2_loss0.5000739991664886.pypots +2024-09-13 12:33:41 [INFO]: Epoch 003 - training loss: 13.4167, validation loss: 0.4392 +2024-09-13 12:33:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch3_loss0.439183764732801.pypots +2024-09-13 12:33:47 [INFO]: Epoch 004 - training loss: 12.5427, validation loss: 0.4077 +2024-09-13 12:33:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch4_loss0.4077342335994427.pypots +2024-09-13 12:33:53 [INFO]: Epoch 005 - training loss: 11.8881, validation loss: 0.3863 +2024-09-13 12:33:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch5_loss0.3862775449569409.pypots +2024-09-13 12:33:59 [INFO]: Epoch 006 - training loss: 11.3413, validation loss: 0.3734 +2024-09-13 12:33:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch6_loss0.37343067618516773.pypots +2024-09-13 12:34:05 [INFO]: Epoch 007 - training loss: 10.8510, validation loss: 0.3654 +2024-09-13 12:34:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch7_loss0.36537456856324124.pypots +2024-09-13 12:34:11 [INFO]: Epoch 008 - training loss: 10.4839, validation loss: 0.3588 +2024-09-13 12:34:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch8_loss0.3588316383270117.pypots +2024-09-13 12:34:17 [INFO]: Epoch 009 - training loss: 10.2113, validation loss: 0.3533 +2024-09-13 12:34:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch9_loss0.353282379416319.pypots +2024-09-13 12:34:23 [INFO]: Epoch 010 - training loss: 9.9950, validation loss: 0.3514 +2024-09-13 12:34:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch10_loss0.35142794251441956.pypots +2024-09-13 12:34:29 [INFO]: Epoch 011 - training loss: 9.8098, validation loss: 0.3498 +2024-09-13 12:34:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch11_loss0.34977063078146714.pypots +2024-09-13 12:34:35 [INFO]: Epoch 012 - training loss: 9.6576, validation loss: 0.3436 +2024-09-13 12:34:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch12_loss0.34363453204815203.pypots +2024-09-13 12:34:41 [INFO]: Epoch 013 - training loss: 9.5010, validation loss: 0.3449 +2024-09-13 12:34:47 [INFO]: Epoch 014 - training loss: 9.3412, validation loss: 0.3416 +2024-09-13 12:34:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch14_loss0.3415501346954933.pypots +2024-09-13 12:34:53 [INFO]: Epoch 015 - training loss: 9.2051, validation loss: 0.3424 +2024-09-13 12:34:59 [INFO]: Epoch 016 - training loss: 9.0703, validation loss: 0.3383 +2024-09-13 12:34:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch16_loss0.3383355484558986.pypots +2024-09-13 12:35:05 [INFO]: Epoch 017 - training loss: 8.9579, validation loss: 0.3362 +2024-09-13 12:35:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch17_loss0.3362494432009183.pypots +2024-09-13 12:35:11 [INFO]: Epoch 018 - training loss: 8.8809, validation loss: 0.3359 +2024-09-13 12:35:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch18_loss0.3359452130702826.pypots +2024-09-13 12:35:17 [INFO]: Epoch 019 - training loss: 8.8047, validation loss: 0.3369 +2024-09-13 12:35:23 [INFO]: Epoch 020 - training loss: 8.7390, validation loss: 0.3335 +2024-09-13 12:35:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch20_loss0.33350417705682606.pypots +2024-09-13 12:35:29 [INFO]: Epoch 021 - training loss: 8.6853, validation loss: 0.3305 +2024-09-13 12:35:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch21_loss0.33045172920593846.pypots +2024-09-13 12:35:35 [INFO]: Epoch 022 - training loss: 8.6308, validation loss: 0.3337 +2024-09-13 12:35:41 [INFO]: Epoch 023 - training loss: 8.5754, validation loss: 0.3325 +2024-09-13 12:35:47 [INFO]: Epoch 024 - training loss: 8.5298, validation loss: 0.3270 +2024-09-13 12:35:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch24_loss0.327012836933136.pypots +2024-09-13 12:35:53 [INFO]: Epoch 025 - training loss: 8.4729, validation loss: 0.3304 +2024-09-13 12:35:59 [INFO]: Epoch 026 - training loss: 8.4130, validation loss: 0.3274 +2024-09-13 12:36:05 [INFO]: Epoch 027 - training loss: 8.3748, validation loss: 0.3235 +2024-09-13 12:36:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch27_loss0.32347696790328395.pypots +2024-09-13 12:36:11 [INFO]: Epoch 028 - training loss: 8.3357, validation loss: 0.3245 +2024-09-13 12:36:17 [INFO]: Epoch 029 - training loss: 8.3109, validation loss: 0.3227 +2024-09-13 12:36:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch29_loss0.32266572461678433.pypots +2024-09-13 12:36:23 [INFO]: Epoch 030 - training loss: 8.2417, validation loss: 0.3274 +2024-09-13 12:36:29 [INFO]: Epoch 031 - training loss: 8.2205, validation loss: 0.3212 +2024-09-13 12:36:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch31_loss0.3212323280481192.pypots +2024-09-13 12:36:35 [INFO]: Epoch 032 - training loss: 8.1723, validation loss: 0.3183 +2024-09-13 12:36:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch32_loss0.3183346207325275.pypots +2024-09-13 12:36:41 [INFO]: Epoch 033 - training loss: 8.1288, validation loss: 0.3178 +2024-09-13 12:36:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch33_loss0.317815356529676.pypots +2024-09-13 12:36:47 [INFO]: Epoch 034 - training loss: 8.0817, validation loss: 0.3177 +2024-09-13 12:36:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch34_loss0.31765339695490324.pypots +2024-09-13 12:36:53 [INFO]: Epoch 035 - training loss: 8.0366, validation loss: 0.3223 +2024-09-13 12:36:59 [INFO]: Epoch 036 - training loss: 7.9959, validation loss: 0.3221 +2024-09-13 12:37:05 [INFO]: Epoch 037 - training loss: 7.9807, validation loss: 0.3146 +2024-09-13 12:37:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch37_loss0.3145997386712294.pypots +2024-09-13 12:37:11 [INFO]: Epoch 038 - training loss: 7.9427, validation loss: 0.3162 +2024-09-13 12:37:17 [INFO]: Epoch 039 - training loss: 7.9166, validation loss: 0.3179 +2024-09-13 12:37:23 [INFO]: Epoch 040 - training loss: 7.8947, validation loss: 0.3206 +2024-09-13 12:37:29 [INFO]: Epoch 041 - training loss: 7.9263, validation loss: 0.3181 +2024-09-13 12:37:35 [INFO]: Epoch 042 - training loss: 7.8412, validation loss: 0.3203 +2024-09-13 12:37:35 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:37:35 [INFO]: Finished training. The best model is from epoch#37. +2024-09-13 12:37:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 0 ended ........................................ + +INFO:__main__: + Fold 0 Results: +{ + "MAE": 0.2522227483799602, + "MRE": 0.3491908942436348 +} + +INFO:__main__: + Fold 1 started ...................................... + +2024-09-13 12:37:38 [INFO]: Using the given device: cuda +2024-09-13 12:37:38 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123738 +2024-09-13 12:37:38 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123738/tensorboard +INFO:__main__: + Training..................................... + +/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) + warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") +2024-09-13 12:37:47 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 +2024-09-13 12:37:53 [INFO]: Epoch 001 - training loss: 18.5686, validation loss: 0.5924 +2024-09-13 12:37:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch1_loss0.5923694670200348.pypots +2024-09-13 12:37:59 [INFO]: Epoch 002 - training loss: 15.0152, validation loss: 0.4383 +2024-09-13 12:37:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch2_loss0.4382930329212776.pypots +2024-09-13 12:38:05 [INFO]: Epoch 003 - training loss: 13.4660, validation loss: 0.3992 +2024-09-13 12:38:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch3_loss0.39923181671362656.pypots +2024-09-13 12:38:11 [INFO]: Epoch 004 - training loss: 12.6009, validation loss: 0.3811 +2024-09-13 12:38:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch4_loss0.38111079656160796.pypots +2024-09-13 12:38:17 [INFO]: Epoch 005 - training loss: 11.9217, validation loss: 0.3800 +2024-09-13 12:38:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch5_loss0.37996633236224836.pypots +2024-09-13 12:38:23 [INFO]: Epoch 006 - training loss: 11.3803, validation loss: 0.3808 +2024-09-13 12:38:29 [INFO]: Epoch 007 - training loss: 10.8692, validation loss: 0.3803 +2024-09-13 12:38:35 [INFO]: Epoch 008 - training loss: 10.4516, validation loss: 0.3602 +2024-09-13 12:38:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch8_loss0.36019767362337846.pypots +2024-09-13 12:38:41 [INFO]: Epoch 009 - training loss: 10.1624, validation loss: 0.3302 +2024-09-13 12:38:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch9_loss0.33023632031220657.pypots +2024-09-13 12:38:48 [INFO]: Epoch 010 - training loss: 9.9167, validation loss: 0.3116 +2024-09-13 12:38:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch10_loss0.31161352075063264.pypots +2024-09-13 12:38:54 [INFO]: Epoch 011 - training loss: 9.6946, validation loss: 0.2856 +2024-09-13 12:38:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch11_loss0.2856314709553352.pypots +2024-09-13 12:39:00 [INFO]: Epoch 012 - training loss: 9.4901, validation loss: 0.2794 +2024-09-13 12:39:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch12_loss0.2794186300956286.pypots +2024-09-13 12:39:06 [INFO]: Epoch 013 - training loss: 9.3343, validation loss: 0.2694 +2024-09-13 12:39:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch13_loss0.2694310419834577.pypots +2024-09-13 12:39:12 [INFO]: Epoch 014 - training loss: 9.2215, validation loss: 0.2649 +2024-09-13 12:39:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch14_loss0.2648839056491852.pypots +2024-09-13 12:39:18 [INFO]: Epoch 015 - training loss: 9.0909, validation loss: 0.2580 +2024-09-13 12:39:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch15_loss0.25795977963851047.pypots +2024-09-13 12:39:24 [INFO]: Epoch 016 - training loss: 8.9995, validation loss: 0.2535 +2024-09-13 12:39:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch16_loss0.25348290457175326.pypots +2024-09-13 12:39:30 [INFO]: Epoch 017 - training loss: 8.9097, validation loss: 0.2494 +2024-09-13 12:39:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch17_loss0.24935718568471762.pypots +2024-09-13 12:39:36 [INFO]: Epoch 018 - training loss: 8.8485, validation loss: 0.2479 +2024-09-13 12:39:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch18_loss0.247948285478812.pypots +2024-09-13 12:39:42 [INFO]: Epoch 019 - training loss: 8.7699, validation loss: 0.2461 +2024-09-13 12:39:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch19_loss0.24605009303643152.pypots +2024-09-13 12:39:48 [INFO]: Epoch 020 - training loss: 8.7146, validation loss: 0.2440 +2024-09-13 12:39:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch20_loss0.2440407402240313.pypots +2024-09-13 12:39:54 [INFO]: Epoch 021 - training loss: 8.6635, validation loss: 0.2430 +2024-09-13 12:39:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch21_loss0.2429531319783284.pypots +2024-09-13 12:40:00 [INFO]: Epoch 022 - training loss: 8.6202, validation loss: 0.2408 +2024-09-13 12:40:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch22_loss0.24082150826087365.pypots +2024-09-13 12:40:06 [INFO]: Epoch 023 - training loss: 8.5857, validation loss: 0.2401 +2024-09-13 12:40:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch23_loss0.24012872118216294.pypots +2024-09-13 12:40:12 [INFO]: Epoch 024 - training loss: 8.5168, validation loss: 0.2401 +2024-09-13 12:40:18 [INFO]: Epoch 025 - training loss: 8.4674, validation loss: 0.2396 +2024-09-13 12:40:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch25_loss0.23958969345459571.pypots +2024-09-13 12:40:24 [INFO]: Epoch 026 - training loss: 8.4349, validation loss: 0.2395 +2024-09-13 12:40:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch26_loss0.23954786016390875.pypots +2024-09-13 12:40:30 [INFO]: Epoch 027 - training loss: 8.3812, validation loss: 0.2396 +2024-09-13 12:40:36 [INFO]: Epoch 028 - training loss: 8.3438, validation loss: 0.2398 +2024-09-13 12:40:42 [INFO]: Epoch 029 - training loss: 8.3128, validation loss: 0.2389 +2024-09-13 12:40:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch29_loss0.23891761898994446.pypots +2024-09-13 12:40:48 [INFO]: Epoch 030 - training loss: 8.2741, validation loss: 0.2383 +2024-09-13 12:40:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch30_loss0.23825732102760902.pypots +2024-09-13 12:40:54 [INFO]: Epoch 031 - training loss: 8.2236, validation loss: 0.2386 +2024-09-13 12:41:00 [INFO]: Epoch 032 - training loss: 8.2065, validation loss: 0.2377 +2024-09-13 12:41:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch32_loss0.23773047442619616.pypots +2024-09-13 12:41:06 [INFO]: Epoch 033 - training loss: 8.1779, validation loss: 0.2386 +2024-09-13 12:41:12 [INFO]: Epoch 034 - training loss: 8.1342, validation loss: 0.2371 +2024-09-13 12:41:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch34_loss0.23709593025537637.pypots +2024-09-13 12:41:18 [INFO]: Epoch 035 - training loss: 8.1144, validation loss: 0.2378 +2024-09-13 12:41:24 [INFO]: Epoch 036 - training loss: 8.0583, validation loss: 0.2379 +2024-09-13 12:41:30 [INFO]: Epoch 037 - training loss: 8.0278, validation loss: 0.2382 +2024-09-13 12:41:36 [INFO]: Epoch 038 - training loss: 8.0119, validation loss: 0.2364 +2024-09-13 12:41:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch38_loss0.236399219586299.pypots +2024-09-13 12:41:42 [INFO]: Epoch 039 - training loss: 7.9644, validation loss: 0.2372 +2024-09-13 12:41:48 [INFO]: Epoch 040 - training loss: 7.9454, validation loss: 0.2380 +2024-09-13 12:41:54 [INFO]: Epoch 041 - training loss: 7.9328, validation loss: 0.2370 +2024-09-13 12:42:00 [INFO]: Epoch 042 - training loss: 7.8796, validation loss: 0.2375 +2024-09-13 12:42:06 [INFO]: Epoch 043 - training loss: 7.8528, validation loss: 0.2375 +2024-09-13 12:42:06 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:42:06 [INFO]: Finished training. The best model is from epoch#38. +2024-09-13 12:42:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 1 ended ........................................ + +INFO:__main__: + Fold 1 Results: +{ + "MAE": 0.24833315072947296, + "MRE": 0.34538290796537663 +} + +INFO:__main__: + Fold 2 started ...................................... + +2024-09-13 12:42:09 [INFO]: Using the given device: cuda +2024-09-13 12:42:09 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124209 +2024-09-13 12:42:09 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124209/tensorboard +INFO:__main__: + Training..................................... + +2024-09-13 12:42:19 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 +2024-09-13 12:42:25 [INFO]: Epoch 001 - training loss: 18.5348, validation loss: 0.5612 +2024-09-13 12:42:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch1_loss0.5611514747142792.pypots +2024-09-13 12:42:31 [INFO]: Epoch 002 - training loss: 14.9343, validation loss: 0.4211 +2024-09-13 12:42:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch2_loss0.4211423442913936.pypots +2024-09-13 12:42:37 [INFO]: Epoch 003 - training loss: 13.4568, validation loss: 0.3764 +2024-09-13 12:42:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch3_loss0.3763529383219205.pypots +2024-09-13 12:42:43 [INFO]: Epoch 004 - training loss: 12.5687, validation loss: 0.3459 +2024-09-13 12:42:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch4_loss0.3459321971123035.pypots +2024-09-13 12:42:49 [INFO]: Epoch 005 - training loss: 11.9092, validation loss: 0.3259 +2024-09-13 12:42:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch5_loss0.32591839363941777.pypots +2024-09-13 12:42:55 [INFO]: Epoch 006 - training loss: 11.3498, validation loss: 0.3121 +2024-09-13 12:42:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch6_loss0.3120722415355536.pypots +2024-09-13 12:43:01 [INFO]: Epoch 007 - training loss: 10.8595, validation loss: 0.2990 +2024-09-13 12:43:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch7_loss0.29896558362704057.pypots +2024-09-13 12:43:07 [INFO]: Epoch 008 - training loss: 10.4851, validation loss: 0.2914 +2024-09-13 12:43:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch8_loss0.29137278176271.pypots +2024-09-13 12:43:13 [INFO]: Epoch 009 - training loss: 10.1950, validation loss: 0.2870 +2024-09-13 12:43:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch9_loss0.2870428814337804.pypots +2024-09-13 12:43:19 [INFO]: Epoch 010 - training loss: 9.9764, validation loss: 0.2847 +2024-09-13 12:43:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch10_loss0.28473271200290096.pypots +2024-09-13 12:43:25 [INFO]: Epoch 011 - training loss: 9.7934, validation loss: 0.2787 +2024-09-13 12:43:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch11_loss0.2786905009012956.pypots +2024-09-13 12:43:31 [INFO]: Epoch 012 - training loss: 9.6277, validation loss: 0.2773 +2024-09-13 12:43:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch12_loss0.277261625115688.pypots +2024-09-13 12:43:37 [INFO]: Epoch 013 - training loss: 9.4717, validation loss: 0.2765 +2024-09-13 12:43:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch13_loss0.276475286254516.pypots +2024-09-13 12:43:43 [INFO]: Epoch 014 - training loss: 9.3222, validation loss: 0.2745 +2024-09-13 12:43:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch14_loss0.27448664605617523.pypots +2024-09-13 12:43:49 [INFO]: Epoch 015 - training loss: 9.1564, validation loss: 0.2720 +2024-09-13 12:43:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch15_loss0.2719874611267677.pypots +2024-09-13 12:43:55 [INFO]: Epoch 016 - training loss: 9.0493, validation loss: 0.2713 +2024-09-13 12:43:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch16_loss0.2712785303592682.pypots +2024-09-13 12:44:01 [INFO]: Epoch 017 - training loss: 8.9550, validation loss: 0.2690 +2024-09-13 12:44:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch17_loss0.2689772821389712.pypots +2024-09-13 12:44:08 [INFO]: Epoch 018 - training loss: 8.8579, validation loss: 0.2715 +2024-09-13 12:44:14 [INFO]: Epoch 019 - training loss: 8.7816, validation loss: 0.2696 +2024-09-13 12:44:19 [INFO]: Epoch 020 - training loss: 8.7212, validation loss: 0.2689 +2024-09-13 12:44:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch20_loss0.26889700385240406.pypots +2024-09-13 12:44:25 [INFO]: Epoch 021 - training loss: 8.6609, validation loss: 0.2689 +2024-09-13 12:44:31 [INFO]: Epoch 022 - training loss: 8.6029, validation loss: 0.2689 +2024-09-13 12:44:37 [INFO]: Epoch 023 - training loss: 8.5666, validation loss: 0.2684 +2024-09-13 12:44:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch23_loss0.26841238141059875.pypots +2024-09-13 12:44:43 [INFO]: Epoch 024 - training loss: 8.5122, validation loss: 0.2685 +2024-09-13 12:44:49 [INFO]: Epoch 025 - training loss: 8.4428, validation loss: 0.2680 +2024-09-13 12:44:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch25_loss0.268048128256431.pypots +2024-09-13 12:44:55 [INFO]: Epoch 026 - training loss: 8.4081, validation loss: 0.2683 +2024-09-13 12:45:01 [INFO]: Epoch 027 - training loss: 8.3657, validation loss: 0.2681 +2024-09-13 12:45:07 [INFO]: Epoch 028 - training loss: 8.3273, validation loss: 0.2670 +2024-09-13 12:45:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch28_loss0.2669859161743751.pypots +2024-09-13 12:45:13 [INFO]: Epoch 029 - training loss: 8.2955, validation loss: 0.2668 +2024-09-13 12:45:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch29_loss0.2668392062187195.pypots +2024-09-13 12:45:20 [INFO]: Epoch 030 - training loss: 8.2595, validation loss: 0.2669 +2024-09-13 12:45:26 [INFO]: Epoch 031 - training loss: 8.2161, validation loss: 0.2672 +2024-09-13 12:45:32 [INFO]: Epoch 032 - training loss: 8.1760, validation loss: 0.2663 +2024-09-13 12:45:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch32_loss0.26633317883198077.pypots +2024-09-13 12:45:38 [INFO]: Epoch 033 - training loss: 8.1431, validation loss: 0.2676 +2024-09-13 12:45:43 [INFO]: Epoch 034 - training loss: 8.1101, validation loss: 0.2676 +2024-09-13 12:45:49 [INFO]: Epoch 035 - training loss: 8.0758, validation loss: 0.2663 +2024-09-13 12:45:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch35_loss0.26626667036459994.pypots +2024-09-13 12:45:55 [INFO]: Epoch 036 - training loss: 8.0328, validation loss: 0.2677 +2024-09-13 12:46:01 [INFO]: Epoch 037 - training loss: 8.0061, validation loss: 0.2661 +2024-09-13 12:46:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch37_loss0.26614915980742526.pypots +2024-09-13 12:46:07 [INFO]: Epoch 038 - training loss: 7.9972, validation loss: 0.2665 +2024-09-13 12:46:13 [INFO]: Epoch 039 - training loss: 7.9451, validation loss: 0.2648 +2024-09-13 12:46:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch39_loss0.26480964170052457.pypots +2024-09-13 12:46:19 [INFO]: Epoch 040 - training loss: 7.8999, validation loss: 0.2667 +2024-09-13 12:46:25 [INFO]: Epoch 041 - training loss: 7.8679, validation loss: 0.2649 +2024-09-13 12:46:31 [INFO]: Epoch 042 - training loss: 7.8303, validation loss: 0.2666 +2024-09-13 12:46:37 [INFO]: Epoch 043 - training loss: 7.8062, validation loss: 0.2663 +2024-09-13 12:46:43 [INFO]: Epoch 044 - training loss: 7.7753, validation loss: 0.2670 +2024-09-13 12:46:43 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:46:43 [INFO]: Finished training. The best model is from epoch#39. +2024-09-13 12:46:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 2 ended ........................................ + +INFO:__main__: + Fold 2 Results: +{ + "MAE": 0.25268375496555934, + "MRE": 0.3468272181807543 +} + +INFO:__main__: + Fold 3 started ...................................... + +2024-09-13 12:46:47 [INFO]: Using the given device: cuda +2024-09-13 12:46:47 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124647 +2024-09-13 12:46:47 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124647/tensorboard +INFO:__main__: + Training..................................... + +2024-09-13 12:46:56 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 +2024-09-13 12:47:02 [INFO]: Epoch 001 - training loss: 18.6528, validation loss: 0.5896 +2024-09-13 12:47:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch1_loss0.5896110672217149.pypots +2024-09-13 12:47:08 [INFO]: Epoch 002 - training loss: 15.0505, validation loss: 0.4486 +2024-09-13 12:47:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch2_loss0.448551744222641.pypots +2024-09-13 12:47:14 [INFO]: Epoch 003 - training loss: 13.5125, validation loss: 0.3941 +2024-09-13 12:47:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch3_loss0.39414907648013187.pypots +2024-09-13 12:47:20 [INFO]: Epoch 004 - training loss: 12.6204, validation loss: 0.3625 +2024-09-13 12:47:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch4_loss0.36246832746725816.pypots +2024-09-13 12:47:26 [INFO]: Epoch 005 - training loss: 11.9558, validation loss: 0.3467 +2024-09-13 12:47:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch5_loss0.3466833738180307.pypots +2024-09-13 12:47:32 [INFO]: Epoch 006 - training loss: 11.3833, validation loss: 0.3337 +2024-09-13 12:47:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch6_loss0.33370912762788624.pypots +2024-09-13 12:47:38 [INFO]: Epoch 007 - training loss: 10.9050, validation loss: 0.3241 +2024-09-13 12:47:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch7_loss0.32408049473395717.pypots +2024-09-13 12:47:44 [INFO]: Epoch 008 - training loss: 10.5231, validation loss: 0.3188 +2024-09-13 12:47:44 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch8_loss0.3188299639866902.pypots +2024-09-13 12:47:50 [INFO]: Epoch 009 - training loss: 10.2466, validation loss: 0.3137 +2024-09-13 12:47:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch9_loss0.3137410913522427.pypots +2024-09-13 12:47:57 [INFO]: Epoch 010 - training loss: 10.0112, validation loss: 0.3114 +2024-09-13 12:47:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch10_loss0.31143705317607295.pypots +2024-09-13 12:48:03 [INFO]: Epoch 011 - training loss: 9.8428, validation loss: 0.3094 +2024-09-13 12:48:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch11_loss0.30936380418447346.pypots +2024-09-13 12:48:09 [INFO]: Epoch 012 - training loss: 9.6768, validation loss: 0.3065 +2024-09-13 12:48:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch12_loss0.30649459935151613.pypots +2024-09-13 12:48:15 [INFO]: Epoch 013 - training loss: 9.5489, validation loss: 0.3050 +2024-09-13 12:48:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch13_loss0.30499985126348644.pypots +2024-09-13 12:48:21 [INFO]: Epoch 014 - training loss: 9.4256, validation loss: 0.3047 +2024-09-13 12:48:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch14_loss0.3046507560289823.pypots +2024-09-13 12:48:27 [INFO]: Epoch 015 - training loss: 9.3041, validation loss: 0.3019 +2024-09-13 12:48:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch15_loss0.30189058299248034.pypots +2024-09-13 12:48:33 [INFO]: Epoch 016 - training loss: 9.1878, validation loss: 0.3004 +2024-09-13 12:48:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch16_loss0.3004143008818993.pypots +2024-09-13 12:48:39 [INFO]: Epoch 017 - training loss: 9.0926, validation loss: 0.3016 +2024-09-13 12:48:45 [INFO]: Epoch 018 - training loss: 8.9575, validation loss: 0.3015 +2024-09-13 12:48:51 [INFO]: Epoch 019 - training loss: 8.8735, validation loss: 0.3013 +2024-09-13 12:48:58 [INFO]: Epoch 020 - training loss: 8.7711, validation loss: 0.3012 +2024-09-13 12:49:04 [INFO]: Epoch 021 - training loss: 8.7145, validation loss: 0.3002 +2024-09-13 12:49:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch21_loss0.3002306039516742.pypots +2024-09-13 12:49:10 [INFO]: Epoch 022 - training loss: 8.6676, validation loss: 0.3003 +2024-09-13 12:49:16 [INFO]: Epoch 023 - training loss: 8.6097, validation loss: 0.3006 +2024-09-13 12:49:22 [INFO]: Epoch 024 - training loss: 8.5589, validation loss: 0.3004 +2024-09-13 12:49:28 [INFO]: Epoch 025 - training loss: 8.5222, validation loss: 0.2997 +2024-09-13 12:49:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch25_loss0.29971028979008013.pypots +2024-09-13 12:49:34 [INFO]: Epoch 026 - training loss: 8.4555, validation loss: 0.2996 +2024-09-13 12:49:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch26_loss0.29962454506984126.pypots +2024-09-13 12:49:40 [INFO]: Epoch 027 - training loss: 8.4317, validation loss: 0.2967 +2024-09-13 12:49:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch27_loss0.2967267162524737.pypots +2024-09-13 12:49:46 [INFO]: Epoch 028 - training loss: 8.3893, validation loss: 0.3005 +2024-09-13 12:49:52 [INFO]: Epoch 029 - training loss: 8.3394, validation loss: 0.2990 +2024-09-13 12:49:58 [INFO]: Epoch 030 - training loss: 8.2851, validation loss: 0.2989 +2024-09-13 12:50:04 [INFO]: Epoch 031 - training loss: 8.2500, validation loss: 0.2962 +2024-09-13 12:50:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch31_loss0.29616755361740404.pypots +2024-09-13 12:50:10 [INFO]: Epoch 032 - training loss: 8.1912, validation loss: 0.2978 +2024-09-13 12:50:16 [INFO]: Epoch 033 - training loss: 8.1711, validation loss: 0.2964 +2024-09-13 12:50:22 [INFO]: Epoch 034 - training loss: 8.1256, validation loss: 0.2972 +2024-09-13 12:50:28 [INFO]: Epoch 035 - training loss: 8.0841, validation loss: 0.2959 +2024-09-13 12:50:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch35_loss0.29589974880218506.pypots +2024-09-13 12:50:34 [INFO]: Epoch 036 - training loss: 8.0485, validation loss: 0.2961 +2024-09-13 12:50:40 [INFO]: Epoch 037 - training loss: 8.0247, validation loss: 0.2954 +2024-09-13 12:50:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch37_loss0.2953845365689351.pypots +2024-09-13 12:50:46 [INFO]: Epoch 038 - training loss: 7.9824, validation loss: 0.2962 +2024-09-13 12:50:52 [INFO]: Epoch 039 - training loss: 7.9632, validation loss: 0.2947 +2024-09-13 12:50:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch39_loss0.2946692028870949.pypots +2024-09-13 12:50:58 [INFO]: Epoch 040 - training loss: 7.9081, validation loss: 0.2953 +2024-09-13 12:51:04 [INFO]: Epoch 041 - training loss: 7.8854, validation loss: 0.2940 +2024-09-13 12:51:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch41_loss0.2939985004755167.pypots +2024-09-13 12:51:10 [INFO]: Epoch 042 - training loss: 7.8589, validation loss: 0.2953 +2024-09-13 12:51:16 [INFO]: Epoch 043 - training loss: 7.8427, validation loss: 0.2925 +2024-09-13 12:51:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch43_loss0.2925326961737413.pypots +2024-09-13 12:51:22 [INFO]: Epoch 044 - training loss: 7.8126, validation loss: 0.2938 +2024-09-13 12:51:28 [INFO]: Epoch 045 - training loss: 7.7695, validation loss: 0.2936 +2024-09-13 12:51:34 [INFO]: Epoch 046 - training loss: 7.7545, validation loss: 0.2948 +2024-09-13 12:51:40 [INFO]: Epoch 047 - training loss: 7.7470, validation loss: 0.2961 +2024-09-13 12:51:46 [INFO]: Epoch 048 - training loss: 7.7173, validation loss: 0.2942 +2024-09-13 12:51:46 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:51:46 [INFO]: Finished training. The best model is from epoch#43. +2024-09-13 12:51:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 3 ended ........................................ + +INFO:__main__: + Fold 3 Results: +{ + "MAE": 0.25894437333366543, + "MRE": 0.34814777479039904 +} + +INFO:__main__: + Fold 4 started ...................................... + +2024-09-13 12:51:49 [INFO]: Using the given device: cuda +2024-09-13 12:51:49 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T125149 +2024-09-13 12:51:49 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T125149/tensorboard +INFO:__main__: + Training..................................... + +2024-09-13 12:51:59 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 +2024-09-13 12:52:05 [INFO]: Epoch 001 - training loss: 18.5119, validation loss: 0.5635 +2024-09-13 12:52:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch1_loss0.563519686460495.pypots +2024-09-13 12:52:11 [INFO]: Epoch 002 - training loss: 14.9026, validation loss: 0.4182 +2024-09-13 12:52:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch2_loss0.41821371821256786.pypots +2024-09-13 12:52:17 [INFO]: Epoch 003 - training loss: 13.4091, validation loss: 0.3636 +2024-09-13 12:52:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch3_loss0.36356426316958207.pypots +2024-09-13 12:52:23 [INFO]: Epoch 004 - training loss: 12.5351, validation loss: 0.3342 +2024-09-13 12:52:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch4_loss0.3342179312155797.pypots +2024-09-13 12:52:29 [INFO]: Epoch 005 - training loss: 11.8907, validation loss: 0.3143 +2024-09-13 12:52:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch5_loss0.314338546532851.pypots +2024-09-13 12:52:35 [INFO]: Epoch 006 - training loss: 11.3183, validation loss: 0.2987 +2024-09-13 12:52:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch6_loss0.2986761675431178.pypots +2024-09-13 12:52:41 [INFO]: Epoch 007 - training loss: 10.8321, validation loss: 0.2843 +2024-09-13 12:52:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch7_loss0.28431492126905.pypots +2024-09-13 12:52:47 [INFO]: Epoch 008 - training loss: 10.4464, validation loss: 0.2780 +2024-09-13 12:52:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch8_loss0.2780210788433368.pypots +2024-09-13 12:52:53 [INFO]: Epoch 009 - training loss: 10.1411, validation loss: 0.2717 +2024-09-13 12:52:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch9_loss0.2716936022043228.pypots +2024-09-13 12:52:59 [INFO]: Epoch 010 - training loss: 9.8886, validation loss: 0.2675 +2024-09-13 12:52:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch10_loss0.26754155640418714.pypots +2024-09-13 12:53:05 [INFO]: Epoch 011 - training loss: 9.6573, validation loss: 0.2617 +2024-09-13 12:53:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch11_loss0.26171908470300526.pypots +2024-09-13 12:53:11 [INFO]: Epoch 012 - training loss: 9.4534, validation loss: 0.2588 +2024-09-13 12:53:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch12_loss0.25877043719475085.pypots +2024-09-13 12:53:18 [INFO]: Epoch 013 - training loss: 9.3149, validation loss: 0.2558 +2024-09-13 12:53:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch13_loss0.2557584700676111.pypots +2024-09-13 12:53:24 [INFO]: Epoch 014 - training loss: 9.1776, validation loss: 0.2542 +2024-09-13 12:53:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch14_loss0.25421963746731097.pypots +2024-09-13 12:53:30 [INFO]: Epoch 015 - training loss: 9.0721, validation loss: 0.2537 +2024-09-13 12:53:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch15_loss0.25367432144971996.pypots +2024-09-13 12:53:36 [INFO]: Epoch 016 - training loss: 8.9740, validation loss: 0.2508 +2024-09-13 12:53:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch16_loss0.25082376713936144.pypots +2024-09-13 12:53:41 [INFO]: Epoch 017 - training loss: 8.8739, validation loss: 0.2499 +2024-09-13 12:53:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch17_loss0.24994707909914163.pypots +2024-09-13 12:53:47 [INFO]: Epoch 018 - training loss: 8.8084, validation loss: 0.2496 +2024-09-13 12:53:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch18_loss0.24957519884292895.pypots +2024-09-13 12:53:54 [INFO]: Epoch 019 - training loss: 8.7477, validation loss: 0.2480 +2024-09-13 12:53:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch19_loss0.24800925988417405.pypots +2024-09-13 12:54:00 [INFO]: Epoch 020 - training loss: 8.6866, validation loss: 0.2475 +2024-09-13 12:54:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch20_loss0.24747206958440635.pypots +2024-09-13 12:54:06 [INFO]: Epoch 021 - training loss: 8.6250, validation loss: 0.2478 +2024-09-13 12:54:12 [INFO]: Epoch 022 - training loss: 8.5700, validation loss: 0.2473 +2024-09-13 12:54:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch22_loss0.24734074679704812.pypots +2024-09-13 12:54:18 [INFO]: Epoch 023 - training loss: 8.5134, validation loss: 0.2455 +2024-09-13 12:54:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch23_loss0.24554229699648344.pypots +2024-09-13 12:54:24 [INFO]: Epoch 024 - training loss: 8.4657, validation loss: 0.2468 +2024-09-13 12:54:30 [INFO]: Epoch 025 - training loss: 8.4304, validation loss: 0.2437 +2024-09-13 12:54:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch25_loss0.2436517637509566.pypots +2024-09-13 12:54:36 [INFO]: Epoch 026 - training loss: 8.3759, validation loss: 0.2423 +2024-09-13 12:54:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch26_loss0.24229151927507842.pypots +2024-09-13 12:54:42 [INFO]: Epoch 027 - training loss: 8.3343, validation loss: 0.2414 +2024-09-13 12:54:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch27_loss0.24144030878177056.pypots +2024-09-13 12:54:48 [INFO]: Epoch 028 - training loss: 8.2927, validation loss: 0.2431 +2024-09-13 12:54:54 [INFO]: Epoch 029 - training loss: 8.2643, validation loss: 0.2409 +2024-09-13 12:54:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch29_loss0.24094540912371415.pypots +2024-09-13 12:55:00 [INFO]: Epoch 030 - training loss: 8.2241, validation loss: 0.2415 +2024-09-13 12:55:06 [INFO]: Epoch 031 - training loss: 8.1760, validation loss: 0.2402 +2024-09-13 12:55:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch31_loss0.24021976383832785.pypots +2024-09-13 12:55:12 [INFO]: Epoch 032 - training loss: 8.1481, validation loss: 0.2374 +2024-09-13 12:55:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch32_loss0.23736257048753592.pypots +2024-09-13 12:55:18 [INFO]: Epoch 033 - training loss: 8.0922, validation loss: 0.2385 +2024-09-13 12:55:24 [INFO]: Epoch 034 - training loss: 8.0782, validation loss: 0.2379 +2024-09-13 12:55:30 [INFO]: Epoch 035 - training loss: 8.0490, validation loss: 0.2359 +2024-09-13 12:55:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch35_loss0.2358845346249067.pypots +2024-09-13 12:55:36 [INFO]: Epoch 036 - training loss: 8.0172, validation loss: 0.2365 +2024-09-13 12:55:42 [INFO]: Epoch 037 - training loss: 7.9829, validation loss: 0.2358 +2024-09-13 12:55:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch37_loss0.23576750892859238.pypots +2024-09-13 12:55:48 [INFO]: Epoch 038 - training loss: 7.9402, validation loss: 0.2373 +2024-09-13 12:55:54 [INFO]: Epoch 039 - training loss: 7.9054, validation loss: 0.2348 +2024-09-13 12:55:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch39_loss0.23477741502798521.pypots +2024-09-13 12:56:01 [INFO]: Epoch 040 - training loss: 7.8854, validation loss: 0.2360 +2024-09-13 12:56:06 [INFO]: Epoch 041 - training loss: 7.8602, validation loss: 0.2339 +2024-09-13 12:56:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch41_loss0.23388041785130134.pypots +2024-09-13 12:56:12 [INFO]: Epoch 042 - training loss: 7.8280, validation loss: 0.2352 +2024-09-13 12:56:18 [INFO]: Epoch 043 - training loss: 7.8045, validation loss: 0.2325 +2024-09-13 12:56:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch43_loss0.23245421395852015.pypots +2024-09-13 12:56:24 [INFO]: Epoch 044 - training loss: 7.7597, validation loss: 0.2319 +2024-09-13 12:56:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch44_loss0.23189380421088293.pypots +2024-09-13 12:56:30 [INFO]: Epoch 045 - training loss: 7.7357, validation loss: 0.2328 +2024-09-13 12:56:36 [INFO]: Epoch 046 - training loss: 7.7160, validation loss: 0.2325 +2024-09-13 12:56:42 [INFO]: Epoch 047 - training loss: 7.6981, validation loss: 0.2306 +2024-09-13 12:56:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch47_loss0.23061815133461586.pypots +2024-09-13 12:56:49 [INFO]: Epoch 048 - training loss: 7.6576, validation loss: 0.2305 +2024-09-13 12:56:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch48_loss0.2304995426764855.pypots +2024-09-13 12:56:55 [INFO]: Epoch 049 - training loss: 7.6384, validation loss: 0.2316 +2024-09-13 12:57:01 [INFO]: Epoch 050 - training loss: 7.6140, validation loss: 0.2277 +2024-09-13 12:57:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch50_loss0.22768438091644874.pypots +2024-09-13 12:57:07 [INFO]: Epoch 051 - training loss: 7.6261, validation loss: 0.2285 +2024-09-13 12:57:13 [INFO]: Epoch 052 - training loss: 7.5924, validation loss: 0.2279 +2024-09-13 12:57:19 [INFO]: Epoch 053 - training loss: 7.5728, validation loss: 0.2291 +2024-09-13 12:57:25 [INFO]: Epoch 054 - training loss: 7.5471, validation loss: 0.2270 +2024-09-13 12:57:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch54_loss0.2269772761143171.pypots +2024-09-13 12:57:31 [INFO]: Epoch 055 - training loss: 7.5182, validation loss: 0.2271 +2024-09-13 12:57:37 [INFO]: Epoch 056 - training loss: 7.5176, validation loss: 0.2246 +2024-09-13 12:57:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch56_loss0.22458868760329026.pypots +2024-09-13 12:57:43 [INFO]: Epoch 057 - training loss: 7.4998, validation loss: 0.2250 +2024-09-13 12:57:49 [INFO]: Epoch 058 - training loss: 7.4922, validation loss: 0.2253 +2024-09-13 12:57:55 [INFO]: Epoch 059 - training loss: 7.4583, validation loss: 0.2281 +2024-09-13 12:58:01 [INFO]: Epoch 060 - training loss: 7.4547, validation loss: 0.2245 +2024-09-13 12:58:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch60_loss0.22450509094274962.pypots +2024-09-13 12:58:07 [INFO]: Epoch 061 - training loss: 7.4415, validation loss: 0.2279 +2024-09-13 12:58:13 [INFO]: Epoch 062 - training loss: 7.4168, validation loss: 0.2254 +2024-09-13 12:58:19 [INFO]: Epoch 063 - training loss: 7.4041, validation loss: 0.2238 +2024-09-13 12:58:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch63_loss0.22382787672372964.pypots +2024-09-13 12:58:25 [INFO]: Epoch 064 - training loss: 7.3849, validation loss: 0.2233 +2024-09-13 12:58:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch64_loss0.22330084557716662.pypots +2024-09-13 12:58:31 [INFO]: Epoch 065 - training loss: 7.3857, validation loss: 0.2248 +2024-09-13 12:58:37 [INFO]: Epoch 066 - training loss: 7.3641, validation loss: 0.2274 +2024-09-13 12:58:43 [INFO]: Epoch 067 - training loss: 7.3423, validation loss: 0.2228 +2024-09-13 12:58:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch67_loss0.22279441815156203.pypots +2024-09-13 12:58:49 [INFO]: Epoch 068 - training loss: 7.3196, validation loss: 0.2237 +2024-09-13 12:58:55 [INFO]: Epoch 069 - training loss: 7.3225, validation loss: 0.2245 +2024-09-13 12:59:01 [INFO]: Epoch 070 - training loss: 7.3039, validation loss: 0.2239 +2024-09-13 12:59:07 [INFO]: Epoch 071 - training loss: 7.2925, validation loss: 0.2214 +2024-09-13 12:59:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch71_loss0.22138233253589043.pypots +2024-09-13 12:59:13 [INFO]: Epoch 072 - training loss: 7.2945, validation loss: 0.2230 +2024-09-13 12:59:19 [INFO]: Epoch 073 - training loss: 7.2789, validation loss: 0.2222 +2024-09-13 12:59:25 [INFO]: Epoch 074 - training loss: 7.2574, validation loss: 0.2228 +2024-09-13 12:59:31 [INFO]: Epoch 075 - training loss: 7.2429, validation loss: 0.2223 +2024-09-13 12:59:37 [INFO]: Epoch 076 - training loss: 7.2428, validation loss: 0.2227 +2024-09-13 12:59:37 [INFO]: Exceeded the training patience. Terminating the training procedure... +2024-09-13 12:59:37 [INFO]: Finished training. The best model is from epoch#71. +2024-09-13 12:59:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI.pypots +INFO:__main__: + Testing...................................... + +INFO:__main__: + Fold 4 ended ........................................ + +INFO:__main__: + Fold 4 Results: +{ + "MAE": 0.24529347374725138, + "MRE": 0.3424914307207086 +} + +INFO:__main__: + Average performance across all folds.......................... + +INFO:__main__: + Imputation performance: + {'MAE': 0.2514955002311819, 'MRE': 0.34640804518017465} + diff --git a/pypots/classification/base.py b/pypots/classification/base.py index c86d2993..2b26cde4 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -16,7 +16,7 @@ from ..base import BaseModel, BaseNNModel from ..utils.logging import logger - +from sklearn.metrics import roc_auc_score try: import nni except ImportError: @@ -287,9 +287,9 @@ def _train_model( try: training_step = 0 - print("Training model......................................") + # print("Training model......................................") for epoch in range(1, self.epochs + 1): - print(f"Epoch {epoch}......................................") + # print(f"Epoch {epoch}......................................") self.model.train() epoch_train_loss_collector = [] for idx, data in enumerate(training_loader): @@ -307,7 +307,8 @@ def _train_model( # mean training loss of the current epoch mean_train_loss = np.mean(epoch_train_loss_collector) - + val_results = [] + val_labels = [] if val_loader is not None: self.model.eval() epoch_val_loss_collector = [] @@ -318,6 +319,8 @@ def _train_model( epoch_val_loss_collector.append( results["loss"].sum().item() ) + val_results.append(results['classification_pred']) + val_labels.append(inputs['labels']) mean_val_loss = np.mean(epoch_val_loss_collector) @@ -327,11 +330,14 @@ def _train_model( "classification_loss": mean_val_loss, } self._save_log_into_tb_file(epoch, "validating", val_loss_dict) - + val_classification = torch.cat(val_results).cpu().detach().numpy() + val_labels = torch.cat(val_labels).cpu().detach().numpy() + val_auc = roc_auc_score(val_labels, val_classification) logger.info( f"Epoch {epoch:03d} - " f"training loss: {mean_train_loss:.4f}, " - f"validation loss: {mean_val_loss:.4f}" + f"validation loss: {mean_val_loss:.4f}, " + f"validation AUC: {val_auc}" ) mean_loss = mean_val_loss else: diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py index 35e24890..1f465286 100644 --- a/pypots/classification/csai/core.py +++ b/pypots/classification/csai/core.py @@ -60,7 +60,7 @@ def __init__( # create models - self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) self.classifier = nn.Linear(self.rnn_hidden_size, n_classes) self.imputer = nn.Linear(self.rnn_hidden_size, n_features) self.dropout = nn.Dropout(dropout) diff --git a/pypots/classification/csai/data.py b/pypots/classification/csai/data.py index 3766b4f2..1e5d145b 100644 --- a/pypots/classification/csai/data.py +++ b/pypots/classification/csai/data.py @@ -5,7 +5,7 @@ class DatasetForCSAI(DatasetForCSAI_Imputation): def __init__(self, - data: dict | str, + data: Union[dict, str], file_type: str = "hdf5", return_y: bool = True, removal_percent: float = 0.0, @@ -27,6 +27,7 @@ def __init__(self, replacement_probabilities=replacement_probabilities, normalise_mean=normalise_mean, normalise_std=normalise_std, + impute_only=False, training=training ) \ No newline at end of file diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index e3e077f8..84c0465c 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -27,12 +27,12 @@ def __init__(self, batch_size: int, epochs: int, dropout: float = 0.5, - patience: int | None = None, + patience: Union[int, None] = None, optimizer: Optimizer = Adam(), num_workers: int = 0, device: Optional[Union[str, torch.device, list]] = None, saving_path: str = None, - model_saving_strategy: str | None = "best", + model_saving_strategy: Union[str, None] = "best", verbose: bool = True): super().__init__( n_classes, diff --git a/pypots/data/utils.py b/pypots/data/utils.py index f89cc18a..4b2b0f87 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -361,7 +361,7 @@ def non_uniform_sample_loader_bidirectional(data, removal_percent, pre_replaceme replacement_probabilities = np.full(D, removal_percent / 100) if increase_factor > 0: - print('The increase_factor is {}!'.format(increase_factor)) + # print('The increase_factor is {}!'.format(increase_factor)) for feature_idx in range(D): replacement_probabilities[feature_idx] = adjust_probability_vectorized( observations_per_feature[feature_idx], diff --git a/pypots/imputation/base.py b/pypots/imputation/base.py index 5366e74a..6ca8bcb2 100644 --- a/pypots/imputation/base.py +++ b/pypots/imputation/base.py @@ -275,9 +275,7 @@ def _train_model( try: training_step = 0 - print("Training model......................................") for epoch in range(1, self.epochs + 1): - print(f"Epoch {epoch}......................................") self.model.train() epoch_train_loss_collector = [] for idx, data in enumerate(training_loader): diff --git a/pypots/imputation/csai/core.py b/pypots/imputation/csai/core.py index efc9d9df..c7e6f1ce 100644 --- a/pypots/imputation/csai/core.py +++ b/pypots/imputation/csai/core.py @@ -10,7 +10,8 @@ def __init__(self, n_steps, step_channels, intervals, consistency_weight, - imputation_weight) : + imputation_weight, + device) : super().__init__() self.n_steps = n_steps self.n_features = n_features @@ -19,8 +20,9 @@ def __init__(self, n_steps, self.intervals = intervals self.consistency_weight = consistency_weight self.imputation_weight = imputation_weight + self.device = device - self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) def forward(self, inputs:dict, training:bool = True) -> dict: ( diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index 5d8b9288..59058e02 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -3,9 +3,10 @@ import numpy as np import torch from ...data.utils import collate_fn_bidirectional, non_uniform_sample_loader_bidirectional, normalize_csai +from typing import Union class DatasetForCSAI(BaseDataset): - def __init__(self, data: dict | str, + def __init__(self, data: Union[dict, str], return_X_ori: bool, return_y: bool, file_type: str = "hdf5", @@ -15,6 +16,7 @@ def __init__(self, data: dict | str, replacement_probabilities = None, normalise_mean : list = [], normalise_std: list = [], + impute_only: bool = True, training: bool = True ): super().__init__(data = data, @@ -28,6 +30,7 @@ def __init__(self, data: dict | str, self.replacement_probabilities = replacement_probabilities self.normalise_mean = normalise_mean self.normalise_std = normalise_std + self.impute_only = impute_only self.training = training if not isinstance(self.data, str): @@ -94,8 +97,8 @@ def _fetch_data_from_array(self, idx: int) -> Iterable: self.processed_data["last_obs_b"][idx], ] - # if not self.training: - # sample.extend([self.X_ori[idx], self.indicating_mask[idx]]) + if not self.training and self.impute_only: + sample.extend([self.X_ori[idx], self.indicating_mask[idx]]) if self.return_y: sample.append(self.y[idx].to(torch.long)) diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index 10348fa2..dd8bc16c 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -27,12 +27,12 @@ def __init__(self, step_channels:int, batch_size: int, epochs: int, - patience: int | None = None, + patience: Union[int, None ]= None, optimizer: Optional[Optimizer] = Adam(), num_workers: int = 0, - device: str | torch.device | list | None = None, + device: Union[str, torch.device, list, None ]= None, saving_path: str = None, - model_saving_strategy: str | None = "best", + model_saving_strategy: Union[str, None] = "best", verbose: bool = True): super().__init__(batch_size, epochs, patience, num_workers, device, saving_path, model_saving_strategy, verbose) self.n_steps = n_steps @@ -149,7 +149,7 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): file_type, self.removal_percent, self.increase_factor, self.compute_intervals, self.training_set.replacement_probabilities, - self.training_set.mean_set, self.training_set.std_set, False + self.training_set.mean_set, self.training_set.std_set, True, False ) val_loader = DataLoader( val_set, @@ -166,7 +166,8 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): self.step_channels, self.training_set.intervals, self.consistency_weight, - self.imputation_weight) + self.imputation_weight, + self.device) self._send_model_to_given_device() self._print_model_size() @@ -182,7 +183,7 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): # Step 3: save the model if necessary self._auto_save_model_if_necessary(confirm_saving=True) - def predict(self, test_set: dict | str, file_type: str = "hdf5") -> dict: + def predict(self, test_set: Union[dict, str], file_type: str = "hdf5") -> dict: if self.model == None: raise ValueError("Training must be run before predict") @@ -193,7 +194,7 @@ def predict(self, test_set: dict | str, file_type: str = "hdf5") -> dict: file_type, self.removal_percent, self.increase_factor, self.compute_intervals, self.training_set.replacement_probabilities, - self.training_set.mean_set, self.training_set.std_set, False + self.training_set.mean_set, self.training_set.std_set,True, False ) test_loader = DataLoader( test_set, diff --git a/run_pypots.sh b/run_pypots.sh new file mode 100644 index 00000000..b36543e0 --- /dev/null +++ b/run_pypots.sh @@ -0,0 +1,25 @@ +#!/bin/bash -l + + + +#SBATCH --output=/scratch/users/k23031260/PyPOTS/output_logs/Imputation.out +#SBATCH --job-name=pypots +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --mem=256G +#SBATCH --signal=USR2 +# Load required modules +module load anaconda3/2021.05-gcc-9.4.0 +module load cuda/11.1.1-gcc-9.4.0 +module load cudnn/8.0.5.39-11.1-gcc-9.4.0 +nvidia-smi + +# Activate the conda environment +source activate imputation + +# Navigate to the directory containing the Python script +cd /scratch/users/k23031260/PyPOTS + +python csai.py + + From bd113f19fbd1686924c1cd6ffca5cb93fa73ae97 Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Mon, 16 Sep 2024 07:51:28 +0100 Subject: [PATCH 03/22] Fix merge conflicts --- pypots/classification/base.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pypots/classification/base.py b/pypots/classification/base.py index 2b26cde4..b67ecbe9 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -287,9 +287,7 @@ def _train_model( try: training_step = 0 - # print("Training model......................................") for epoch in range(1, self.epochs + 1): - # print(f"Epoch {epoch}......................................") self.model.train() epoch_train_loss_collector = [] for idx, data in enumerate(training_loader): @@ -307,8 +305,7 @@ def _train_model( # mean training loss of the current epoch mean_train_loss = np.mean(epoch_train_loss_collector) - val_results = [] - val_labels = [] + if val_loader is not None: self.model.eval() epoch_val_loss_collector = [] @@ -319,8 +316,7 @@ def _train_model( epoch_val_loss_collector.append( results["loss"].sum().item() ) - val_results.append(results['classification_pred']) - val_labels.append(inputs['labels']) + mean_val_loss = np.mean(epoch_val_loss_collector) @@ -330,14 +326,11 @@ def _train_model( "classification_loss": mean_val_loss, } self._save_log_into_tb_file(epoch, "validating", val_loss_dict) - val_classification = torch.cat(val_results).cpu().detach().numpy() - val_labels = torch.cat(val_labels).cpu().detach().numpy() - val_auc = roc_auc_score(val_labels, val_classification) + logger.info( f"Epoch {epoch:03d} - " f"training loss: {mean_train_loss:.4f}, " f"validation loss: {mean_val_loss:.4f}, " - f"validation AUC: {val_auc}" ) mean_loss = mean_val_loss else: From c2143adfc35ef0ca7353ad9d748307181580cd95 Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Mon, 16 Sep 2024 07:54:58 +0100 Subject: [PATCH 04/22] Fix merge conflicts --- pypots/classification/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pypots/classification/base.py b/pypots/classification/base.py index b67ecbe9..7b6a5416 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -313,11 +313,8 @@ def _train_model( for idx, data in enumerate(val_loader): inputs = self._assemble_input_for_validating(data) results = self.model.forward(inputs) - epoch_val_loss_collector.append( - results["loss"].sum().item() - ) + epoch_val_loss_collector.append(results["loss"].sum().item()) - mean_val_loss = np.mean(epoch_val_loss_collector) # save validation loss logs into the tensorboard file for every epoch if in need From 97a7c61bc7cc8fbc65e2f3de203f67095541f404 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Thu, 3 Oct 2024 09:44:54 +0100 Subject: [PATCH 05/22] update --- pypots/nn/modules/csai/__init__.py | 13 +++ pypots/nn/modules/csai/backbone.py | 165 +++++++++++++++++++++++++++++ pypots/nn/modules/csai/layers.py | 128 ++++++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 pypots/nn/modules/csai/__init__.py create mode 100644 pypots/nn/modules/csai/backbone.py create mode 100644 pypots/nn/modules/csai/layers.py diff --git a/pypots/nn/modules/csai/__init__.py b/pypots/nn/modules/csai/__init__.py new file mode 100644 index 00000000..84e04f4f --- /dev/null +++ b/pypots/nn/modules/csai/__init__.py @@ -0,0 +1,13 @@ +from .backbone import BackboneCSAI, BackboneBCSAI +from .layers import FeatureRegression + +__all__ = [ + "BackboneCSAI", + "BackboneBCSAI", + "FeatureRegression", + "Decay", + "Decay_obs", + "PositionalEncoding", + "Conv1dWithInit", + "TorchTransformerEncoder" +] diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py new file mode 100644 index 00000000..d71b8f9c --- /dev/null +++ b/pypots/nn/modules/csai/backbone.py @@ -0,0 +1,165 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from .layers import FeatureRegression, Decay, Decay_obs, PositionalEncoding, Conv1dWithInit, TorchTransformerEncoder + + +class BackboneCSAI(nn.Module): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + super(BackboneCSAI, self).__init__() + + + if medians_df is not None: + self.medians_tensor = torch.tensor(list(medians_df.values())).float() + else: + self.medians_tensor = None + + self.n_steps = n_steps + self.step_channels = step_channels + self.input_size = n_features + self.hidden_size = rnn_hidden_size + self.temp_decay_h = Decay(input_size=self.input_size, output_size=self.hidden_size, diag = False) + self.temp_decay_x = Decay(input_size=self.input_size, output_size=self.input_size, diag = True) + self.hist = nn.Linear(self.hidden_size, self.input_size) + self.feat_reg_v = FeatureRegression(self.input_size) + self.weight_combine = nn.Linear(self.input_size * 2, self.input_size) + self.weighted_obs = Decay_obs(self.input_size, self.input_size) + self.gru = nn.GRUCell(self.input_size * 2, self.hidden_size) + + self.pos_encoder = PositionalEncoding(self.step_channels) + self.input_projection = Conv1dWithInit(self.input_size, self.step_channels, 1) + self.output_projection1 = Conv1dWithInit(self.step_channels, self.hidden_size, 1) + self.output_projection2 = Conv1dWithInit(self.n_steps*2, 1, 1) + self.time_layer = TorchTransformerEncoder(channels=self.step_channels) + self.device = device + + self.reset_parameters() + + def reset_parameters(self): + for weight in self.parameters(): + if len(weight.size()) == 1: + continue + stv = 1. / math.sqrt(weight.size(1)) + nn.init.uniform_(weight, -stv, stv) + + def forward(self, x, mask, deltas, last_obs, h=None): + + # Get dimensionality + [B, _, _] = x.shape + + if self.medians_tensor is not None: + medians_t = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(self.device) + + decay_factor = self.weighted_obs(deltas - medians_t.unsqueeze(1)) + + if h == None: + data_last_obs = self.input_projection(last_obs.permute(0, 2, 1)).permute(0, 2, 1) + data_decay_factor = self.input_projection(decay_factor.permute(0, 2, 1)).permute(0, 2, 1) + + data_last_obs = self.pos_encoder(data_last_obs.permute(1, 0, 2)).permute(1, 0, 2) + data_decay_factor = self.pos_encoder(data_decay_factor.permute(1, 0, 2)).permute(1, 0, 2) + + data = torch.cat([data_last_obs, data_decay_factor], dim=1) + + data = self.time_layer(data) + data = self.output_projection1(data.permute(0, 2, 1)).permute(0, 2, 1) + h = self.output_projection2(data).squeeze() + + x_loss = 0 + x_imp = x.clone() + Hiddens = [] + reconstruction = [] + for t in range(self.n_steps): + x_t = x[:, t, :] + d_t = deltas[:, t, :] + m_t = mask[:, t, :] + + # Decayed Hidden States + gamma_h = self.temp_decay_h(d_t) + h = h * gamma_h + + # history based estimation + x_h = self.hist(h) + + x_r_t = (m_t * x_t) + ((1 - m_t) * x_h) + + # feature based estimation + xu = self.feat_reg_v(x_r_t) + gamma_x = self.temp_decay_x(d_t) + + beta = self.weight_combine(torch.cat([gamma_x, m_t], dim=1)) + x_comb_t = beta * xu + (1 - beta) * x_h + + x_loss += torch.sum(torch.abs(x_t - x_comb_t) * m_t) / (torch.sum(m_t) + 1e-5) + + # Final Imputation Estimates + x_imp[:, t, :] = (m_t * x_t) + ((1 - m_t) * x_comb_t) + + # Set input the RNN + input_t = torch.cat([x_imp[:, t, :], m_t], dim=1) + + h = self.gru(input_t, h) + Hiddens.append(h.unsqueeze(dim=1)) + reconstruction.append(x_comb_t.unsqueeze(dim=1)) + Hiddens = torch.cat(Hiddens, dim=1) + + return x_imp, reconstruction, h, x_loss + + +class BackboneBCSAI(nn.Module): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + super(BackboneBCSAI, self).__init__() + + self.model_f = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) + self.model_b = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) + + + def forward(self, xdata): + + # Fetching forward data from xdata + x = xdata['forward']['X'] + m = xdata['forward']['missing_mask'] + d_f = xdata['forward']['deltas'] + last_obs_f = xdata['forward']['last_obs'] + + # Fetching backward data from xdata + x_b = xdata['backward']['X'] + m_b = xdata['backward']['missing_mask'] + d_b = xdata['backward']['deltas'] + last_obs_b = xdata['backward']['last_obs'] + + # Call forward model + ( + f_imputed_data, + f_reconstruction, + f_hidden_states, + f_reconstruction_loss, + ) = self.model_f(x, m, d_f, last_obs_f) + + # Call backward model + ( + b_imputed_data, + b_reconstruction, + b_hidden_states, + b_reconstruction_loss, + ) = self.model_b(x_b, m_b, d_b, last_obs_b) + + # Averaging the imputations and prediction + x_imp = (f_imputed_data + b_imputed_data.flip(dims=[1])) / 2 + imputed_data = (x * m)+ ((1-m) * x_imp) + + # average consistency loss + consistency_loss = torch.abs(f_imputed_data - b_imputed_data.flip(dims=[1])).mean() * 1e-1 + + # Merge the regression loss + reconstruction_loss = f_reconstruction_loss + b_reconstruction_loss + return ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) diff --git a/pypots/nn/modules/csai/layers.py b/pypots/nn/modules/csai/layers.py new file mode 100644 index 00000000..ecfadd7f --- /dev/null +++ b/pypots/nn/modules/csai/layers.py @@ -0,0 +1,128 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +from torch.nn.parameter import Parameter +import math +import numpy as np +import os +import copy +import pandas as pd +from torch.nn.modules import TransformerEncoderLayer + + +class FeatureRegression(nn.Module): + def __init__(self, input_size): + super(FeatureRegression, self).__init__() + self.build(input_size) + + def build(self, input_size): + self.W = Parameter(torch.Tensor(input_size, input_size)) + self.b = Parameter(torch.Tensor(input_size)) + m = torch.ones(input_size, input_size) - torch.eye(input_size, input_size) + self.register_buffer('m', m) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.W.size(0)) + self.W.data.uniform_(-stdv, stdv) + if self.b is not None: + self.b.data.uniform_(-stdv, stdv) + + def forward(self, x): + z_h = F.linear(x, self.W * Variable(self.m), self.b) + return z_h + +class Decay(nn.Module): + def __init__(self, input_size, output_size, diag=False): + super(Decay, self).__init__() + self.diag = diag + self.build(input_size, output_size) + + def build(self, input_size, output_size): + self.W = Parameter(torch.Tensor(output_size, input_size)) + self.b = Parameter(torch.Tensor(output_size)) + + if self.diag == True: + assert(input_size == output_size) + m = torch.eye(input_size, input_size) + self.register_buffer('m', m) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.W.size(0)) + self.W.data.uniform_(-stdv, stdv) + if self.b is not None: + self.b.data.uniform_(-stdv, stdv) + + def forward(self, d): + if self.diag == True: + gamma = F.relu(F.linear(d, self.W * Variable(self.m), self.b)) + else: + gamma = F.relu(F.linear(d, self.W, self.b)) + gamma = torch.exp(-gamma) + return gamma + +class Decay_obs(nn.Module): + def __init__(self, input_size, output_size): + super(Decay_obs, self).__init__() + self.linear = nn.Linear(input_size, output_size) + + def forward(self, delta_diff): + # When delta_diff is negative, weight tends to 1. + # When delta_diff is positive, weight tends to 0. + sign = torch.sign(delta_diff) + weight_diff = self.linear(delta_diff) + # weight_diff can be either positive or negative for each delta_diff + positive_part = F.relu(weight_diff) + negative_part = F.relu(-weight_diff) + weight_diff = positive_part + negative_part + weight_diff = sign * weight_diff + # Using a tanh activation to squeeze values between -1 and 1 + weight_diff = torch.tanh(weight_diff) + # This will move the weight values towards 1 if delta_diff is negative + # and towards 0 if delta_diff is positive + weight = 0.5 * (1 - weight_diff) + + return weight + +class TorchTransformerEncoder(nn.Module): + def __init__(self, heads=8, layers=1, channels=64): + super(TorchTransformerEncoder, self).__init__() + self.encoder_layer = nn.TransformerEncoderLayer( + d_model=channels, nhead=heads, dim_feedforward=64, activation="gelu" + ) + self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=layers) + + def forward(self, x): + return self.transformer_encoder(x) + +class Conv1dWithInit(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size): + super(Conv1dWithInit, self).__init__() + self.conv = nn.Conv1d(in_channels, out_channels, kernel_size) + nn.init.kaiming_normal_(self.conv.weight) + + def forward(self, x): + return self.conv(x) + +class PositionalEncoding(nn.Module): + + def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): + super().__init__() + self.dropout = nn.Dropout(p=dropout) + + position = torch.arange(max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) + pe = torch.zeros(max_len, 1, d_model) + pe[:, 0, 0::2] = torch.sin(position * div_term) + pe[:, 0, 1::2] = torch.cos(position * div_term) + self.register_buffer('pe', pe) + + def forward(self, x): + """ + Arguments: + x: Tensor, shape ``[seq_len, batch_size, embedding_dim]`` + """ + x = x + self.pe[:x.size(0)] + return self.dropout(x) \ No newline at end of file From 2c2b32f96e4f2ae5d53578cc903617928ab95aa0 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Thu, 3 Oct 2024 10:19:40 +0100 Subject: [PATCH 06/22] update --- pypots/nn/modules/csai/__init__.py | 20 +++++++++++++++++++- pypots/nn/modules/csai/backbone.py | 7 +++++++ pypots/nn/modules/csai/layers.py | 7 +++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pypots/nn/modules/csai/__init__.py b/pypots/nn/modules/csai/__init__.py index 84e04f4f..b4ff5295 100644 --- a/pypots/nn/modules/csai/__init__.py +++ b/pypots/nn/modules/csai/__init__.py @@ -1,5 +1,23 @@ +""" +The package including the modules of CSDI. + +Refer to the paper +`Linglong Qian, Zina Ibrahim, Hugh Logan Ellis, Ao Zhang, Yuezhou Zhang, Tao Wang, Richard Dobson. +Knowledge Enhanced Conditional Imputation for Healthcare Time-series. +In Arxiv, 2024. +`_ + +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/LinglongQian/CSAI. + +""" + +# Created by Joseph Arul Raj +# License: BSD-3-Clause + from .backbone import BackboneCSAI, BackboneBCSAI -from .layers import FeatureRegression +from .layers import FeatureRegression, Decay, Decay_obs, PositionalEncoding, Conv1dWithInit, TorchTransformerEncoder __all__ = [ "BackboneCSAI", diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py index d71b8f9c..bdea8013 100644 --- a/pypots/nn/modules/csai/backbone.py +++ b/pypots/nn/modules/csai/backbone.py @@ -1,3 +1,10 @@ +""" + +""" + +# Created by Joseph Arul Raj +# License: BSD-3-Clause + import torch import torch.nn as nn import torch.nn.functional as F diff --git a/pypots/nn/modules/csai/layers.py b/pypots/nn/modules/csai/layers.py index ecfadd7f..d603eef1 100644 --- a/pypots/nn/modules/csai/layers.py +++ b/pypots/nn/modules/csai/layers.py @@ -1,3 +1,10 @@ +""" + +""" + +# Created by Joseph Arul Raj +# License: BSD-3-Clause + import torch import torch.nn as nn import torch.nn.functional as F From 29aec0079032756c354ab9b03068bba5e02560b7 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Thu, 3 Oct 2024 11:46:41 +0100 Subject: [PATCH 07/22] csai update --- pypots/imputation/csai/__init__.py | 23 +++ pypots/imputation/csai/core.py | 78 +++++++++ pypots/imputation/csai/data.py | 186 +++++++++++++++++++++ pypots/imputation/csai/model.py | 257 +++++++++++++++++++++++++++++ pypots/nn/modules/csai/__init__.py | 2 +- pypots/nn/modules/csai/backbone.py | 4 +- 6 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 pypots/imputation/csai/__init__.py create mode 100644 pypots/imputation/csai/core.py create mode 100644 pypots/imputation/csai/data.py create mode 100644 pypots/imputation/csai/model.py diff --git a/pypots/imputation/csai/__init__.py b/pypots/imputation/csai/__init__.py new file mode 100644 index 00000000..529cb0ce --- /dev/null +++ b/pypots/imputation/csai/__init__.py @@ -0,0 +1,23 @@ +""" +The package including the modules of CSAI. + +Refer to the paper +`Linglong Qian, Zina Ibrahim, Hugh Logan Ellis, Ao Zhang, Yuezhou Zhang, Tao Wang, Richard Dobson. +Knowledge Enhanced Conditional Imputation for Healthcare Time-series. +In Arxiv, 2024. +`_ + +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/LinglongQian/CSAI. + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +from .model import CSAI + +__all__ = [ + "CSAI", +] \ No newline at end of file diff --git a/pypots/imputation/csai/core.py b/pypots/imputation/csai/core.py new file mode 100644 index 00000000..fb247163 --- /dev/null +++ b/pypots/imputation/csai/core.py @@ -0,0 +1,78 @@ +""" + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +import torch.nn as nn +from ...nn.modules.csai.backbone import BackboneBCSAI + + +class _BCSAI(nn.Module): + """model BRITS: Bidirectional RITS + BRITS consists of two RITS, which take time-series data from two directions (forward/backward) respectively. + + Parameters + ---------- + n_steps : + sequence length (number of time steps) + + n_features : + number of features (input dimensions) + + rnn_hidden_size : + the hidden size of the RNN cell + + """ + def __init__(self, n_steps, + n_features, + rnn_hidden_size, + step_channels, + intervals, + consistency_weight, + imputation_weight, + device) : + super().__init__() + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.step_channels = step_channels + self.intervals = intervals + self.consistency_weight = consistency_weight + self.imputation_weight = imputation_weight + self.device = device + + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) + + def forward(self, inputs:dict, training:bool = True) -> dict: + ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) = self.model(inputs) + + results = { + "imputed_data": imputed_data, + } + + # if in training mode, return results with losses + if training: + results["consistency_loss"] = consistency_loss + results["reconstruction_loss"] = reconstruction_loss + loss = self.consistency_weight * consistency_loss + self.imputation_weight * reconstruction_loss + + # `loss` is always the item for backward propagating to update the model + results["loss"] = loss + # results["reconstruction"] = (f_reconstruction + b_reconstruction) / 2 + results["f_reconstruction"] = f_reconstruction + results["b_reconstruction"] = b_reconstruction + if not training: + results["X_ori"] = inputs["X_ori"] + results["indicating_mask"] = inputs["indicating_mask"] + + return results diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py new file mode 100644 index 00000000..3cd80cd0 --- /dev/null +++ b/pypots/imputation/csai/data.py @@ -0,0 +1,186 @@ +""" + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +from typing import Iterable +from ...data.dataset import BaseDataset +import numpy as np +import torch +from ...data.utils import collate_fn_bidirectional, non_uniform_sample_loader_bidirectional, normalize_csai +from typing import Union + +class DatasetForCSAI(BaseDataset): + def __init__(self, data: Union[dict, str], + return_X_ori: bool, + return_y: bool, + file_type: str = "hdf5", + removal_percent: float = 0.0, + increase_factor: float = 0.1, + compute_intervals: bool = False, + replacement_probabilities = None, + normalise_mean : list = [], + normalise_std: list = [], + impute_only: bool = True, + training: bool = True + ): + super().__init__(data = data, + return_X_ori = return_X_ori, + return_X_pred = False, + return_y = return_y, + file_type = file_type) + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.compute_intervals = compute_intervals + self.replacement_probabilities = replacement_probabilities + self.normalise_mean = normalise_mean + self.normalise_std = normalise_std + self.impute_only = impute_only + self.training = training + + if not isinstance(self.data, str): + self.normalized_data, self.mean_set, self.std_set, self.intervals = normalize_csai(self.data['X'], normalise_mean, + normalise_std, compute_intervals) + _data, self.replacement_probabilities = non_uniform_sample_loader_bidirectional(self.normalized_data, + removal_percent, + replacement_probabilities, + increase_factor) + self.processed_data = collate_fn_bidirectional(_data) + self.forward_X = self.processed_data['values'] + self.forward_missing_mask = self.processed_data['masks'] + self.backward_X = torch.flip(self.forward_X, dims=[1]) + self.backward_missing_mask = torch.flip(self.forward_missing_mask, dims=[1]) + + self.X_ori = self.processed_data['evals'] + self.indicating_mask = self.processed_data['eval_masks'] + # if self.return_y: + # self.y = self.processed_data['labels'] + + + + def _fetch_data_from_array(self, idx: int) -> Iterable: + """Fetch data from self.X if it is given. + + Parameters + ---------- + idx : + The index of the sample to be return. + + Returns + ------- + sample : + A list contains + + index : int tensor, + The index of the sample. + + X : tensor, + The feature vector for model input. + + missing_mask : tensor, + The mask indicates all missing values in X. + + delta : tensor, + The delta matrix contains time gaps of missing values. + + label (optional) : tensor, + The target label of the time-series sample. + """ + + + sample = [ + torch.tensor(idx), + # for forward + self.forward_X[idx], + self.forward_missing_mask[idx], + self.processed_data["deltas_f"][idx], + self.processed_data["last_obs_f"][idx], + # for backward + self.backward_X[idx], + self.backward_missing_mask[idx], + self.processed_data["deltas_b"][idx], + self.processed_data["last_obs_b"][idx], + ] + + if not self.training and self.impute_only: + sample.extend([self.X_ori[idx], self.indicating_mask[idx]]) + + if self.return_y: + sample.append(self.y[idx].to(torch.long)) + + return { + 'sample': sample, + 'replacement_probabilities': self.replacement_probabilities, + 'mean_set': self.mean_set, + 'std_set': self.std_set, + 'intervals': self.intervals + } + + def _fetch_data_from_file(self, idx: int) -> Iterable: + """Fetch data with the lazy-loading strategy, i.e. only loading data from the file while requesting for samples. + Here the opened file handle doesn't load the entire dataset into RAM but only load the currently accessed slice. + + Parameters + ---------- + idx : + The index of the sample to be return. + + Returns + ------- + sample : + The collated data sample, a list including all necessary sample info. + """ + + if self.file_handle is None: + self.file_handle = self._open_file_handle() + + X = torch.from_numpy(self.file_handle["X"][idx]) + normalized_data, mean_set, std_set, intervals = normalize_csai(X, self.normalise_mean, + self.normalise_std, + self.compute_intervals) + processed_data, replacement_probabilities = non_uniform_sample_loader_bidirectional(normalized_data, + self.removal_percent, + self.replacement_probabilities, + self.increase_factor) + forward_X = processed_data['values'] + forward_missing_mask = processed_data['masks'] + backward_X = torch.flip(forward_X, dims=[1]) + backward_missing_mask = torch.flip(forward_missing_mask, dims=[1]) + + X_ori = self.processed_data['evals'] + indicating_mask = self.processed_data['eval_masks'] + if self.return_y: + y = self.processed_data['labels'] + + + sample = [ + torch.tensor(idx), + # for forward + forward_X, + forward_missing_mask, + processed_data["deltas_f"], + processed_data["last_obs_f"], + # for backward + backward_X, + backward_missing_mask, + processed_data["deltas_b"], + processed_data["last_obs_b"] + ] + + if self.return_X_ori: + sample.extend([X_ori, indicating_mask]) + + # if the dataset has labels and is for training, then fetch it from the file + if self.return_y: + sample.append(y) + + return { + 'sample': sample, + 'replacement_probabilities': replacement_probabilities, + 'mean_set': mean_set, + 'std_set': std_set, + 'intervals': intervals + } + diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py new file mode 100644 index 00000000..67d406ae --- /dev/null +++ b/pypots/imputation/csai/model.py @@ -0,0 +1,257 @@ +""" + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +from typing import Union, Optional + +import numpy as np +import torch +from torch.utils.data import DataLoader + + +from .core import _BCSAI +from .data import DatasetForCSAI +from ..base import BaseNNImputer +from ...data.checking import key_in_data_set +from ...optim.adam import Adam +from ...optim.base import Optimizer + + +class CSAI(BaseNNImputer): + def __init__(self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + removal_percent: int, + increase_factor: float, + compute_intervals: bool, + step_channels:int, + batch_size: int, + epochs: int, + patience: Union[int, None ]= None, + optimizer: Optional[Optimizer] = Adam(), + num_workers: int = 0, + device: Union[str, torch.device, list, None ]= None, + saving_path: str = None, + model_saving_strategy: Union[str, None] = "best", + verbose: bool = True): + super().__init__(batch_size, epochs, patience, num_workers, device, saving_path, model_saving_strategy, verbose) + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.step_channels = step_channels + self.compute_intervals = compute_intervals + self.intervals = None + + # Initialise empty model + self.model = None + self.optimizer = optimizer + + def _assemble_input_for_training(self, data: list, training=True) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + } + + + return inputs + + def _assemble_input_for_validating(self, data: list) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + X_ori, + indicating_mask, + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + "X_ori": X_ori, + "indicating_mask": indicating_mask, + } + return inputs + + def _assemble_input_for_testing(self, data: list) -> dict: + return self._assemble_input_for_validating(data) + + def fit(self, train_set, val_set, file_type: str = "hdf5",): + + self.training_set = DatasetForCSAI( + train_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals + ) + + training_loader = DataLoader( + self.training_set, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + + if val_set is not None: + # if not key_in_data_set("X_ori", val_set): + # raise ValueError("val_set must contain 'X_ori' for model validation.") + val_set = DatasetForCSAI( + val_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals, + self.training_set.replacement_probabilities, + self.training_set.mean_set, self.training_set.std_set, True, False + ) + val_loader = DataLoader( + val_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + + # set up the model + self.model = _BCSAI(self.n_steps, + self.n_features, + self.rnn_hidden_size, + self.step_channels, + self.training_set.intervals, + self.consistency_weight, + self.imputation_weight, + self.device) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer + self.optimizer = self.optimizer + self.optimizer.init_optimizer(self.model.parameters()) + + # train the model + self._train_model(training_loader, val_loader) + self.model.load_state_dict(self.best_model_dict) + self.model.eval() # set the model as eval status to freeze it. + + # Step 3: save the model if necessary + self._auto_save_model_if_necessary(confirm_saving=True) + + def predict(self, test_set: Union[dict, str], file_type: str = "hdf5") -> dict: + + if self.model == None: + raise ValueError("Training must be run before predict") + + self.model.eval() + test_set = DatasetForCSAI( + test_set, False, False, + file_type, self.removal_percent, + self.increase_factor, self.compute_intervals, + self.training_set.replacement_probabilities, + self.training_set.mean_set, self.training_set.std_set,True, False + ) + test_loader = DataLoader( + test_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + # collate_fn=collate_fn_bidirectional + ) + imputation_collector = [] + x_ori_collector = [] + indicating_mask_collector = [] + with torch.no_grad(): + for idx, data in enumerate(test_loader): + inputs = self._assemble_input_for_testing(data) + results = self.model.forward(inputs, training=False) + imputed_data = results["imputed_data"] + imputation_collector.append(imputed_data) + x_ori_collector.append(inputs["X_ori"]) + indicating_mask_collector.append(inputs["indicating_mask"]) + + imputation = torch.cat(imputation_collector).cpu().detach().numpy() + result_dict = { + "imputation": imputation, + "X_ori": torch.cat(x_ori_collector).cpu().detach().numpy(), + "indicating_mask": torch.cat(indicating_mask_collector).cpu().detach().numpy(), + } + return result_dict + + def impute( + self, + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> np.ndarray: + """Impute missing values in the given data with the trained model. + + Parameters + ---------- + test_set : + The data samples for testing, should be array-like of shape [n_samples, sequence length (n_steps), + n_features], or a path string locating a data file, e.g. h5 file. + + file_type : + The type of the given file if X is a path string. + + Returns + ------- + array-like, shape [n_samples, sequence length (n_steps), n_features], + Imputed data. + """ + + result_dict = self.predict(test_set, file_type=file_type) + return result_dict["imputation"] + + \ No newline at end of file diff --git a/pypots/nn/modules/csai/__init__.py b/pypots/nn/modules/csai/__init__.py index b4ff5295..64c57392 100644 --- a/pypots/nn/modules/csai/__init__.py +++ b/pypots/nn/modules/csai/__init__.py @@ -1,5 +1,5 @@ """ -The package including the modules of CSDI. +The package including the modules of CSAI. Refer to the paper `Linglong Qian, Zina Ibrahim, Hugh Logan Ellis, Ao Zhang, Yuezhou Zhang, Tao Wang, Richard Dobson. diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py index bdea8013..59a65487 100644 --- a/pypots/nn/modules/csai/backbone.py +++ b/pypots/nn/modules/csai/backbone.py @@ -2,7 +2,7 @@ """ -# Created by Joseph Arul Raj +# Created by Linglong Qian, Joseph Arul Raj # License: BSD-3-Clause import torch @@ -10,7 +10,7 @@ import torch.nn.functional as F import math from .layers import FeatureRegression, Decay, Decay_obs, PositionalEncoding, Conv1dWithInit, TorchTransformerEncoder - +from ....utils.metrics import calc_mae class BackboneCSAI(nn.Module): def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): From 60e0c3533acfc1dcbc81d933ad7c8e15104b0e14 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Thu, 3 Oct 2024 12:05:36 +0100 Subject: [PATCH 08/22] csai update --- pypots/nn/modules/csai/backbone.py | 84 ++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py index 59a65487..608440be 100644 --- a/pypots/nn/modules/csai/backbone.py +++ b/pypots/nn/modules/csai/backbone.py @@ -13,10 +13,86 @@ from ....utils.metrics import calc_mae class BackboneCSAI(nn.Module): - def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): - super(BackboneCSAI, self).__init__() + """ + Attributes + ---------- + n_steps : + sequence length (number of time steps) + + n_features : + number of features (input dimensions) + + rnn_hidden_size : + the hidden size of the GRU cell + + step_channels : + number of channels for each step in the sequence + + medians_tensor : + tensor of median values for features, used to adjust decayed observations + + temp_decay_h : + the temporal decay module to decay the hidden state of the GRU + + temp_decay_x : + the temporal decay module to decay data in the raw feature space + + hist : + the temporal-regression module that projects the GRU hidden state into the raw feature space + + feat_reg_v : + the feature-regression module used for feature-based estimation + + weight_combine : + the module that generates the weight to combine history regression and feature regression + + weighted_obs : + the decay module that computes weighted decay based on observed data and deltas + + gru : + the GRU cell that models temporal data for imputation + + pos_encoder : + the positional encoding module that adds temporal information to the sequence data + input_projection : + the convolutional module used to project input features into a higher-dimensional space + output_projection1 : + the convolutional module used to project the output from the Transformer layer + + output_projection2 : + the final convolutional module used to generate the hidden state from the time-layer's output + + time_layer : + the Transformer encoder layer used to model complex temporal dependencies within the sequence + + device : + the device (CPU/GPU) used for model computations + + Parameters + ---------- + n_steps : + sequence length (number of time steps) + + n_features : + number of features (input dimensions) + + rnn_hidden_size : + the hidden size of the GRU cell + + step_channels : + number of channels for each step in the sequence + + medians_df : + dataframe of median values for each feature, optional + + device : + the device (CPU/GPU) for running the model, optional + """ + + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + super(BackboneCSAI, self).__init__() if medians_df is not None: self.medians_tensor = torch.tensor(list(medians_df.values())).float() else: @@ -98,7 +174,8 @@ def forward(self, x, mask, deltas, last_obs, h=None): beta = self.weight_combine(torch.cat([gamma_x, m_t], dim=1)) x_comb_t = beta * xu + (1 - beta) * x_h - x_loss += torch.sum(torch.abs(x_t - x_comb_t) * m_t) / (torch.sum(m_t) + 1e-5) + # x_loss += torch.sum(torch.abs(x_t - x_comb_t) * m_t) / (torch.sum(m_t) + 1e-5) + x_loss += calc_mae(x_comb_t, x_t, m_t) # Final Imputation Estimates x_imp[:, t, :] = (m_t * x_t) + ((1 - m_t) * x_comb_t) @@ -121,7 +198,6 @@ def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_ self.model_f = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) self.model_b = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) - def forward(self, xdata): # Fetching forward data from xdata From 4311c14ad3c0c81f0d1edf197f397e42f1a19953 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 01:11:49 +0100 Subject: [PATCH 09/22] csai update --- pypots/data/utils.py | 5 +- pypots/imputation/csai/core.py | 62 +++++- pypots/imputation/csai/data.py | 357 ++++++++++++++++++++++++++++++-- pypots/imputation/csai/model.py | 180 +++++++++++++--- 4 files changed, 544 insertions(+), 60 deletions(-) diff --git a/pypots/data/utils.py b/pypots/data/utils.py index 40919fa8..56b915c1 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -10,7 +10,7 @@ import benchpots import numpy as np import torch - +from sklearn.preprocessing import StandardScaler def turn_data_into_specified_dtype( data: Union[np.ndarray, torch.Tensor, list], @@ -106,7 +106,6 @@ def cal_delta_for_single_sample(mask: np.ndarray) -> np.ndarray: """calculate single sample's delta. The sample's shape is [n_steps, n_features].""" # the first step in the delta matrix is all 0 d = [np.zeros(n_features)] - for step in range(1, seq_len): d.append(np.ones(n_features) + (1 - mask[step - 1]) * d[-1]) d = np.asarray(d) @@ -224,4 +223,4 @@ def inverse_sliding_window(X, sliding_len): return benchpots.utils.inverse_sliding_window( X, sliding_len, - ) + ) \ No newline at end of file diff --git a/pypots/imputation/csai/core.py b/pypots/imputation/csai/core.py index fb247163..aa7afe9e 100644 --- a/pypots/imputation/csai/core.py +++ b/pypots/imputation/csai/core.py @@ -10,8 +10,35 @@ class _BCSAI(nn.Module): - """model BRITS: Bidirectional RITS - BRITS consists of two RITS, which take time-series data from two directions (forward/backward) respectively. + """ + Attributes + ---------- + n_steps : + sequence length (number of time steps) + + n_features : + number of features (input dimensions) + + rnn_hidden_size : + the hidden size of the GRU cell + + step_channels : + number of channels for each step in the sequence + + intervals : + time intervals between the observations, used for handling irregular time-series + + consistency_weight : + weight assigned to the consistency loss during training + + imputation_weight : + weight assigned to the reconstruction loss during training + + model : + the underlying BackboneBCSAI model that handles forward and backward pass imputation + + device : + the device (CPU/GPU) used for model computations Parameters ---------- @@ -22,17 +49,38 @@ class _BCSAI(nn.Module): number of features (input dimensions) rnn_hidden_size : - the hidden size of the RNN cell + the hidden size of the GRU cell + + step_channels : + number of channels for each step in the sequence + + intervals : + time intervals between observations + + consistency_weight : + weight assigned to the consistency loss + + imputation_weight : + weight assigned to the reconstruction loss + + device : + the device (CPU/GPU) for running the model + + Notes + ----- + BCSAI is a bidirectional imputation model that uses forward and backward GRU cells to handle time-series data. It computes consistency and reconstruction losses to improve imputation accuracy. During training, the forward and backward reconstructions are combined, and losses are used to update the model. In evaluation mode, the model also outputs original data and indicating masks for further analysis. """ - def __init__(self, n_steps, + def __init__(self, + n_steps, n_features, rnn_hidden_size, step_channels, - intervals, consistency_weight, imputation_weight, - device) : + device=None, + intervals=None, + ): super().__init__() self.n_steps = n_steps self.n_features = n_features @@ -43,7 +91,7 @@ def __init__(self, n_steps, self.imputation_weight = imputation_weight self.device = device - self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, device) def forward(self, inputs:dict, training:bool = True) -> dict: ( diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index 3cd80cd0..ad38ca6d 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -9,10 +9,321 @@ from ...data.dataset import BaseDataset import numpy as np import torch -from ...data.utils import collate_fn_bidirectional, non_uniform_sample_loader_bidirectional, normalize_csai from typing import Union +import copy + +def normalize_csai( + data: np.ndarray, + mean: float, + std: float, + compute_intervals: bool = False +) -> : + """ + Normalize the data based on the given mean and standard deviation, and optionally compute time intervals between observations. + + Parameters + ---------- + data : np.ndarray + The input time-series data of shape [n_patients, n_hours, n_variables], which may contain missing values (NaNs). + + mean : list of float + The mean values for each variable, used for normalization. If empty, means will be computed from the data. + + std : list of float + The standard deviation values for each variable, used for normalization. If empty, std values will be computed from the data. + + compute_intervals : bool, optional, default=False + Whether to compute the time intervals between observations for each variable. + + Returns + ------- + data : np.ndarray + The normalized time-series data with the same shape as the input data. + + mean_set : np.ndarray + The mean values for each variable after normalization, either computed from the data or passed as input. + + std_set : np.ndarray + The standard deviation values for each variable after normalization, either computed from the data or passed as input. + + intervals_list : dict of int to float, optional + If `compute_intervals` is True, this will return the median time intervals between observations for each variable. + """ + + n_patients, n_hours, n_variables = data.shape + + # Flatten data for easier computation of statistics + reshaped_data = copy.deepcopy(data).reshape(-1, n_variables) + + # Use StandardScaler for normalization + scaler = StandardScaler() + + if mean is None or std is None: + # Fit the scaler on the data (ignores NaNs during the fitting process) + scaled_data = scaler.fit_transform(reshaped_data) + mean_set = scaler.mean_ + std_set = scaler.scale_ + else: + # Use provided mean and std by directly setting them in the scaler + scaler.mean_ = np.array(mean) + scaler.scale_ = np.array(std) + # Transform data using scaler, which ignores NaNs + scaled_data = scaler.transform(reshaped_data) + + # Reshape back to original shape [n_patients, n_hours, n_variables] + normalized_data = scaled_data.reshape(n_patients, n_hours, n_variables) + + # Optimized interval calculation considering NaNs in each patient + if compute_intervals: + intervals_list = {} + + for v in range(n_variables): + all_intervals = [] + # Loop over each patient + for p in range(n_patients): + # Get non-NaN observation indices for the current patient and variable + valid_time_points = np.where(~np.isnan(data[p, :, v]))[0] + + # If the patient has more than one valid observation, compute time intervals + if len(valid_time_points) > 1: + # Calculate time differences between consecutive observations + intervals = np.diff(valid_time_points) + all_intervals.extend(intervals) + + # Compute the median interval for the current variable + intervals_list[v] = np.median(all_intervals) if all_intervals else np.nan + else: + intervals_list = None + + return normalized_data, mean_set, std_set, intervals_list + + +def compute_last_obs(data, masks): + """ + Compute the last observed values for each time step. + + Parameters: + - data (np.array): Original data array of shape [T, D]. + - masks (np.array): Binary masks indicating where data is not NaN, of shape [T, D]. + + Returns: + - last_obs (np.array): Array of the last observed values, of shape [T, D]. + """ + T, D = masks.shape + last_obs = np.full((T, D), np.nan) # Initialize last observed values with NaNs + last_obs_val = np.full(D, np.nan) # Initialize last observed values for first time step with NaNs + + for t in range(1, T): # Start from t=1, keeping first row as NaN + mask = masks[t - 1] + # Update last observed values based on previous time step + last_obs_val[mask] = data[t - 1, mask] + # Assign last observed values to the current time step + last_obs[t] = last_obs_val + + return last_obs + +def adjust_probability_vectorized( + obs_count: Union[int, float], + avg_count: Union[int, float], + base_prob: float, + increase_factor: float = 0.5 +) -> float: + """ + Adjusts the base probability based on observed and average counts using a scaling factor. + + Parameters + ---------- + obs_count : int or float + The observed count of an event or observation in the dataset. + + avg_count : int or float + The average count of the event or observation across the dataset. + + base_prob : float + The base probability of the event or observation occurring. + + increase_factor : float, optional, default=0.5 + A scaling factor applied to adjust the probability when `obs_count` is below `avg_count`. + This factor influences how much to increase or decrease the probability. + + Returns + ------- + float + The adjusted probability, scaled based on the ratio between the observed count and the average count. + The adjusted probability will be within the range [0, 1]. + + Notes + ----- + This function adjusts a base probability based on the observed count (`obs_count`) compared to the average count + (`avg_count`). If the observed count is lower than the average, the probability is increased proportionally, + but capped at a maximum of 1.0. Conversely, if the observed count exceeds the average, the probability is reduced, + but not below 0. The `increase_factor` controls the sensitivity of the probability adjustment when the observed + count is less than the average count. + """ + if obs_count < avg_count: + # Increase probability when observed count is lower than average count + return min(base_prob * (avg_count / obs_count) * increase_factor, 1.0) + else: + # Decrease probability when observed count exceeds average count + return max(base_prob * (obs_count / avg_count) / increase_factor, 0.0) + +def non_uniform_sample_loader_bidirectional(data, removal_percent, pre_replacement_probabilities=None, increase_factor=0.5): + """ + Process time-series data by randomly removing a certain percentage of observed values based on pre-defined + replacement probabilities, and compute the necessary features such as forward and backward deltas, masks, + and last observed values. + + This function generates records for each time series and returns them as PyTorch tensors for further usage. + + Parameters + ---------- + data : np.ndarray + The input data with shape [N, T, D], where N is the number of samples, T is the number of time steps, + and D is the number of features. Missing values should be indicated with NaNs. + + removal_percent : float + The percentage of observed values to be removed randomly from the dataset. + + pre_replacement_probabilities : np.ndarray, optional + Pre-defined replacement probabilities for each feature. If provided, this will be used to determine + which values to remove. + + increase_factor : float, default=0.5 + A factor to adjust replacement probabilities based on the observation count for each feature. + + Returns + ------- + tensor_dict : dict of torch.Tensors + A dictionary of PyTorch tensors including 'values', 'last_obs_f', 'last_obs_b', 'masks', 'deltas_f', + 'deltas_b', 'evals', and 'eval_masks'. + + replacement_probabilities : np.ndarray + The computed or provided replacement probabilities for each feature. + """ + # Get dimensionality + [N, T, D] = data.shape + + # Compute replacement probabilities if not provided + if pre_replacement_probabilities is None: + observations_per_feature = np.sum(~np.isnan(data), axis=(0, 1)) + average_observations = np.mean(observations_per_feature) + replacement_probabilities = np.full(D, removal_percent / 100) + + if increase_factor > 0: + for feature_idx in range(D): + replacement_probabilities[feature_idx] = adjust_probability_vectorized( + observations_per_feature[feature_idx], + average_observations, + replacement_probabilities[feature_idx], + increase_factor=increase_factor + ) + + total_observations = np.sum(observations_per_feature) + total_replacement_target = total_observations * removal_percent / 100 + + for _ in range(1000): # Limit iterations to prevent infinite loop + total_replacement = np.sum(replacement_probabilities * observations_per_feature) + if np.isclose(total_replacement, total_replacement_target, rtol=1e-3): + break + adjustment_factor = total_replacement_target / total_replacement + replacement_probabilities *= adjustment_factor + else: + replacement_probabilities = pre_replacement_probabilities + + # Prepare data structures + recs = [] + values = copy.deepcopy(data) + + # Randomly remove data points based on replacement probabilities + random_matrix = np.random.rand(N, T, D) + values[(~np.isnan(values)) & (random_matrix < replacement_probabilities)] = np.nan + + # Generate records and features for each sample + for i in range(N): + masks = ~np.isnan(values[i, :, :]) + eval_masks = (~np.isnan(values[i, :, :])) ^ (~np.isnan(data[i, :, :])) + evals = data[i, :, :] + + # Compute forward and backward deltas + deltas_f = parse_delta(masks) + deltas_b = parse_delta(masks[::-1, :]) + + # Compute last observations for forward and backward directions + last_obs_f = compute_last_obs(values[i, :, :], masks) + last_obs_b = compute_last_obs(values[i, ::-1, :], masks[::-1, :]) + + # Append the record for this sample + recs.append({ + 'values': np.nan_to_num(values[i, :, :]), + 'last_obs_f': np.nan_to_num(last_obs_f), + 'last_obs_b': np.nan_to_num(last_obs_b), + 'masks': masks.astype('int32'), + 'evals': np.nan_to_num(evals), + 'eval_masks': eval_masks.astype('int32'), + 'deltas_f': deltas_f, + 'deltas_b': deltas_b + }) + + # Convert records to PyTorch tensors + tensor_dict = { + 'values': torch.FloatTensor(np.array([r['values'] for r in recs])), + 'last_obs_f': torch.FloatTensor(np.array([r['last_obs_f'] for r in recs])), + 'last_obs_b': torch.FloatTensor(np.array([r['last_obs_b'] for r in recs])), + 'masks': torch.FloatTensor(np.array([r['masks'] for r in recs])), + 'deltas_f': torch.FloatTensor(np.array([r['deltas_f'] for r in recs])), + 'deltas_b': torch.FloatTensor(np.array([r['deltas_b'] for r in recs])), + 'evals': torch.FloatTensor(np.array([r['evals'] for r in recs])), + 'eval_masks': torch.FloatTensor(np.array([r['eval_masks'] for r in recs])) + } + + return tensor_dict, replacement_probabilities + class DatasetForCSAI(BaseDataset): + """" + Parameters + ---------- + data : + The dataset for model input, which can be either a dictionary or a path string to a data file. If it's a dictionary, `X` should be an array-like structure with shape [n_samples, sequence length (n_steps), n_features], containing the time-series data, and it can have missing values. Optionally, the dictionary can include `y`, an array-like structure with shape [n_samples], representing the labels of `X`. If `data` is a path string, it should point to a data file (e.g., h5 file) that contains key-value pairs like a dictionary, including keys for `X` and possibly `y`. + + return_X_ori : + Whether to return the original time-series data (`X_ori`) when fetching data samples, useful for evaluation purposes. + + return_y : + Whether to return classification labels in the `__getitem__()` method if they exist in the dataset. If `True`, labels will be included in the returned data samples, which is useful for training classification models. If `False`, the labels won't be returned, suitable for testing or validation stages. + + file_type : + The type of the data file if `data` is a path string, such as "hdf5". + + removal_percent : + The percentage of data to be removed for simulating missing values during training. + + increase_factor : + A scaling factor to increase the probability of missing data during training. + + compute_intervals : + Whether to compute time intervals between observations for handling irregular time-series data. + + replacement_probabilities : + Optional precomputed probabilities for sampling missing values. If not provided, they will be calculated during the initialization of the dataset. + + normalise_mean : + A list of mean values for normalizing the input features. If not provided, they will be computed during initialization. + + normalise_std : + A list of standard deviation values for normalizing the input features. If not provided, they will be computed during initialization. + + impute_only : + Whether the dataset is used for imputation tasks only. If `True`, additional label-related operations will be skipped. + + training : + Whether the dataset is used for training. If `False`, it will adjust how data is processed, particularly for evaluation and testing phases. + + Notes + ----- + The DatasetForCSAI class is designed for bidirectional imputation of time-series data, handling both forward and backward directions to improve imputation accuracy. It supports on-the-fly data normalization and missing value simulation, making it suitable for training and evaluating deep learning models like CSAI. The class can work with large datasets stored on disk, leveraging lazy-loading to minimize memory usage, and supports both training and testing scenarios, adjusting data handling as needed. + + """ def __init__(self, data: Union[dict, str], return_X_ori: bool, return_y: bool, @@ -31,6 +342,7 @@ def __init__(self, data: Union[dict, str], return_X_pred = False, return_y = return_y, file_type = file_type) + self.removal_percent = removal_percent self.increase_factor = increase_factor self.compute_intervals = compute_intervals @@ -41,13 +353,19 @@ def __init__(self, data: Union[dict, str], self.training = training if not isinstance(self.data, str): - self.normalized_data, self.mean_set, self.std_set, self.intervals = normalize_csai(self.data['X'], normalise_mean, - normalise_std, compute_intervals) - _data, self.replacement_probabilities = non_uniform_sample_loader_bidirectional(self.normalized_data, - removal_percent, - replacement_probabilities, - increase_factor) - self.processed_data = collate_fn_bidirectional(_data) + self.normalized_data, self.mean_set, self.std_set, self.intervals = normalize_csai( + self.data['X'], + normalise_mean, + normalise_std, + compute_intervals, + ) + + self.processed_data, self.replacement_probabilities = non_uniform_sample_loader_bidirectional( + self.normalized_data, + removal_percent, + replacement_probabilities, + increase_factor, + ) self.forward_X = self.processed_data['values'] self.forward_missing_mask = self.processed_data['masks'] self.backward_X = torch.flip(self.forward_X, dims=[1]) @@ -57,8 +375,6 @@ def __init__(self, data: Union[dict, str], self.indicating_mask = self.processed_data['eval_masks'] # if self.return_y: # self.y = self.processed_data['labels'] - - def _fetch_data_from_array(self, idx: int) -> Iterable: """Fetch data from self.X if it is given. @@ -137,13 +453,19 @@ def _fetch_data_from_file(self, idx: int) -> Iterable: self.file_handle = self._open_file_handle() X = torch.from_numpy(self.file_handle["X"][idx]) - normalized_data, mean_set, std_set, intervals = normalize_csai(X, self.normalise_mean, - self.normalise_std, - self.compute_intervals) - processed_data, replacement_probabilities = non_uniform_sample_loader_bidirectional(normalized_data, - self.removal_percent, - self.replacement_probabilities, - self.increase_factor) + normalized_data, mean_set, std_set, intervals = normalize_csai( + X, + self.normalise_mean, + self.normalise_std, + self.compute_intervals, + ) + + processed_data, replacement_probabilities = non_uniform_sample_loader_bidirectional( + normalized_data, + self.removal_percent, + self.replacement_probabilities, + self.increase_factor, + ) forward_X = processed_data['values'] forward_missing_mask = processed_data['masks'] backward_X = torch.flip(forward_X, dims=[1]) @@ -154,7 +476,6 @@ def _fetch_data_from_file(self, idx: int) -> Iterable: if self.return_y: y = self.processed_data['labels'] - sample = [ torch.tensor(idx), # for forward diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index 67d406ae..6355e15e 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -21,6 +21,69 @@ class CSAI(BaseNNImputer): + """ + Parameters + ---------- + n_steps : + The number of time steps in the time-series data sample. + + n_features : + The number of features in the time-series data sample. + + rnn_hidden_size : + The size of the GRU hidden state, also the number of hidden units in the GRU cell. + + imputation_weight : + The weight assigned to the reconstruction loss during training. + + consistency_weight : + The weight assigned to the consistency loss during training. + + removal_percent : + The percentage of data to be removed during training for imputation tasks. + + increase_factor : + A scaling factor used to adjust the amount of missing data during training. + + compute_intervals : + Whether to compute time intervals between observations for handling irregular time-series. + + step_channels : + The number of channels for each step in the sequence. + + batch_size : + The batch size for training and evaluating the model. + + epochs : + The number of epochs for training the model. + + patience : + The patience for the early-stopping mechanism. Given a positive integer, training will stop when no improvement is observed after the specified number of epochs. If set to None, early-stopping is disabled. + + optimizer : + The optimizer used for model training. Defaults to the Adam optimizer if not specified. + + num_workers : + The number of subprocesses used for data loading. Setting this to `0` means that data loading is performed in the main process without using subprocesses. + + device : + The device for the model to run on, which can be a string, a :class:`torch.device` object, or a list of devices. If not provided, the model will attempt to use available CUDA devices first, then default to CPUs. + + saving_path : + The path for saving model checkpoints and tensorboard files during training. If not provided, models will not be saved automatically. + + model_saving_strategy : + The strategy for saving model checkpoints. Can be one of [None, "best", "better", "all"]. "best" saves the best model after training, "better" saves any model that improves during training, and "all" saves models after each epoch. If set to None, no models will be saved. + + verbose : + Whether to print training logs during the training process. + + Notes + ----- + CSAI (Consistent Sequential Imputation) is a bidirectional model designed for time-series imputation. It employs a forward and backward GRU network to handle missing data, using consistency and reconstruction losses to improve accuracy. The model supports various training configurations, such as interval computations, early-stopping, and multiple devices for training. Results can be saved based on the specified saving strategy, and tensorboard files are generated for tracking the model's performance over time. + + """ + def __init__(self, n_steps: int, n_features: int, @@ -39,8 +102,19 @@ def __init__(self, device: Union[str, torch.device, list, None ]= None, saving_path: str = None, model_saving_strategy: Union[str, None] = "best", - verbose: bool = True): - super().__init__(batch_size, epochs, patience, num_workers, device, saving_path, model_saving_strategy, verbose) + verbose: bool = True, + ): + super().__init__( + batch_size, + epochs, + patience, + num_workers, + device, + saving_path, + model_saving_strategy, + verbose, + ) + self.n_steps = n_steps self.n_features = n_features self.rnn_hidden_size = rnn_hidden_size @@ -53,7 +127,20 @@ def __init__(self, self.intervals = None # Initialise empty model - self.model = None + self.model = _BCSAI( + self.n_steps, + self.n_features, + self.rnn_hidden_size, + self.step_channels, + self.consistency_weight, + self.imputation_weight, + self.device, + None, + ) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer self.optimizer = optimizer def _assemble_input_for_training(self, data: list, training=True) -> dict: @@ -88,7 +175,6 @@ def _assemble_input_for_training(self, data: list, training=True) -> dict: }, } - return inputs def _assemble_input_for_validating(self, data: list) -> dict: @@ -131,12 +217,21 @@ def _assemble_input_for_validating(self, data: list) -> dict: def _assemble_input_for_testing(self, data: list) -> dict: return self._assemble_input_for_validating(data) - def fit(self, train_set, val_set, file_type: str = "hdf5",): + def fit( + self, + train_set, + val_set, + file_type: str = "hdf5", + )-> None: self.training_set = DatasetForCSAI( - train_set, False, False, - file_type, self.removal_percent, - self.increase_factor, self.compute_intervals + train_set, + False, + False, + file_type, + self.removal_percent, + self.increase_factor, + self.compute_intervals ) training_loader = DataLoader( @@ -151,11 +246,18 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): # if not key_in_data_set("X_ori", val_set): # raise ValueError("val_set must contain 'X_ori' for model validation.") val_set = DatasetForCSAI( - val_set, False, False, - file_type, self.removal_percent, - self.increase_factor, self.compute_intervals, + val_set, + False, + False, + file_type, + self.removal_percent, + self.increase_factor, + self.compute_intervals, self.training_set.replacement_probabilities, - self.training_set.mean_set, self.training_set.std_set, True, False + self.training_set.mean_set, + self.training_set.std_set, + True, + False, ) val_loader = DataLoader( val_set, @@ -165,20 +267,22 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): # collate_fn=collate_fn_bidirectional ) - # set up the model - self.model = _BCSAI(self.n_steps, - self.n_features, - self.rnn_hidden_size, - self.step_channels, - self.training_set.intervals, - self.consistency_weight, - self.imputation_weight, - self.device) + # Reset the model + self.model = _BCSAI( + self.n_steps, + self.n_features, + self.rnn_hidden_size, + self.step_channels, + self.consistency_weight, + self.imputation_weight, + self.device, + self.training_set.intervals, + ) + self._send_model_to_given_device() self._print_model_size() # set up the optimizer - self.optimizer = self.optimizer self.optimizer.init_optimizer(self.model.parameters()) # train the model @@ -187,21 +291,33 @@ def fit(self, train_set, val_set, file_type: str = "hdf5",): self.model.eval() # set the model as eval status to freeze it. # Step 3: save the model if necessary - self._auto_save_model_if_necessary(confirm_saving=True) + self._auto_save_model_if_necessary(confirm_saving=self.model_saving_strategy == "best") - def predict(self, test_set: Union[dict, str], file_type: str = "hdf5") -> dict: + def predict( + self, + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> dict: - if self.model == None: - raise ValueError("Training must be run before predict") + # if self.model == None: + # raise ValueError("Training must be run before predict") self.model.eval() test_set = DatasetForCSAI( - test_set, False, False, - file_type, self.removal_percent, - self.increase_factor, self.compute_intervals, + test_set, + False, + False, + file_type, + self.removal_percent, + self.increase_factor, + self.compute_intervals, self.training_set.replacement_probabilities, - self.training_set.mean_set, self.training_set.std_set,True, False + self.training_set.mean_set, + self.training_set.std_set, + True, + False, ) + test_loader = DataLoader( test_set, batch_size=self.batch_size, @@ -209,9 +325,11 @@ def predict(self, test_set: Union[dict, str], file_type: str = "hdf5") -> dict: num_workers=self.num_workers, # collate_fn=collate_fn_bidirectional ) + imputation_collector = [] x_ori_collector = [] indicating_mask_collector = [] + with torch.no_grad(): for idx, data in enumerate(test_loader): inputs = self._assemble_input_for_testing(data) @@ -253,5 +371,3 @@ def impute( result_dict = self.predict(test_set, file_type=file_type) return result_dict["imputation"] - - \ No newline at end of file From 7b0b487eb4c52a7c9d620317b74ff3ef9f21854f Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 01:29:29 +0100 Subject: [PATCH 10/22] csai update --- pypots/imputation/csai/data.py | 56 +++++++++++++++------------------ pypots/imputation/csai/model.py | 54 +++++++++++++++---------------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index ad38ca6d..b4437ed7 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -167,7 +167,7 @@ def adjust_probability_vectorized( # Decrease probability when observed count exceeds average count return max(base_prob * (obs_count / avg_count) / increase_factor, 0.0) -def non_uniform_sample_loader_bidirectional(data, removal_percent, pre_replacement_probabilities=None, increase_factor=0.5): +def non_uniform_sample(data, removal_percent, pre_replacement_probabilities=None, increase_factor=0.5): """ Process time-series data by randomly removing a certain percentage of observed values based on pre-defined replacement probabilities, and compute the necessary features such as forward and backward deltas, masks, @@ -313,9 +313,6 @@ class DatasetForCSAI(BaseDataset): normalise_std : A list of standard deviation values for normalizing the input features. If not provided, they will be computed during initialization. - impute_only : - Whether the dataset is used for imputation tasks only. If `True`, additional label-related operations will be skipped. - training : Whether the dataset is used for training. If `False`, it will adjust how data is processed, particularly for evaluation and testing phases. @@ -324,18 +321,18 @@ class DatasetForCSAI(BaseDataset): The DatasetForCSAI class is designed for bidirectional imputation of time-series data, handling both forward and backward directions to improve imputation accuracy. It supports on-the-fly data normalization and missing value simulation, making it suitable for training and evaluating deep learning models like CSAI. The class can work with large datasets stored on disk, leveraging lazy-loading to minimize memory usage, and supports both training and testing scenarios, adjusting data handling as needed. """ - def __init__(self, data: Union[dict, str], - return_X_ori: bool, - return_y: bool, - file_type: str = "hdf5", - removal_percent: float = 0.0, - increase_factor: float = 0.1, - compute_intervals: bool = False, - replacement_probabilities = None, - normalise_mean : list = [], - normalise_std: list = [], - impute_only: bool = True, - training: bool = True + def __init__(self, + data: Union[dict, str], + return_X_ori: bool, + return_y: bool, + file_type: str = "hdf5", + removal_percent: float = 0.0, + increase_factor: float = 0.1, + compute_intervals: bool = False, + replacement_probabilities = None, + normalise_mean : list = [], + normalise_std: list = [], + training: bool = True ): super().__init__(data = data, return_X_ori = return_X_ori, @@ -349,7 +346,6 @@ def __init__(self, data: Union[dict, str], self.replacement_probabilities = replacement_probabilities self.normalise_mean = normalise_mean self.normalise_std = normalise_std - self.impute_only = impute_only self.training = training if not isinstance(self.data, str): @@ -360,12 +356,12 @@ def __init__(self, data: Union[dict, str], compute_intervals, ) - self.processed_data, self.replacement_probabilities = non_uniform_sample_loader_bidirectional( - self.normalized_data, - removal_percent, - replacement_probabilities, - increase_factor, - ) + self.processed_data, self.replacement_probabilities = non_uniform_sample( + self.normalized_data, + removal_percent, + replacement_probabilities, + increase_factor, + ) self.forward_X = self.processed_data['values'] self.forward_missing_mask = self.processed_data['masks'] self.backward_X = torch.flip(self.forward_X, dims=[1]) @@ -420,7 +416,7 @@ def _fetch_data_from_array(self, idx: int) -> Iterable: self.processed_data["last_obs_b"][idx], ] - if not self.training and self.impute_only: + if not self.training: sample.extend([self.X_ori[idx], self.indicating_mask[idx]]) if self.return_y: @@ -460,12 +456,12 @@ def _fetch_data_from_file(self, idx: int) -> Iterable: self.compute_intervals, ) - processed_data, replacement_probabilities = non_uniform_sample_loader_bidirectional( - normalized_data, - self.removal_percent, - self.replacement_probabilities, - self.increase_factor, - ) + processed_data, replacement_probabilities = non_uniform_sample( + normalized_data, + self.removal_percent, + self.replacement_probabilities, + self.increase_factor, + ) forward_X = processed_data['values'] forward_missing_mask = processed_data['masks'] backward_X = torch.flip(forward_X, dims=[1]) diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index 6355e15e..d76cc923 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -1,5 +1,5 @@ """ - +The implementation of CSAI """ # Created by Linglong Qian, Joseph Arul Raj @@ -135,7 +135,7 @@ def __init__(self, self.consistency_weight, self.imputation_weight, self.device, - None, + self.intervals, ) self._send_model_to_given_device() self._print_model_size() @@ -146,6 +146,7 @@ def __init__(self, def _assemble_input_for_training(self, data: list, training=True) -> dict: # extract data sample = data['sample'] + ( indices, X, @@ -225,14 +226,18 @@ def fit( )-> None: self.training_set = DatasetForCSAI( - train_set, - False, - False, - file_type, - self.removal_percent, - self.increase_factor, - self.compute_intervals - ) + train_set, + False, + False, + file_type, + self.removal_percent, + self.increase_factor, + self.compute_intervals + ) + self.intervals = self.training_set.intervals + self.replacement_probabilities = self.training_set.replacement_probabilities + self.mean_set = self.training_set.mean_set + self.std_set = self.training_set.std_set training_loader = DataLoader( self.training_set, @@ -243,20 +248,17 @@ def fit( ) if val_set is not None: - # if not key_in_data_set("X_ori", val_set): - # raise ValueError("val_set must contain 'X_ori' for model validation.") val_set = DatasetForCSAI( val_set, - False, + True, False, file_type, self.removal_percent, self.increase_factor, - self.compute_intervals, - self.training_set.replacement_probabilities, - self.training_set.mean_set, - self.training_set.std_set, - True, + False, + self.replacement_probabilities, + self.mean_set, + self.std_set, False, ) val_loader = DataLoader( @@ -276,7 +278,7 @@ def fit( self.consistency_weight, self.imputation_weight, self.device, - self.training_set.intervals, + self.intervals, ) self._send_model_to_given_device() @@ -299,22 +301,18 @@ def predict( file_type: str = "hdf5", ) -> dict: - # if self.model == None: - # raise ValueError("Training must be run before predict") - self.model.eval() test_set = DatasetForCSAI( test_set, - False, + True, False, file_type, self.removal_percent, self.increase_factor, - self.compute_intervals, - self.training_set.replacement_probabilities, - self.training_set.mean_set, - self.training_set.std_set, - True, + False, + self.replacement_probabilities, + self.mean_set, + self.std_set, False, ) From 276ae3b535da740497cb62ba03d38a4b563622e0 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 01:44:55 +0100 Subject: [PATCH 11/22] csai update --- pypots/classification/csai/__init__.py | 20 ++ pypots/classification/csai/core.py | 122 +++++++++++ pypots/classification/csai/data.py | 40 ++++ pypots/classification/csai/model.py | 267 +++++++++++++++++++++++++ pypots/imputation/csai/data.py | 3 +- 5 files changed, 450 insertions(+), 2 deletions(-) create mode 100644 pypots/classification/csai/__init__.py create mode 100644 pypots/classification/csai/core.py create mode 100644 pypots/classification/csai/data.py create mode 100644 pypots/classification/csai/model.py diff --git a/pypots/classification/csai/__init__.py b/pypots/classification/csai/__init__.py new file mode 100644 index 00000000..5ea14ae3 --- /dev/null +++ b/pypots/classification/csai/__init__.py @@ -0,0 +1,20 @@ +""" +The package including the modules of CSAI. + +Refer to the paper +`Linglong Qian, Zina Ibrahim, Hugh Logan Ellis, Ao Zhang, Yuezhou Zhang, Tao Wang, Richard Dobson. +Knowledge Enhanced Conditional Imputation for Healthcare Time-series. +In Arxiv, 2024. +`_ + +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/LinglongQian/CSAI. + +""" + +from .model import CSAI + +__all__ = [ + "CSAI", +] \ No newline at end of file diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py new file mode 100644 index 00000000..f1ca08e0 --- /dev/null +++ b/pypots/classification/csai/core.py @@ -0,0 +1,122 @@ +""" + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ...nn.modules.csai import BackboneBCSAI + +class DiceBCELoss(nn.Module): + def __init__(self, weight=None, size_average=True): + super(DiceBCELoss, self).__init__() + self.bcelogits = nn.BCEWithLogitsLoss() + + def forward(self, y_score, y_out, targets, smooth=1): + + #comment out if your model contains a sigmoid or equivalent activation layer + # inputs = F.sigmoid(inputs) + + #flatten label and prediction tensors + BCE = self.bcelogits(y_out, targets) + + y_score = y_score.view(-1) + targets = targets.view(-1) + intersection = (y_score * targets).sum() + dice_loss = 1 - (2.*intersection + smooth)/(y_score.sum() + targets.sum() + smooth) + + Dice_BCE = BCE + dice_loss + + return BCE, Dice_BCE + + +class _BCSAI(nn.Module): + def __init__( + self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + classification_weight: float, + n_classes: int, + step_channels: int, + dropout: float = 0.5, + intervals: list = None, + device = None, + + ): + super().__init__() + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.classification_weight = classification_weight + self.n_classes = n_classes + self.step_channels = step_channels + self.intervals = intervals + self.device = device + + # create models + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) + self.f_classifier = nn.Linear(self.rnn_hidden_size, n_classes) + self.b_classifier = nn.Linear(self.rnn_hidden_size, n_classes) + self.imputer = nn.Linear(self.rnn_hidden_size, n_features) + self.dropout = nn.Dropout(dropout) + + def forward(self, inputs: dict, training: bool = True) -> dict: + + ( + imputed_data, + f_reconstruction, + b_reconstruction, + f_hidden_states, + b_hidden_states, + consistency_loss, + reconstruction_loss, + ) = self.model(inputs) + + results = { + "imputed_data": imputed_data, + } + + f_logits = self.f_classifier(self.dropout(f_hidden_states)) + b_logits = self.b_classifier(self.dropout(b_hidden_states)) + + f_prediction = torch.sigmoid(f_logits) + b_prediction = torch.sigmoid(b_logits) + + classification_pred = (f_prediction + b_prediction) / 2 + + results = { + "imputed_data": imputed_data, + "classification_pred": classification_pred, + } + + # if in training mode, return results with losses + if training: + criterion = DiceBCELoss().to(self.device) + results["consistency_loss"] = consistency_loss + results["reconstruction_loss"] = reconstruction_loss + # print(inputs["labels"].unsqueeze(1)) + f_classification_loss, _ = criterion(f_prediction, f_logits, inputs["labels"].unsqueeze(1).float()) + b_classification_loss, _ = criterion(b_prediction, b_logits, inputs["labels"].unsqueeze(1).float()) + classification_loss = (f_classification_loss + b_classification_loss) + + loss = ( + self.consistency_weight * consistency_loss + + self.imputation_weight * reconstruction_loss + + self.classification_weight * classification_loss + ) + + results["loss"] = loss + results["classification_loss"] = classification_loss + results["f_reconstruction"] = f_reconstruction + results["b_reconstruction"] = b_reconstruction + + return results \ No newline at end of file diff --git a/pypots/classification/csai/data.py b/pypots/classification/csai/data.py new file mode 100644 index 00000000..f4a36c48 --- /dev/null +++ b/pypots/classification/csai/data.py @@ -0,0 +1,40 @@ +""" + +""" + +# Created by Joseph Arul Raj +# License: BSD-3-Clause + +from typing import Union +from ...imputation.csai.data import DatasetForCSAI as DatasetForCSAI_Imputation + + + +class DatasetForCSAI(DatasetForCSAI_Imputation): + def __init__(self, + data: Union[dict, str], + file_type: str = "hdf5", + return_y: bool = True, + removal_percent: float = 0.0, + increase_factor: float = 0.1, + compute_intervals: bool = False, + replacement_probabilities = None, + normalise_mean : list = [], + normalise_std: list = [], + training: bool = True + ): + super().__init__( + data=data, + return_X_ori=False, + return_y=return_y, + file_type=file_type, + removal_percent=removal_percent, + increase_factor=increase_factor, + compute_intervals=compute_intervals, + replacement_probabilities=replacement_probabilities, + normalise_mean=normalise_mean, + normalise_std=normalise_std, + impute_only=False, + training=training + ) + \ No newline at end of file diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py new file mode 100644 index 00000000..cd20757a --- /dev/null +++ b/pypots/classification/csai/model.py @@ -0,0 +1,267 @@ + +""" + +""" + +# Created by Linglong Qian, Joseph Arul Raj +# License: BSD-3-Clause + +from typing import Optional, Union +import numpy as np +import torch +from torch.utils.data import DataLoader + +from .core import _BCSAI +from .data import DatasetForCSAI +from ..base import BaseNNClassifier +from ...optim.adam import Adam +from ...optim.base import Optimizer + + +class CSAI(BaseNNClassifier): + def __init__(self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + classification_weight: float, + n_classes: int, + removal_percent: int, + increase_factor: float, + compute_intervals: bool, + step_channels:int, + batch_size: int, + epochs: int, + dropout: float = 0.5, + patience: Union[int, None] = None, + optimizer: Optimizer = Adam(), + num_workers: int = 0, + device: Optional[Union[str, torch.device, list]] = None, + saving_path: str = None, + model_saving_strategy: Union[str, None] = "best", + verbose: bool = True): + super().__init__( + n_classes, + batch_size, + epochs, + patience, + num_workers, + device, + saving_path, + model_saving_strategy, + verbose) + + self.n_steps = n_steps + self.n_features = n_features + self.rnn_hidden_size = rnn_hidden_size + self.imputation_weight = imputation_weight + self.consistency_weight = consistency_weight + self.classification_weight = classification_weight + self.removal_percent = removal_percent + self.increase_factor = increase_factor + self.step_channels = step_channels + self.compute_intervals = compute_intervals + self.intervals = None + self.dropout = dropout + self.device = device + + # Initialise empty model + self.model = None + self.optimizer = optimizer + + def _assemble_input_for_training(self, data: list, training=True) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + labels + ) = self._send_data_to_given_device(sample) + + inputs = { + "indices": indices, + "labels": labels, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + } + + + return inputs + def _assemble_input_for_validating(self, data: list) -> dict: + return self._assemble_input_for_training(data) + + def _assemble_input_for_testing(self, data: list) -> dict: + # extract data + sample = data['sample'] + ( + indices, + X, + missing_mask, + deltas, + last_obs, + back_X, + back_missing_mask, + back_deltas, + back_last_obs, + # X_ori, + # indicating_mask, + ) = self._send_data_to_given_device(sample) + + # assemble input data + inputs = { + "indices": indices, + "forward": { + "X": X, + "missing_mask": missing_mask, + "deltas": deltas, + "last_obs": last_obs, + }, + "backward": { + "X": back_X, + "missing_mask": back_missing_mask, + "deltas": back_deltas, + "last_obs": back_last_obs, + }, + # "X_ori": X_ori, + # "indicating_mask": indicating_mask, + } + + return inputs + + def fit( + self, + train_set, + val_set=None, + file_type: str = "hdf5", + ): + # Create dataset + self.training_set = DatasetForCSAI( + data=train_set, + file_type=file_type, + return_y=True, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + ) + + train_loader = DataLoader( + self.training_set, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + ) + + val_loader = None + if val_set is not None: + val_set = DatasetForCSAI( + data=val_set, + file_type=file_type, + return_y=True, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + replacement_probabilities=self.training_set.replacement_probabilities, + normalise_mean=self.training_set.normalise_mean, + normalise_std=self.training_set.normalise_std, + training=False + + ) + val_loader = DataLoader( + val_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + # Create model + self.model = _BCSAI( + n_steps=self.n_steps, + n_features=self.n_features, + rnn_hidden_size=self.rnn_hidden_size, + imputation_weight=self.imputation_weight, + consistency_weight=self.consistency_weight, + classification_weight=self.classification_weight, + n_classes=self.n_classes, + step_channels=self.step_channels, + dropout=self.dropout, + intervals=self.training_set.intervals, + device=self.device, + ) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer + self.optimizer.init_optimizer(self.model.parameters()) + + # train the model + self._train_model(train_loader, val_loader) + self.model.load_state_dict(self.best_model_dict) + self.model.eval() + + self._auto_save_model_if_necessary(confirm_saving=True) + + + def predict( + self, + test_set, + file_type: str = "hdf5"): + + self.model.eval() + test_set = DatasetForCSAI( + data=test_set, + file_type=file_type, + return_y=False, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + replacement_probabilities=self.training_set.replacement_probabilities, + normalise_mean=self.training_set.normalise_mean, + normalise_std=self.training_set.normalise_std, + training=False + ) + test_loader = DataLoader( + test_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + + classificaion_results = [] + + with torch.no_grad(): + for idx, data in enumerate(test_loader): + inputs = self._assemble_input_for_testing(data) + results = self.model.forward(inputs, training=False) + classificaion_results.append(results['classification_pred']) + + + classification = torch.cat(classificaion_results).cpu().detach().numpy() + result_dict = { + "classification": classification, + } + return result_dict + + def classify( + self, + test_set, + file_type): + + result_dict = self.predict(test_set, file_type) + return result_dict['classification'] \ No newline at end of file diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index b4437ed7..b2d282fa 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -369,8 +369,7 @@ def __init__(self, self.X_ori = self.processed_data['evals'] self.indicating_mask = self.processed_data['eval_masks'] - # if self.return_y: - # self.y = self.processed_data['labels'] + def _fetch_data_from_array(self, idx: int) -> Iterable: """Fetch data from self.X if it is given. From 95b715b7e1f1a59099821704ddaa285113b2f95d Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 02:08:10 +0100 Subject: [PATCH 12/22] csai update --- pypots/classification/__init__.py | 2 ++ pypots/classification/csai/model.py | 45 +++++++++++++++++++++-------- pypots/imputation/__init__.py | 2 ++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/pypots/classification/__init__.py b/pypots/classification/__init__.py index 73f6524d..4344ad92 100644 --- a/pypots/classification/__init__.py +++ b/pypots/classification/__init__.py @@ -6,10 +6,12 @@ # License: BSD-3-Clause from .brits import BRITS +from .csai import CSAI from .grud import GRUD from .raindrop import Raindrop __all__ = [ + "CSAI", "BRITS", "GRUD", "Raindrop", diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index cd20757a..c26dad39 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -62,12 +62,29 @@ def __init__(self, self.increase_factor = increase_factor self.step_channels = step_channels self.compute_intervals = compute_intervals - self.intervals = None self.dropout = dropout self.device = device + self.intervals = None # Initialise empty model - self.model = None + self.model = _BCSAI( + n_steps=self.n_steps, + n_features=self.n_features, + rnn_hidden_size=self.rnn_hidden_size, + imputation_weight=self.imputation_weight, + consistency_weight=self.consistency_weight, + classification_weight=self.classification_weight, + n_classes=self.n_classes, + step_channels=self.step_channels, + dropout=self.dropout, + intervals=self.intervals, + device=self.device, + ) + + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer self.optimizer = optimizer def _assemble_input_for_training(self, data: list, training=True) -> dict: @@ -102,9 +119,8 @@ def _assemble_input_for_training(self, data: list, training=True) -> dict: "last_obs": back_last_obs, }, } - - return inputs + def _assemble_input_for_validating(self, data: list) -> dict: return self._assemble_input_for_training(data) @@ -162,6 +178,11 @@ def fit( compute_intervals=self.compute_intervals, ) + self.intervals = self.training_set.intervals + self.replacement_probabilities = self.training_set.replacement_probabilities + self.mean_set = self.training_set.mean_set + self.std_set = self.training_set.std_set + train_loader = DataLoader( self.training_set, batch_size=self.batch_size, @@ -178,9 +199,9 @@ def fit( removal_percent=self.removal_percent, increase_factor=self.increase_factor, compute_intervals=self.compute_intervals, - replacement_probabilities=self.training_set.replacement_probabilities, - normalise_mean=self.training_set.normalise_mean, - normalise_std=self.training_set.normalise_std, + replacement_probabilities=self.replacement_probabilities, + normalise_mean=self.normalise_mean, + normalise_std=self.normalise_std, training=False ) @@ -201,7 +222,7 @@ def fit( n_classes=self.n_classes, step_channels=self.step_channels, dropout=self.dropout, - intervals=self.training_set.intervals, + intervals=self.intervals, device=self.device, ) self._send_model_to_given_device() @@ -215,7 +236,7 @@ def fit( self.model.load_state_dict(self.best_model_dict) self.model.eval() - self._auto_save_model_if_necessary(confirm_saving=True) + self._auto_save_model_if_necessary(confirm_saving=self.model_saving_strategy == "best") def predict( @@ -231,9 +252,9 @@ def predict( removal_percent=self.removal_percent, increase_factor=self.increase_factor, compute_intervals=self.compute_intervals, - replacement_probabilities=self.training_set.replacement_probabilities, - normalise_mean=self.training_set.normalise_mean, - normalise_std=self.training_set.normalise_std, + replacement_probabilities=self.replacement_probabilities, + normalise_mean=self.normalise_mean, + normalise_std=self.normalise_std, training=False ) test_loader = DataLoader( diff --git a/pypots/imputation/__init__.py b/pypots/imputation/__init__.py index 19a7e2c6..6600dcfd 100644 --- a/pypots/imputation/__init__.py +++ b/pypots/imputation/__init__.py @@ -6,6 +6,7 @@ # License: BSD-3-Clause from .brits import BRITS +from .csai import CSAI from .csdi import CSDI from .gpvae import GPVAE from .mrnn import MRNN @@ -85,4 +86,5 @@ "Median", "Lerp", "TEFN", + "CSAI", ] From bc010e03fcc39656ffff28d9b787dd34028c0825 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 02:09:10 +0100 Subject: [PATCH 13/22] csai update --- pypots/classification/csai/model.py | 71 +++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index c26dad39..13bcdd97 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -19,6 +19,77 @@ class CSAI(BaseNNClassifier): + + """ + The PyTorch implementation of the CSAI model. + + Parameters + + n_steps : + The number of time steps in the time-series data sample. + + n_features : + The number of features in the time-series data sample. + + rnn_hidden_size : + The size of the RNN hidden state. + + imputation_weight : + The loss weight for the imputation task. + + consistency_weight : + The loss weight for the consistency task. + + classification_weight : + The loss weight for the classification task. + + n_classes : + The number of classes in the classification task. + + removal_percent : + The percentage of data to be removed during training for simulating missingness. + + increase_factor : + The factor to increase the frequency of missing value occurrences. + + compute_intervals : + Whether to compute time intervals between observations during data processing. + + step_channels : + The number of step channels for the model. + + batch_size : + The batch size for training and evaluating the model. + + epochs : + The number of epochs for training the model. + + dropout : + The dropout rate for the model to prevent overfitting. Default is 0.5. + + patience : + The patience for the early-stopping mechanism. Given a positive integer, the training process will be stopped when the model does not perform better after that number of epochs. Leaving it default as None will disable the early-stopping. + + optimizer : + The optimizer for model training. If not given, will use a default Adam optimizer. + + num_workers : + The number of subprocesses to use for data loading. 0 means data loading will be in the main process, i.e. there won't be subprocesses. + + device : + The device for the model to run on. It can be a string, a :class:torch.device object, or a list of them. If not given, will try to use CUDA devices first (will use the default CUDA device if there are multiple), then CPUs, considering CUDA and CPU are so far the main devices for people to train ML models. If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')], the model will be parallely trained on the multiple devices (so far only support parallel training on CUDA devices). Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. + + saving_path : + The path for automatically saving model checkpoints and tensorboard files (i.e. loss values recorded during training into a tensorboard file). Will not save if not given. + + model_saving_strategy : + The strategy to save model checkpoints. It has to be one of [None, "best", "better", "all"]. No model will be saved when it is set as None. The "best" strategy will only automatically save the best model after the training finished. The "better" strategy will automatically save the model during training whenever the model performs better than in previous epochs. The "all" strategy will save every model after each epoch training. + + verbose : + Whether to print out the training logs during the training process. + + """ + def __init__(self, n_steps: int, n_features: int, From 686b53c0cf31c33f53370966e1f78d38c1760570 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 16:13:21 +0100 Subject: [PATCH 14/22] csai update --- pypots/classification/csai/data.py | 1 - pypots/classification/csai/model.py | 35 ++++++++--------- pypots/data/utils.py | 1 - pypots/imputation/csai/data.py | 59 +++++++++++++++++++---------- pypots/imputation/csai/model.py | 18 ++++----- pypots/nn/modules/csai/backbone.py | 8 ++-- 6 files changed, 69 insertions(+), 53 deletions(-) diff --git a/pypots/classification/csai/data.py b/pypots/classification/csai/data.py index f4a36c48..caeb5005 100644 --- a/pypots/classification/csai/data.py +++ b/pypots/classification/csai/data.py @@ -34,7 +34,6 @@ def __init__(self, replacement_probabilities=replacement_probabilities, normalise_mean=normalise_mean, normalise_std=normalise_std, - impute_only=False, training=training ) \ No newline at end of file diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index 13bcdd97..23488c15 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -236,18 +236,18 @@ def _assemble_input_for_testing(self, data: list) -> dict: def fit( self, train_set, - val_set=None, + val_set= None, file_type: str = "hdf5", - ): + )-> None: # Create dataset self.training_set = DatasetForCSAI( - data=train_set, - file_type=file_type, - return_y=True, - removal_percent=self.removal_percent, - increase_factor=self.increase_factor, - compute_intervals=self.compute_intervals, - ) + data=train_set, + file_type=file_type, + return_y=True, + removal_percent=self.removal_percent, + increase_factor=self.increase_factor, + compute_intervals=self.compute_intervals, + ) self.intervals = self.training_set.intervals self.replacement_probabilities = self.training_set.replacement_probabilities @@ -260,7 +260,6 @@ def fit( shuffle=True, num_workers=self.num_workers, ) - val_loader = None if val_set is not None: val_set = DatasetForCSAI( @@ -271,8 +270,8 @@ def fit( increase_factor=self.increase_factor, compute_intervals=self.compute_intervals, replacement_probabilities=self.replacement_probabilities, - normalise_mean=self.normalise_mean, - normalise_std=self.normalise_std, + normalise_mean=self.mean_set, + normalise_std=self.std_set, training=False ) @@ -312,8 +311,9 @@ def fit( def predict( self, - test_set, - file_type: str = "hdf5"): + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> dict: self.model.eval() test_set = DatasetForCSAI( @@ -324,8 +324,8 @@ def predict( increase_factor=self.increase_factor, compute_intervals=self.compute_intervals, replacement_probabilities=self.replacement_probabilities, - normalise_mean=self.normalise_mean, - normalise_std=self.normalise_std, + normalise_mean=self.mean_set, + normalise_std=self.std_set, training=False ) test_loader = DataLoader( @@ -353,7 +353,8 @@ def predict( def classify( self, test_set, - file_type): + file_type: str = "hdf5", + ): result_dict = self.predict(test_set, file_type) return result_dict['classification'] \ No newline at end of file diff --git a/pypots/data/utils.py b/pypots/data/utils.py index 56b915c1..dd7ef994 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -10,7 +10,6 @@ import benchpots import numpy as np import torch -from sklearn.preprocessing import StandardScaler def turn_data_into_specified_dtype( data: Union[np.ndarray, torch.Tensor, list], diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index b2d282fa..eb8b8a7e 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -11,13 +11,15 @@ import torch from typing import Union import copy +from ...data.utils import parse_delta +from sklearn.preprocessing import StandardScaler def normalize_csai( - data: np.ndarray, - mean: float, - std: float, - compute_intervals: bool = False -) -> : + data, + mean: list = None, + std: list = None, + compute_intervals: bool = False, +): """ Normalize the data based on the given mean and standard deviation, and optionally compute time intervals between observations. @@ -26,19 +28,19 @@ def normalize_csai( data : np.ndarray The input time-series data of shape [n_patients, n_hours, n_variables], which may contain missing values (NaNs). - mean : list of float + mean : list of float, optional The mean values for each variable, used for normalization. If empty, means will be computed from the data. - std : list of float + std : list of float, optional The standard deviation values for each variable, used for normalization. If empty, std values will be computed from the data. compute_intervals : bool, optional, default=False - Whether to compute the time intervals between observations for each variable. + Whether to compute the time intervals between observations for each variable. Returns ------- - data : np.ndarray - The normalized time-series data with the same shape as the input data. + data : torch.Tensor + The normalized time-series data with the same shape as the input data, moved to the specified device. mean_set : np.ndarray The mean values for each variable after normalization, either computed from the data or passed as input. @@ -50,25 +52,33 @@ def normalize_csai( If `compute_intervals` is True, this will return the median time intervals between observations for each variable. """ + # Convert data to numpy array if it is a torch tensor + if isinstance(data, torch.Tensor): + data = data.cpu().numpy() + n_patients, n_hours, n_variables = data.shape # Flatten data for easier computation of statistics - reshaped_data = copy.deepcopy(data).reshape(-1, n_variables) + reshaped_data = data.reshape(-1, n_variables) # Use StandardScaler for normalization scaler = StandardScaler() - if mean is None or std is None: + # Update condition to handle empty list as well + if mean is None or std is None or len(mean) == 0 or len(std) == 0: # Fit the scaler on the data (ignores NaNs during the fitting process) - scaled_data = scaler.fit_transform(reshaped_data) + scaler.fit(reshaped_data) mean_set = scaler.mean_ std_set = scaler.scale_ else: # Use provided mean and std by directly setting them in the scaler scaler.mean_ = np.array(mean) scaler.scale_ = np.array(std) - # Transform data using scaler, which ignores NaNs - scaled_data = scaler.transform(reshaped_data) + mean_set = np.array(mean) + std_set = np.array(std) + + # Transform data using scaler, which ignores NaNs + scaled_data = scaler.transform(reshaped_data) # Reshape back to original shape [n_patients, n_hours, n_variables] normalized_data = scaled_data.reshape(n_patients, n_hours, n_variables) @@ -76,20 +86,20 @@ def normalize_csai( # Optimized interval calculation considering NaNs in each patient if compute_intervals: intervals_list = {} - + for v in range(n_variables): all_intervals = [] # Loop over each patient for p in range(n_patients): # Get non-NaN observation indices for the current patient and variable valid_time_points = np.where(~np.isnan(data[p, :, v]))[0] - + # If the patient has more than one valid observation, compute time intervals if len(valid_time_points) > 1: # Calculate time differences between consecutive observations intervals = np.diff(valid_time_points) all_intervals.extend(intervals) - + # Compute the median interval for the current variable intervals_list[v] = np.median(all_intervals) if all_intervals else np.nan else: @@ -167,7 +177,13 @@ def adjust_probability_vectorized( # Decrease probability when observed count exceeds average count return max(base_prob * (obs_count / avg_count) / increase_factor, 0.0) -def non_uniform_sample(data, removal_percent, pre_replacement_probabilities=None, increase_factor=0.5): +def non_uniform_sample( + data, + removal_percent, + pre_replacement_probabilities=None, + increase_factor=0.5 + ): + """ Process time-series data by randomly removing a certain percentage of observed values based on pre-defined replacement probabilities, and compute the necessary features such as forward and backward deltas, masks, @@ -351,8 +367,8 @@ def __init__(self, if not isinstance(self.data, str): self.normalized_data, self.mean_set, self.std_set, self.intervals = normalize_csai( self.data['X'], - normalise_mean, - normalise_std, + self.normalise_mean, + self.normalise_std, compute_intervals, ) @@ -468,6 +484,7 @@ def _fetch_data_from_file(self, idx: int) -> Iterable: X_ori = self.processed_data['evals'] indicating_mask = self.processed_data['eval_masks'] + if self.return_y: y = self.processed_data['labels'] diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index d76cc923..14b9623d 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -126,7 +126,7 @@ def __init__(self, self.compute_intervals = compute_intervals self.intervals = None - # Initialise empty model + # Initialise model self.model = _BCSAI( self.n_steps, self.n_features, @@ -137,6 +137,7 @@ def __init__(self, self.device, self.intervals, ) + self._send_model_to_given_device() self._print_model_size() @@ -219,11 +220,11 @@ def _assemble_input_for_testing(self, data: list) -> dict: return self._assemble_input_for_validating(data) def fit( - self, - train_set, - val_set, - file_type: str = "hdf5", - )-> None: + self, + train_set, + val_set=None, + file_type: str = "hdf5", + )-> None: self.training_set = DatasetForCSAI( train_set, @@ -246,7 +247,6 @@ def fit( num_workers=self.num_workers, # collate_fn=collate_fn_bidirectional ) - if val_set is not None: val_set = DatasetForCSAI( val_set, @@ -255,7 +255,7 @@ def fit( file_type, self.removal_percent, self.increase_factor, - False, + self.compute_intervals, self.replacement_probabilities, self.mean_set, self.std_set, @@ -309,7 +309,7 @@ def predict( file_type, self.removal_percent, self.increase_factor, - False, + self.compute_intervals, self.replacement_probabilities, self.mean_set, self.std_set, diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py index 608440be..6c6327f0 100644 --- a/pypots/nn/modules/csai/backbone.py +++ b/pypots/nn/modules/csai/backbone.py @@ -93,10 +93,11 @@ class BackboneCSAI(nn.Module): def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): super(BackboneCSAI, self).__init__() + if medians_df is not None: self.medians_tensor = torch.tensor(list(medians_df.values())).float() else: - self.medians_tensor = None + self.medians_tensor = torch.zeros(n_features).float() self.n_steps = n_steps self.step_channels = step_channels @@ -131,10 +132,9 @@ def forward(self, x, mask, deltas, last_obs, h=None): # Get dimensionality [B, _, _] = x.shape - if self.medians_tensor is not None: - medians_t = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(self.device) + medians = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(self.device) - decay_factor = self.weighted_obs(deltas - medians_t.unsqueeze(1)) + decay_factor = self.weighted_obs(deltas - medians.unsqueeze(1)) if h == None: data_last_obs = self.input_projection(last_obs.permute(0, 2, 1)).permute(0, 2, 1) From 0f2dea82e234532ad9e52668325ee9e343106fac Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 16:39:52 +0100 Subject: [PATCH 15/22] csai update --- pypots/classification/csai/core.py | 7 ++----- pypots/classification/csai/model.py | 2 -- pypots/imputation/csai/core.py | 10 +--------- pypots/imputation/csai/model.py | 2 -- pypots/nn/modules/csai/backbone.py | 13 +++++-------- 5 files changed, 8 insertions(+), 26 deletions(-) diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py index f1ca08e0..5b21e363 100644 --- a/pypots/classification/csai/core.py +++ b/pypots/classification/csai/core.py @@ -46,9 +46,7 @@ def __init__( n_classes: int, step_channels: int, dropout: float = 0.5, - intervals: list = None, - device = None, - + intervals=None, ): super().__init__() self.n_steps = n_steps @@ -60,10 +58,9 @@ def __init__( self.n_classes = n_classes self.step_channels = step_channels self.intervals = intervals - self.device = device # create models - self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, self.device) + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) self.f_classifier = nn.Linear(self.rnn_hidden_size, n_classes) self.b_classifier = nn.Linear(self.rnn_hidden_size, n_classes) self.imputer = nn.Linear(self.rnn_hidden_size, n_features) diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index 23488c15..40998533 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -149,7 +149,6 @@ def __init__(self, step_channels=self.step_channels, dropout=self.dropout, intervals=self.intervals, - device=self.device, ) self._send_model_to_given_device() @@ -293,7 +292,6 @@ def fit( step_channels=self.step_channels, dropout=self.dropout, intervals=self.intervals, - device=self.device, ) self._send_model_to_given_device() self._print_model_size() diff --git a/pypots/imputation/csai/core.py b/pypots/imputation/csai/core.py index aa7afe9e..fc9ea6f9 100644 --- a/pypots/imputation/csai/core.py +++ b/pypots/imputation/csai/core.py @@ -37,9 +37,6 @@ class _BCSAI(nn.Module): model : the underlying BackboneBCSAI model that handles forward and backward pass imputation - device : - the device (CPU/GPU) used for model computations - Parameters ---------- n_steps : @@ -63,9 +60,6 @@ class _BCSAI(nn.Module): imputation_weight : weight assigned to the reconstruction loss - device : - the device (CPU/GPU) for running the model - Notes ----- BCSAI is a bidirectional imputation model that uses forward and backward GRU cells to handle time-series data. It computes consistency and reconstruction losses to improve imputation accuracy. During training, the forward and backward reconstructions are combined, and losses are used to update the model. In evaluation mode, the model also outputs original data and indicating masks for further analysis. @@ -78,7 +72,6 @@ def __init__(self, step_channels, consistency_weight, imputation_weight, - device=None, intervals=None, ): super().__init__() @@ -89,9 +82,8 @@ def __init__(self, self.intervals = intervals self.consistency_weight = consistency_weight self.imputation_weight = imputation_weight - self.device = device - self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals, device) + self.model = BackboneBCSAI(n_steps, n_features, rnn_hidden_size, step_channels, intervals) def forward(self, inputs:dict, training:bool = True) -> dict: ( diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index 14b9623d..8afd9d23 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -134,7 +134,6 @@ def __init__(self, self.step_channels, self.consistency_weight, self.imputation_weight, - self.device, self.intervals, ) @@ -277,7 +276,6 @@ def fit( self.step_channels, self.consistency_weight, self.imputation_weight, - self.device, self.intervals, ) diff --git a/pypots/nn/modules/csai/backbone.py b/pypots/nn/modules/csai/backbone.py index 6c6327f0..57600db9 100644 --- a/pypots/nn/modules/csai/backbone.py +++ b/pypots/nn/modules/csai/backbone.py @@ -87,11 +87,9 @@ class BackboneCSAI(nn.Module): medians_df : dataframe of median values for each feature, optional - device : - the device (CPU/GPU) for running the model, optional """ - def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None): super(BackboneCSAI, self).__init__() if medians_df is not None: @@ -116,7 +114,6 @@ def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_ self.output_projection1 = Conv1dWithInit(self.step_channels, self.hidden_size, 1) self.output_projection2 = Conv1dWithInit(self.n_steps*2, 1, 1) self.time_layer = TorchTransformerEncoder(channels=self.step_channels) - self.device = device self.reset_parameters() @@ -132,7 +129,7 @@ def forward(self, x, mask, deltas, last_obs, h=None): # Get dimensionality [B, _, _] = x.shape - medians = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(self.device) + medians = self.medians_tensor.unsqueeze(0).repeat(B, 1).to(x.device) decay_factor = self.weighted_obs(deltas - medians.unsqueeze(1)) @@ -192,11 +189,11 @@ def forward(self, x, mask, deltas, last_obs, h=None): class BackboneBCSAI(nn.Module): - def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None, device=None): + def __init__(self, n_steps, n_features, rnn_hidden_size, step_channels, medians_df=None): super(BackboneBCSAI, self).__init__() - self.model_f = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) - self.model_b = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df, device) + self.model_f = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df) + self.model_b = BackboneCSAI(n_steps, n_features, rnn_hidden_size, step_channels, medians_df) def forward(self, xdata): From 15f7f652c49231f2fc92da320a7d654cf1534e39 Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 16:45:45 +0100 Subject: [PATCH 16/22] csai update --- pypots/classification/csai/core.py | 2 +- pypots/classification/csai/model.py | 50 +++++++++++++++-------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py index 5b21e363..c372a331 100644 --- a/pypots/classification/csai/core.py +++ b/pypots/classification/csai/core.py @@ -97,7 +97,7 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # if in training mode, return results with losses if training: - criterion = DiceBCELoss().to(self.device) + criterion = DiceBCELoss().to(imputed_data.device) results["consistency_loss"] = consistency_loss results["reconstruction_loss"] = reconstruction_loss # print(inputs["labels"].unsqueeze(1)) diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index 40998533..220c0eda 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -90,28 +90,30 @@ class CSAI(BaseNNClassifier): """ - def __init__(self, - n_steps: int, - n_features: int, - rnn_hidden_size: int, - imputation_weight: float, - consistency_weight: float, - classification_weight: float, - n_classes: int, - removal_percent: int, - increase_factor: float, - compute_intervals: bool, - step_channels:int, - batch_size: int, - epochs: int, - dropout: float = 0.5, - patience: Union[int, None] = None, - optimizer: Optimizer = Adam(), - num_workers: int = 0, - device: Optional[Union[str, torch.device, list]] = None, - saving_path: str = None, - model_saving_strategy: Union[str, None] = "best", - verbose: bool = True): + def __init__( + self, + n_steps: int, + n_features: int, + rnn_hidden_size: int, + imputation_weight: float, + consistency_weight: float, + classification_weight: float, + n_classes: int, + removal_percent: int, + increase_factor: float, + compute_intervals: bool, + step_channels:int, + batch_size: int, + epochs: int, + dropout: float = 0.5, + patience: Union[int, None] = None, + optimizer: Optimizer = Adam(), + num_workers: int = 0, + device: Optional[Union[str, torch.device, list]] = None, + saving_path: str = None, + model_saving_strategy: Union[str, None] = "best", + verbose: bool = True + ): super().__init__( n_classes, batch_size, @@ -121,7 +123,8 @@ def __init__(self, device, saving_path, model_saving_strategy, - verbose) + verbose, + ) self.n_steps = n_steps self.n_features = n_features @@ -134,7 +137,6 @@ def __init__(self, self.step_channels = step_channels self.compute_intervals = compute_intervals self.dropout = dropout - self.device = device self.intervals = None # Initialise empty model From 92f6a639b04f1a5510e0117a4e393fc3618814ec Mon Sep 17 00:00:00 2001 From: LINGLONGQIAN <15869023990@163.com> Date: Fri, 4 Oct 2024 17:16:21 +0100 Subject: [PATCH 17/22] csai update --- pypots/classification/csai/core.py | 44 ++++++++++++++++------------- pypots/classification/csai/model.py | 8 ++---- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/pypots/classification/csai/core.py b/pypots/classification/csai/core.py index c372a331..30f052f6 100644 --- a/pypots/classification/csai/core.py +++ b/pypots/classification/csai/core.py @@ -11,27 +11,27 @@ from ...nn.modules.csai import BackboneBCSAI -class DiceBCELoss(nn.Module): - def __init__(self, weight=None, size_average=True): - super(DiceBCELoss, self).__init__() - self.bcelogits = nn.BCEWithLogitsLoss() +# class DiceBCELoss(nn.Module): +# def __init__(self, weight=None, size_average=True): +# super(DiceBCELoss, self).__init__() +# self.bcelogits = nn.BCEWithLogitsLoss() - def forward(self, y_score, y_out, targets, smooth=1): +# def forward(self, y_score, y_out, targets, smooth=1): - #comment out if your model contains a sigmoid or equivalent activation layer - # inputs = F.sigmoid(inputs) +# #comment out if your model contains a sigmoid or equivalent activation layer +# # inputs = F.sigmoid(inputs) - #flatten label and prediction tensors - BCE = self.bcelogits(y_out, targets) +# #flatten label and prediction tensors +# BCE = self.bcelogits(y_out, targets) - y_score = y_score.view(-1) - targets = targets.view(-1) - intersection = (y_score * targets).sum() - dice_loss = 1 - (2.*intersection + smooth)/(y_score.sum() + targets.sum() + smooth) +# y_score = y_score.view(-1) +# targets = targets.view(-1) +# intersection = (y_score * targets).sum() +# dice_loss = 1 - (2.*intersection + smooth)/(y_score.sum() + targets.sum() + smooth) - Dice_BCE = BCE + dice_loss +# Dice_BCE = BCE + dice_loss - return BCE, Dice_BCE +# return BCE, Dice_BCE class _BCSAI(nn.Module): @@ -85,9 +85,11 @@ def forward(self, inputs: dict, training: bool = True) -> dict: f_logits = self.f_classifier(self.dropout(f_hidden_states)) b_logits = self.b_classifier(self.dropout(b_hidden_states)) - f_prediction = torch.sigmoid(f_logits) - b_prediction = torch.sigmoid(b_logits) + # f_prediction = torch.sigmoid(f_logits) + # b_prediction = torch.sigmoid(b_logits) + f_prediction = torch.softmax(f_logits, dim=1) + b_prediction = torch.softmax(b_logits, dim=1) classification_pred = (f_prediction + b_prediction) / 2 results = { @@ -97,12 +99,14 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # if in training mode, return results with losses if training: - criterion = DiceBCELoss().to(imputed_data.device) + # criterion = DiceBCELoss().to(imputed_data.device) results["consistency_loss"] = consistency_loss results["reconstruction_loss"] = reconstruction_loss # print(inputs["labels"].unsqueeze(1)) - f_classification_loss, _ = criterion(f_prediction, f_logits, inputs["labels"].unsqueeze(1).float()) - b_classification_loss, _ = criterion(b_prediction, b_logits, inputs["labels"].unsqueeze(1).float()) + f_classification_loss = F.nll_loss(torch.log(f_prediction), inputs["labels"]) + b_classification_loss = F.nll_loss(torch.log(b_prediction), inputs["labels"]) + # f_classification_loss, _ = criterion(f_prediction, f_logits, inputs["labels"].unsqueeze(1).float()) + # b_classification_loss, _ = criterion(b_prediction, b_logits, inputs["labels"].unsqueeze(1).float()) classification_loss = (f_classification_loss + b_classification_loss) loss = ( diff --git a/pypots/classification/csai/model.py b/pypots/classification/csai/model.py index 220c0eda..fb9bd5b5 100644 --- a/pypots/classification/csai/model.py +++ b/pypots/classification/csai/model.py @@ -209,8 +209,8 @@ def _assemble_input_for_testing(self, data: list) -> dict: back_missing_mask, back_deltas, back_last_obs, - # X_ori, - # indicating_mask, + X_ori, + indicating_mask, ) = self._send_data_to_given_device(sample) # assemble input data @@ -273,8 +273,6 @@ def fit( replacement_probabilities=self.replacement_probabilities, normalise_mean=self.mean_set, normalise_std=self.std_set, - training=False - ) val_loader = DataLoader( val_set, @@ -326,7 +324,7 @@ def predict( replacement_probabilities=self.replacement_probabilities, normalise_mean=self.mean_set, normalise_std=self.std_set, - training=False + training=False, ) test_loader = DataLoader( test_set, From 4ed4ef160b6ca4e5ab938d4201597de12ce5083e Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Fri, 4 Oct 2024 17:59:36 +0100 Subject: [PATCH 18/22] Fixed conflicts --- csai.py | 175 -------- output_logs/Classification.out | 636 ------------------------------ output_logs/Imputation.out | 605 ---------------------------- pypots/classification/__init__.py | 2 - pypots/classification/base.py | 4 +- pypots/data/utils.py | 2 +- pypots/imputation/__init__.py | 2 - pypots/imputation/csai/data.py | 1 + pypots/imputation/csai/model.py | 1 + run_pypots.sh | 25 -- 10 files changed, 4 insertions(+), 1449 deletions(-) delete mode 100644 csai.py delete mode 100644 output_logs/Classification.out delete mode 100644 output_logs/Imputation.out delete mode 100644 run_pypots.sh diff --git a/csai.py b/csai.py deleted file mode 100644 index d987955d..00000000 --- a/csai.py +++ /dev/null @@ -1,175 +0,0 @@ -import pickle -import logging -import numpy as np -import torch -from sklearn.metrics import ( - confusion_matrix, roc_auc_score, accuracy_score, precision_recall_curve, - balanced_accuracy_score, recall_score, precision_score, f1_score, - PrecisionRecallDisplay, ConfusionMatrixDisplay, roc_curve, RocCurveDisplay -) -from pypots.classification import CSAI as classify -from pypots.imputation import CSAI as imputer -from pypots.optim import Adam -from pypots.utils.metrics import calc_mae, calc_mre, calc_binary_classification_metrics -import json -# Setup logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def calculate_performance_brits(y, y_score, y_pred, pr_display=False, cm_display=False, roc_display=False): - - acc = accuracy_score(y, y_pred) - auc = roc_auc_score(y, y_score) - prec_macro = precision_score(y, y_pred, average='macro') - recall_macro = recall_score(y, y_pred, average='macro') - f1_macro = f1_score(y, y_pred, average='macro') - bal_acc = balanced_accuracy_score(y, y_pred) - - if pr_display: - prec, recall, _ = precision_recall_curve(y, y_score) - PrecisionRecallDisplay(precision=prec, recall=recall).plot() - - if cm_display: - cm = confusion_matrix(y, y_pred) - ConfusionMatrixDisplay(confusion_matrix=cm).plot() - - if roc_display: - fpr, tpr, _ = roc_curve(y, y_score) - RocCurveDisplay(fpr=fpr, tpr=tpr).plot() - - return acc, auc, prec_macro, recall_macro, f1_macro, bal_acc - - -def load_data(kfold_data_path, kfold_label_path, fold=2): - - kfold_data = pickle.load(open(kfold_data_path, 'rb')) - kfold_label = pickle.load(open(kfold_label_path, 'rb')) - - return { - "train": (kfold_data[fold][0], kfold_label[fold][0]), - "valid": (kfold_data[fold][1], kfold_label[fold][1]), - "test": (kfold_data[fold][2], kfold_label[fold][2]) - } - - -def get_csai_model(mode, **kwargs): - - if mode == 'classify': - return classify(**kwargs) - else: - return imputer(**kwargs) - - -def run_pypots(mode, data): - - - dataset_for_training = {"X": data["train"][0], "y": data["train"][1]} - dataset_for_validating = {"X": data["valid"][0], "y": data["valid"][1]} - dataset_for_testing = {"X": data["test"][0], "y": data["test"][1]} - - model_params = { - "n_steps": 48, - "n_features": 35, - "rnn_hidden_size": 108, - "imputation_weight": 0.3, - "consistency_weight": 0.1, - "removal_percent": 10, - "increase_factor": 0.1, - "compute_intervals": True, - "step_channels": 512, - "batch_size": 64, - "epochs": 200, - "patience": 5, - "optimizer": Adam(lr=0.0005, weight_decay=0.00001), - "num_workers": 0, - "device": torch.device("cuda" if torch.cuda.is_available() else "cpu"), - "saving_path": '/scratch/users/k23031260/PyPOTS', - "model_saving_strategy": "best", - "verbose": True - } - - if mode == 'classify': - model_params["classification_weight"] = 1 - model_params["n_classes"] = 1 - - csai_model = get_csai_model(mode, **model_params) - - logger.info("\n Training..................................... \n") - csai_model.fit(dataset_for_training, dataset_for_validating) - - logger.info("\n Testing...................................... \n") - csai_results = csai_model.predict(dataset_for_testing) - - if mode == 'classify': - classification_predictions = csai_results['classification'] - metrics = calc_binary_classification_metrics(classification_predictions, dataset_for_testing["y"]) - - y_pred = np.round(classification_predictions) - acc, auc, prec_macro, recall_macro, f1_macro, bal_acc = calculate_performance_brits( - dataset_for_testing["y"], classification_predictions, y_pred - ) - - return { - "PyPots_evaluation": { - "ROC_AUC": metrics['roc_auc'], - "PR_AUC": metrics['pr_auc'], - "F1": metrics['f1'], - "Precision": metrics['precision'], - "Recall": metrics['recall'] - }, - "CSAI_evaluation": { - "ACC": acc, - "AUC": auc, - "Prec_Macro": prec_macro, - "Recall_Macro": recall_macro, - "F1_Macro": f1_macro, - "Bal_ACC": bal_acc - } - } - - else: - - original_data = csai_results["X_ori"] - indicating_mask = csai_results["indicating_mask"] - imputed_data = csai_results['imputation'] - - mae = calc_mae(imputed_data, original_data, indicating_mask) - mre = calc_mre(imputed_data, original_data, indicating_mask) - - return { - "MAE": mae, - "MRE": mre - } - - - -if __name__ == "__main__": - - mode = 'impute' - data_path = "/scratch/users/k23031260/data/physionet/data_nan.pkl" - label_path = "/scratch/users/k23031260/data/physionet/label.pkl" - - all_fold_results = [] - for fold in range(5): - logger.info(f"\n Fold {fold} started ...................................... \n") - data = load_data(data_path, label_path, fold) - results = run_pypots(mode, data) - all_fold_results.append(results) - logger.info(f"\n Fold {fold} ended ........................................ \n") - logger.info(f"\n Fold {fold} Results: \n{json.dumps(results, indent=4)} \n") - - logger.info(f"\n Average performance across all folds.......................... \n") - if mode == 'impute': - avg_result = {metric: np.mean([fold[metric] for fold in all_fold_results]) for metric in all_fold_results[0].keys()} - logger.info(f"\n Imputation performance: \n {avg_result} \n") - else: - PyPots_evaluation = [{k: v for k,v in fold_result['PyPots_evaluation'].items()} for fold_result in all_fold_results] - CSAI_evaluation = [{k: v for k,v in fold_result['CSAI_evaluation'].items()} for fold_result in all_fold_results] - - avg_result = {metric: np.mean([fold[metric] for fold in PyPots_evaluation]) for metric in PyPots_evaluation[0].keys()} - logger.info(f"\n Classification performance using Pypots evaluation: \n {avg_result} \n") - - avg_result = {metric: np.mean([fold[metric] for fold in CSAI_evaluation]) for metric in CSAI_evaluation[0].keys()} - logger.info(f"\n Classification performance using CSAI evaluation: \n {avg_result} \n") - - diff --git a/output_logs/Classification.out b/output_logs/Classification.out deleted file mode 100644 index 06fd4f28..00000000 --- a/output_logs/Classification.out +++ /dev/null @@ -1,636 +0,0 @@ -Lmod has detected the following error: The following module(s) are unknown: -"anaconda3/2021.05-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "anaconda3/2021.05-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Lmod has detected the following error: The following module(s) are unknown: -"cuda/11.1.1-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "cuda/11.1.1-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Lmod has detected the following error: The following module(s) are unknown: -"cudnn/8.0.5.39-11.1-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "cudnn/8.0.5.39-11.1-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Fri Sep 13 12:02:41 2024 -+---------------------------------------------------------------------------------------+ -| NVIDIA-SMI 535.183.01 Driver Version: 535.183.01 CUDA Version: 12.2 | -|-----------------------------------------+----------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+======================+======================| -| 0 NVIDIA A100-SXM4-40GB On | 00000000:17:00.0 Off | 0 | -| N/A 33C P0 53W / 400W | 0MiB / 40960MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+----------------------+----------------------+ - -+---------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=======================================================================================| -| No running processes found | -+---------------------------------------------------------------------------------------+ -/var/lib/slurm/slurmd/job21076806/slurm_script: line 18: activate: No such file or directory -2024-09-13 12:04:04.605532: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. -2024-09-13 12:04:08.365187: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. -To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. -2024-09-13 12:04:30.558164: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT -INFO:__main__: - Fold 0 started ...................................... - -2024-09-13 12:05:06 [INFO]: Using the given device: cuda -2024-09-13 12:05:06 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120506 -2024-09-13 12:05:06 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120506/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:05:16 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 -2024-09-13 12:05:32 [INFO]: Epoch 001 - training loss: 19.8551, validation loss: 17.4215, validation AUC: 0.4966053925763644 -2024-09-13 12:05:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch1_loss17.421496171217697.pypots -2024-09-13 12:05:38 [INFO]: Epoch 002 - training loss: 15.9493, validation loss: 14.9795, validation AUC: 0.6574584374536451 -2024-09-13 12:05:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch2_loss14.979489913353554.pypots -2024-09-13 12:05:44 [INFO]: Epoch 003 - training loss: 14.2548, validation loss: 13.8747, validation AUC: 0.7760471936010224 -2024-09-13 12:05:44 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch3_loss13.874650735121508.pypots -2024-09-13 12:05:50 [INFO]: Epoch 004 - training loss: 13.3428, validation loss: 13.1115, validation AUC: 0.8137358938372186 -2024-09-13 12:05:50 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch4_loss13.111526049100435.pypots -2024-09-13 12:05:56 [INFO]: Epoch 005 - training loss: 12.6785, validation loss: 12.5040, validation AUC: 0.8340807174887893 -2024-09-13 12:05:56 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch5_loss12.50400081047645.pypots -2024-09-13 12:06:02 [INFO]: Epoch 006 - training loss: 12.1627, validation loss: 12.0352, validation AUC: 0.8508198404819771 -2024-09-13 12:06:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch6_loss12.035209068885216.pypots -2024-09-13 12:06:09 [INFO]: Epoch 007 - training loss: 11.6513, validation loss: 11.5828, validation AUC: 0.8498499526466526 -2024-09-13 12:06:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch7_loss11.582774309011606.pypots -2024-09-13 12:06:15 [INFO]: Epoch 008 - training loss: 11.3052, validation loss: 11.2321, validation AUC: 0.8583621447072651 -2024-09-13 12:06:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch8_loss11.232126529400166.pypots -2024-09-13 12:06:21 [INFO]: Epoch 009 - training loss: 10.9663, validation loss: 10.9465, validation AUC: 0.8650828968838074 -2024-09-13 12:06:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch9_loss10.946472608126127.pypots -2024-09-13 12:06:27 [INFO]: Epoch 010 - training loss: 10.7618, validation loss: 10.7389, validation AUC: 0.8638049270302034 -2024-09-13 12:06:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch10_loss10.738909794734074.pypots -2024-09-13 12:06:33 [INFO]: Epoch 011 - training loss: 10.5480, validation loss: 10.6198, validation AUC: 0.8506258629149124 -2024-09-13 12:06:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch11_loss10.619767115666317.pypots -2024-09-13 12:06:39 [INFO]: Epoch 012 - training loss: 10.3870, validation loss: 10.4472, validation AUC: 0.8605643606157076 -2024-09-13 12:06:39 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch12_loss10.44724391056941.pypots -2024-09-13 12:06:45 [INFO]: Epoch 013 - training loss: 10.2086, validation loss: 10.3096, validation AUC: 0.8556350483232352 -2024-09-13 12:06:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch13_loss10.309552999643179.pypots -2024-09-13 12:06:51 [INFO]: Epoch 014 - training loss: 10.0798, validation loss: 10.1739, validation AUC: 0.8625269571765994 -2024-09-13 12:06:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch14_loss10.173876102154072.pypots -2024-09-13 12:06:57 [INFO]: Epoch 015 - training loss: 9.9415, validation loss: 10.0565, validation AUC: 0.8666461278654479 -2024-09-13 12:06:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch15_loss10.05653770153339.pypots -2024-09-13 12:07:03 [INFO]: Epoch 016 - training loss: 9.8283, validation loss: 9.9481, validation AUC: 0.863325688335102 -2024-09-13 12:07:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch16_loss9.94806275000939.pypots -2024-09-13 12:07:09 [INFO]: Epoch 017 - training loss: 9.6922, validation loss: 9.9070, validation AUC: 0.8664635607435046 -2024-09-13 12:07:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch17_loss9.907046978290264.pypots -2024-09-13 12:07:15 [INFO]: Epoch 018 - training loss: 9.6060, validation loss: 9.8097, validation AUC: 0.8598911443535412 -2024-09-13 12:07:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch18_loss9.809694950397198.pypots -2024-09-13 12:07:21 [INFO]: Epoch 019 - training loss: 9.5409, validation loss: 9.7606, validation AUC: 0.869989388286037 -2024-09-13 12:07:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch19_loss9.760592313913198.pypots -2024-09-13 12:07:27 [INFO]: Epoch 020 - training loss: 9.4575, validation loss: 9.7574, validation AUC: 0.8667716427617842 -2024-09-13 12:07:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch20_loss9.757406088022085.pypots -2024-09-13 12:07:34 [INFO]: Epoch 021 - training loss: 9.3948, validation loss: 9.6642, validation AUC: 0.8601535845913348 -2024-09-13 12:07:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch21_loss9.664151485149677.pypots -2024-09-13 12:07:40 [INFO]: Epoch 022 - training loss: 9.3306, validation loss: 9.6316, validation AUC: 0.8560458243476078 -2024-09-13 12:07:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch22_loss9.631555557250977.pypots -2024-09-13 12:07:46 [INFO]: Epoch 023 - training loss: 9.3060, validation loss: 9.6051, validation AUC: 0.854722212713518 -2024-09-13 12:07:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch23_loss9.605094029353214.pypots -2024-09-13 12:07:52 [INFO]: Epoch 024 - training loss: 9.2244, validation loss: 9.6119, validation AUC: 0.854699391823275 -2024-09-13 12:07:57 [INFO]: Epoch 025 - training loss: 9.1648, validation loss: 9.5212, validation AUC: 0.8517554969819372 -2024-09-13 12:07:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch25_loss9.521164967463566.pypots -2024-09-13 12:08:03 [INFO]: Epoch 026 - training loss: 9.1255, validation loss: 9.4918, validation AUC: 0.8550873469574047 -2024-09-13 12:08:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch26_loss9.491754311781664.pypots -2024-09-13 12:08:10 [INFO]: Epoch 027 - training loss: 9.1028, validation loss: 9.4686, validation AUC: 0.8524743550245896 -2024-09-13 12:08:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch27_loss9.468608709482046.pypots -2024-09-13 12:08:16 [INFO]: Epoch 028 - training loss: 9.0499, validation loss: 9.4682, validation AUC: 0.8566277570488026 -2024-09-13 12:08:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch28_loss9.468163930452787.pypots -2024-09-13 12:08:22 [INFO]: Epoch 029 - training loss: 9.0216, validation loss: 9.5157, validation AUC: 0.8501238033295678 -2024-09-13 12:08:28 [INFO]: Epoch 030 - training loss: 8.9307, validation loss: 9.4084, validation AUC: 0.8450119239151519 -2024-09-13 12:08:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI_epoch30_loss9.408410145686222.pypots -2024-09-13 12:08:34 [INFO]: Epoch 031 - training loss: 8.8653, validation loss: 9.4911, validation AUC: 0.8337612250253883 -2024-09-13 12:08:40 [INFO]: Epoch 032 - training loss: 8.8738, validation loss: 9.4475, validation AUC: 0.8396604251531853 -2024-09-13 12:08:46 [INFO]: Epoch 033 - training loss: 8.8115, validation loss: 9.4653, validation AUC: 0.8333048072205297 -2024-09-13 12:08:52 [INFO]: Epoch 034 - training loss: 8.7482, validation loss: 9.4411, validation AUC: 0.8224877052453815 -2024-09-13 12:08:58 [INFO]: Epoch 035 - training loss: 8.7117, validation loss: 9.4449, validation AUC: 0.8189504672577277 -2024-09-13 12:08:58 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:08:58 [INFO]: Finished training. The best model is from epoch#30. -2024-09-13 12:08:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120506/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 0 ended ........................................ - -INFO:__main__: - Fold 0 Results: -{ - "PyPots_evaluation": { - "ROC_AUC": 0.7999868247694335, - "PR_AUC": 0.40923534439915027, - "F1": 0.326797385620915, - "Precision": 0.5813953488372093, - "Recall": 0.22727272727272727 - }, - "CSAI_evaluation": { - "ACC": 0.87125, - "AUC": 0.7999868247694335, - "Prec_Macro": 0.7345550059905994, - "Recall_Macro": 0.600592885375494, - "F1_Macro": 0.627807815132503, - "Bal_ACC": 0.600592885375494 - } -} - -INFO:__main__: - Fold 1 started ...................................... - -2024-09-13 12:09:02 [INFO]: Using the given device: cuda -2024-09-13 12:09:02 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120902 -2024-09-13 12:09:02 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T120902/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:09:11 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 -2024-09-13 12:09:17 [INFO]: Epoch 001 - training loss: 19.8380, validation loss: 17.5980, validation AUC: 0.5338787497377806 -2024-09-13 12:09:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch1_loss17.59797653785119.pypots -2024-09-13 12:09:23 [INFO]: Epoch 002 - training loss: 15.9641, validation loss: 15.1263, validation AUC: 0.6651488789348339 -2024-09-13 12:09:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch2_loss15.12628489274245.pypots -2024-09-13 12:09:29 [INFO]: Epoch 003 - training loss: 14.3313, validation loss: 14.0248, validation AUC: 0.7368674341983489 -2024-09-13 12:09:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch3_loss14.024825316209059.pypots -2024-09-13 12:09:35 [INFO]: Epoch 004 - training loss: 13.4015, validation loss: 13.3206, validation AUC: 0.7668036377546614 -2024-09-13 12:09:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch4_loss13.320613787724422.pypots -2024-09-13 12:09:41 [INFO]: Epoch 005 - training loss: 12.7353, validation loss: 12.7163, validation AUC: 0.7872012241019756 -2024-09-13 12:09:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch5_loss12.716337717496431.pypots -2024-09-13 12:09:47 [INFO]: Epoch 006 - training loss: 12.1400, validation loss: 12.1862, validation AUC: 0.8018608324387023 -2024-09-13 12:09:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch6_loss12.186163535484901.pypots -2024-09-13 12:09:54 [INFO]: Epoch 007 - training loss: 11.6714, validation loss: 11.7651, validation AUC: 0.8000839102160688 -2024-09-13 12:09:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch7_loss11.765064973097582.pypots -2024-09-13 12:10:00 [INFO]: Epoch 008 - training loss: 11.2592, validation loss: 11.4473, validation AUC: 0.7982823085181209 -2024-09-13 12:10:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch8_loss11.447268706101637.pypots -2024-09-13 12:10:06 [INFO]: Epoch 009 - training loss: 10.9614, validation loss: 11.1697, validation AUC: 0.8008366342131565 -2024-09-13 12:10:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch9_loss11.169659467843863.pypots -2024-09-13 12:10:12 [INFO]: Epoch 010 - training loss: 10.7048, validation loss: 10.9466, validation AUC: 0.8111032959439283 -2024-09-13 12:10:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch10_loss10.946574284480167.pypots -2024-09-13 12:10:18 [INFO]: Epoch 011 - training loss: 10.4913, validation loss: 10.7678, validation AUC: 0.8016510568985304 -2024-09-13 12:10:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch11_loss10.767754774827223.pypots -2024-09-13 12:10:24 [INFO]: Epoch 012 - training loss: 10.2789, validation loss: 10.5824, validation AUC: 0.8090795789681512 -2024-09-13 12:10:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch12_loss10.582400542039137.pypots -2024-09-13 12:10:30 [INFO]: Epoch 013 - training loss: 10.1285, validation loss: 10.4989, validation AUC: 0.8066116314367157 -2024-09-13 12:10:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch13_loss10.498880899869478.pypots -2024-09-13 12:10:36 [INFO]: Epoch 014 - training loss: 10.0109, validation loss: 10.3582, validation AUC: 0.806290798257629 -2024-09-13 12:10:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch14_loss10.358169629023624.pypots -2024-09-13 12:10:42 [INFO]: Epoch 015 - training loss: 9.8685, validation loss: 10.2522, validation AUC: 0.8099557003418107 -2024-09-13 12:10:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch15_loss10.252222941471981.pypots -2024-09-13 12:10:48 [INFO]: Epoch 016 - training loss: 9.7759, validation loss: 10.2081, validation AUC: 0.8008119547378423 -2024-09-13 12:10:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch16_loss10.208140519949106.pypots -2024-09-13 12:10:54 [INFO]: Epoch 017 - training loss: 9.6913, validation loss: 10.1220, validation AUC: 0.7995656412344674 -2024-09-13 12:10:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch17_loss10.121957192054161.pypots -2024-09-13 12:11:00 [INFO]: Epoch 018 - training loss: 9.6082, validation loss: 10.0452, validation AUC: 0.7976900011105763 -2024-09-13 12:11:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch18_loss10.04515633216271.pypots -2024-09-13 12:11:07 [INFO]: Epoch 019 - training loss: 9.5294, validation loss: 9.9853, validation AUC: 0.8068707659275163 -2024-09-13 12:11:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch19_loss9.985262870788574.pypots -2024-09-13 12:11:13 [INFO]: Epoch 020 - training loss: 9.4589, validation loss: 10.0217, validation AUC: 0.8058218882266563 -2024-09-13 12:11:19 [INFO]: Epoch 021 - training loss: 9.3960, validation loss: 9.9241, validation AUC: 0.8132750897715915 -2024-09-13 12:11:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch21_loss9.924118335430439.pypots -2024-09-13 12:11:25 [INFO]: Epoch 022 - training loss: 9.3173, validation loss: 9.8431, validation AUC: 0.8125100260368465 -2024-09-13 12:11:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch22_loss9.843071497403658.pypots -2024-09-13 12:11:31 [INFO]: Epoch 023 - training loss: 9.2848, validation loss: 9.8281, validation AUC: 0.8046989720998532 -2024-09-13 12:11:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch23_loss9.828058609595665.pypots -2024-09-13 12:11:37 [INFO]: Epoch 024 - training loss: 9.2376, validation loss: 9.8259, validation AUC: 0.8040943249546514 -2024-09-13 12:11:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch24_loss9.825871027432955.pypots -2024-09-13 12:11:43 [INFO]: Epoch 025 - training loss: 9.1740, validation loss: 9.7791, validation AUC: 0.8060193240291711 -2024-09-13 12:11:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch25_loss9.779062491196852.pypots -2024-09-13 12:11:49 [INFO]: Epoch 026 - training loss: 9.1133, validation loss: 9.7563, validation AUC: 0.8030207677784771 -2024-09-13 12:11:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch26_loss9.756314864525429.pypots -2024-09-13 12:11:55 [INFO]: Epoch 027 - training loss: 9.0885, validation loss: 9.8235, validation AUC: 0.804402818396081 -2024-09-13 12:12:01 [INFO]: Epoch 028 - training loss: 9.0216, validation loss: 9.7160, validation AUC: 0.8045015362973382 -2024-09-13 12:12:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch28_loss9.71602880037748.pypots -2024-09-13 12:12:07 [INFO]: Epoch 029 - training loss: 8.9750, validation loss: 9.7326, validation AUC: 0.8029467293525341 -2024-09-13 12:12:13 [INFO]: Epoch 030 - training loss: 8.9495, validation loss: 9.7374, validation AUC: 0.8016140376855588 -2024-09-13 12:12:19 [INFO]: Epoch 031 - training loss: 8.9167, validation loss: 9.6785, validation AUC: 0.8000839102160688 -2024-09-13 12:12:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch31_loss9.678529225862944.pypots -2024-09-13 12:12:25 [INFO]: Epoch 032 - training loss: 8.8744, validation loss: 9.6473, validation AUC: 0.8043904786584237 -2024-09-13 12:12:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch32_loss9.647251716026894.pypots -2024-09-13 12:12:31 [INFO]: Epoch 033 - training loss: 8.8384, validation loss: 9.6853, validation AUC: 0.7965917644590875 -2024-09-13 12:12:37 [INFO]: Epoch 034 - training loss: 8.8245, validation loss: 9.6900, validation AUC: 0.8031194856797345 -2024-09-13 12:12:43 [INFO]: Epoch 035 - training loss: 8.7635, validation loss: 9.7357, validation AUC: 0.7963079504929726 -2024-09-13 12:12:49 [INFO]: Epoch 036 - training loss: 8.7447, validation loss: 9.7541, validation AUC: 0.7911129209393007 -2024-09-13 12:12:55 [INFO]: Epoch 037 - training loss: 8.6846, validation loss: 9.6365, validation AUC: 0.7853996224040277 -2024-09-13 12:12:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI_epoch37_loss9.636484439556416.pypots -2024-09-13 12:13:01 [INFO]: Epoch 038 - training loss: 8.6647, validation loss: 9.7143, validation AUC: 0.7943089129925098 -2024-09-13 12:13:07 [INFO]: Epoch 039 - training loss: 8.7310, validation loss: 9.6946, validation AUC: 0.7955922457088561 -2024-09-13 12:13:14 [INFO]: Epoch 040 - training loss: 8.6248, validation loss: 9.6612, validation AUC: 0.7932600352916497 -2024-09-13 12:13:20 [INFO]: Epoch 041 - training loss: 8.6021, validation loss: 9.7136, validation AUC: 0.7877318328212342 -2024-09-13 12:13:26 [INFO]: Epoch 042 - training loss: 8.5472, validation loss: 9.6806, validation AUC: 0.7863497822036304 -2024-09-13 12:13:26 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:13:26 [INFO]: Finished training. The best model is from epoch#37. -2024-09-13 12:13:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T120902/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 1 ended ........................................ - -INFO:__main__: - Fold 1 Results: -{ - "PyPots_evaluation": { - "ROC_AUC": 0.7708333333333334, - "PR_AUC": 0.4004018527922971, - "F1": 0.39263803680981596, - "Precision": 0.5423728813559322, - "Recall": 0.3076923076923077 - }, - "CSAI_evaluation": { - "ACC": 0.87625, - "AUC": 0.7708333333333334, - "Prec_Macro": 0.722603444726549, - "Recall_Macro": 0.6344496021220158, - "F1_Macro": 0.6618722543130499, - "Bal_ACC": 0.6344496021220158 - } -} - -INFO:__main__: - Fold 2 started ...................................... - -2024-09-13 12:13:29 [INFO]: Using the given device: cuda -2024-09-13 12:13:29 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121329 -2024-09-13 12:13:29 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121329/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:13:39 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 -2024-09-13 12:13:45 [INFO]: Epoch 001 - training loss: 19.8475, validation loss: 17.6167, validation AUC: 0.5809640640785105 -2024-09-13 12:13:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch1_loss17.61674191401555.pypots -2024-09-13 12:13:51 [INFO]: Epoch 002 - training loss: 15.9218, validation loss: 15.1607, validation AUC: 0.6880502236975032 -2024-09-13 12:13:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch2_loss15.160728748028095.pypots -2024-09-13 12:13:57 [INFO]: Epoch 003 - training loss: 14.2607, validation loss: 13.9902, validation AUC: 0.7676793188050224 -2024-09-13 12:13:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch3_loss13.990155880267803.pypots -2024-09-13 12:14:03 [INFO]: Epoch 004 - training loss: 13.3474, validation loss: 13.2878, validation AUC: 0.7781666426131718 -2024-09-13 12:14:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch4_loss13.287820229163536.pypots -2024-09-13 12:14:09 [INFO]: Epoch 005 - training loss: 12.6907, validation loss: 12.7066, validation AUC: 0.7864650983787944 -2024-09-13 12:14:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch5_loss12.70658830495981.pypots -2024-09-13 12:14:15 [INFO]: Epoch 006 - training loss: 12.1345, validation loss: 12.2049, validation AUC: 0.7893394910280463 -2024-09-13 12:14:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch6_loss12.204926637502817.pypots -2024-09-13 12:14:21 [INFO]: Epoch 007 - training loss: 11.6593, validation loss: 11.7718, validation AUC: 0.7989248087747151 -2024-09-13 12:14:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch7_loss11.771849412184496.pypots -2024-09-13 12:14:28 [INFO]: Epoch 008 - training loss: 11.2531, validation loss: 11.4333, validation AUC: 0.7916125463029777 -2024-09-13 12:14:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch8_loss11.43331028864934.pypots -2024-09-13 12:14:33 [INFO]: Epoch 009 - training loss: 10.9356, validation loss: 11.1864, validation AUC: 0.7990210227546063 -2024-09-13 12:14:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch9_loss11.186383614173302.pypots -2024-09-13 12:14:39 [INFO]: Epoch 010 - training loss: 10.6980, validation loss: 11.0176, validation AUC: 0.8005243661904073 -2024-09-13 12:14:39 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch10_loss11.017565727233887.pypots -2024-09-13 12:14:45 [INFO]: Epoch 011 - training loss: 10.5164, validation loss: 10.8152, validation AUC: 0.8005724731803532 -2024-09-13 12:14:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch11_loss10.81515957758977.pypots -2024-09-13 12:14:52 [INFO]: Epoch 012 - training loss: 10.3187, validation loss: 10.6378, validation AUC: 0.8013421850194833 -2024-09-13 12:14:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch12_loss10.63784591968243.pypots -2024-09-13 12:14:58 [INFO]: Epoch 013 - training loss: 10.1152, validation loss: 10.5081, validation AUC: 0.8024486457882332 -2024-09-13 12:14:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch13_loss10.508104764498198.pypots -2024-09-13 12:15:04 [INFO]: Epoch 014 - training loss: 9.9899, validation loss: 10.3729, validation AUC: 0.796459325540001 -2024-09-13 12:15:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch14_loss10.372918495765099.pypots -2024-09-13 12:15:10 [INFO]: Epoch 015 - training loss: 9.8594, validation loss: 10.3116, validation AUC: 0.8013782652619426 -2024-09-13 12:15:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch15_loss10.311591001657339.pypots -2024-09-13 12:15:16 [INFO]: Epoch 016 - training loss: 9.7446, validation loss: 10.2354, validation AUC: 0.8044811661134363 -2024-09-13 12:15:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch16_loss10.235374890840971.pypots -2024-09-13 12:15:22 [INFO]: Epoch 017 - training loss: 9.6591, validation loss: 10.1343, validation AUC: 0.8100134699571848 -2024-09-13 12:15:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch17_loss10.134286440335787.pypots -2024-09-13 12:15:28 [INFO]: Epoch 018 - training loss: 9.5688, validation loss: 10.0724, validation AUC: 0.8039880694664934 -2024-09-13 12:15:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch18_loss10.072449977581318.pypots -2024-09-13 12:15:34 [INFO]: Epoch 019 - training loss: 9.5057, validation loss: 10.0097, validation AUC: 0.8036272670419011 -2024-09-13 12:15:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch19_loss10.009667323185848.pypots -2024-09-13 12:15:40 [INFO]: Epoch 020 - training loss: 9.4289, validation loss: 9.9999, validation AUC: 0.8061769375090201 -2024-09-13 12:15:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch20_loss9.999937864450308.pypots -2024-09-13 12:15:46 [INFO]: Epoch 021 - training loss: 9.3561, validation loss: 9.9410, validation AUC: 0.8058762688218599 -2024-09-13 12:15:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch21_loss9.940957436194786.pypots -2024-09-13 12:15:52 [INFO]: Epoch 022 - training loss: 9.2925, validation loss: 9.9246, validation AUC: 0.8167604752970608 -2024-09-13 12:15:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch22_loss9.924552550682655.pypots -2024-09-13 12:15:58 [INFO]: Epoch 023 - training loss: 9.2463, validation loss: 9.8549, validation AUC: 0.8096165872901333 -2024-09-13 12:15:58 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch23_loss9.854878132159893.pypots -2024-09-13 12:16:05 [INFO]: Epoch 024 - training loss: 9.1737, validation loss: 9.8140, validation AUC: 0.808558233511329 -2024-09-13 12:16:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch24_loss9.814040257380558.pypots -2024-09-13 12:16:11 [INFO]: Epoch 025 - training loss: 9.1330, validation loss: 9.7736, validation AUC: 0.8207653822100353 -2024-09-13 12:16:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch25_loss9.773633883549618.pypots -2024-09-13 12:16:17 [INFO]: Epoch 026 - training loss: 9.0987, validation loss: 9.7521, validation AUC: 0.8032784913647953 -2024-09-13 12:16:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch26_loss9.752125373253456.pypots -2024-09-13 12:16:23 [INFO]: Epoch 027 - training loss: 9.0337, validation loss: 9.7578, validation AUC: 0.8066580074084766 -2024-09-13 12:16:29 [INFO]: Epoch 028 - training loss: 8.9899, validation loss: 9.7796, validation AUC: 0.7995622263914947 -2024-09-13 12:16:35 [INFO]: Epoch 029 - training loss: 8.9836, validation loss: 9.7480, validation AUC: 0.795665560205898 -2024-09-13 12:16:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch29_loss9.748024500333345.pypots -2024-09-13 12:16:41 [INFO]: Epoch 030 - training loss: 8.9059, validation loss: 9.7315, validation AUC: 0.7934165584259394 -2024-09-13 12:16:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch30_loss9.731545301584097.pypots -2024-09-13 12:16:47 [INFO]: Epoch 031 - training loss: 8.8881, validation loss: 9.7377, validation AUC: 0.8008370616250542 -2024-09-13 12:16:53 [INFO]: Epoch 032 - training loss: 8.8780, validation loss: 9.7357, validation AUC: 0.8071511040554193 -2024-09-13 12:16:59 [INFO]: Epoch 033 - training loss: 8.8250, validation loss: 9.7180, validation AUC: 0.7788281137249242 -2024-09-13 12:16:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch33_loss9.717986547029936.pypots -2024-09-13 12:17:05 [INFO]: Epoch 034 - training loss: 8.7778, validation loss: 9.6552, validation AUC: 0.8032063308798769 -2024-09-13 12:17:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI_epoch34_loss9.655178436866173.pypots -2024-09-13 12:17:11 [INFO]: Epoch 035 - training loss: 8.7297, validation loss: 9.7163, validation AUC: 0.8014744792418339 -2024-09-13 12:17:17 [INFO]: Epoch 036 - training loss: 8.7866, validation loss: 9.6828, validation AUC: 0.7946432866695531 -2024-09-13 12:17:23 [INFO]: Epoch 037 - training loss: 8.6777, validation loss: 9.6914, validation AUC: 0.7854668783374225 -2024-09-13 12:17:29 [INFO]: Epoch 038 - training loss: 8.7008, validation loss: 9.7403, validation AUC: 0.796988502429403 -2024-09-13 12:17:35 [INFO]: Epoch 039 - training loss: 8.6678, validation loss: 9.6835, validation AUC: 0.7959421753980854 -2024-09-13 12:17:35 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:17:35 [INFO]: Finished training. The best model is from epoch#34. -2024-09-13 12:17:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121329/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 2 ended ........................................ - -INFO:__main__: - Fold 2 Results: -{ - "PyPots_evaluation": { - "ROC_AUC": 0.8132865385489072, - "PR_AUC": 0.4924055309152301, - "F1": 0.32954545454545453, - "Precision": 0.6590909090909091, - "Recall": 0.2196969696969697 - }, - "CSAI_evaluation": { - "ACC": 0.8523153942428036, - "AUC": 0.8132865385489072, - "Prec_Macro": 0.7613335340156532, - "Recall_Macro": 0.5986041070373904, - "F1_Macro": 0.623281869326173, - "Bal_ACC": 0.5986041070373904 - } -} - -INFO:__main__: - Fold 3 started ...................................... - -2024-09-13 12:17:39 [INFO]: Using the given device: cuda -2024-09-13 12:17:39 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121739 -2024-09-13 12:17:39 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T121739/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:17:48 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 -2024-09-13 12:17:54 [INFO]: Epoch 001 - training loss: 19.8825, validation loss: 17.4137, validation AUC: 0.5158057340583293 -2024-09-13 12:17:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch1_loss17.413655647864708.pypots -2024-09-13 12:18:01 [INFO]: Epoch 002 - training loss: 16.0221, validation loss: 14.9347, validation AUC: 0.6432773109243698 -2024-09-13 12:18:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch2_loss14.93472458766057.pypots -2024-09-13 12:18:07 [INFO]: Epoch 003 - training loss: 14.3395, validation loss: 13.8289, validation AUC: 0.7859861591695502 -2024-09-13 12:18:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch3_loss13.82885162646954.pypots -2024-09-13 12:18:13 [INFO]: Epoch 004 - training loss: 13.4141, validation loss: 13.1185, validation AUC: 0.8242956005931786 -2024-09-13 12:18:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch4_loss13.118463369516226.pypots -2024-09-13 12:18:19 [INFO]: Epoch 005 - training loss: 12.7445, validation loss: 12.5076, validation AUC: 0.831957488877904 -2024-09-13 12:18:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch5_loss12.50760790017935.pypots -2024-09-13 12:18:24 [INFO]: Epoch 006 - training loss: 12.1803, validation loss: 12.0181, validation AUC: 0.8430548690064261 -2024-09-13 12:18:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch6_loss12.018106460571289.pypots -2024-09-13 12:18:31 [INFO]: Epoch 007 - training loss: 11.7076, validation loss: 11.6133, validation AUC: 0.8420044488383589 -2024-09-13 12:18:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch7_loss11.613336343031664.pypots -2024-09-13 12:18:37 [INFO]: Epoch 008 - training loss: 11.3546, validation loss: 11.3158, validation AUC: 0.8478868017795355 -2024-09-13 12:18:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch8_loss11.315820840688852.pypots -2024-09-13 12:18:43 [INFO]: Epoch 009 - training loss: 11.0431, validation loss: 11.1116, validation AUC: 0.845304003954523 -2024-09-13 12:18:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch9_loss11.111641370333158.pypots -2024-09-13 12:18:49 [INFO]: Epoch 010 - training loss: 10.8137, validation loss: 10.9210, validation AUC: 0.8418685121107267 -2024-09-13 12:18:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch10_loss10.920994465167706.pypots -2024-09-13 12:18:55 [INFO]: Epoch 011 - training loss: 10.6497, validation loss: 10.7272, validation AUC: 0.8377656945130993 -2024-09-13 12:18:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch11_loss10.727198380690355.pypots -2024-09-13 12:19:01 [INFO]: Epoch 012 - training loss: 10.4457, validation loss: 10.5835, validation AUC: 0.846280276816609 -2024-09-13 12:19:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch12_loss10.583506510807918.pypots -2024-09-13 12:19:07 [INFO]: Epoch 013 - training loss: 10.2859, validation loss: 10.4547, validation AUC: 0.8366658428077114 -2024-09-13 12:19:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch13_loss10.454722331120418.pypots -2024-09-13 12:19:13 [INFO]: Epoch 014 - training loss: 10.1591, validation loss: 10.3023, validation AUC: 0.8379263470093921 -2024-09-13 12:19:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch14_loss10.302282993610088.pypots -2024-09-13 12:19:20 [INFO]: Epoch 015 - training loss: 10.0102, validation loss: 10.2317, validation AUC: 0.8472936233316857 -2024-09-13 12:19:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch15_loss10.23169524853046.pypots -2024-09-13 12:19:26 [INFO]: Epoch 016 - training loss: 9.8710, validation loss: 10.1016, validation AUC: 0.8339841819080575 -2024-09-13 12:19:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch16_loss10.101593971252441.pypots -2024-09-13 12:19:32 [INFO]: Epoch 017 - training loss: 9.7457, validation loss: 10.0257, validation AUC: 0.834268413247652 -2024-09-13 12:19:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch17_loss10.025650537931002.pypots -2024-09-13 12:19:38 [INFO]: Epoch 018 - training loss: 9.6776, validation loss: 9.9535, validation AUC: 0.8422639644092931 -2024-09-13 12:19:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch18_loss9.953543516305777.pypots -2024-09-13 12:19:44 [INFO]: Epoch 019 - training loss: 9.5949, validation loss: 9.9773, validation AUC: 0.8271131982204647 -2024-09-13 12:19:50 [INFO]: Epoch 020 - training loss: 9.5266, validation loss: 9.9032, validation AUC: 0.8192535837864557 -2024-09-13 12:19:50 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch20_loss9.903202497042143.pypots -2024-09-13 12:19:56 [INFO]: Epoch 021 - training loss: 9.4495, validation loss: 9.8674, validation AUC: 0.8209218981710331 -2024-09-13 12:19:56 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch21_loss9.867419976454515.pypots -2024-09-13 12:20:02 [INFO]: Epoch 022 - training loss: 9.3996, validation loss: 9.8199, validation AUC: 0.8238012852199703 -2024-09-13 12:20:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch22_loss9.81988026545598.pypots -2024-09-13 12:20:08 [INFO]: Epoch 023 - training loss: 9.3449, validation loss: 9.8067, validation AUC: 0.8128769154720712 -2024-09-13 12:20:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch23_loss9.806729390070988.pypots -2024-09-13 12:20:14 [INFO]: Epoch 024 - training loss: 9.2911, validation loss: 9.7355, validation AUC: 0.8299678695007415 -2024-09-13 12:20:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch24_loss9.735456540034367.pypots -2024-09-13 12:20:21 [INFO]: Epoch 025 - training loss: 9.2479, validation loss: 9.7462, validation AUC: 0.8284602076124568 -2024-09-13 12:20:27 [INFO]: Epoch 026 - training loss: 9.1890, validation loss: 9.7657, validation AUC: 0.8208601087493821 -2024-09-13 12:20:33 [INFO]: Epoch 027 - training loss: 9.1731, validation loss: 9.6901, validation AUC: 0.82194760257044 -2024-09-13 12:20:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch27_loss9.690123631403996.pypots -2024-09-13 12:20:39 [INFO]: Epoch 028 - training loss: 9.1152, validation loss: 9.6960, validation AUC: 0.8299555116164113 -2024-09-13 12:20:45 [INFO]: Epoch 029 - training loss: 9.0335, validation loss: 9.6730, validation AUC: 0.8254695996045477 -2024-09-13 12:20:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch29_loss9.672953972449669.pypots -2024-09-13 12:20:51 [INFO]: Epoch 030 - training loss: 9.0020, validation loss: 9.6380, validation AUC: 0.827805239742956 -2024-09-13 12:20:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch30_loss9.637961314274715.pypots -2024-09-13 12:20:57 [INFO]: Epoch 031 - training loss: 8.9571, validation loss: 9.6237, validation AUC: 0.8336875926841324 -2024-09-13 12:20:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch31_loss9.62369889479417.pypots -2024-09-13 12:21:03 [INFO]: Epoch 032 - training loss: 8.9231, validation loss: 9.6379, validation AUC: 0.8148912506178941 -2024-09-13 12:21:09 [INFO]: Epoch 033 - training loss: 8.8739, validation loss: 9.6433, validation AUC: 0.8089594661393971 -2024-09-13 12:21:15 [INFO]: Epoch 034 - training loss: 8.8328, validation loss: 9.6885, validation AUC: 0.804695996045477 -2024-09-13 12:21:21 [INFO]: Epoch 035 - training loss: 8.8352, validation loss: 9.5828, validation AUC: 0.8196613939693524 -2024-09-13 12:21:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI_epoch35_loss9.58282654102032.pypots -2024-09-13 12:21:27 [INFO]: Epoch 036 - training loss: 8.7549, validation loss: 9.6210, validation AUC: 0.7901631240731587 -2024-09-13 12:21:33 [INFO]: Epoch 037 - training loss: 8.7074, validation loss: 9.5951, validation AUC: 0.7956376668314384 -2024-09-13 12:21:39 [INFO]: Epoch 038 - training loss: 8.6545, validation loss: 9.6687, validation AUC: 0.7702916460701927 -2024-09-13 12:21:45 [INFO]: Epoch 039 - training loss: 8.6706, validation loss: 9.6737, validation AUC: 0.7659540286702917 -2024-09-13 12:21:51 [INFO]: Epoch 040 - training loss: 8.6064, validation loss: 9.5919, validation AUC: 0.7831562036579338 -2024-09-13 12:21:51 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:21:51 [INFO]: Finished training. The best model is from epoch#35. -2024-09-13 12:21:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T121739/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 3 ended ........................................ - -INFO:__main__: - Fold 3 Results: -{ - "PyPots_evaluation": { - "ROC_AUC": 0.8221998842486825, - "PR_AUC": 0.42611023661860026, - "F1": 0.39999999999999997, - "Precision": 0.4583333333333333, - "Recall": 0.3548387096774194 - }, - "CSAI_evaluation": { - "ACC": 0.8760951188986232, - "AUC": 0.8221998842486825, - "Prec_Macro": 0.6879011921137093, - "Recall_Macro": 0.6497989582381432, - "F1_Macro": 0.665457083042568, - "Bal_ACC": 0.6497989582381432 - } -} - -INFO:__main__: - Fold 4 started ...................................... - -2024-09-13 12:21:55 [INFO]: Using the given device: cuda -2024-09-13 12:21:55 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T122155 -2024-09-13 12:21:55 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T122155/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:22:04 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,771,502 -2024-09-13 12:22:10 [INFO]: Epoch 001 - training loss: 19.7550, validation loss: 17.4351, validation AUC: 0.5277519903226714 -2024-09-13 12:22:10 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch1_loss17.435132100031925.pypots -2024-09-13 12:22:16 [INFO]: Epoch 002 - training loss: 15.7833, validation loss: 15.0071, validation AUC: 0.642318620418038 -2024-09-13 12:22:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch2_loss15.007051687974196.pypots -2024-09-13 12:22:22 [INFO]: Epoch 003 - training loss: 14.2049, validation loss: 13.9272, validation AUC: 0.7292879849213717 -2024-09-13 12:22:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch3_loss13.927179703345665.pypots -2024-09-13 12:22:28 [INFO]: Epoch 004 - training loss: 13.3239, validation loss: 13.2162, validation AUC: 0.7507103271724759 -2024-09-13 12:22:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch4_loss13.21615402515118.pypots -2024-09-13 12:22:34 [INFO]: Epoch 005 - training loss: 12.6727, validation loss: 12.6155, validation AUC: 0.7631445691619545 -2024-09-13 12:22:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch5_loss12.615543512197641.pypots -2024-09-13 12:22:40 [INFO]: Epoch 006 - training loss: 12.0835, validation loss: 12.0658, validation AUC: 0.7773370467268688 -2024-09-13 12:22:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch6_loss12.065805288461538.pypots -2024-09-13 12:22:47 [INFO]: Epoch 007 - training loss: 11.6361, validation loss: 11.6526, validation AUC: 0.7797704447632713 -2024-09-13 12:22:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch7_loss11.652607697706957.pypots -2024-09-13 12:22:53 [INFO]: Epoch 008 - training loss: 11.2179, validation loss: 11.2851, validation AUC: 0.7865923987959602 -2024-09-13 12:22:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch8_loss11.285062423119179.pypots -2024-09-13 12:22:59 [INFO]: Epoch 009 - training loss: 10.9033, validation loss: 11.0406, validation AUC: 0.7826539511069851 -2024-09-13 12:22:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch9_loss11.040606351999136.pypots -2024-09-13 12:23:05 [INFO]: Epoch 010 - training loss: 10.6604, validation loss: 10.8240, validation AUC: 0.7906574394463667 -2024-09-13 12:23:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch10_loss10.823973142183744.pypots -2024-09-13 12:23:12 [INFO]: Epoch 011 - training loss: 10.4568, validation loss: 10.6639, validation AUC: 0.785649984527527 -2024-09-13 12:23:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch11_loss10.663858340336727.pypots -2024-09-13 12:23:18 [INFO]: Epoch 012 - training loss: 10.2834, validation loss: 10.4640, validation AUC: 0.7900666722930204 -2024-09-13 12:23:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch12_loss10.463965269235464.pypots -2024-09-13 12:23:24 [INFO]: Epoch 013 - training loss: 10.1037, validation loss: 10.3295, validation AUC: 0.7903198582158832 -2024-09-13 12:23:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch13_loss10.329508854792667.pypots -2024-09-13 12:23:31 [INFO]: Epoch 014 - training loss: 9.9464, validation loss: 10.2101, validation AUC: 0.7876754719104284 -2024-09-13 12:23:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch14_loss10.210116239694448.pypots -2024-09-13 12:23:37 [INFO]: Epoch 015 - training loss: 9.8494, validation loss: 10.1288, validation AUC: 0.792556333867837 -2024-09-13 12:23:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch15_loss10.128803693331205.pypots -2024-09-13 12:23:43 [INFO]: Epoch 016 - training loss: 9.7545, validation loss: 10.0486, validation AUC: 0.7848060314513179 -2024-09-13 12:23:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch16_loss10.048647660475512.pypots -2024-09-13 12:23:49 [INFO]: Epoch 017 - training loss: 9.6540, validation loss: 10.0037, validation AUC: 0.7942301741356514 -2024-09-13 12:23:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch17_loss10.003671866196852.pypots -2024-09-13 12:23:55 [INFO]: Epoch 018 - training loss: 9.5533, validation loss: 9.9426, validation AUC: 0.781486482684896 -2024-09-13 12:23:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch18_loss9.942629153911884.pypots -2024-09-13 12:24:01 [INFO]: Epoch 019 - training loss: 9.4786, validation loss: 9.8861, validation AUC: 0.7929923762905449 -2024-09-13 12:24:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch19_loss9.886068124037523.pypots -2024-09-13 12:24:08 [INFO]: Epoch 020 - training loss: 9.4157, validation loss: 9.7993, validation AUC: 0.7877317354488423 -2024-09-13 12:24:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch20_loss9.799341935377855.pypots -2024-09-13 12:24:14 [INFO]: Epoch 021 - training loss: 9.3380, validation loss: 9.8065, validation AUC: 0.786789321180409 -2024-09-13 12:24:20 [INFO]: Epoch 022 - training loss: 9.2901, validation loss: 9.7783, validation AUC: 0.7809097814161531 -2024-09-13 12:24:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch22_loss9.778309235206017.pypots -2024-09-13 12:24:26 [INFO]: Epoch 023 - training loss: 9.2254, validation loss: 9.7417, validation AUC: 0.7812051649928263 -2024-09-13 12:24:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch23_loss9.74165703700139.pypots -2024-09-13 12:24:32 [INFO]: Epoch 024 - training loss: 9.1778, validation loss: 9.7390, validation AUC: 0.7845669114130588 -2024-09-13 12:24:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch24_loss9.739037880530724.pypots -2024-09-13 12:24:38 [INFO]: Epoch 025 - training loss: 9.1469, validation loss: 9.7079, validation AUC: 0.7792781388021494 -2024-09-13 12:24:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch25_loss9.707893738379845.pypots -2024-09-13 12:24:44 [INFO]: Epoch 026 - training loss: 9.1043, validation loss: 9.6506, validation AUC: 0.78469350437449 -2024-09-13 12:24:45 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch26_loss9.650553336510292.pypots -2024-09-13 12:24:50 [INFO]: Epoch 027 - training loss: 9.0213, validation loss: 9.6380, validation AUC: 0.7773511126114723 -2024-09-13 12:24:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch27_loss9.63803085914025.pypots -2024-09-13 12:24:57 [INFO]: Epoch 028 - training loss: 8.9983, validation loss: 9.6047, validation AUC: 0.7793625341097702 -2024-09-13 12:24:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch28_loss9.604698988107534.pypots -2024-09-13 12:25:03 [INFO]: Epoch 029 - training loss: 8.9608, validation loss: 9.6082, validation AUC: 0.785213942104819 -2024-09-13 12:25:09 [INFO]: Epoch 030 - training loss: 8.8889, validation loss: 9.5917, validation AUC: 0.7713590457703885 -2024-09-13 12:25:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch30_loss9.591749264643742.pypots -2024-09-13 12:25:15 [INFO]: Epoch 031 - training loss: 8.8505, validation loss: 9.5793, validation AUC: 0.7748614510366558 -2024-09-13 12:25:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch31_loss9.579335065988394.pypots -2024-09-13 12:25:21 [INFO]: Epoch 032 - training loss: 8.8151, validation loss: 9.5944, validation AUC: 0.7682645511576223 -2024-09-13 12:25:27 [INFO]: Epoch 033 - training loss: 8.7853, validation loss: 9.5422, validation AUC: 0.772624975384702 -2024-09-13 12:25:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch33_loss9.54223764859713.pypots -2024-09-13 12:25:33 [INFO]: Epoch 034 - training loss: 8.7152, validation loss: 9.5936, validation AUC: 0.7766618842659014 -2024-09-13 12:25:40 [INFO]: Epoch 035 - training loss: 8.6958, validation loss: 9.6095, validation AUC: 0.7647340141221483 -2024-09-13 12:25:46 [INFO]: Epoch 036 - training loss: 8.7181, validation loss: 9.5777, validation AUC: 0.7849185585281458 -2024-09-13 12:25:52 [INFO]: Epoch 037 - training loss: 8.6481, validation loss: 9.4928, validation AUC: 0.7674065321968098 -2024-09-13 12:25:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch37_loss9.492766306950497.pypots -2024-09-13 12:25:58 [INFO]: Epoch 038 - training loss: 8.5793, validation loss: 9.4996, validation AUC: 0.7613019382788984 -2024-09-13 12:26:04 [INFO]: Epoch 039 - training loss: 8.5310, validation loss: 9.5502, validation AUC: 0.7683067488114328 -2024-09-13 12:26:10 [INFO]: Epoch 040 - training loss: 8.5103, validation loss: 9.6960, validation AUC: 0.7804596731088417 -2024-09-13 12:26:16 [INFO]: Epoch 041 - training loss: 8.6236, validation loss: 9.4729, validation AUC: 0.7637494021999043 -2024-09-13 12:26:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch41_loss9.472880730262169.pypots -2024-09-13 12:26:22 [INFO]: Epoch 042 - training loss: 8.4435, validation loss: 9.4721, validation AUC: 0.7571665682054746 -2024-09-13 12:26:22 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch42_loss9.472069373497597.pypots -2024-09-13 12:26:28 [INFO]: Epoch 043 - training loss: 8.4412, validation loss: 9.5390, validation AUC: 0.7747911216136383 -2024-09-13 12:26:34 [INFO]: Epoch 044 - training loss: 8.3844, validation loss: 9.4883, validation AUC: 0.7643261034686472 -2024-09-13 12:26:40 [INFO]: Epoch 045 - training loss: 8.3603, validation loss: 9.4581, validation AUC: 0.7577995330126311 -2024-09-13 12:26:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch45_loss9.458125554598295.pypots -2024-09-13 12:26:46 [INFO]: Epoch 046 - training loss: 8.3310, validation loss: 9.4998, validation AUC: 0.7775902326497314 -2024-09-13 12:26:53 [INFO]: Epoch 047 - training loss: 8.2575, validation loss: 9.5115, validation AUC: 0.7640166540073705 -2024-09-13 12:26:59 [INFO]: Epoch 048 - training loss: 8.2970, validation loss: 9.4353, validation AUC: 0.7679691675809491 -2024-09-13 12:26:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI_epoch48_loss9.435322248018705.pypots -2024-09-13 12:27:05 [INFO]: Epoch 049 - training loss: 8.2504, validation loss: 9.4680, validation AUC: 0.7643542352378541 -2024-09-13 12:27:11 [INFO]: Epoch 050 - training loss: 8.2382, validation loss: 9.4753, validation AUC: 0.7467015500604832 -2024-09-13 12:27:17 [INFO]: Epoch 051 - training loss: 8.2924, validation loss: 9.4827, validation AUC: 0.7728640954229612 -2024-09-13 12:27:23 [INFO]: Epoch 052 - training loss: 8.1721, validation loss: 9.4675, validation AUC: 0.7588685402424958 -2024-09-13 12:27:29 [INFO]: Epoch 053 - training loss: 8.1054, validation loss: 9.5912, validation AUC: 0.7507947224800968 -2024-09-13 12:27:29 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:27:29 [INFO]: Finished training. The best model is from epoch#48. -2024-09-13 12:27:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T122155/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 4 ended ........................................ - -INFO:__main__: - Fold 4 Results: -{ - "PyPots_evaluation": { - "ROC_AUC": 0.839816933638444, - "PR_AUC": 0.4424882640898875, - "F1": 0.3870967741935485, - "Precision": 0.5070422535211268, - "Recall": 0.3130434782608696 - }, - "CSAI_evaluation": { - "ACC": 0.8573216520650814, - "AUC": 0.839816933638444, - "Prec_Macro": 0.6992628850023217, - "Recall_Macro": 0.6309369438087974, - "F1_Macro": 0.6531801151421001, - "Bal_ACC": 0.6309369438087974 - } -} - -INFO:__main__: - Average performance across all folds.......................... - -INFO:__main__: - Classification performance using Pypots evaluation: - {'ROC_AUC': 0.8092247029077602, 'PR_AUC': 0.434128245763033, 'F1': 0.36721553023394676, 'Precision': 0.5496469452277022, 'Recall': 0.28450883852005876} - -INFO:__main__: - Classification performance using CSAI evaluation: - {'ACC': 0.8666464330413017, 'AUC': 0.8092247029077602, 'Prec_Macro': 0.7211312123697666, 'Recall_Macro': 0.6228764993163682, 'F1_Macro': 0.6463198273912789, 'Bal_ACC': 0.6228764993163682} - diff --git a/output_logs/Imputation.out b/output_logs/Imputation.out deleted file mode 100644 index f67a58c1..00000000 --- a/output_logs/Imputation.out +++ /dev/null @@ -1,605 +0,0 @@ -Lmod has detected the following error: The following module(s) are unknown: -"anaconda3/2021.05-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "anaconda3/2021.05-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Lmod has detected the following error: The following module(s) are unknown: -"cuda/11.1.1-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "cuda/11.1.1-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Lmod has detected the following error: The following module(s) are unknown: -"cudnn/8.0.5.39-11.1-gcc-9.4.0" - -Please check the spelling or version number. Also try "module spider ..." -It is also possible your cache file is out-of-date; it may help to try: - $ module --ignore_cache load "cudnn/8.0.5.39-11.1-gcc-9.4.0" - -Also make sure that all modulefiles written in TCL start with the string -#%Module - - - -Fri Sep 13 12:32:45 2024 -+---------------------------------------------------------------------------------------+ -| NVIDIA-SMI 535.183.01 Driver Version: 535.183.01 CUDA Version: 12.2 | -|-----------------------------------------+----------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+======================+======================| -| 0 NVIDIA A100-SXM4-40GB On | 00000000:17:00.0 Off | 0 | -| N/A 33C P0 53W / 400W | 0MiB / 40960MiB | 0% Default | -| | | Disabled | -+-----------------------------------------+----------------------+----------------------+ - -+---------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=======================================================================================| -| No running processes found | -+---------------------------------------------------------------------------------------+ -/var/lib/slurm/slurmd/job21076914/slurm_script: line 18: activate: No such file or directory -2024-09-13 12:32:51.319467: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. -2024-09-13 12:32:51.361282: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. -To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. -2024-09-13 12:32:52.037976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT -INFO:__main__: - Fold 0 started ...................................... - -2024-09-13 12:33:09 [INFO]: Using the given device: cuda -2024-09-13 12:33:09 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123309 -2024-09-13 12:33:09 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123309/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:33:19 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 -2024-09-13 12:33:29 [INFO]: Epoch 001 - training loss: 18.5395, validation loss: 0.6407 -2024-09-13 12:33:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch1_loss0.6407435054962451.pypots -2024-09-13 12:33:35 [INFO]: Epoch 002 - training loss: 14.9314, validation loss: 0.5001 -2024-09-13 12:33:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch2_loss0.5000739991664886.pypots -2024-09-13 12:33:41 [INFO]: Epoch 003 - training loss: 13.4167, validation loss: 0.4392 -2024-09-13 12:33:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch3_loss0.439183764732801.pypots -2024-09-13 12:33:47 [INFO]: Epoch 004 - training loss: 12.5427, validation loss: 0.4077 -2024-09-13 12:33:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch4_loss0.4077342335994427.pypots -2024-09-13 12:33:53 [INFO]: Epoch 005 - training loss: 11.8881, validation loss: 0.3863 -2024-09-13 12:33:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch5_loss0.3862775449569409.pypots -2024-09-13 12:33:59 [INFO]: Epoch 006 - training loss: 11.3413, validation loss: 0.3734 -2024-09-13 12:33:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch6_loss0.37343067618516773.pypots -2024-09-13 12:34:05 [INFO]: Epoch 007 - training loss: 10.8510, validation loss: 0.3654 -2024-09-13 12:34:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch7_loss0.36537456856324124.pypots -2024-09-13 12:34:11 [INFO]: Epoch 008 - training loss: 10.4839, validation loss: 0.3588 -2024-09-13 12:34:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch8_loss0.3588316383270117.pypots -2024-09-13 12:34:17 [INFO]: Epoch 009 - training loss: 10.2113, validation loss: 0.3533 -2024-09-13 12:34:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch9_loss0.353282379416319.pypots -2024-09-13 12:34:23 [INFO]: Epoch 010 - training loss: 9.9950, validation loss: 0.3514 -2024-09-13 12:34:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch10_loss0.35142794251441956.pypots -2024-09-13 12:34:29 [INFO]: Epoch 011 - training loss: 9.8098, validation loss: 0.3498 -2024-09-13 12:34:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch11_loss0.34977063078146714.pypots -2024-09-13 12:34:35 [INFO]: Epoch 012 - training loss: 9.6576, validation loss: 0.3436 -2024-09-13 12:34:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch12_loss0.34363453204815203.pypots -2024-09-13 12:34:41 [INFO]: Epoch 013 - training loss: 9.5010, validation loss: 0.3449 -2024-09-13 12:34:47 [INFO]: Epoch 014 - training loss: 9.3412, validation loss: 0.3416 -2024-09-13 12:34:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch14_loss0.3415501346954933.pypots -2024-09-13 12:34:53 [INFO]: Epoch 015 - training loss: 9.2051, validation loss: 0.3424 -2024-09-13 12:34:59 [INFO]: Epoch 016 - training loss: 9.0703, validation loss: 0.3383 -2024-09-13 12:34:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch16_loss0.3383355484558986.pypots -2024-09-13 12:35:05 [INFO]: Epoch 017 - training loss: 8.9579, validation loss: 0.3362 -2024-09-13 12:35:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch17_loss0.3362494432009183.pypots -2024-09-13 12:35:11 [INFO]: Epoch 018 - training loss: 8.8809, validation loss: 0.3359 -2024-09-13 12:35:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch18_loss0.3359452130702826.pypots -2024-09-13 12:35:17 [INFO]: Epoch 019 - training loss: 8.8047, validation loss: 0.3369 -2024-09-13 12:35:23 [INFO]: Epoch 020 - training loss: 8.7390, validation loss: 0.3335 -2024-09-13 12:35:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch20_loss0.33350417705682606.pypots -2024-09-13 12:35:29 [INFO]: Epoch 021 - training loss: 8.6853, validation loss: 0.3305 -2024-09-13 12:35:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch21_loss0.33045172920593846.pypots -2024-09-13 12:35:35 [INFO]: Epoch 022 - training loss: 8.6308, validation loss: 0.3337 -2024-09-13 12:35:41 [INFO]: Epoch 023 - training loss: 8.5754, validation loss: 0.3325 -2024-09-13 12:35:47 [INFO]: Epoch 024 - training loss: 8.5298, validation loss: 0.3270 -2024-09-13 12:35:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch24_loss0.327012836933136.pypots -2024-09-13 12:35:53 [INFO]: Epoch 025 - training loss: 8.4729, validation loss: 0.3304 -2024-09-13 12:35:59 [INFO]: Epoch 026 - training loss: 8.4130, validation loss: 0.3274 -2024-09-13 12:36:05 [INFO]: Epoch 027 - training loss: 8.3748, validation loss: 0.3235 -2024-09-13 12:36:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch27_loss0.32347696790328395.pypots -2024-09-13 12:36:11 [INFO]: Epoch 028 - training loss: 8.3357, validation loss: 0.3245 -2024-09-13 12:36:17 [INFO]: Epoch 029 - training loss: 8.3109, validation loss: 0.3227 -2024-09-13 12:36:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch29_loss0.32266572461678433.pypots -2024-09-13 12:36:23 [INFO]: Epoch 030 - training loss: 8.2417, validation loss: 0.3274 -2024-09-13 12:36:29 [INFO]: Epoch 031 - training loss: 8.2205, validation loss: 0.3212 -2024-09-13 12:36:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch31_loss0.3212323280481192.pypots -2024-09-13 12:36:35 [INFO]: Epoch 032 - training loss: 8.1723, validation loss: 0.3183 -2024-09-13 12:36:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch32_loss0.3183346207325275.pypots -2024-09-13 12:36:41 [INFO]: Epoch 033 - training loss: 8.1288, validation loss: 0.3178 -2024-09-13 12:36:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch33_loss0.317815356529676.pypots -2024-09-13 12:36:47 [INFO]: Epoch 034 - training loss: 8.0817, validation loss: 0.3177 -2024-09-13 12:36:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch34_loss0.31765339695490324.pypots -2024-09-13 12:36:53 [INFO]: Epoch 035 - training loss: 8.0366, validation loss: 0.3223 -2024-09-13 12:36:59 [INFO]: Epoch 036 - training loss: 7.9959, validation loss: 0.3221 -2024-09-13 12:37:05 [INFO]: Epoch 037 - training loss: 7.9807, validation loss: 0.3146 -2024-09-13 12:37:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI_epoch37_loss0.3145997386712294.pypots -2024-09-13 12:37:11 [INFO]: Epoch 038 - training loss: 7.9427, validation loss: 0.3162 -2024-09-13 12:37:17 [INFO]: Epoch 039 - training loss: 7.9166, validation loss: 0.3179 -2024-09-13 12:37:23 [INFO]: Epoch 040 - training loss: 7.8947, validation loss: 0.3206 -2024-09-13 12:37:29 [INFO]: Epoch 041 - training loss: 7.9263, validation loss: 0.3181 -2024-09-13 12:37:35 [INFO]: Epoch 042 - training loss: 7.8412, validation loss: 0.3203 -2024-09-13 12:37:35 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:37:35 [INFO]: Finished training. The best model is from epoch#37. -2024-09-13 12:37:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123309/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 0 ended ........................................ - -INFO:__main__: - Fold 0 Results: -{ - "MAE": 0.2522227483799602, - "MRE": 0.3491908942436348 -} - -INFO:__main__: - Fold 1 started ...................................... - -2024-09-13 12:37:38 [INFO]: Using the given device: cuda -2024-09-13 12:37:38 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123738 -2024-09-13 12:37:38 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T123738/tensorboard -INFO:__main__: - Training..................................... - -/scratch/users/k23031260/.conda/envs/imputation/lib/python3.8/site-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance) - warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}") -2024-09-13 12:37:47 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 -2024-09-13 12:37:53 [INFO]: Epoch 001 - training loss: 18.5686, validation loss: 0.5924 -2024-09-13 12:37:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch1_loss0.5923694670200348.pypots -2024-09-13 12:37:59 [INFO]: Epoch 002 - training loss: 15.0152, validation loss: 0.4383 -2024-09-13 12:37:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch2_loss0.4382930329212776.pypots -2024-09-13 12:38:05 [INFO]: Epoch 003 - training loss: 13.4660, validation loss: 0.3992 -2024-09-13 12:38:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch3_loss0.39923181671362656.pypots -2024-09-13 12:38:11 [INFO]: Epoch 004 - training loss: 12.6009, validation loss: 0.3811 -2024-09-13 12:38:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch4_loss0.38111079656160796.pypots -2024-09-13 12:38:17 [INFO]: Epoch 005 - training loss: 11.9217, validation loss: 0.3800 -2024-09-13 12:38:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch5_loss0.37996633236224836.pypots -2024-09-13 12:38:23 [INFO]: Epoch 006 - training loss: 11.3803, validation loss: 0.3808 -2024-09-13 12:38:29 [INFO]: Epoch 007 - training loss: 10.8692, validation loss: 0.3803 -2024-09-13 12:38:35 [INFO]: Epoch 008 - training loss: 10.4516, validation loss: 0.3602 -2024-09-13 12:38:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch8_loss0.36019767362337846.pypots -2024-09-13 12:38:41 [INFO]: Epoch 009 - training loss: 10.1624, validation loss: 0.3302 -2024-09-13 12:38:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch9_loss0.33023632031220657.pypots -2024-09-13 12:38:48 [INFO]: Epoch 010 - training loss: 9.9167, validation loss: 0.3116 -2024-09-13 12:38:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch10_loss0.31161352075063264.pypots -2024-09-13 12:38:54 [INFO]: Epoch 011 - training loss: 9.6946, validation loss: 0.2856 -2024-09-13 12:38:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch11_loss0.2856314709553352.pypots -2024-09-13 12:39:00 [INFO]: Epoch 012 - training loss: 9.4901, validation loss: 0.2794 -2024-09-13 12:39:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch12_loss0.2794186300956286.pypots -2024-09-13 12:39:06 [INFO]: Epoch 013 - training loss: 9.3343, validation loss: 0.2694 -2024-09-13 12:39:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch13_loss0.2694310419834577.pypots -2024-09-13 12:39:12 [INFO]: Epoch 014 - training loss: 9.2215, validation loss: 0.2649 -2024-09-13 12:39:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch14_loss0.2648839056491852.pypots -2024-09-13 12:39:18 [INFO]: Epoch 015 - training loss: 9.0909, validation loss: 0.2580 -2024-09-13 12:39:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch15_loss0.25795977963851047.pypots -2024-09-13 12:39:24 [INFO]: Epoch 016 - training loss: 8.9995, validation loss: 0.2535 -2024-09-13 12:39:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch16_loss0.25348290457175326.pypots -2024-09-13 12:39:30 [INFO]: Epoch 017 - training loss: 8.9097, validation loss: 0.2494 -2024-09-13 12:39:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch17_loss0.24935718568471762.pypots -2024-09-13 12:39:36 [INFO]: Epoch 018 - training loss: 8.8485, validation loss: 0.2479 -2024-09-13 12:39:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch18_loss0.247948285478812.pypots -2024-09-13 12:39:42 [INFO]: Epoch 019 - training loss: 8.7699, validation loss: 0.2461 -2024-09-13 12:39:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch19_loss0.24605009303643152.pypots -2024-09-13 12:39:48 [INFO]: Epoch 020 - training loss: 8.7146, validation loss: 0.2440 -2024-09-13 12:39:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch20_loss0.2440407402240313.pypots -2024-09-13 12:39:54 [INFO]: Epoch 021 - training loss: 8.6635, validation loss: 0.2430 -2024-09-13 12:39:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch21_loss0.2429531319783284.pypots -2024-09-13 12:40:00 [INFO]: Epoch 022 - training loss: 8.6202, validation loss: 0.2408 -2024-09-13 12:40:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch22_loss0.24082150826087365.pypots -2024-09-13 12:40:06 [INFO]: Epoch 023 - training loss: 8.5857, validation loss: 0.2401 -2024-09-13 12:40:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch23_loss0.24012872118216294.pypots -2024-09-13 12:40:12 [INFO]: Epoch 024 - training loss: 8.5168, validation loss: 0.2401 -2024-09-13 12:40:18 [INFO]: Epoch 025 - training loss: 8.4674, validation loss: 0.2396 -2024-09-13 12:40:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch25_loss0.23958969345459571.pypots -2024-09-13 12:40:24 [INFO]: Epoch 026 - training loss: 8.4349, validation loss: 0.2395 -2024-09-13 12:40:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch26_loss0.23954786016390875.pypots -2024-09-13 12:40:30 [INFO]: Epoch 027 - training loss: 8.3812, validation loss: 0.2396 -2024-09-13 12:40:36 [INFO]: Epoch 028 - training loss: 8.3438, validation loss: 0.2398 -2024-09-13 12:40:42 [INFO]: Epoch 029 - training loss: 8.3128, validation loss: 0.2389 -2024-09-13 12:40:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch29_loss0.23891761898994446.pypots -2024-09-13 12:40:48 [INFO]: Epoch 030 - training loss: 8.2741, validation loss: 0.2383 -2024-09-13 12:40:48 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch30_loss0.23825732102760902.pypots -2024-09-13 12:40:54 [INFO]: Epoch 031 - training loss: 8.2236, validation loss: 0.2386 -2024-09-13 12:41:00 [INFO]: Epoch 032 - training loss: 8.2065, validation loss: 0.2377 -2024-09-13 12:41:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch32_loss0.23773047442619616.pypots -2024-09-13 12:41:06 [INFO]: Epoch 033 - training loss: 8.1779, validation loss: 0.2386 -2024-09-13 12:41:12 [INFO]: Epoch 034 - training loss: 8.1342, validation loss: 0.2371 -2024-09-13 12:41:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch34_loss0.23709593025537637.pypots -2024-09-13 12:41:18 [INFO]: Epoch 035 - training loss: 8.1144, validation loss: 0.2378 -2024-09-13 12:41:24 [INFO]: Epoch 036 - training loss: 8.0583, validation loss: 0.2379 -2024-09-13 12:41:30 [INFO]: Epoch 037 - training loss: 8.0278, validation loss: 0.2382 -2024-09-13 12:41:36 [INFO]: Epoch 038 - training loss: 8.0119, validation loss: 0.2364 -2024-09-13 12:41:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI_epoch38_loss0.236399219586299.pypots -2024-09-13 12:41:42 [INFO]: Epoch 039 - training loss: 7.9644, validation loss: 0.2372 -2024-09-13 12:41:48 [INFO]: Epoch 040 - training loss: 7.9454, validation loss: 0.2380 -2024-09-13 12:41:54 [INFO]: Epoch 041 - training loss: 7.9328, validation loss: 0.2370 -2024-09-13 12:42:00 [INFO]: Epoch 042 - training loss: 7.8796, validation loss: 0.2375 -2024-09-13 12:42:06 [INFO]: Epoch 043 - training loss: 7.8528, validation loss: 0.2375 -2024-09-13 12:42:06 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:42:06 [INFO]: Finished training. The best model is from epoch#38. -2024-09-13 12:42:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T123738/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 1 ended ........................................ - -INFO:__main__: - Fold 1 Results: -{ - "MAE": 0.24833315072947296, - "MRE": 0.34538290796537663 -} - -INFO:__main__: - Fold 2 started ...................................... - -2024-09-13 12:42:09 [INFO]: Using the given device: cuda -2024-09-13 12:42:09 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124209 -2024-09-13 12:42:09 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124209/tensorboard -INFO:__main__: - Training..................................... - -2024-09-13 12:42:19 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 -2024-09-13 12:42:25 [INFO]: Epoch 001 - training loss: 18.5348, validation loss: 0.5612 -2024-09-13 12:42:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch1_loss0.5611514747142792.pypots -2024-09-13 12:42:31 [INFO]: Epoch 002 - training loss: 14.9343, validation loss: 0.4211 -2024-09-13 12:42:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch2_loss0.4211423442913936.pypots -2024-09-13 12:42:37 [INFO]: Epoch 003 - training loss: 13.4568, validation loss: 0.3764 -2024-09-13 12:42:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch3_loss0.3763529383219205.pypots -2024-09-13 12:42:43 [INFO]: Epoch 004 - training loss: 12.5687, validation loss: 0.3459 -2024-09-13 12:42:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch4_loss0.3459321971123035.pypots -2024-09-13 12:42:49 [INFO]: Epoch 005 - training loss: 11.9092, validation loss: 0.3259 -2024-09-13 12:42:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch5_loss0.32591839363941777.pypots -2024-09-13 12:42:55 [INFO]: Epoch 006 - training loss: 11.3498, validation loss: 0.3121 -2024-09-13 12:42:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch6_loss0.3120722415355536.pypots -2024-09-13 12:43:01 [INFO]: Epoch 007 - training loss: 10.8595, validation loss: 0.2990 -2024-09-13 12:43:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch7_loss0.29896558362704057.pypots -2024-09-13 12:43:07 [INFO]: Epoch 008 - training loss: 10.4851, validation loss: 0.2914 -2024-09-13 12:43:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch8_loss0.29137278176271.pypots -2024-09-13 12:43:13 [INFO]: Epoch 009 - training loss: 10.1950, validation loss: 0.2870 -2024-09-13 12:43:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch9_loss0.2870428814337804.pypots -2024-09-13 12:43:19 [INFO]: Epoch 010 - training loss: 9.9764, validation loss: 0.2847 -2024-09-13 12:43:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch10_loss0.28473271200290096.pypots -2024-09-13 12:43:25 [INFO]: Epoch 011 - training loss: 9.7934, validation loss: 0.2787 -2024-09-13 12:43:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch11_loss0.2786905009012956.pypots -2024-09-13 12:43:31 [INFO]: Epoch 012 - training loss: 9.6277, validation loss: 0.2773 -2024-09-13 12:43:31 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch12_loss0.277261625115688.pypots -2024-09-13 12:43:37 [INFO]: Epoch 013 - training loss: 9.4717, validation loss: 0.2765 -2024-09-13 12:43:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch13_loss0.276475286254516.pypots -2024-09-13 12:43:43 [INFO]: Epoch 014 - training loss: 9.3222, validation loss: 0.2745 -2024-09-13 12:43:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch14_loss0.27448664605617523.pypots -2024-09-13 12:43:49 [INFO]: Epoch 015 - training loss: 9.1564, validation loss: 0.2720 -2024-09-13 12:43:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch15_loss0.2719874611267677.pypots -2024-09-13 12:43:55 [INFO]: Epoch 016 - training loss: 9.0493, validation loss: 0.2713 -2024-09-13 12:43:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch16_loss0.2712785303592682.pypots -2024-09-13 12:44:01 [INFO]: Epoch 017 - training loss: 8.9550, validation loss: 0.2690 -2024-09-13 12:44:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch17_loss0.2689772821389712.pypots -2024-09-13 12:44:08 [INFO]: Epoch 018 - training loss: 8.8579, validation loss: 0.2715 -2024-09-13 12:44:14 [INFO]: Epoch 019 - training loss: 8.7816, validation loss: 0.2696 -2024-09-13 12:44:19 [INFO]: Epoch 020 - training loss: 8.7212, validation loss: 0.2689 -2024-09-13 12:44:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch20_loss0.26889700385240406.pypots -2024-09-13 12:44:25 [INFO]: Epoch 021 - training loss: 8.6609, validation loss: 0.2689 -2024-09-13 12:44:31 [INFO]: Epoch 022 - training loss: 8.6029, validation loss: 0.2689 -2024-09-13 12:44:37 [INFO]: Epoch 023 - training loss: 8.5666, validation loss: 0.2684 -2024-09-13 12:44:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch23_loss0.26841238141059875.pypots -2024-09-13 12:44:43 [INFO]: Epoch 024 - training loss: 8.5122, validation loss: 0.2685 -2024-09-13 12:44:49 [INFO]: Epoch 025 - training loss: 8.4428, validation loss: 0.2680 -2024-09-13 12:44:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch25_loss0.268048128256431.pypots -2024-09-13 12:44:55 [INFO]: Epoch 026 - training loss: 8.4081, validation loss: 0.2683 -2024-09-13 12:45:01 [INFO]: Epoch 027 - training loss: 8.3657, validation loss: 0.2681 -2024-09-13 12:45:07 [INFO]: Epoch 028 - training loss: 8.3273, validation loss: 0.2670 -2024-09-13 12:45:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch28_loss0.2669859161743751.pypots -2024-09-13 12:45:13 [INFO]: Epoch 029 - training loss: 8.2955, validation loss: 0.2668 -2024-09-13 12:45:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch29_loss0.2668392062187195.pypots -2024-09-13 12:45:20 [INFO]: Epoch 030 - training loss: 8.2595, validation loss: 0.2669 -2024-09-13 12:45:26 [INFO]: Epoch 031 - training loss: 8.2161, validation loss: 0.2672 -2024-09-13 12:45:32 [INFO]: Epoch 032 - training loss: 8.1760, validation loss: 0.2663 -2024-09-13 12:45:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch32_loss0.26633317883198077.pypots -2024-09-13 12:45:38 [INFO]: Epoch 033 - training loss: 8.1431, validation loss: 0.2676 -2024-09-13 12:45:43 [INFO]: Epoch 034 - training loss: 8.1101, validation loss: 0.2676 -2024-09-13 12:45:49 [INFO]: Epoch 035 - training loss: 8.0758, validation loss: 0.2663 -2024-09-13 12:45:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch35_loss0.26626667036459994.pypots -2024-09-13 12:45:55 [INFO]: Epoch 036 - training loss: 8.0328, validation loss: 0.2677 -2024-09-13 12:46:01 [INFO]: Epoch 037 - training loss: 8.0061, validation loss: 0.2661 -2024-09-13 12:46:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch37_loss0.26614915980742526.pypots -2024-09-13 12:46:07 [INFO]: Epoch 038 - training loss: 7.9972, validation loss: 0.2665 -2024-09-13 12:46:13 [INFO]: Epoch 039 - training loss: 7.9451, validation loss: 0.2648 -2024-09-13 12:46:13 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI_epoch39_loss0.26480964170052457.pypots -2024-09-13 12:46:19 [INFO]: Epoch 040 - training loss: 7.8999, validation loss: 0.2667 -2024-09-13 12:46:25 [INFO]: Epoch 041 - training loss: 7.8679, validation loss: 0.2649 -2024-09-13 12:46:31 [INFO]: Epoch 042 - training loss: 7.8303, validation loss: 0.2666 -2024-09-13 12:46:37 [INFO]: Epoch 043 - training loss: 7.8062, validation loss: 0.2663 -2024-09-13 12:46:43 [INFO]: Epoch 044 - training loss: 7.7753, validation loss: 0.2670 -2024-09-13 12:46:43 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:46:43 [INFO]: Finished training. The best model is from epoch#39. -2024-09-13 12:46:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124209/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 2 ended ........................................ - -INFO:__main__: - Fold 2 Results: -{ - "MAE": 0.25268375496555934, - "MRE": 0.3468272181807543 -} - -INFO:__main__: - Fold 3 started ...................................... - -2024-09-13 12:46:47 [INFO]: Using the given device: cuda -2024-09-13 12:46:47 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124647 -2024-09-13 12:46:47 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T124647/tensorboard -INFO:__main__: - Training..................................... - -2024-09-13 12:46:56 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 -2024-09-13 12:47:02 [INFO]: Epoch 001 - training loss: 18.6528, validation loss: 0.5896 -2024-09-13 12:47:02 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch1_loss0.5896110672217149.pypots -2024-09-13 12:47:08 [INFO]: Epoch 002 - training loss: 15.0505, validation loss: 0.4486 -2024-09-13 12:47:08 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch2_loss0.448551744222641.pypots -2024-09-13 12:47:14 [INFO]: Epoch 003 - training loss: 13.5125, validation loss: 0.3941 -2024-09-13 12:47:14 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch3_loss0.39414907648013187.pypots -2024-09-13 12:47:20 [INFO]: Epoch 004 - training loss: 12.6204, validation loss: 0.3625 -2024-09-13 12:47:20 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch4_loss0.36246832746725816.pypots -2024-09-13 12:47:26 [INFO]: Epoch 005 - training loss: 11.9558, validation loss: 0.3467 -2024-09-13 12:47:26 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch5_loss0.3466833738180307.pypots -2024-09-13 12:47:32 [INFO]: Epoch 006 - training loss: 11.3833, validation loss: 0.3337 -2024-09-13 12:47:32 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch6_loss0.33370912762788624.pypots -2024-09-13 12:47:38 [INFO]: Epoch 007 - training loss: 10.9050, validation loss: 0.3241 -2024-09-13 12:47:38 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch7_loss0.32408049473395717.pypots -2024-09-13 12:47:44 [INFO]: Epoch 008 - training loss: 10.5231, validation loss: 0.3188 -2024-09-13 12:47:44 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch8_loss0.3188299639866902.pypots -2024-09-13 12:47:50 [INFO]: Epoch 009 - training loss: 10.2466, validation loss: 0.3137 -2024-09-13 12:47:51 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch9_loss0.3137410913522427.pypots -2024-09-13 12:47:57 [INFO]: Epoch 010 - training loss: 10.0112, validation loss: 0.3114 -2024-09-13 12:47:57 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch10_loss0.31143705317607295.pypots -2024-09-13 12:48:03 [INFO]: Epoch 011 - training loss: 9.8428, validation loss: 0.3094 -2024-09-13 12:48:03 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch11_loss0.30936380418447346.pypots -2024-09-13 12:48:09 [INFO]: Epoch 012 - training loss: 9.6768, validation loss: 0.3065 -2024-09-13 12:48:09 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch12_loss0.30649459935151613.pypots -2024-09-13 12:48:15 [INFO]: Epoch 013 - training loss: 9.5489, validation loss: 0.3050 -2024-09-13 12:48:15 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch13_loss0.30499985126348644.pypots -2024-09-13 12:48:21 [INFO]: Epoch 014 - training loss: 9.4256, validation loss: 0.3047 -2024-09-13 12:48:21 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch14_loss0.3046507560289823.pypots -2024-09-13 12:48:27 [INFO]: Epoch 015 - training loss: 9.3041, validation loss: 0.3019 -2024-09-13 12:48:27 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch15_loss0.30189058299248034.pypots -2024-09-13 12:48:33 [INFO]: Epoch 016 - training loss: 9.1878, validation loss: 0.3004 -2024-09-13 12:48:33 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch16_loss0.3004143008818993.pypots -2024-09-13 12:48:39 [INFO]: Epoch 017 - training loss: 9.0926, validation loss: 0.3016 -2024-09-13 12:48:45 [INFO]: Epoch 018 - training loss: 8.9575, validation loss: 0.3015 -2024-09-13 12:48:51 [INFO]: Epoch 019 - training loss: 8.8735, validation loss: 0.3013 -2024-09-13 12:48:58 [INFO]: Epoch 020 - training loss: 8.7711, validation loss: 0.3012 -2024-09-13 12:49:04 [INFO]: Epoch 021 - training loss: 8.7145, validation loss: 0.3002 -2024-09-13 12:49:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch21_loss0.3002306039516742.pypots -2024-09-13 12:49:10 [INFO]: Epoch 022 - training loss: 8.6676, validation loss: 0.3003 -2024-09-13 12:49:16 [INFO]: Epoch 023 - training loss: 8.6097, validation loss: 0.3006 -2024-09-13 12:49:22 [INFO]: Epoch 024 - training loss: 8.5589, validation loss: 0.3004 -2024-09-13 12:49:28 [INFO]: Epoch 025 - training loss: 8.5222, validation loss: 0.2997 -2024-09-13 12:49:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch25_loss0.29971028979008013.pypots -2024-09-13 12:49:34 [INFO]: Epoch 026 - training loss: 8.4555, validation loss: 0.2996 -2024-09-13 12:49:34 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch26_loss0.29962454506984126.pypots -2024-09-13 12:49:40 [INFO]: Epoch 027 - training loss: 8.4317, validation loss: 0.2967 -2024-09-13 12:49:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch27_loss0.2967267162524737.pypots -2024-09-13 12:49:46 [INFO]: Epoch 028 - training loss: 8.3893, validation loss: 0.3005 -2024-09-13 12:49:52 [INFO]: Epoch 029 - training loss: 8.3394, validation loss: 0.2990 -2024-09-13 12:49:58 [INFO]: Epoch 030 - training loss: 8.2851, validation loss: 0.2989 -2024-09-13 12:50:04 [INFO]: Epoch 031 - training loss: 8.2500, validation loss: 0.2962 -2024-09-13 12:50:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch31_loss0.29616755361740404.pypots -2024-09-13 12:50:10 [INFO]: Epoch 032 - training loss: 8.1912, validation loss: 0.2978 -2024-09-13 12:50:16 [INFO]: Epoch 033 - training loss: 8.1711, validation loss: 0.2964 -2024-09-13 12:50:22 [INFO]: Epoch 034 - training loss: 8.1256, validation loss: 0.2972 -2024-09-13 12:50:28 [INFO]: Epoch 035 - training loss: 8.0841, validation loss: 0.2959 -2024-09-13 12:50:28 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch35_loss0.29589974880218506.pypots -2024-09-13 12:50:34 [INFO]: Epoch 036 - training loss: 8.0485, validation loss: 0.2961 -2024-09-13 12:50:40 [INFO]: Epoch 037 - training loss: 8.0247, validation loss: 0.2954 -2024-09-13 12:50:40 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch37_loss0.2953845365689351.pypots -2024-09-13 12:50:46 [INFO]: Epoch 038 - training loss: 7.9824, validation loss: 0.2962 -2024-09-13 12:50:52 [INFO]: Epoch 039 - training loss: 7.9632, validation loss: 0.2947 -2024-09-13 12:50:52 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch39_loss0.2946692028870949.pypots -2024-09-13 12:50:58 [INFO]: Epoch 040 - training loss: 7.9081, validation loss: 0.2953 -2024-09-13 12:51:04 [INFO]: Epoch 041 - training loss: 7.8854, validation loss: 0.2940 -2024-09-13 12:51:04 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch41_loss0.2939985004755167.pypots -2024-09-13 12:51:10 [INFO]: Epoch 042 - training loss: 7.8589, validation loss: 0.2953 -2024-09-13 12:51:16 [INFO]: Epoch 043 - training loss: 7.8427, validation loss: 0.2925 -2024-09-13 12:51:16 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI_epoch43_loss0.2925326961737413.pypots -2024-09-13 12:51:22 [INFO]: Epoch 044 - training loss: 7.8126, validation loss: 0.2938 -2024-09-13 12:51:28 [INFO]: Epoch 045 - training loss: 7.7695, validation loss: 0.2936 -2024-09-13 12:51:34 [INFO]: Epoch 046 - training loss: 7.7545, validation loss: 0.2948 -2024-09-13 12:51:40 [INFO]: Epoch 047 - training loss: 7.7470, validation loss: 0.2961 -2024-09-13 12:51:46 [INFO]: Epoch 048 - training loss: 7.7173, validation loss: 0.2942 -2024-09-13 12:51:46 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:51:46 [INFO]: Finished training. The best model is from epoch#43. -2024-09-13 12:51:46 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T124647/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 3 ended ........................................ - -INFO:__main__: - Fold 3 Results: -{ - "MAE": 0.25894437333366543, - "MRE": 0.34814777479039904 -} - -INFO:__main__: - Fold 4 started ...................................... - -2024-09-13 12:51:49 [INFO]: Using the given device: cuda -2024-09-13 12:51:49 [INFO]: Model files will be saved to /scratch/users/k23031260/PyPOTS/20240913_T125149 -2024-09-13 12:51:49 [INFO]: Tensorboard file will be saved to /scratch/users/k23031260/PyPOTS/20240913_T125149/tensorboard -INFO:__main__: - Training..................................... - -2024-09-13 12:51:59 [INFO]: CSAI initialized with the given hyperparameters, the number of trainable parameters: 4,767,578 -2024-09-13 12:52:05 [INFO]: Epoch 001 - training loss: 18.5119, validation loss: 0.5635 -2024-09-13 12:52:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch1_loss0.563519686460495.pypots -2024-09-13 12:52:11 [INFO]: Epoch 002 - training loss: 14.9026, validation loss: 0.4182 -2024-09-13 12:52:11 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch2_loss0.41821371821256786.pypots -2024-09-13 12:52:17 [INFO]: Epoch 003 - training loss: 13.4091, validation loss: 0.3636 -2024-09-13 12:52:17 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch3_loss0.36356426316958207.pypots -2024-09-13 12:52:23 [INFO]: Epoch 004 - training loss: 12.5351, validation loss: 0.3342 -2024-09-13 12:52:23 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch4_loss0.3342179312155797.pypots -2024-09-13 12:52:29 [INFO]: Epoch 005 - training loss: 11.8907, validation loss: 0.3143 -2024-09-13 12:52:29 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch5_loss0.314338546532851.pypots -2024-09-13 12:52:35 [INFO]: Epoch 006 - training loss: 11.3183, validation loss: 0.2987 -2024-09-13 12:52:35 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch6_loss0.2986761675431178.pypots -2024-09-13 12:52:41 [INFO]: Epoch 007 - training loss: 10.8321, validation loss: 0.2843 -2024-09-13 12:52:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch7_loss0.28431492126905.pypots -2024-09-13 12:52:47 [INFO]: Epoch 008 - training loss: 10.4464, validation loss: 0.2780 -2024-09-13 12:52:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch8_loss0.2780210788433368.pypots -2024-09-13 12:52:53 [INFO]: Epoch 009 - training loss: 10.1411, validation loss: 0.2717 -2024-09-13 12:52:53 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch9_loss0.2716936022043228.pypots -2024-09-13 12:52:59 [INFO]: Epoch 010 - training loss: 9.8886, validation loss: 0.2675 -2024-09-13 12:52:59 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch10_loss0.26754155640418714.pypots -2024-09-13 12:53:05 [INFO]: Epoch 011 - training loss: 9.6573, validation loss: 0.2617 -2024-09-13 12:53:05 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch11_loss0.26171908470300526.pypots -2024-09-13 12:53:11 [INFO]: Epoch 012 - training loss: 9.4534, validation loss: 0.2588 -2024-09-13 12:53:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch12_loss0.25877043719475085.pypots -2024-09-13 12:53:18 [INFO]: Epoch 013 - training loss: 9.3149, validation loss: 0.2558 -2024-09-13 12:53:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch13_loss0.2557584700676111.pypots -2024-09-13 12:53:24 [INFO]: Epoch 014 - training loss: 9.1776, validation loss: 0.2542 -2024-09-13 12:53:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch14_loss0.25421963746731097.pypots -2024-09-13 12:53:30 [INFO]: Epoch 015 - training loss: 9.0721, validation loss: 0.2537 -2024-09-13 12:53:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch15_loss0.25367432144971996.pypots -2024-09-13 12:53:36 [INFO]: Epoch 016 - training loss: 8.9740, validation loss: 0.2508 -2024-09-13 12:53:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch16_loss0.25082376713936144.pypots -2024-09-13 12:53:41 [INFO]: Epoch 017 - training loss: 8.8739, validation loss: 0.2499 -2024-09-13 12:53:41 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch17_loss0.24994707909914163.pypots -2024-09-13 12:53:47 [INFO]: Epoch 018 - training loss: 8.8084, validation loss: 0.2496 -2024-09-13 12:53:47 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch18_loss0.24957519884292895.pypots -2024-09-13 12:53:54 [INFO]: Epoch 019 - training loss: 8.7477, validation loss: 0.2480 -2024-09-13 12:53:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch19_loss0.24800925988417405.pypots -2024-09-13 12:54:00 [INFO]: Epoch 020 - training loss: 8.6866, validation loss: 0.2475 -2024-09-13 12:54:00 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch20_loss0.24747206958440635.pypots -2024-09-13 12:54:06 [INFO]: Epoch 021 - training loss: 8.6250, validation loss: 0.2478 -2024-09-13 12:54:12 [INFO]: Epoch 022 - training loss: 8.5700, validation loss: 0.2473 -2024-09-13 12:54:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch22_loss0.24734074679704812.pypots -2024-09-13 12:54:18 [INFO]: Epoch 023 - training loss: 8.5134, validation loss: 0.2455 -2024-09-13 12:54:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch23_loss0.24554229699648344.pypots -2024-09-13 12:54:24 [INFO]: Epoch 024 - training loss: 8.4657, validation loss: 0.2468 -2024-09-13 12:54:30 [INFO]: Epoch 025 - training loss: 8.4304, validation loss: 0.2437 -2024-09-13 12:54:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch25_loss0.2436517637509566.pypots -2024-09-13 12:54:36 [INFO]: Epoch 026 - training loss: 8.3759, validation loss: 0.2423 -2024-09-13 12:54:36 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch26_loss0.24229151927507842.pypots -2024-09-13 12:54:42 [INFO]: Epoch 027 - training loss: 8.3343, validation loss: 0.2414 -2024-09-13 12:54:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch27_loss0.24144030878177056.pypots -2024-09-13 12:54:48 [INFO]: Epoch 028 - training loss: 8.2927, validation loss: 0.2431 -2024-09-13 12:54:54 [INFO]: Epoch 029 - training loss: 8.2643, validation loss: 0.2409 -2024-09-13 12:54:54 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch29_loss0.24094540912371415.pypots -2024-09-13 12:55:00 [INFO]: Epoch 030 - training loss: 8.2241, validation loss: 0.2415 -2024-09-13 12:55:06 [INFO]: Epoch 031 - training loss: 8.1760, validation loss: 0.2402 -2024-09-13 12:55:06 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch31_loss0.24021976383832785.pypots -2024-09-13 12:55:12 [INFO]: Epoch 032 - training loss: 8.1481, validation loss: 0.2374 -2024-09-13 12:55:12 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch32_loss0.23736257048753592.pypots -2024-09-13 12:55:18 [INFO]: Epoch 033 - training loss: 8.0922, validation loss: 0.2385 -2024-09-13 12:55:24 [INFO]: Epoch 034 - training loss: 8.0782, validation loss: 0.2379 -2024-09-13 12:55:30 [INFO]: Epoch 035 - training loss: 8.0490, validation loss: 0.2359 -2024-09-13 12:55:30 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch35_loss0.2358845346249067.pypots -2024-09-13 12:55:36 [INFO]: Epoch 036 - training loss: 8.0172, validation loss: 0.2365 -2024-09-13 12:55:42 [INFO]: Epoch 037 - training loss: 7.9829, validation loss: 0.2358 -2024-09-13 12:55:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch37_loss0.23576750892859238.pypots -2024-09-13 12:55:48 [INFO]: Epoch 038 - training loss: 7.9402, validation loss: 0.2373 -2024-09-13 12:55:54 [INFO]: Epoch 039 - training loss: 7.9054, validation loss: 0.2348 -2024-09-13 12:55:55 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch39_loss0.23477741502798521.pypots -2024-09-13 12:56:01 [INFO]: Epoch 040 - training loss: 7.8854, validation loss: 0.2360 -2024-09-13 12:56:06 [INFO]: Epoch 041 - training loss: 7.8602, validation loss: 0.2339 -2024-09-13 12:56:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch41_loss0.23388041785130134.pypots -2024-09-13 12:56:12 [INFO]: Epoch 042 - training loss: 7.8280, validation loss: 0.2352 -2024-09-13 12:56:18 [INFO]: Epoch 043 - training loss: 7.8045, validation loss: 0.2325 -2024-09-13 12:56:18 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch43_loss0.23245421395852015.pypots -2024-09-13 12:56:24 [INFO]: Epoch 044 - training loss: 7.7597, validation loss: 0.2319 -2024-09-13 12:56:24 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch44_loss0.23189380421088293.pypots -2024-09-13 12:56:30 [INFO]: Epoch 045 - training loss: 7.7357, validation loss: 0.2328 -2024-09-13 12:56:36 [INFO]: Epoch 046 - training loss: 7.7160, validation loss: 0.2325 -2024-09-13 12:56:42 [INFO]: Epoch 047 - training loss: 7.6981, validation loss: 0.2306 -2024-09-13 12:56:42 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch47_loss0.23061815133461586.pypots -2024-09-13 12:56:49 [INFO]: Epoch 048 - training loss: 7.6576, validation loss: 0.2305 -2024-09-13 12:56:49 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch48_loss0.2304995426764855.pypots -2024-09-13 12:56:55 [INFO]: Epoch 049 - training loss: 7.6384, validation loss: 0.2316 -2024-09-13 12:57:01 [INFO]: Epoch 050 - training loss: 7.6140, validation loss: 0.2277 -2024-09-13 12:57:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch50_loss0.22768438091644874.pypots -2024-09-13 12:57:07 [INFO]: Epoch 051 - training loss: 7.6261, validation loss: 0.2285 -2024-09-13 12:57:13 [INFO]: Epoch 052 - training loss: 7.5924, validation loss: 0.2279 -2024-09-13 12:57:19 [INFO]: Epoch 053 - training loss: 7.5728, validation loss: 0.2291 -2024-09-13 12:57:25 [INFO]: Epoch 054 - training loss: 7.5471, validation loss: 0.2270 -2024-09-13 12:57:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch54_loss0.2269772761143171.pypots -2024-09-13 12:57:31 [INFO]: Epoch 055 - training loss: 7.5182, validation loss: 0.2271 -2024-09-13 12:57:37 [INFO]: Epoch 056 - training loss: 7.5176, validation loss: 0.2246 -2024-09-13 12:57:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch56_loss0.22458868760329026.pypots -2024-09-13 12:57:43 [INFO]: Epoch 057 - training loss: 7.4998, validation loss: 0.2250 -2024-09-13 12:57:49 [INFO]: Epoch 058 - training loss: 7.4922, validation loss: 0.2253 -2024-09-13 12:57:55 [INFO]: Epoch 059 - training loss: 7.4583, validation loss: 0.2281 -2024-09-13 12:58:01 [INFO]: Epoch 060 - training loss: 7.4547, validation loss: 0.2245 -2024-09-13 12:58:01 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch60_loss0.22450509094274962.pypots -2024-09-13 12:58:07 [INFO]: Epoch 061 - training loss: 7.4415, validation loss: 0.2279 -2024-09-13 12:58:13 [INFO]: Epoch 062 - training loss: 7.4168, validation loss: 0.2254 -2024-09-13 12:58:19 [INFO]: Epoch 063 - training loss: 7.4041, validation loss: 0.2238 -2024-09-13 12:58:19 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch63_loss0.22382787672372964.pypots -2024-09-13 12:58:25 [INFO]: Epoch 064 - training loss: 7.3849, validation loss: 0.2233 -2024-09-13 12:58:25 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch64_loss0.22330084557716662.pypots -2024-09-13 12:58:31 [INFO]: Epoch 065 - training loss: 7.3857, validation loss: 0.2248 -2024-09-13 12:58:37 [INFO]: Epoch 066 - training loss: 7.3641, validation loss: 0.2274 -2024-09-13 12:58:43 [INFO]: Epoch 067 - training loss: 7.3423, validation loss: 0.2228 -2024-09-13 12:58:43 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch67_loss0.22279441815156203.pypots -2024-09-13 12:58:49 [INFO]: Epoch 068 - training loss: 7.3196, validation loss: 0.2237 -2024-09-13 12:58:55 [INFO]: Epoch 069 - training loss: 7.3225, validation loss: 0.2245 -2024-09-13 12:59:01 [INFO]: Epoch 070 - training loss: 7.3039, validation loss: 0.2239 -2024-09-13 12:59:07 [INFO]: Epoch 071 - training loss: 7.2925, validation loss: 0.2214 -2024-09-13 12:59:07 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI_epoch71_loss0.22138233253589043.pypots -2024-09-13 12:59:13 [INFO]: Epoch 072 - training loss: 7.2945, validation loss: 0.2230 -2024-09-13 12:59:19 [INFO]: Epoch 073 - training loss: 7.2789, validation loss: 0.2222 -2024-09-13 12:59:25 [INFO]: Epoch 074 - training loss: 7.2574, validation loss: 0.2228 -2024-09-13 12:59:31 [INFO]: Epoch 075 - training loss: 7.2429, validation loss: 0.2223 -2024-09-13 12:59:37 [INFO]: Epoch 076 - training loss: 7.2428, validation loss: 0.2227 -2024-09-13 12:59:37 [INFO]: Exceeded the training patience. Terminating the training procedure... -2024-09-13 12:59:37 [INFO]: Finished training. The best model is from epoch#71. -2024-09-13 12:59:37 [INFO]: Saved the model to /scratch/users/k23031260/PyPOTS/20240913_T125149/CSAI.pypots -INFO:__main__: - Testing...................................... - -INFO:__main__: - Fold 4 ended ........................................ - -INFO:__main__: - Fold 4 Results: -{ - "MAE": 0.24529347374725138, - "MRE": 0.3424914307207086 -} - -INFO:__main__: - Average performance across all folds.......................... - -INFO:__main__: - Imputation performance: - {'MAE': 0.2514955002311819, 'MRE': 0.34640804518017465} - diff --git a/pypots/classification/__init__.py b/pypots/classification/__init__.py index 4f3c66d9..4344ad92 100644 --- a/pypots/classification/__init__.py +++ b/pypots/classification/__init__.py @@ -9,12 +9,10 @@ from .csai import CSAI from .grud import GRUD from .raindrop import Raindrop -from .csai import CSAI __all__ = [ "CSAI", "BRITS", "GRUD", "Raindrop", - "CSAI", ] diff --git a/pypots/classification/base.py b/pypots/classification/base.py index 13f7ab0c..8713e8d4 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -16,7 +16,6 @@ from ..base import BaseModel, BaseNNModel from ..utils.logging import logger -from sklearn.metrics import roc_auc_score try: import nni except ImportError: @@ -323,11 +322,10 @@ def _train_model( "classification_loss": mean_val_loss, } self._save_log_into_tb_file(epoch, "validating", val_loss_dict) - logger.info( f"Epoch {epoch:03d} - " f"training loss: {mean_train_loss:.4f}, " - f"validation loss: {mean_val_loss:.4f}, " + f"validation loss: {mean_val_loss:.4f}," ) mean_loss = mean_val_loss else: diff --git a/pypots/data/utils.py b/pypots/data/utils.py index 2d7c0ef6..7dcf8d93 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -import copy from typing import Union import benchpots @@ -223,4 +222,5 @@ def inverse_sliding_window(X, sliding_len): return benchpots.utils.inverse_sliding_window( X, sliding_len, + ) diff --git a/pypots/imputation/__init__.py b/pypots/imputation/__init__.py index be50fb43..6600dcfd 100644 --- a/pypots/imputation/__init__.py +++ b/pypots/imputation/__init__.py @@ -38,7 +38,6 @@ from .imputeformer import ImputeFormer from .timemixer import TimeMixer from .moderntcn import ModernTCN -from .csai import CSAI # naive imputation methods from .locf import LOCF @@ -49,7 +48,6 @@ __all__ = [ # neural network imputation methods - "CSAI", "SAITS", "Transformer", "iTransformer", diff --git a/pypots/imputation/csai/data.py b/pypots/imputation/csai/data.py index 80a32fd5..6e4e481a 100644 --- a/pypots/imputation/csai/data.py +++ b/pypots/imputation/csai/data.py @@ -355,6 +355,7 @@ def __init__(self, return_X_pred = False, return_y = return_y, file_type = file_type) + self.removal_percent = removal_percent self.increase_factor = increase_factor self.compute_intervals = compute_intervals diff --git a/pypots/imputation/csai/model.py b/pypots/imputation/csai/model.py index d7f154a1..b61286c5 100644 --- a/pypots/imputation/csai/model.py +++ b/pypots/imputation/csai/model.py @@ -146,6 +146,7 @@ def __init__(self, def _assemble_input_for_training(self, data: list, training=True) -> dict: # extract data sample = data['sample'] + ( indices, X, diff --git a/run_pypots.sh b/run_pypots.sh deleted file mode 100644 index b36543e0..00000000 --- a/run_pypots.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -l - - - -#SBATCH --output=/scratch/users/k23031260/PyPOTS/output_logs/Imputation.out -#SBATCH --job-name=pypots -#SBATCH --partition=gpu -#SBATCH --gres=gpu:1 -#SBATCH --mem=256G -#SBATCH --signal=USR2 -# Load required modules -module load anaconda3/2021.05-gcc-9.4.0 -module load cuda/11.1.1-gcc-9.4.0 -module load cudnn/8.0.5.39-11.1-gcc-9.4.0 -nvidia-smi - -# Activate the conda environment -source activate imputation - -# Navigate to the directory containing the Python script -cd /scratch/users/k23031260/PyPOTS - -python csai.py - - From e493b98bc9820bd2e07140b5076f189657329b6e Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Fri, 4 Oct 2024 18:24:02 +0100 Subject: [PATCH 19/22] removed unnecessary commits --- pypots/classification/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pypots/classification/base.py b/pypots/classification/base.py index 8713e8d4..283fb17d 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -16,6 +16,7 @@ from ..base import BaseModel, BaseNNModel from ..utils.logging import logger + try: import nni except ImportError: @@ -312,7 +313,7 @@ def _train_model( for idx, data in enumerate(val_loader): inputs = self._assemble_input_for_validating(data) results = self.model.forward(inputs) - epoch_val_loss_collector.append(results["loss"].sum().item()) + epoch_val_loss_collector.append(results["loss"].sum().item()) mean_val_loss = np.mean(epoch_val_loss_collector) @@ -322,10 +323,11 @@ def _train_model( "classification_loss": mean_val_loss, } self._save_log_into_tb_file(epoch, "validating", val_loss_dict) + logger.info( f"Epoch {epoch:03d} - " f"training loss: {mean_train_loss:.4f}, " - f"validation loss: {mean_val_loss:.4f}," + f"validation loss: {mean_val_loss:.4f}" ) mean_loss = mean_val_loss else: @@ -446,4 +448,4 @@ def classify( Classification results of the given samples. """ - raise NotImplementedError + raise NotImplementedError \ No newline at end of file From 5d925f51173f3e27370727c8286b61667600426c Mon Sep 17 00:00:00 2001 From: Joseph Arul Raj Patterson Kulandai Raj Date: Fri, 4 Oct 2024 18:26:30 +0100 Subject: [PATCH 20/22] removed unnecessary commits --- pypots/classification/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypots/classification/base.py b/pypots/classification/base.py index 283fb17d..c5a6679d 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -448,4 +448,5 @@ def classify( Classification results of the given samples. """ - raise NotImplementedError \ No newline at end of file + raise NotImplementedError + \ No newline at end of file From 6ff365ae148e89b77f15a57dd153fac00c51d613 Mon Sep 17 00:00:00 2001 From: joseph-arulraj Date: Fri, 4 Oct 2024 18:29:45 +0100 Subject: [PATCH 21/22] update base file --- pypots/classification/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pypots/classification/base.py b/pypots/classification/base.py index c5a6679d..75a3a3bb 100644 --- a/pypots/classification/base.py +++ b/pypots/classification/base.py @@ -449,4 +449,3 @@ def classify( """ raise NotImplementedError - \ No newline at end of file From 02c97de0ace133963efef799d435b1d1ff8d2167 Mon Sep 17 00:00:00 2001 From: joseph-arulraj Date: Fri, 4 Oct 2024 18:32:51 +0100 Subject: [PATCH 22/22] removed space --- pypots/data/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypots/data/utils.py b/pypots/data/utils.py index 7dcf8d93..40919fa8 100644 --- a/pypots/data/utils.py +++ b/pypots/data/utils.py @@ -11,6 +11,7 @@ import numpy as np import torch + def turn_data_into_specified_dtype( data: Union[np.ndarray, torch.Tensor, list], dtype: str = "tensor", @@ -105,6 +106,7 @@ def cal_delta_for_single_sample(mask: np.ndarray) -> np.ndarray: """calculate single sample's delta. The sample's shape is [n_steps, n_features].""" # the first step in the delta matrix is all 0 d = [np.zeros(n_features)] + for step in range(1, seq_len): d.append(np.ones(n_features) + (1 - mask[step - 1]) * d[-1]) d = np.asarray(d) @@ -222,5 +224,4 @@ def inverse_sliding_window(X, sliding_len): return benchpots.utils.inverse_sliding_window( X, sliding_len, - )