USPPM-deberta-v3-large (Attention head).ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import and EDA","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport pandas as pd\nimport numpy as np\nimport scipy as sp\nfrom datasets import Dataset,DatasetDict\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import Adam,SGD,AdamW\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nimport shutil\nimport random,os,math, time\nfrom tqdm.auto import tqdm\n\nimport tokenizers\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n%env TOKENIZERS_PARALLELISM =true\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:25.755172Z","iopub.execute_input":"2022-06-03T18:41:25.755906Z","iopub.status.idle":"2022-06-03T18:41:32.670843Z","shell.execute_reply.started":"2022-06-03T18:41:25.755813Z","shell.execute_reply":"2022-06-03T18:41:32.670061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    debug = False\n    train = True\n    n_fold = 4\n    trn_fold = [0,2]\n    apex = True\n    print_freq = 100\n    fc_dropout = 0.2\n    target_size = 1\n    num_workers = 2\n    scheduler = 'cosine' # 'linear'\n    batch_scheduler = True\n    num_cycles = 0.5\n    num_warmup_steps = 0\n    epochs = 4\n    encoder_lr = 1e-5\n    decoder_lr = 1e-5\n    min_lr = 1e-6\n    eps = 1e-6\n    betas = (0.9, 0.999)\n    batch_size = 16\n    fc_dropout = 0.2\n    max_len = 512\n    weight_decay = 0.01\n    gradient_accumulation_steps =1\n    max_grad_norm = 1000\n    # model_nm = '../input/deberta-v3-large/deberta-v3-large'\n    # model_nm = 'anferico/bert-for-patents'\n    # model_nm = '../input/bert-for-patents/bert-for-patents'\n    model_nm = 'microsoft/deberta-v3-large'","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:32.672561Z","iopub.execute_input":"2022-06-03T18:41:32.672962Z","iopub.status.idle":"2022-06-03T18:41:32.680278Z","shell.execute_reply.started":"2022-06-03T18:41:32.672924Z","shell.execute_reply":"2022-06-03T18:41:32.679453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_title = True\noutput_dir = \"./\"\nsep = \"[SEP]\"\nuse_sep = False\nuse_full_dataset = True\nuse_cross_val = True","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:32.681329Z","iopub.execute_input":"2022-06-03T18:41:32.681577Z","iopub.status.idle":"2022-06-03T18:41:32.689605Z","shell.execute_reply.started":"2022-06-03T18:41:32.681544Z","shell.execute_reply":"2022-06-03T18:41:32.688818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:32.691499Z","iopub.execute_input":"2022-06-03T18:41:32.691741Z","iopub.status.idle":"2022-06-03T18:41:32.703229Z","shell.execute_reply.started":"2022-06-03T18:41:32.691709Z","shell.execute_reply":"2022-06-03T18:41:32.702604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('../input/us-patent-phrase-to-phrase-matching')\n! pip install --no-index --find-links ../input/huggingface-datasets datasets -q\ndf = pd.read_csv(path/'train.csv')\ntitles = pd.read_csv('../input/cpc-codes/titles.csv', dtype=str)\ndf = df.merge(titles, left_on='context', right_on='code')\n\ndf_anchors = pd.DataFrame(df['anchor'].unique(),columns = ['anchor'])\ndf_anchors['anchor_val'] = np.arange(len(df_anchors))\n\ndf = df.merge(df_anchors, left_on='anchor', right_on='anchor')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:32.704527Z","iopub.execute_input":"2022-06-03T18:41:32.704895Z","iopub.status.idle":"2022-06-03T18:41:42.595842Z","shell.execute_reply.started":"2022-06-03T18:41:32.704861Z","shell.execute_reply":"2022-06-03T18:41:42.595018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if use_title:\n    if use_sep:\n        df['input'] = df.context + sep + df.title + sep + df.target + sep + df.anchor\n    else:\n        df['input'] = 'TEXT1: ' + df.context + ';TEXT2: ' + df.title + '; TEXT3: ' + df.target + '; ANC1: ' + df.anchor\nelse:\n    if use_sep:\n        df['input'] = df.context + sep + df.target + sep + df.anchor\n    else:\n        df['input'] = 'TEXT1: ' + df.context + '; TEXT2: ' + df.target + '; ANC1: ' + df.anchor\n\nif not use_full_dataset:\n    df = df.iloc[0:2000]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:42.597596Z","iopub.execute_input":"2022-06-03T18:41:42.598549Z","iopub.status.idle":"2022-06-03T18:41:42.645736Z","shell.execute_reply.started":"2022-06-03T18:41:42.598507Z","shell.execute_reply":"2022-06-03T18:41:42.644749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################Tokenizer################################\ntokz = AutoTokenizer.from_pretrained(CFG.model_nm)\ntokz.save_pretrained(output_dir+'tokenizer/')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:42.647061Z","iopub.execute_input":"2022-06-03T18:41:42.647327Z","iopub.status.idle":"2022-06-03T18:41:48.801055Z","shell.execute_reply.started":"2022-06-03T18:41:42.647295Z","shell.execute_reply":"2022-06-03T18:41:48.800357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_anchor = []\nlen_target = []\nlen_title = []\nfor text in df['anchor'].values:\n    len_anchor.append(len(tokz(text, add_special_tokens=False)['input_ids']))\nfor text in df['target'].values:\n    len_target.append(len(tokz(text, add_special_tokens=False)['input_ids']))\nfor text in df['title'].values:\n    len_title.append(len(tokz(text, add_special_tokens=False)['input_ids']))\n    \nCFG.max_len = max(len_anchor)+ max(len_target)+ max(len_title)+ 20 #(for safety)\nprint(CFG.max_len)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:41:50.033663Z","iopub.execute_input":"2022-06-03T18:41:50.034399Z","iopub.status.idle":"2022-06-03T18:42:00.848879Z","shell.execute_reply.started":"2022-06-03T18:41:50.034356Z","shell.execute_reply":"2022-06-03T18:42:00.84748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################Dataset################################\ndef tok_func(x):\n    inputs = tokz(x,\n                  add_special_tokens=True,\n                  max_length=CFG.max_len,\n                  padding=\"max_length\",\n                  return_offsets_mapping=False)\n    for k, v in inputs.items():\n        inputs[k] = torch.tensor(v, dtype=torch.long)\n    return inputs\n\n\nclass TrainDataset(Dataset):\n    def __init__(self,df):\n        self.texts = df['input'].values\n        self.labels = df['score'].values\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, item):\n        inputs = tok_func(self.texts[item])\n        label = torch.tensor(self.labels[item], dtype=torch.float)\n        return inputs, label","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:00.850321Z","iopub.execute_input":"2022-06-03T18:42:00.85058Z","iopub.status.idle":"2022-06-03T18:42:00.858918Z","shell.execute_reply.started":"2022-06-03T18:42:00.850544Z","shell.execute_reply":"2022-06-03T18:42:00.858182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=CFG.n_fold)\ndf = df.sample(frac =1 , random_state = CFG.seed)\nscores = (df.score*100).astype(int)\nidxs = np.arange(len(df))\nfolds = list(cv.split(idxs,scores,df.anchor_val))\n\ndef get_fold(folds,fold_num):\n    trn,val = folds[fold_num]\n    return trn,val","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:00.860504Z","iopub.execute_input":"2022-06-03T18:42:00.861025Z","iopub.status.idle":"2022-06-03T18:42:01.208259Z","shell.execute_reply.started":"2022-06-03T18:42:00.860986Z","shell.execute_reply":"2022-06-03T18:42:01.207416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################Model################################\nclass CustomModel(nn.Module):\n    def __init__(self,cfg,config_path = None , pretrained = False):\n        super().__init__()\n        self.cfg = cfg\n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model_nm, output_hidden_states = True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model_nm, config = self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        self.fc_dropout = nn.Dropout(cfg.fc_dropout)\n        self.fc = nn.Linear(self.config.hidden_size, self.cfg.target_size)\n        self._init_weights(self.fc)\n        self.attention = nn.Sequential(\n            nn.Linear(self.config.hidden_size, 512),\n            nn.Tanh(),\n            nn.Linear(512,1),\n            nn.Softmax(dim=1))\n        self._init_weights(self.attention)\n    \n    def _init_weights(self,module):\n        if isinstance(module,nn.Linear):\n            module.weight.data.normal_(mean = 0.0, std = self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module,nn.Embedding):\n            module.weight.data.normal_(mean = 0.0, std = self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module,nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n    \n    def feature(self, inputs):\n        outputs = self.model(**inputs)\n        last_hidden_states = outputs[0]\n        weights = self.attention(last_hidden_states)\n        feature = torch.sum(weights * last_hidden_states, dim =1)\n        return feature\n    \n    def forward(self,inputs):\n        feature = self.feature(inputs)\n        output = self.fc(self.fc_dropout(feature))\n        return output\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.209653Z","iopub.execute_input":"2022-06-03T18:42:01.210077Z","iopub.status.idle":"2022-06-03T18:42:01.226239Z","shell.execute_reply.started":"2022-06-03T18:42:01.210037Z","shell.execute_reply":"2022-06-03T18:42:01.225384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################HELPERS################################\nclass AverageMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n    def update(self, val ,n=1):\n        self.val = val\n        self.sum +=val*n\n        self.count +=n\n        self.avg = self.sum/self.count\n        \ndef asMinutes(s):\n    m = math.floor(s/60)\n    s -=m*60\n    return '%dm %ds' %(m,s)\n\ndef timeSince(since,percent):\n    now = time.time()\n    s = now-since\n    es = s/(percent)\n    rs = es-s\n    return '%s(remain %s)' %(asMinutes(s), asMinutes(rs))\n\ndef get_score(y_true, y_pred):\n    score = sp.stats.pearsonr(y_true,y_pred)[0]\n    return score\n\ndef get_logger(filename=output_dir+'train'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = get_logger()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.227725Z","iopub.execute_input":"2022-06-03T18:42:01.228032Z","iopub.status.idle":"2022-06-03T18:42:01.242555Z","shell.execute_reply.started":"2022-06-03T18:42:01.227991Z","shell.execute_reply":"2022-06-03T18:42:01.241757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################Train ,Validation,Inference function################################\ndef train_fn(fold, train_loader,model, criterion,optimizer, epoch,scheduler,device):\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled = CFG.apex)\n    losses = AverageMeter()\n    start = end = time.time()\n    global_step = 0\n\n    for step,(inputs,labels) in enumerate(train_loader):\n        for k,v in inputs.items():\n            inputs[k] = v.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.cuda.amp.autocast(enabled = CFG.apex):\n            y_preds = model(inputs)\n            loss = criterion(y_preds.view(-1,1),labels.view(-1,1))\n        if CFG.gradient_accumulation_steps >1:\n            loss = loss/CFG.gradient_accumulation_steps\n        losses.update(loss.item(),batch_size)\n        scaler.scale(loss).backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),CFG.max_grad_norm)\n        if (step+1) %CFG.gradient_accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            global_step +=1\n            if CFG.batch_scheduler:\n                scheduler.step()\n        end = time.time()\n    \n        if step% CFG.print_freq == 0 or step ==(len(train_loader)-1):\n            print('EPOCH : [{0}][{1}/{2}]'\n                 'Elapsed : {remain:s}'\n                 'Loss: {loss.val:.4f}({loss.avg:.4f})'\n                 'Grad: {grad_norm: .4f}'\n                 'LR: {lr:.8f}'\n                 .format(epoch+1,step,len(train_loader),\n                        remain =timeSince(start,float(step+1)/len(train_loader)),\n                        loss = losses,\n                        grad_norm = grad_norm,\n                        lr = scheduler.get_lr()[0]))\n    return losses.avg\n\ndef valid_fn(valid_loader, model,criterion,device):\n    model.eval()\n    losses = AverageMeter()\n    preds = []\n    start = end = time.time()\n    \n    for step, (inputs,labels) in enumerate(valid_loader):\n        for k,v in inputs.items():\n            inputs[k] = v.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        loss = criterion(y_preds.view(-1,1), labels.view(-1,1))\n        if CFG.gradient_accumulation_steps>1:\n            loss = loss/ CFG.gradient_accumulation_steps\n        losses.update(loss.item(),batch_size)\n        preds.append(y_preds.sigmoid().to('cpu').numpy()) #???\n        end = time.time()\n        if step % CFG.print_freq ==0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}]'\n                 'Elapsed {remain:s}'\n                  'Loss: {loss.val:.4f} ({loss.avg:.4f})'\n                 .format(step, len(valid_loader),\n                        loss = losses,\n                        remain = timeSince(start, float(step+1)/len(valid_loader))))\n    predictions = np.concatenate(preds)\n    predictions = np.concatenate(predictions)\n    return losses.avg,predictions\n\ndef inference_fn(test_loader,model,device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader,total = len(test_loader))\n    \n    for inputs in tk0:\n        for k,v in inputs.item():\n            inputs[k] = v.to(device)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    return predictions\n        ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.245688Z","iopub.execute_input":"2022-06-03T18:42:01.245899Z","iopub.status.idle":"2022-06-03T18:42:01.270075Z","shell.execute_reply.started":"2022-06-03T18:42:01.245874Z","shell.execute_reply":"2022-06-03T18:42:01.26928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###################Train loop################################\ndef train_loop(df,folds,fold):\n    \n    LOGGER.info(f\"-----------fold : {fold} training -----------\")\n    \n    train_idx,val_idx = get_fold(folds,fold)\n    train_folds = df.iloc[train_idx]\n    valid_folds = df.iloc[val_idx]\n    \n#     train_folds = folds[folds['fold'] != fold].reset_index(drop = True)\n#     valid_folds = folds[folds['fold'] == fold].reset_index(drop = True)\n    valid_labels = valid_folds['score'].values\n    \n    train_dataset = TrainDataset( train_folds) #---\n    valid_dataset = TrainDataset( valid_folds)\n    \n    train_loader = DataLoader(train_dataset,\n                             batch_size = CFG.batch_size,\n                             shuffle = True,\n                             num_workers = CFG.num_workers, pin_memory = True, drop_last = True)\n    valid_loader = DataLoader(valid_dataset,\n                             batch_size = CFG.batch_size,\n                             shuffle = False,\n                             num_workers = CFG.num_workers, pin_memory = True, drop_last = False)\n    \n    model = CustomModel(CFG,config_path = None, pretrained = True)\n    torch.save(model.config, output_dir+'config.pth')\n    model.to(device)\n    \n    def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay = 0.0):\n        param_optimizer = list(model.named_parameters())\n        no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n        optimizer_parameters = [\n            {'params' : [p for n,p in model.model.named_parameters() if not any(nd in n for nd in no_decay)],\n            'lr': encoder_lr,'weight_decay' : weight_decay},\n            {'params' : [p for n,p in model.model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"lr\": encoder_lr, 'weight_decay' : 0.0},\n            {'params' : [p for n,p in model.named_parameters() if \"model\" not in n],\n            'lr' : decoder_lr, 'weight_decay': 0.0}\n        ]\n        return optimizer_parameters\n    \n    optimizer_parameters = get_optimizer_params(model,\n                                               encoder_lr = CFG.encoder_lr,\n                                               decoder_lr = CFG.decoder_lr,\n                                               weight_decay = CFG.weight_decay)\n    optimizer = AdamW(optimizer_parameters, lr= CFG.encoder_lr, eps = CFG.eps, betas = CFG.betas)\n    \n    def get_scheduler(cfg, optimizer,num_train_steps):\n        if cfg.scheduler == 'linear':\n            scheduler = get_linear_schedule_with_warmup(\n            optimizer, num_warmup_steps = cfg.num_warmup_steps, num_training_steps = num_train_steps)\n        elif cfg.scheduler == 'cosine':\n            scheduler = get_cosine_schedule_with_warmup(\n            optimizer, num_warmup_steps = cfg.num_warmup_steps, num_training_steps = num_train_steps, num_cycles = cfg.num_cycles)\n        \n        return scheduler\n    \n    num_train_steps = int(len(train_folds)/CFG.batch_size * CFG.epochs)\n    scheduler = get_scheduler(CFG,optimizer,num_train_steps)\n    \n#     criterion = nn.BCEWithLogitsLoss(reduction = 'mean')\n    criterion = nn.MSELoss(reduction = 'mean')\n    \n    best_score = 0.0\n    \n    for epoch in range(CFG.epochs):\n        start_time = time.time()\n        \n        avg_loss = train_fn(fold,train_loader,model,criterion,optimizer,epoch,scheduler,device)\n        \n        avg_val_loss, predictions = valid_fn(valid_loader,model,criterion,device)\n        print(predictions.shape)\n        print(valid_labels.shape)\n        score = get_score(valid_labels,predictions)\n        \n        elapsed = time.time()-start_time\n        \n        LOGGER.info(f'EPOCH {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss:{avg_val_loss:.4f} time:{elapsed:.0f}s')\n        LOGGER.info(f'EPOCH {epoch+1} - SCORE: {score:.4f}')\n        \n        if best_score <score:\n            best_score = score\n            LOGGER.info(f'EPOCH {epoch+1} - Save best score: {best_score: .4f} Model')\n            torch.save({'model' : model.state_dict(),\n                       'predictions': predictions},\n                      output_dir + f\"{CFG.model_nm.replace('/','-')}_fold{fold}_best.pth\")\n    predictions = torch.load(output_dir + f\"{CFG.model_nm.replace('/','-')}_fold{fold}_best.pth\",\n                            map_location = torch.device('cpu'))['predictions']\n    \n    valid_folds['pred'] = predictions\n    \n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    return valid_folds          ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.273537Z","iopub.execute_input":"2022-06-03T18:42:01.273743Z","iopub.status.idle":"2022-06-03T18:42:01.297649Z","shell.execute_reply.started":"2022-06-03T18:42:01.273719Z","shell.execute_reply":"2022-06-03T18:42:01.296806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.298981Z","iopub.execute_input":"2022-06-03T18:42:01.29952Z","iopub.status.idle":"2022-06-03T18:42:01.49333Z","shell.execute_reply.started":"2022-06-03T18:42:01.299466Z","shell.execute_reply":"2022-06-03T18:42:01.492574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    def get_result(oof_df):\n        labels = oof_df['score'].values\n        preds = oof_df['pred'].values\n        score = get_score(labels,preds)\n        LOGGER.info(f'Score: {score:<.4f}')\n    \n    if CFG.train:\n        oof_df = pd.DataFrame()\n        for fold in CFG.trn_fold:\n            _oof_df = train_loop(df,folds,fold)\n            oof_df = pd.concat([oof_df,_oof_df])\n            LOGGER.info(f\"----------fold:{fold} result----------\")\n            get_result(_oof_df)\n        oof_df = oof_df.reset_index(drop = True)\n        LOGGER.info(f\"======== CV ========\")\n        get_result(oof_df)\n        oof_df.to_pickle(output_dir+'oof_df.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T18:42:01.517516Z","iopub.execute_input":"2022-06-03T18:42:01.518575Z"},"trusted":true},"execution_count":null,"outputs":[]}]}