baseline-fuji-sfm-dataset-model-trainer-and-evaluator.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Model Training, Inference and Evaluation Module\n",
        "\n",
        "In this notebook we first train the segmentation model, plot logging plots and perform evaluation on the trained apple segmentation task.\n",
        "\n",
        "#### 1. Open3D's RanLA-Net Model Setup and Training\n",
        "\n",
        "In this section we declare a custom dataloader object for Open3D's segmentation model training and then plot the tensorboard logger plots for better model learning understanding.\n",
        "\n",
        "#### 2. Model Performance Inference and Evaluation\n",
        "\n",
        "In this section we evaluate performance of the trained segmentation model, and also plot the final segmentation outputs for better qualitative understanding."
      ],
      "metadata": {
        "id": "pUYaXtWg4bvR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0CyA1iEpKFMc"
      },
      "outputs": [],
      "source": [
        "# for loading the dataset into the runtime from google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# installing open3d library for importing RandLa-Net model implementation\n",
        "!pip install open3d"
      ],
      "metadata": {
        "id": "cfYhpU0ncu5L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# cloning repository for building the creating custom dataloaders\n",
        "!git clone https://github.com/isl-org/Open3D-ML"
      ],
      "metadata": {
        "id": "JUMzZnFA42DH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# installing compatible pytorch version\n",
        "!pip install -r Open3D-ML/requirements-torch-cuda.txt\n",
        "# restart the runtime before execution of further code"
      ],
      "metadata": {
        "id": "JFY3CkUo44zm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# checking the torch installed version\n",
        "# latest version 1.13.0+cu116 is not compatible with the \n",
        "import torch\n",
        "print(torch.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wZWq0-B349MS",
        "outputId": "c181cbb2-61a8-4b08-abeb-e5bc87a42f4a"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.13.1+cu116\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# setting up for the Open3D-ML package after all the environment restarts\n",
        "import sys\n",
        "sys.path.insert(0,'Open3D-ML')"
      ],
      "metadata": {
        "id": "neYjLUsu5A5j"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# checking the path variables for the Open3D-ML package source code\n",
        "from ml3d.datasets.base_dataset import BaseDataset, BaseDatasetSplit\n",
        "from ml3d.utils import make_dir, DATASET"
      ],
      "metadata": {
        "id": "AWXZYs3L5A2y"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# further open3d-ml's ml3d related import statements\n",
        "import open3d.ml as _ml3d\n",
        "import open3d.ml.torch as ml3d"
      ],
      "metadata": {
        "id": "_a4xk4Nj9o3r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# loading the already extracted fuji-sfm dataset for model training into current runtime\n",
        "# TODO: update the corresponding path variables based on your project setup\n",
        "!ls drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R-p6wBRy5Az2",
        "outputId": "cb96d2ca-8b6b-4f93-8397-4ecf7eb0b317"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test  train  valid\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Open3D's RanLA-Net Model Setup and Training"
      ],
      "metadata": {
        "id": "DOYaO58w-IZj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "6VHBL629izoq"
      },
      "outputs": [],
      "source": [
        "# custom data set loader code, loads the '.npy' files from train, val & test directories\n",
        "import numpy as np\n",
        "import os, sys, glob, pickle\n",
        "from pathlib import Path\n",
        "from os.path import join, exists, dirname, abspath\n",
        "from sklearn.neighbors import KDTree\n",
        "from tqdm import tqdm\n",
        "import logging\n",
        "\n",
        "# from .base_dataset import BaseDataset, BaseDatasetSplit\n",
        "# from ..utils import make_dir, DATASET\n",
        "# updated paths for the ml3d github source code usage\n",
        "from ml3d.datasets.base_dataset import BaseDataset, BaseDatasetSplit\n",
        "from ml3d.utils import make_dir, DATASET\n",
        "\n",
        "log = logging.getLogger(__name__)\n",
        "# Expect point clouds to be in npy format with train, val and test files in separate folders.\n",
        "# Expected format of npy files : ['x', 'y', 'z', 'class', 'feat_1', 'feat_2', ........,'feat_n'].\n",
        "# For test files, format should be : ['x', 'y', 'z', 'feat_1', 'feat_2', ........,'feat_n'].\n",
        "\n",
        "class Custom3DSplit(BaseDatasetSplit):\n",
        "    \"\"\"This class is used to create a custom dataset split.\n",
        "    Initialize the class.\n",
        "    Args:\n",
        "        dataset: The dataset to split.\n",
        "        split: A string identifying the dataset split that is usually one of\n",
        "        'training', 'test', 'validation', or 'all'.\n",
        "        **kwargs: The configuration of the model as keyword arguments.\n",
        "    Returns:\n",
        "        A dataset split object providing the requested subset of the data.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, dataset, split='training'):\n",
        "        super().__init__(dataset, split=split)\n",
        "        self.cfg = dataset.cfg\n",
        "        path_list = dataset.get_split_list(split)\n",
        "        log.info(\"Found {} pointclouds for {}\".format(len(path_list), split))\n",
        "        self.path_list = path_list\n",
        "        self.split = split\n",
        "        self.dataset = dataset\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.path_list)\n",
        "\n",
        "    def get_data(self, idx):\n",
        "        pc_path = self.path_list[idx]\n",
        "        data = np.load(pc_path)\n",
        "        \n",
        "        points = np.array(data[:, :3], dtype=np.float32)\n",
        "        feat = np.array(data[:, 3:6], dtype=np.float32)\n",
        "        labels = np.array(data[:, 6], dtype=np.int32).reshape((-1,))\n",
        "        # no normal estimate fetures only points, rgb features and labels as inputs\n",
        "        data = {'point': points,  'feat': feat, 'label': labels} \n",
        "        return data\n",
        "\n",
        "    def get_attr(self, idx):\n",
        "        pc_path = Path(self.path_list[idx])\n",
        "        name = pc_path.name.replace('.npy', '')\n",
        "        attr = {'name': name, 'path': str(pc_path), 'split': self.split}\n",
        "        return attr\n",
        "\n",
        "\n",
        "class Custom3D(BaseDataset):\n",
        "    \"\"\"A template for customized dataset that you can use with a dataloader to\n",
        "    feed data when training a model. This inherits all functions from the base\n",
        "    dataset and can be modified by users. Initialize the function by passing the\n",
        "    dataset and other details.\n",
        "    Args:\n",
        "        dataset_path: The path to the dataset to use.\n",
        "        name: The name of the dataset.\n",
        "        cache_dir: The directory where the cache is stored.\n",
        "        use_cache: Indicates if the dataset should be cached.\n",
        "        num_points: The maximum number of points to use when splitting the dataset.\n",
        "        ignored_label_inds: A list of labels that should be ignored in the dataset.\n",
        "        test_result_folder: The folder where the test results should be stored.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 dataset_path,\n",
        "                 name='Custom3D',\n",
        "                 cache_dir='./logs/cache',\n",
        "                 use_cache=False,\n",
        "                 num_points=65536,\n",
        "                 ignored_label_inds=[],\n",
        "                 test_result_folder='./test',\n",
        "                 **kwargs):\n",
        "\n",
        "        super().__init__(dataset_path=dataset_path,\n",
        "                         name=name,\n",
        "                         cache_dir=cache_dir,\n",
        "                         use_cache=use_cache,\n",
        "                         num_points=num_points,\n",
        "                         ignored_label_inds=ignored_label_inds,\n",
        "                         test_result_folder=test_result_folder,\n",
        "                         **kwargs)\n",
        "\n",
        "        cfg = self.cfg\n",
        "\n",
        "        self.dataset_path = cfg.dataset_path\n",
        "\n",
        "        self.label_to_names = self.get_label_to_names()\n",
        "\n",
        "        self.num_classes = len(self.label_to_names)\n",
        "        self.label_values = np.sort([k for k, v in self.label_to_names.items()])\n",
        "        self.label_to_idx = {l: i for i, l in enumerate(self.label_values)}\n",
        "        self.ignored_labels = np.array(cfg.ignored_label_inds)\n",
        "\n",
        "        self.train_dir = str(Path(cfg.dataset_path) / cfg.train_dir)\n",
        "        self.val_dir = str(Path(cfg.dataset_path) / cfg.val_dir)\n",
        "        self.test_dir = str(Path(cfg.dataset_path) / cfg.test_dir)\n",
        "        print(self.train_dir, self.val_dir, self.test_dir)\n",
        "        self.train_files = [f for f in glob.glob(self.train_dir + \"/*.npy\")]\n",
        "        self.val_files = [f for f in glob.glob(self.val_dir + \"/*.npy\")]\n",
        "        self.test_files = [f for f in glob.glob(self.test_dir + \"/*.npy\")]\n",
        "\n",
        "        print(\"Training Data List: \" ,self.train_files, \"\\nValidation Data List: \", self.val_files, \"\\nTesting Data List: \", self.test_files)\n",
        "\n",
        "    @staticmethod\n",
        "    def get_label_to_names():\n",
        "        \"\"\"Returns a label to names dictionary object.\n",
        "        Returns:\n",
        "            A dict where keys are label numbers and\n",
        "            values are the corresponding names.\n",
        "        \"\"\"\n",
        "        label_to_names = {0: 'background', 1: 'apple'}\n",
        "        return label_to_names\n",
        "\n",
        "    def get_split(self, split):\n",
        "        \"\"\"Returns a dataset split.\n",
        "        Args:\n",
        "            split: A string identifying the dataset split that is usually one of\n",
        "            'training', 'test', 'validation', or 'all'.\n",
        "        Returns:\n",
        "            A dataset split object providing the requested subset of the data.\n",
        "        \"\"\"\n",
        "        return Custom3DSplit(self, split=split)\n",
        "\n",
        "    def get_split_list(self, split):\n",
        "        \"\"\"Returns a dataset split.\n",
        "        Args:\n",
        "            split: A string identifying the dataset split that is usually one of\n",
        "            'training', 'test', 'validation', or 'all'.\n",
        "        Returns:\n",
        "            A dataset split object providing the requested subset of the data.\n",
        "        Raises:\n",
        "             ValueError: Indicates that the split name passed is incorrect. The\n",
        "             split name should be one of 'training', 'test', 'validation', or\n",
        "             'all'.\n",
        "        \"\"\"\n",
        "        if split in ['test', 'testing']:\n",
        "            self.rng.shuffle(self.test_files)\n",
        "            return self.test_files\n",
        "        elif split in ['val', 'validation']:\n",
        "            self.rng.shuffle(self.val_files)\n",
        "            return self.val_files\n",
        "        elif split in ['train', 'training']:\n",
        "            self.rng.shuffle(self.train_files)\n",
        "            return self.train_files\n",
        "        elif split in ['all']:\n",
        "            files = self.val_files + self.train_files + self.test_files\n",
        "            return files\n",
        "        else:\n",
        "            raise ValueError(\"Invalid split {}\".format(split))\n",
        "\n",
        "    def is_tested(self, attr):\n",
        "        \"\"\"Checks if a datum in the dataset has been tested.\n",
        "        Args:\n",
        "            dataset: The current dataset to which the datum belongs to.\n",
        "            attr: The attribute that needs to be checked.\n",
        "        Returns:\n",
        "            If the dataum attribute is tested, then return the path where the\n",
        "            attribute is stored; else, returns false.\n",
        "        \"\"\"\n",
        "        cfg = self.cfg\n",
        "        name = attr['name']\n",
        "        path = cfg.test_result_folder\n",
        "        store_path = join(path, self.name, name + '.npy')\n",
        "        if exists(store_path):\n",
        "            print(\"{} already exists.\".format(store_path))\n",
        "            return True\n",
        "        else:\n",
        "            return False\n",
        "\n",
        "    def save_test_result(self, results, attr):\n",
        "        \"\"\"Saves the output of a model.\n",
        "        Args:\n",
        "            results: The output of a model for the datum associated with the attribute passed.\n",
        "            attr: The attributes that correspond to the outputs passed in results.\n",
        "        \"\"\"\n",
        "        cfg = self.cfg\n",
        "        name = attr['name']\n",
        "        path = cfg.test_result_folder\n",
        "        make_dir(path)\n",
        "        pred = results['predict_labels']\n",
        "        pred = np.array(self.label_to_names[pred])\n",
        "        store_path = join(path, name + '.npy')\n",
        "        np.save(store_path, pred)\n",
        "\n",
        "DATASET._register_module(Custom3D)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch related import statements\n",
        "import os\n",
        "import random\n",
        "import open3d.ml as _ml3d\n",
        "import open3d.ml.torch as ml3d\n",
        "# general import statements\n",
        "import logging\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "from pathlib import Path\n",
        "import os, sys, glob, pickle\n",
        "from sklearn.neighbors import KDTree\n",
        "from os.path import join, exists, dirname, abspath"
      ],
      "metadata": {
        "id": "_w2CzeWO9hee"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "U5Nq1SsOizqk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80d62c84-8256-4e95-ccaf-eb4430e68acd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/RandLANet_fuji-apple-segmentation_torch/checkpoint/ckpt_00018.pth\n"
          ]
        }
      ],
      "source": [
        "# loading the previous checkpoint for further training after runtime disconnect\n",
        "!ls drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/RandLANet_fuji-apple-segmentation_torch/checkpoint/ckpt_00018.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I-afEm3riztZ"
      },
      "outputs": [],
      "source": [
        "# RandLA-Net model configuration file\n",
        "!cat Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n",
        "# modifying this file based on the pipe dataset requirements with %%writefile command\n",
        "# for custom dataset we are modifying configurations like, data subset paths, classes, class weights etc."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A81p9No8kD8a",
        "outputId": "a3e3955f-2b5e-45e8-b197-79dacc472254"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n"
          ]
        }
      ],
      "source": [
        "%%writefile Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n",
        "dataset:\n",
        "  name: fuji-apple-segmentation\n",
        "  dataset_path: drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset\n",
        "  train_dir: train\n",
        "  val_dir: valid\n",
        "  test_dir: test\n",
        "  cache_dir: drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/cache\n",
        "  class_weights: []\n",
        "  ignored_label_inds: []\n",
        "  num_points: 40960\n",
        "  test_area_idx: 1\n",
        "  test_result_folder: ./test\n",
        "  use_cache: False\n",
        "model:\n",
        "  name: RandLANet\n",
        "  batcher: DefaultBatcher\n",
        "  ckpt_path: drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/RandLANet_fuji-apple-segmentation_torch/checkpoint/ckpt_00018.pth\n",
        "  num_neighbors: 16\n",
        "  num_layers: 5\n",
        "  num_points: 40960\n",
        "  num_classes: 2\n",
        "  ignored_label_inds: []\n",
        "  sub_sampling_ratio: [4, 4, 4, 4, 2]\n",
        "  in_channels: 6\n",
        "  dim_features: 8\n",
        "  dim_output: [16, 64, 128, 256, 512]\n",
        "  grid_size: 0.04\n",
        "  augment:\n",
        "    recenter:\n",
        "      dim: [0, 1]\n",
        "    rotate:\n",
        "      method: vertical\n",
        "    scale:\n",
        "      min_s: 0.9\n",
        "      max_s: 1.1\n",
        "    noise:\n",
        "      noise_std: 0.001\n",
        "pipeline:\n",
        "  name: SemanticSegmentation\n",
        "  optimizer:\n",
        "    lr: 0.00025\n",
        "  batch_size: 2\n",
        "  main_log_dir: drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs\n",
        "  max_epoch: 45\n",
        "  save_ckpt_freq: 9\n",
        "  scheduler_gamma: 0.99\n",
        "  test_batch_size: 3\n",
        "  train_sum_dir: drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/train_log\n",
        "  val_batch_size: 5\n",
        "  summary:\n",
        "    record_for: []\n",
        "    max_pts:\n",
        "    use_reference: false\n",
        "    max_outputs: 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "HgNGst-JkD-e"
      },
      "outputs": [],
      "source": [
        "# removing 'class_weights' vector for model training, know logged bug\n",
        "# otherwise, [1,3] (or [1, num_classes]) incompatible shape error is produced\n",
        "cfg_file = \"Open3D-ML/ml3d/configs/randlanet_s3dis.yml\"\n",
        "cfg = _ml3d.utils.Config.load_from_file(cfg_file)\n",
        "model = ml3d.models.RandLANet(**cfg.model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y2eHBbpkkEBT",
        "outputId": "6a1805e0-61ad-4cc8-a9e6-1962c9b44ac8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'name': 'fuji-apple-segmentation', 'dataset_path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset', 'train_dir': 'train', 'val_dir': 'valid', 'test_dir': 'test', 'cache_dir': 'drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/cache', 'class_weights': [], 'ignored_label_inds': [], 'num_points': 40960, 'test_area_idx': 1, 'test_result_folder': './test', 'use_cache': False}\n",
            "{'name': 'SemanticSegmentation', 'optimizer': {'lr': 0.00025}, 'batch_size': 2, 'main_log_dir': 'drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs', 'max_epoch': 45, 'save_ckpt_freq': 9, 'scheduler_gamma': 0.99, 'test_batch_size': 3, 'train_sum_dir': 'drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/train_log', 'val_batch_size': 5, 'summary': {'record_for': [], 'max_pts': None, 'use_reference': False, 'max_outputs': 1}}\n"
          ]
        }
      ],
      "source": [
        "# verifying the updated config file for the custom data loader and RandLa-net model\n",
        "print(cfg.dataset)\n",
        "print(cfg.pipeline)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zx3LB-E1iu34",
        "outputId": "884e9804-de21-4ec5-ccf5-277c5d3c2f58"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test\n",
            "Training Data List:  ['drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_0_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_0_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_0_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_0_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_1_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_1_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_1_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_1_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_1_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_2_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_2_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_2_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_2_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_2_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_3_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_3_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_3_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_3_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_4_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_5_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_5_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_5_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_5_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_5_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_6_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_7_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_7_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_7_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_7_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_8_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_8_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_8_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_8_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_8_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_9_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_9_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_9_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_9_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_9_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_10_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_10_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_10_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_10_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_0_11_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_0_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_0_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_0_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_0_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_0_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_1_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_2_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_3_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_4_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_5_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_5_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_5_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_5_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_5_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_6_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_6_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_6_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_6_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_6_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_7_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_8_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_9_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_10_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_4.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_1_11_5.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_0_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_0_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_0_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_1_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_1_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_2_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_2_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/train/data_patch_2_3_0.npy'] \n",
            "Validation Data List:  ['drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_3_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_3_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_4_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_4_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_4_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_5_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_5_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_5_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_5_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_6_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_6_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_6_3.npy'] \n",
            "Testing Data List:  ['drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_7_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_7_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_7_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_3.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_9_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_9_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_9_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_10_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_10_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_10_2.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_11_0.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_11_1.npy', 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_11_2.npy']\n"
          ]
        }
      ],
      "source": [
        "dataset = Custom3D(cfg.dataset.pop('dataset_path', None), **cfg.dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VHFbNWO3lPaE",
        "outputId": "ace5719f-972f-4ac5-88cc-8f83c7071401"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<__main__.Custom3DSplit object at 0x7f0ef6658c70>\n",
            "{'name': 'data_patch_2_3_1', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/valid/data_patch_2_3_1.npy', 'split': 'all'}\n",
            "(24305, 3)\n"
          ]
        }
      ],
      "source": [
        "# exploration: verifying whether loaded point cloud data is correct\n",
        "# get the 'all' split that combines training, validation and test set\n",
        "all_split = dataset.get_split('all')\n",
        "print(all_split)\n",
        "# print the attributes of the first datum\n",
        "print(all_split.get_attr(0))\n",
        "# print the shape of the first point cloud\n",
        "print(all_split.get_data(0)['point'].shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "89TP4tEZirNq"
      },
      "outputs": [],
      "source": [
        "# creating the segmentation pipeline for RandLA-Net model training\n",
        "pipeline = ml3d.pipelines.SemanticSegmentation(model, dataset=dataset, device=\"auto\", **cfg.pipeline)\n",
        "ckpt_path = \"drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/RandLANet_fuji-apple-segmentation_torch/checkpoint/ckpt_00018.pth\"\n",
        "pipeline.load_ckpt(ckpt_path=ckpt_path)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# semseg file updation for adding label smoothing and\n",
        "# additional penalization for apple class misclassification\n",
        "!cat /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py"
      ],
      "metadata": {
        "id": "Em-odrgx90EJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cvi0e_d8bU0Y",
        "outputId": "2722721c-4e68-424f-a529-a8a982114704"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py\n"
          ]
        }
      ],
      "source": [
        "%%writefile /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from ....datasets.utils import DataProcessing\n",
        "\n",
        "\n",
        "def filter_valid_label(scores, labels, num_classes, ignored_label_inds, device):\n",
        "    \"\"\"Loss functions for semantic segmentation.\"\"\"\n",
        "    valid_scores = scores.reshape(-1, num_classes).to(device)\n",
        "    valid_labels = labels.reshape(-1).to(device)\n",
        "\n",
        "    ignored_bool = torch.zeros_like(valid_labels, dtype=torch.bool)\n",
        "    for ign_label in ignored_label_inds:\n",
        "        ignored_bool = torch.logical_or(ignored_bool,\n",
        "                                        torch.eq(valid_labels, ign_label))\n",
        "\n",
        "    valid_idx = torch.where(torch.logical_not(ignored_bool))[0].to(device)\n",
        "\n",
        "    valid_scores = torch.gather(valid_scores, 0,\n",
        "                                valid_idx.unsqueeze(-1).expand(-1, num_classes))\n",
        "    valid_labels = torch.gather(valid_labels, 0, valid_idx)\n",
        "\n",
        "    # Reduce label values in the range of logit shape\n",
        "    reducing_list = torch.arange(0, num_classes, dtype=torch.int64)\n",
        "    inserted_value = torch.zeros([1], dtype=torch.int64)\n",
        "\n",
        "    for ign_label in ignored_label_inds:\n",
        "        if ign_label >= 0:\n",
        "\n",
        "            reducing_list = torch.cat([\n",
        "                reducing_list[:ign_label], inserted_value,\n",
        "                reducing_list[ign_label:]\n",
        "            ], 0)\n",
        "    valid_labels = torch.gather(reducing_list.to(device), 0,\n",
        "                                valid_labels.long())\n",
        "\n",
        "    return valid_scores, valid_labels\n",
        "\n",
        "\n",
        "class SemSegLoss(object):\n",
        "    \"\"\"Loss functions for semantic segmentation.\"\"\"\n",
        "\n",
        "    def __init__(self, pipeline, model, dataset, device):\n",
        "        super(SemSegLoss, self).__init__()\n",
        "        # weighted_CrossEntropyLoss\n",
        "        if 'class_weights' in dataset.cfg.keys() and len(\n",
        "                dataset.cfg.class_weights) != 0:\n",
        "            class_wt = DataProcessing.get_class_weights(\n",
        "                dataset.cfg.class_weights)\n",
        "            weights = torch.tensor(class_wt, dtype=torch.float, device=device)\n",
        "\n",
        "            self.weighted_CrossEntropyLoss = nn.CrossEntropyLoss(weight=weights)\n",
        "        else:\n",
        "            weights = torch.tensor([1.0, 4.0], dtype=torch.float).squeeze(-1)\n",
        "            self.weighted_CrossEntropyLoss = nn.CrossEntropyLoss(weight=weights, ignore_index=2, label_smoothing=0.05)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-m0C2xlIiraQ"
      },
      "outputs": [],
      "source": [
        "# training the model from scratch with the pipe dataset\n",
        "# uncomment during model training stage\n",
        "# pipeline.run_train()\n",
        "# error, incorporation of 'class_weights' from the configuration file give shape head error"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FU5Os4VjjZyd"
      },
      "outputs": [],
      "source": [
        "# for loading the training logs to observe the learning curve during training \n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir drive/MyDrive/point-cloud-prototyping/fuji_sfm_logs/train_log"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "8Mns-G9zjZ1S",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f9d8eaaf-4749-4559-9647-98fe83338ef4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "test 0/16: 100%|██████████| 605/605 [04:38<00:00,  2.17it/s]\n",
            "\n",
            "test 0/1: 100%|██████████| 646/646 [00:13<00:00, 47.26it/s]\n",
            "\n",
            "test 0/1: 100%|██████████| 157/157 [00:10<00:00, 15.11it/s]\n",
            "\n",
            "test 0/1: 100%|██████████| 186/186 [00:13<00:00, 14.24it/s]\n",
            "test 0/1: 100%|██████████| 525/525 [00:06<00:00, 82.70it/s]\n",
            "test 0/1: 100%|██████████| 525/525 [00:12<00:00, 40.50it/s]\n",
            "\n",
            "test 0/1: 100%|██████████| 371/371 [00:07<00:00, 46.42it/s]\u001b[A"
          ]
        }
      ],
      "source": [
        "# generating sample test point cloud outputs from the trained models\n",
        "# for five data patches for more in-depth and systematic analysis\n",
        "# at data patch performance level of system performance\n",
        "test_split = dataset.get_split(\"test\")\n",
        "\n",
        "data_one = test_split.get_data(0)\n",
        "result_one = pipeline.run_inference(data_one)\n",
        "\n",
        "data_two = test_split.get_data(1)\n",
        "result_two = pipeline.run_inference(data_two)\n",
        "\n",
        "data_three = test_split.get_data(2)\n",
        "result_three = pipeline.run_inference(data_three)\n",
        "\n",
        "data_four = test_split.get_data(3)\n",
        "result_four = pipeline.run_inference(data_four)\n",
        "\n",
        "data_five = test_split.get_data(4)\n",
        "result_five = pipeline.run_inference(data_five)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.a Model Performance Inference and Evaluation"
      ],
      "metadata": {
        "id": "6osAVkC49_cn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch sample specific testing\n",
        "# for obtaining basic accuracy related metrics\n",
        "# to measure the initial model performannce quantitatively\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "# jaccard score being used for mIoU calculation\n",
        "from sklearn.metrics import jaccard_score\n",
        "from sklearn.metrics import accuracy_score, f1_score, \\\n",
        "precision_score, recall_score, classification_report, confusion_matrix\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "W5jvTgGs9_Is"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# point cloud size for the data patches under consideration\n",
        "print(result_one['predict_labels'].shape)\n",
        "print(result_two['predict_labels'].shape)\n",
        "print(result_three['predict_labels'].shape)\n",
        "print(result_four['predict_labels'].shape)\n",
        "print(result_five['predict_labels'].shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C_4nzu7QBxRU",
        "outputId": "50e22fcd-9960-4e28-8655-950ddce6e4da"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(44694,)\n",
            "(8943,)\n",
            "(9238,)\n",
            "(30636,)\n",
            "(20633,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch one metric result summary\n",
        "print(\"Average Accuracy: \" + str(accuracy_score(data_one['label'], result_one['predict_labels'])))\n",
        "matrix = confusion_matrix(data_one['label'], result_one['predict_labels'])\n",
        "print(\"Class Level Accuracy: \" + str(matrix.diagonal()/matrix.sum(axis=1)))\n",
        "print(\"Class Level mIoU: \" + str(jaccard_score(data_one['label'], result_one['predict_labels'], average=None)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ctLmfwsj9_GP",
        "outputId": "36cbcce2-589e-48a1-ba23-dd375b0fdd3f"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.12218642323354366\n",
            "Class Level Accuracy: [0.08935862 0.88979815]\n",
            "Class Level mIoU: [0.08893946 0.03991288]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch two metric result summary\n",
        "print(\"Average Accuracy: \" + str(accuracy_score(data_two['label'], result_two['predict_labels'])))\n",
        "matrix = confusion_matrix(data_two['label'], result_two['predict_labels'])\n",
        "print(\"Class Level Accuracy: \" + str(matrix.diagonal()/matrix.sum(axis=1)))\n",
        "print(\"Class Level mIoU: \" + str(jaccard_score(data_two['label'], result_two['predict_labels'], average=None)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "34yq4mIa9_EK",
        "outputId": "ca0aa3e3-8022-43cc-89c2-3da8bda1a6da"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.8397629430839763\n",
            "Class Level Accuracy: [0.85693294 0.75154427]\n",
            "Class Level mIoU: [0.81740571 0.43314873]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch three metric result summary\n",
        "print(\"Average Accuracy: \" + str(accuracy_score(data_three['label'], result_three['predict_labels'])))\n",
        "matrix = confusion_matrix(data_three['label'], result_three['predict_labels'])\n",
        "print(\"Class Level Accuracy: \" + str(matrix.diagonal()/matrix.sum(axis=1)))\n",
        "print(\"Class Level mIoU: \" + str(jaccard_score(data_three['label'], result_three['predict_labels'], average=None)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a2f-CvjN9_CA",
        "outputId": "9db19ce4-9a6c-4087-a09b-6e41f94045cf"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.7444252002597965\n",
            "Class Level Accuracy: [0.71920593 0.98515982]\n",
            "Class Level mIoU: [0.71808955 0.2676799 ]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch two metric result summary\n",
        "print(\"Average Accuracy: \" + str(accuracy_score(data_four['label'], result_four['predict_labels'])))\n",
        "matrix = confusion_matrix(data_four['label'], result_four['predict_labels'])\n",
        "print(\"Class Level Accuracy: \" + str(matrix.diagonal()/matrix.sum(axis=1)))\n",
        "print(\"Class Level mIoU: \" + str(jaccard_score(data_four['label'], result_four['predict_labels'], average=None)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1VDq1j7O9-_6",
        "outputId": "30184518-021d-46d4-eabf-9d8f24881585"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.5123710667188928\n",
            "Class Level Accuracy: [0.3957525  0.97658917]\n",
            "Class Level mIoU: [0.39343863 0.28678507]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# data patch two metric result summary\n",
        "print(\"Average Accuracy: \" + str(accuracy_score(data_five['label'], result_five['predict_labels'])))\n",
        "matrix = confusion_matrix(data_five['label'], result_five['predict_labels'])\n",
        "print(\"Class Level Accuracy: \" + str(matrix.diagonal()/matrix.sum(axis=1)))\n",
        "print(\"Class Level mIoU: \" + str(jaccard_score(data_five['label'], result_five['predict_labels'], average=None)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SnHYNR3y9-9u",
        "outputId": "1541bd7e-b130-4124-e290-cb5f7056c5e3"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average Accuracy: 0.8296418358939562\n",
            "Class Level Accuracy: [0.84440974 0.75328947]\n",
            "Class Level mIoU: [0.8059512  0.41746768]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# utility function to convert the loaded test sample and result prediction\n",
        "# into prediction point cloud '.txt' annotation ground truth and prediction files\n",
        "# for further visualizations with plotly and open3d graphs\n",
        "def gen_downsampled_pc(data_dict, pred_dict, ground_truth_file, prediction_file, ml3d_flg=True):\n",
        "    x_ = list(data_dict['point'][:,0])\n",
        "    y_ = list(data_dict['point'][:,1])\n",
        "    z_ = list(data_dict['point'][:,2])\n",
        "    l_ = list(data_dict['label'])\n",
        "    p_ = list(pred_dict['predict_labels'])\n",
        "    vals_ = pred_dict['predict_labels'].shape[0]\n",
        "    rows_ids_cf = random.sample(range(0,len(x_)-1), int(len(x_)/2))\n",
        "    x_sample = [x_[i] for i in rows_ids_cf]\n",
        "    y_sample = [y_[i] for i in rows_ids_cf]\n",
        "    z_sample = [z_[i] for i in rows_ids_cf]\n",
        "    l_sample = [l_[i] for i in rows_ids_cf]\n",
        "    p_sample = [p_[i] for i in rows_ids_cf]\n",
        "    if ml3d_flg == True:\n",
        "        vals_sample = len(p_sample)\n",
        "    output_lines_gt = []\n",
        "    output_lines_prd = []\n",
        "    if ml3d_flg == True:\n",
        "        output_lines_gt.append(str(vals_sample))\n",
        "        output_lines_prd.append(str(vals_sample))\n",
        "    for i,j,k,l in zip(x_sample, y_sample, z_sample, l_sample):\n",
        "        output_lines_gt.append(str(i)+\" \"+str(j)+\" \"+str(k)+\" \"+str(l))\n",
        "\n",
        "    for i,j,k,p in zip(x_sample, y_sample, z_sample, p_sample):\n",
        "        output_lines_prd.append(str(i)+\" \"+str(j)+\" \"+str(k)+\" \"+str(p))\n",
        "    \n",
        "    gt_i = 0\n",
        "    with open(ground_truth_file, 'w') as f:\n",
        "        for line in output_lines_gt:\n",
        "            gt_i = gt_i + 1\n",
        "            f.write(line)\n",
        "            if gt_i < len(output_lines_gt):\n",
        "                f.write('\\n')\n",
        "    prd_i = 0\n",
        "    with open(prediction_file, 'w') as f:\n",
        "        for line in output_lines_prd:\n",
        "            prd_i = prd_i + 1\n",
        "            f.write(line)\n",
        "            if prd_i < len(output_lines_prd):\n",
        "                f.write('\\n')"
      ],
      "metadata": {
        "id": "VxV7NYR49-74"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# obtaining the names of the test files for visualization\n",
        "# this list will change during replication because of randomization in open3d loaders\n",
        "print(test_split.get_attr(0))\n",
        "print(test_split.get_attr(1))\n",
        "print(test_split.get_attr(2))\n",
        "print(test_split.get_attr(3))\n",
        "print(test_split.get_attr(4))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xrT_5coB9-6P",
        "outputId": "63aaa3f5-5431-4099-bc73-1a1b11368fe1"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'name': 'data_patch_2_8_0', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_0.npy', 'split': 'test'}\n",
            "{'name': 'data_patch_2_7_2', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_7_2.npy', 'split': 'test'}\n",
            "{'name': 'data_patch_2_11_2', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_11_2.npy', 'split': 'test'}\n",
            "{'name': 'data_patch_2_11_1', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_11_1.npy', 'split': 'test'}\n",
            "{'name': 'data_patch_2_8_2', 'path': 'drive/MyDrive/point-cloud-prototyping/datasets/fuji-sfm-dataset/test/data_patch_2_8_2.npy', 'split': 'test'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# saving test samples for visualization with open3d plotly plots\n",
        "# argument ml3d_flg = False specifies that the point cloud will not be directly visualized by open3d ploty function\n",
        "gen_downsampled_pc(data_one, result_one, 'data_patch_2_8_0_gt.txt', 'data_patch_2_8_0_pred.txt', ml3d_flg=False)\n",
        "gen_downsampled_pc(data_two, result_two, 'data_patch_2_7_2_gt.txt', 'data_patch_2_7_2_pred.txt', ml3d_flg=False)\n",
        "gen_downsampled_pc(data_three, result_three, 'data_patch_2_11_2_gt.txt', 'data_patch_2_11_2_pred.txt', ml3d_flg=False)\n",
        "gen_downsampled_pc(data_four, result_four, 'data_patch_2_8_2_gt.txt', 'data_patch_2_8_2_pred.txt', ml3d_flg=False)"
      ],
      "metadata": {
        "id": "yV295ZQN9-4N"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.b Prediction Data Patch Visualization"
      ],
      "metadata": {
        "id": "VdkzQwPuDRuT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# visualizing the segmentation map of the PC\n",
        "# example dict of seg colors and their corresponding hex-values\n",
        "COLOR_TO_HEX = { \"green\" : \"#678286\", \"magenta\" : \"#E990C5\",\n",
        "                 \"blue\" : \"#D8E9F7\", \"yellow\" : \"#F2D4A2\",\n",
        "                \"red\" : \"#CA5047\", \"green\" : \"#689F55\"}\n",
        "\n",
        "def hex_to_rgb(hex):\n",
        "  return list(round(int(hex[i:i+2], 16)/255,2) for i in (1, 3, 5))\n",
        "\n",
        "COLOR_TO_RGB = {}\n",
        "\n",
        "for k,v in COLOR_TO_HEX.items():\n",
        "    COLOR_TO_RGB[k] = hex_to_rgb(v)\n",
        "\n",
        "\n",
        "\n",
        "# PC max size for visualization for plotly\n",
        "MAX_PC_SIZE = 40960 # RandLA-Net's input dimension size"
      ],
      "metadata": {
        "id": "Z7Kg6NM09-2M"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from copy import deepcopy\n",
        "# downsampling the PC size for standardization of visualized 'cls' & 'bg' PC size\n",
        "def downsample_pcl(pcl_arr, downsampling_factor):\n",
        "    assert downsampling_factor >= 0.2 and downsampling_factor < 1.0\n",
        "    downsampled_pc_count = int(downsampling_factor * pcl_arr.shape[0])\n",
        "    if downsampled_pc_count > MAX_PC_SIZE:\n",
        "        downsampled_pc_count = MAX_PC_SIZE\n",
        "    idx_cls = np.random.randint((pcl_arr.shape[0] - 1),\n",
        "                                size = downsampled_pc_count)\n",
        "    pcl_arr = pcl_arr[idx_cls,:] \n",
        "    return pcl_arr\n",
        "     \n",
        "\n",
        "# saving the downsampled segmented PC for visualization with open3d and plotly\n",
        "# function to load the npy array and convert to '.txt' readable\n",
        "# format of open3d's draw_plotly function\n",
        "def segmap_color_generator(pc_arr, save_path):\n",
        "    data_patch = deepcopy(pc_arr)\n",
        "    data_patch = downsample_pcl(data_patch,\n",
        "                                     downsampling_factor = 0.5)\n",
        "    data_patch_list = list(data_patch)\n",
        "    seg_map_list = []\n",
        "    class_colors_list = COLOR_TO_RGB['red']\n",
        "    bg_colors_list = COLOR_TO_RGB['green']\n",
        "    for vals in data_patch_list:\n",
        "        if vals[3] == 1: # updated to 3, since rgb features not in the base file\n",
        "            joined_temp_list = list(vals[:3]) + class_colors_list\n",
        "            seg_map_list.append(joined_temp_list)\n",
        "        else:\n",
        "            joined_temp_list = list(vals[:3]) + bg_colors_list\n",
        "            seg_map_list.append(joined_temp_list)\n",
        "    \n",
        "    seg_map = np.array(seg_map_list)\n",
        "    \n",
        "\n",
        "    np.savetxt(save_path, seg_map, delimiter=' ')\n",
        "\n",
        "# visualization of the stored PC files \n",
        "import open3d as o3d\n",
        "def viz_pcl_o3d(pcl_txt_path):\n",
        "    pcd_viz = o3d.io.read_point_cloud(pcl_txt_path, format='xyzrgb')\n",
        "    o3d.visualization.draw_plotly([pcd_viz])"
      ],
      "metadata": {
        "id": "1QTi-mUX9-0Z"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# listing all files in the directory and making their segmentation map '.txt' file\n",
        "# for the visualization task\n",
        "data_patch_list = ['data_patch_2_11_2_gt.txt', 'data_patch_2_8_0_gt.txt',\n",
        "                    'data_patch_2_11_2_pred.txt', 'data_patch_2_8_0_pred.txt',\n",
        "                    'data_patch_2_7_2_gt.txt', 'data_patch_2_8_2_gt.txt',\n",
        "                    'data_patch_2_7_2_pred.txt', 'data_patch_2_8_2_pred.txt']\n",
        "for data_patch in data_patch_list:\n",
        "    data_patch_arr = np.loadtxt(data_patch)\n",
        "    segmap_color_generator(data_patch_arr, data_patch)"
      ],
      "metadata": {
        "id": "PYrgWrhR9-yV"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_11_2_gt.txt')"
      ],
      "metadata": {
        "id": "0o_U60Vw9-wc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_11_2_pred.txt')"
      ],
      "metadata": {
        "id": "4Y8_VukH9-uO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_8_2_gt.txt')"
      ],
      "metadata": {
        "id": "JgMq5RJq9-oV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_8_2_pred.txt')"
      ],
      "metadata": {
        "id": "e6gxlP_Q9-mR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_7_2_gt.txt')"
      ],
      "metadata": {
        "id": "CgYArsayHtVQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "viz_pcl_o3d('data_patch_2_7_2_pred.txt')"
      ],
      "metadata": {
        "id": "86DJkL8xHdlJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "IK2KkIC69-iB"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}