fix bugs of outdoor optimizaiton and conda environment

README.md

@@ -38,13 +38,14 @@ git clone --recursive https://github.com/aim-uofa/FrozenRecon.git
 cd FrozenRecon
 conda create -y -n frozenrecon python=3.8
 conda activate frozenrecon
-pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html # pytorch 1.7.1 for SegFormer
 pip install -r requirements.txt
 
 # (Optional) For outdoor scenes, we recommand to mask the sky regions and cars (potential dynamic objects)
-pip install --upgrade mmcv-full==1.3.0 -f https://download.openmmlab.com/mmcv/dist/cu111/torch180/index.html
-pip install "mmsegmentation==0.11.0"
 pip install timm==0.3.2
+pip install --upgrade mmcv-full==1.2.7 -f https://download.openmmlab.com/mmcv/dist/cu110/torch171/index.html
+# pip install "mmsegmentation==0.11.0"
+pip install ipython attr 
 git clone https://github.com/NVlabs/SegFormer.git
 cd SegFormer && pip install -e . & cd ..
 # After installing SegFormer, please downlaod segformer.b3.512x512.ade.160k.pth checkpoint following https://github.com/NVlabs/SegFormer, and place it in SegFormer/

optimize.sh

@@ -1,4 +1,4 @@
-conda activate frozenrecon
+# conda activate frozenrecon
 
 # # change GT_ROOT if you would like to use gt priors
 # export GT_ROOT='PATH_TO_GT_DATA_ROOT'
@@ -22,4 +22,4 @@ conda activate frozenrecon
 python src/optimize.py --img_root PATH_TO_IMG_FOLDER --scene_name SCENE_NAME
 
 # # Outdoor Scenes
-# python src/optimize.py --dataset_name NYUDepthVideo --scene_name SCENE_NAME --gt_root $GT_ROOT --gt_intrinsic_flag --scene_name "2011_09_26_drive_0001_sync" --outdoor_scenes 
+# python src/optimize.py --dataset_name KITTI --gt_intrinsic_flag --scene_name 2011_09_26_drive_0001_sync --outdoor_scenes

requirements.txt

@@ -1,15 +1,15 @@
-matplotlib
-opencv-python
-plyfile
+# matplotlib
+# opencv-python
+# plyfile
 scipy
 tqdm
 xlwt
-numpy==1.21.0
+# numpy==1.21.0
 numba
 scikit-image==0.16.2
 pycuda
-logger
-open3d
-path
-kornia==0.6.7
+# logger
+# open3d
+# path
+# kornia==0.6.7
 pykitti

src/optimize.py

@@ -276,7 +276,7 @@ def load_and_init(self):
                 gt_loader = SevenScenes_Loader(osp.join(args.gt_root, args.dataset_name))
             elif self.args.dataset_name == 'TUM':
                 gt_loader = TUM_Loader(osp.join(args.gt_root, args.dataset_name))
-            elif self.args.dataset_name == 'KITTI':
+            elif self.args.dataset_name == 'kitti':
                 gt_loader = KITTIDepthVideo_Loader(osp.join(args.gt_root, args.dataset_name), scene_names_list=[args.scene_name])
             else:
                 raise ValueError
@@ -799,11 +799,11 @@ def recon(args, optimized_params, voxel_size=0.1):
     bs, h, w = depths.shape
     depths = depths.squeeze()
     if args.outdoor_scenes and mmseg_import_flag:
-        invalid_masks = optimized_params['invalid_masks']
+        invalid_masks = torch.from_numpy(optimized_params['invalid_masks'])
         invalid_masks = F.interpolate(invalid_masks, (images.shape[1], images.shape[2]), mode='nearest')
-        invalid_masks = invalid_masks.detach().cpu().numpy()
+        invalid_masks = invalid_masks.detach().cpu().numpy().squeeze()
         assert len(invalid_masks.shape) == 3
-        depths[invalid_masks[optimized_params['optimized_image_indexes']] == 1] = 0
+        depths[invalid_masks == 1] = 0
 
     tsdf = TSDFFusion()
     print('tsdf fusing the pred pcd...')
@@ -839,7 +839,7 @@ def recon(args, optimized_params, voxel_size=0.1):
     parser.add_argument('--loss_gc_weight', help="hyperparameter of losses.", type=float, nargs='+', default=[0.5, 1, 0.1])
     parser.add_argument('--loss_norm_weight', help="hyperparameter of losses.", type=float, nargs='+', default=[0.01, 0.1, 0.1])
 
-    parser.add_argument('--dataset_name', help='Dataset name. Ignore it when input images or videos in the wild.', type=str, choices=['NYUDepthVideo', 'scannet_test_div_5', '7scenes_new_seq1', 'TUM', 'KITTI'], default=None)
+    parser.add_argument('--dataset_name', help='Dataset name. Ignore it when input images or videos in the wild.', type=str, choices=['NYUDepthVideo', 'scannet_test_div_5', '7scenes_new_seq1', 'TUM', 'kitti'], default=None)
     parser.add_argument('--scene_name', help='If None, optimize all scenes. If set, only optimize one scene.', type=str, default=None)
     parser.add_argument('--outdoor_scenes', help='Whether to optimize outdoor scenes. (Used for filtering out sky regions and car (dynamic objects).)', action='store_true')
 
@@ -911,7 +911,7 @@ def recon(args, optimized_params, voxel_size=0.1):
             img_roots.append(osp.join(base_root + scene_name + '/rgb'))
 
     # KITTI
-    elif args.dataset_name == 'KITTI':
+    elif args.dataset_name == 'kitti':
         args.gt_depth_scale = 256.
         base_root = osp.join(args.gt_root, args.dataset_name)
         scene_names = [
@@ -951,6 +951,9 @@ def recon(args, optimized_params, voxel_size=0.1):
     else:
         raise ValueError('Error of args.dataset_name')
 
+    print('scene_names :', scene_names)
+    print('img_roots :', img_roots)
+
     for (scene_name, img_root) in zip(scene_names, img_roots):
         if (args.scene_name is not None) and (scene_name != args.scene_name):
             continue

src/rgbd_fusion/tsdf_fusion.py

@@ -27,7 +27,7 @@ def fusion(self, images, depths, intrinsics, poses, frame_mask=None, save_path=N
             depth_im = np.squeeze(depths[i])
             if edge_mask:
                 # valid_mask
-                valid_mask = np.zeros_like(depth_im).astype(np.bool)
+                valid_mask = np.zeros_like(depth_im).astype(np.bool_)
                 valid_mask[20:h-20, 20:w-20] = True
                 depth_im[~valid_mask] = 0
 
@@ -48,7 +48,7 @@ def fusion(self, images, depths, intrinsics, poses, frame_mask=None, save_path=N
 
             if edge_mask:
                 # valid_mask
-                valid_mask = np.zeros_like(depth_im).astype(np.bool)
+                valid_mask = np.zeros_like(depth_im).astype(np.bool_)
                 valid_mask[20:h-20, 20:w-20] = True
                 depth_im[~valid_mask] = 0