diff --git a/README.md b/README.md index ee974486..1c31c616 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,10 @@ Then download models for _perceptual loss_: ## Places + +⚠️ NB: FID/SSIM/LPIPS metric values for Places that we see in LaMa paper are computed on 30000 images that we produce in evaluation section below. +For more details on evaluation data check [[Section 3. Dataset splits in Supplementary](https://ashukha.com/projects/lama_21/lama_supmat_2021.pdf#subsection.3.1)] ⚠️ + On the host machine: # Download data from http://places2.csail.mit.edu/download.html @@ -170,18 +174,20 @@ On the host machine: wget http://data.csail.mit.edu/places/places365/val_large.tar wget http://data.csail.mit.edu/places/places365/test_large.tar - # Unpack and etc. + # Unpack train/test/val data and create .yaml config for it bash fetch_data/places_standard_train_prepare.sh bash fetch_data/places_standard_test_val_prepare.sh - bash fetch_data/places_standard_evaluation_prepare_data.sh # Sample images for test and viz at the end of epoch bash fetch_data/places_standard_test_val_sample.sh bash fetch_data/places_standard_test_val_gen_masks.sh # Run training - # You can change bs with data.batch_size=10 - python bin/train.py -cn lama-fourier location=places_standard + python3 bin/train.py -cn lama-fourier location=places_standard + + # To evaluate trained model and report metrics as in our paper + # we need to sample previously unseen 30k images and generate masks for them + bash fetch_data/places_standard_evaluation_prepare_data.sh # Infer model on thick/thin/medium masks in 256 and 512 and run evaluation # like this: @@ -191,9 +197,10 @@ On the host machine: outdir=$(pwd)/inference/random_thick_512 model.checkpoint=last.ckpt python3 bin/evaluate_predicts.py \ - $(pwd)/configs/eval_2gpu.yaml \ + $(pwd)/configs/eval2_gpu.yaml \ $(pwd)/places_standard_dataset/evaluation/random_thick_512/ \ - $(pwd)/inference/random_thick_512 $(pwd)/inference/random_thick_512_metrics.csv + $(pwd)/inference/random_thick_512 \ + $(pwd)/inference/random_thick_512_metrics.csv @@ -216,7 +223,7 @@ On the host machine: bash fetch_data/celebahq_gen_masks.sh # Run training - python bin/train.py -cn lama-fourier-celeba data.batch_size=10 + python3 bin/train.py -cn lama-fourier-celeba data.batch_size=10 # Infer model on thick/thin/medium masks in 256 and run evaluation # like this: @@ -335,7 +342,7 @@ On the host machine: # Run training - python bin/train.py -cn lama-fourier location=my_dataset data.batch_size=10 + python3 bin/train.py -cn lama-fourier location=my_dataset data.batch_size=10 # Evaluation: LaMa training procedure picks best few models according to # scores on my_dataset/val/ @@ -353,7 +360,7 @@ On the host machine: # metrics calculation: python3 bin/evaluate_predicts.py \ - $(pwd)/configs/eval_2gpu.yaml \ + $(pwd)/configs/eval2_gpu.yaml \ $(pwd)/my_dataset/eval/random__512/ \ $(pwd)/inference/my_dataset/random__512 \ $(pwd)/inference/my_dataset/random__512_metrics.csv diff --git a/fetch_data/eval_sampler.py b/fetch_data/eval_sampler.py index bf2d70d8..7cffdbc9 100644 --- a/fetch_data/eval_sampler.py +++ b/fetch_data/eval_sampler.py @@ -1,19 +1,18 @@ import os import random - -val_files_path = os.path.abspath('.') + '/places_standard_dataset/original/val/' +val_files_path = os.path.abspath('.') + '/places_standard_dataset/original/val/' +list_of_random_val_files = os.path.abspath('.') + '/places_standard_dataset/original/eval_random_files.txt' val_files = [val_files_path + image for image in os.listdir(val_files_path)] -print(f'found {len(val_files)} images in {val_files_path}') +print(f'Sampling 30000 images out of {len(val_files)} images in {val_files_path}' + \ + f'and put their paths to {list_of_random_val_files}') -random.shuffle(val_files) -val_files_random = val_files[0:2000] +print('In our paper we evaluate trained models on these 30k sampled (mask,image) pairs in our paper (check Sup. mat.)') -list_of_random_val_files = os.path.abspath('.') \ -+ '/places_standard_dataset/original/eval_random_files.txt' +random.shuffle(val_files) +val_files_random = val_files[0:30000] -print(f'copying 2000 random images to {list_of_random_val_files}') with open(list_of_random_val_files, 'w') as fw: for filename in val_files_random: fw.write(filename+'\n') diff --git a/fetch_data/places_standard_evaluation_prepare_data.sh b/fetch_data/places_standard_evaluation_prepare_data.sh index 15d753cc..2962ac8c 100755 --- a/fetch_data/places_standard_evaluation_prepare_data.sh +++ b/fetch_data/places_standard_evaluation_prepare_data.sh @@ -7,7 +7,7 @@ mkdir -p places_standard_dataset/evaluation/random_thick_256/ mkdir -p places_standard_dataset/evaluation/random_thin_256/ mkdir -p places_standard_dataset/evaluation/random_medium_256/ -# 1. sample 2000 new images +# 1. sample 30000 new images OUT=$(python3 fetch_data/eval_sampler.py) echo ${OUT} diff --git a/fetch_data/sampler.py b/fetch_data/sampler.py index b25fa1fe..9bf48b7f 100644 --- a/fetch_data/sampler.py +++ b/fetch_data/sampler.py @@ -1,37 +1,40 @@ import os import random -test_files_path = os.path.abspath('.') + '/places_standard_dataset/original/test/' -test_files = [test_files_path + image for image in os.listdir(test_files_path)] -print(f'found {len(test_files)} images in {test_files_path}') +test_files_path = os.path.abspath('.') + '/places_standard_dataset/original/test/' +list_of_random_test_files = os.path.abspath('.') + '/places_standard_dataset/original/test_random_files.txt' -random.shuffle(test_files) -test_files_random = test_files[0:2000] -#print(test_files_random[0:10]) +test_files = [ + test_files_path + image for image in os.listdir(test_files_path) +] -list_of_random_test_files = os.path.abspath('.') \ -+ '/places_standard_dataset/original/test_random_files.txt' +print(f'Sampling 2000 images out of {len(test_files)} images in {test_files_path}' + \ + f'and put their paths to {list_of_random_test_files}') +print('Our training procedure will pick best checkpoints according to metrics, computed on these images.') -print(f'copying 100 random images to {list_of_random_test_files}') +random.shuffle(test_files) +test_files_random = test_files[0:2000] with open(list_of_random_test_files, 'w') as fw: for filename in test_files_random: fw.write(filename+'\n') print('...done') -# ---------------------------------------------------------------------------------- +# -------------------------------- -val_files_path = os.path.abspath('.') + '/places_standard_dataset/original/val/' -val_files = [val_files_path + image for image in os.listdir(val_files_path)] -print(f'found {len(val_files)} images in {val_files_path}') +val_files_path = os.path.abspath('.') + '/places_standard_dataset/original/val/' +list_of_random_val_files = os.path.abspath('.') + '/places_standard_dataset/original/val_random_files.txt' -random.shuffle(val_files) -val_files_random = val_files[0:100] +val_files = [ + val_files_path + image for image in os.listdir(val_files_path) +] -list_of_random_val_files = os.path.abspath('.') \ -+ '/places_standard_dataset/original/val_random_files.txt' +print(f'Sampling 100 images out of {len(val_files)} in {val_files_path} ' + \ + f'and put their paths to {list_of_random_val_files}') +print('We use these images for visual check up of evolution of inpainting algorithm epoch to epoch' ) -print(f'copying 100 random images to {list_of_random_val_files}') +random.shuffle(val_files) +val_files_random = val_files[0:100] with open(list_of_random_val_files, 'w') as fw: for filename in val_files_random: fw.write(filename+'\n')