-
Notifications
You must be signed in to change notification settings - Fork 38
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
adding generate_dataset_example.ipynb
new file: generate_dataset_example.ipynb
- Loading branch information
Showing
1 changed file
with
197 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,197 @@ | ||
{ | ||
"cells": [ | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 1, | ||
"id": "30d159b1-a72b-433c-98dc-de694462e657", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"signal_image_folder_path = \"<PATH_TO_SPECTROGRAM_IMAGES>\"\n", | ||
"soi_path = \"<PATH_TO_SPECTROGRAM_IMAGES_MARKED_WITH_SIGNAL_OF_INTEREST>\"" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "70ca4d0b-19f2-4a0b-85e1-6f8f795ccb41", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 2, | ||
"id": "d0fa212e-6128-495f-8af6-4bba4629fc76", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"import os\n", | ||
"from torchsig.image_datasets.datasets.synthetic_signals import GeneratorFunctionDataset, rectangle_signal_generator_function, tone_generator_function, repeated_signal_generator_function\n", | ||
"from torchsig.image_datasets.datasets.file_loading_datasets import SOIExtractorDataset, LazyImageDirectoryDataset\n", | ||
"from torchsig.image_datasets.datasets.composites import YOLOImageCompositeDataset, ConcatDataset\n", | ||
"from torchsig.image_datasets.transforms.impairments import BlurTransform, RandomGaussianNoiseTransform, RandomImageResizeTransform, RandomRippleNoiseTransform, ScaleTransform, scale_dynamic_range, normalize_image" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 3, | ||
"id": "cf919892-2a82-4d1e-a4f7-3d4ae3b485d5", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"import cv2\n", | ||
"import numpy as np\n", | ||
"import matplotlib.pyplot as plt\n", | ||
"import torch\n", | ||
"\n", | ||
"def plot_boxes_on_image(image, labels):\n", | ||
" image = image.numpy()\n", | ||
" image = (np.stack([image[0,:,:]]*3).transpose(1,2,0)*255).astype(np.uint8)\n", | ||
" for label in labels:\n", | ||
" cid, cx, cy, w, h = label\n", | ||
" img_h, img_w = image.shape[:2]\n", | ||
" x1 = int((cx - w/2)*img_w)\n", | ||
" x2 = int((cx + w/2)*img_w)\n", | ||
" y1 = int((cy - h/2)*img_h)\n", | ||
" y2 = int((cy + h/2)*img_h)\n", | ||
" image = cv2.rectangle(image.copy(), (x1, y1), (x2, y2), color=(255,0,0), thickness=1)\n", | ||
" plt.imshow(image)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 4, | ||
"id": "d16cc6cc-63e7-4c70-adfd-a40cbb78ef27", | ||
"metadata": {}, | ||
"outputs": [ | ||
{ | ||
"ename": "FileNotFoundError", | ||
"evalue": "[Errno 2] No such file or directory: '<PATH_TO_SPECTROGRAM_IMAGES>'", | ||
"output_type": "error", | ||
"traceback": [ | ||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | ||
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", | ||
"Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m modulations_paths \u001b[38;5;241m=\u001b[39m [signal_image_folder_path \u001b[38;5;241m+\u001b[39m fpath \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m fpath \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignal_image_folder_path\u001b[49m\u001b[43m)\u001b[49m]\n\u001b[1;32m 2\u001b[0m image_datasets \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m path \u001b[38;5;129;01min\u001b[39;00m modulations_paths:\n", | ||
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '<PATH_TO_SPECTROGRAM_IMAGES>'" | ||
] | ||
} | ||
], | ||
"source": [ | ||
"modulations_paths = [signal_image_folder_path + fpath + \"/\" for fpath in os.listdir(signal_image_folder_path)]\n", | ||
"image_datasets = []\n", | ||
"for path in modulations_paths:\n", | ||
" image_datasets += [LazyImageDirectoryDataset(path, 0, read_black_hot=True, transforms = [normalize_image, lambda x: x**30, normalize_image, lambda x: x[0,(x.sum(axis=2)>1)[0]][None,:,:]**(1/10), normalize_image, scale_dynamic_range])]" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "c62857ac-0f35-4dbb-a22b-2f8993b0c002", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"spectrogram_size = (1,2048,2048)\n", | ||
"\n", | ||
"ripple_transform = RandomRippleNoiseTransform((0.1,0.3), num_emitors=8, base_freq=3)\n", | ||
"\n", | ||
"tone_transforms = []\n", | ||
"tone_transforms += [BlurTransform(strength=0.3, blur_shape=(2,1))]\n", | ||
"tone_transforms += [ripple_transform]\n", | ||
"tone_dataset = GeneratorFunctionDataset(tone_generator_function(spectrogram_size[-1], max_height=40), 0, transforms=tone_transforms)\n", | ||
"\n", | ||
"signal_transforms = [normalize_image]\n", | ||
"signal_transforms += [RandomImageResizeTransform([0.95,3])]\n", | ||
"signal_transforms += [BlurTransform(strength=1, blur_shape=(5,1))]\n", | ||
"signal_transforms += [ripple_transform]\n", | ||
"image_ds = ConcatDataset(image_datasets, transforms = signal_transforms)\n", | ||
"\n", | ||
"\n", | ||
"soi_transforms = [normalize_image]\n", | ||
"soi_transforms += [RandomGaussianNoiseTransform(mean=0, range=(0.001,0.002))]\n", | ||
"soi_transforms += [RandomRippleNoiseTransform((0.01,0.03), num_emitors=8, base_freq=3)]\n", | ||
"soi_dataset = SOIExtractorDataset(soi_path,1, transforms=soi_transforms,read_black_hot=True, filter_strength=50)\n", | ||
"\n", | ||
"\n", | ||
"\n", | ||
"repeat_image_ds = ConcatDataset(image_datasets, transforms = [RandomImageResizeTransform([0.3,1.5])])\n", | ||
"repeater_transforms = [BlurTransform(strength=1, blur_shape=(5,1)), ripple_transform]\n", | ||
"repeater_signal_dataset = GeneratorFunctionDataset(repeated_signal_generator_function(lambda: repeat_image_ds[0][0], min_gap=10, max_gap=40, min_repeats=3, max_repeats=10), 0, transforms=repeater_transforms)\n", | ||
"\n", | ||
"composite_transforms = []\n", | ||
"composite_transforms += [normalize_image] # inf norm\n", | ||
"composite_transforms += [RandomRippleNoiseTransform((0.3,0.5), num_emitors=8, base_freq=1)] #light but noticeable ripple in the full image;\n", | ||
"composite_transforms += [RandomGaussianNoiseTransform(mean=0, range=(0.3,0.5))]\n", | ||
"composite_transforms += [scale_dynamic_range] #adjusts the noise floor in each vertical column\n", | ||
"composite_transforms += [normalize_image] # inf norm\n", | ||
"\n", | ||
"composite_spectrogram_dataset = YOLOImageCompositeDataset(spectrogram_size, transforms=composite_transforms, dataset_size=250000)\n", | ||
"composite_spectrogram_dataset.add_component(tone_dataset, min_to_add=0, max_to_add=2)\n", | ||
"composite_spectrogram_dataset.add_component(image_ds, min_to_add=0, max_to_add=3)\n", | ||
"composite_spectrogram_dataset.add_component(soi_dataset, min_to_add=0, max_to_add=2)\n", | ||
"composite_spectrogram_dataset.add_component(repeater_signal_dataset, min_to_add=0, max_to_add=2)\n" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "6f6f004d-cfa4-4130-b497-7d92d4dd0bdb", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"image, labels = composite_spectrogram_dataset[0]\n", | ||
"plt.rcParams[\"figure.figsize\"] = (4,4)\n", | ||
"plt.rcParams[\"figure.figsize\"] = (10,10)\n", | ||
"plot_boxes_on_image(1 - image, labels)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "c5d7c80d-bf90-4cab-b51f-cf031b5a3d8b", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"plt.rcParams[\"figure.figsize\"] = (10,10)\n", | ||
"plt.imshow(image[0])" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "d506c801-bf8d-45ed-bddb-b7edf808ce96", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": null, | ||
"id": "35d4b962-ac26-44b0-8292-806abbce4997", | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [] | ||
} | ||
], | ||
"metadata": { | ||
"kernelspec": { | ||
"display_name": "Python 3 (ipykernel)", | ||
"language": "python", | ||
"name": "python3" | ||
}, | ||
"language_info": { | ||
"codemirror_mode": { | ||
"name": "ipython", | ||
"version": 3 | ||
}, | ||
"file_extension": ".py", | ||
"mimetype": "text/x-python", | ||
"name": "python", | ||
"nbconvert_exporter": "python", | ||
"pygments_lexer": "ipython3", | ||
"version": "3.10.11" | ||
} | ||
}, | ||
"nbformat": 4, | ||
"nbformat_minor": 5 | ||
} |