Skip to content

Commit

Permalink
adding generate_dataset_example.ipynb
Browse files Browse the repository at this point in the history 
	new file:   generate_dataset_example.ipynb
  • Loading branch information
@pvallance
pvallance committed Jun 7, 2024
1 parent 604e6fa commit b57789d
Showing 1 changed file with 197 additions and 0 deletions.
197 changes: 197 additions & 0 deletions torchsig/image_datasets/generate_dataset_example.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "30d159b1-a72b-433c-98dc-de694462e657",
"metadata": {},
"outputs": [],
"source": [
"signal_image_folder_path = \"<PATH_TO_SPECTROGRAM_IMAGES>\"\n",
"soi_path = \"<PATH_TO_SPECTROGRAM_IMAGES_MARKED_WITH_SIGNAL_OF_INTEREST>\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "70ca4d0b-19f2-4a0b-85e1-6f8f795ccb41",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 2,
"id": "d0fa212e-6128-495f-8af6-4bba4629fc76",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from torchsig.image_datasets.datasets.synthetic_signals import GeneratorFunctionDataset, rectangle_signal_generator_function, tone_generator_function, repeated_signal_generator_function\n",
"from torchsig.image_datasets.datasets.file_loading_datasets import SOIExtractorDataset, LazyImageDirectoryDataset\n",
"from torchsig.image_datasets.datasets.composites import YOLOImageCompositeDataset, ConcatDataset\n",
"from torchsig.image_datasets.transforms.impairments import BlurTransform, RandomGaussianNoiseTransform, RandomImageResizeTransform, RandomRippleNoiseTransform, ScaleTransform, scale_dynamic_range, normalize_image"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "cf919892-2a82-4d1e-a4f7-3d4ae3b485d5",
"metadata": {},
"outputs": [],
"source": [
"import cv2\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import torch\n",
"\n",
"def plot_boxes_on_image(image, labels):\n",
" image = image.numpy()\n",
" image = (np.stack([image[0,:,:]]*3).transpose(1,2,0)*255).astype(np.uint8)\n",
" for label in labels:\n",
" cid, cx, cy, w, h = label\n",
" img_h, img_w = image.shape[:2]\n",
" x1 = int((cx - w/2)*img_w)\n",
" x2 = int((cx + w/2)*img_w)\n",
" y1 = int((cy - h/2)*img_h)\n",
" y2 = int((cy + h/2)*img_h)\n",
" image = cv2.rectangle(image.copy(), (x1, y1), (x2, y2), color=(255,0,0), thickness=1)\n",
" plt.imshow(image)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "d16cc6cc-63e7-4c70-adfd-a40cbb78ef27",
"metadata": {},
"outputs": [
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: '<PATH_TO_SPECTROGRAM_IMAGES>'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m modulations_paths \u001b[38;5;241m=\u001b[39m [signal_image_folder_path \u001b[38;5;241m+\u001b[39m fpath \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m fpath \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignal_image_folder_path\u001b[49m\u001b[43m)\u001b[49m]\n\u001b[1;32m 2\u001b[0m image_datasets \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m path \u001b[38;5;129;01min\u001b[39;00m modulations_paths:\n",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '<PATH_TO_SPECTROGRAM_IMAGES>'"
]
}
],
"source": [
"modulations_paths = [signal_image_folder_path + fpath + \"/\" for fpath in os.listdir(signal_image_folder_path)]\n",
"image_datasets = []\n",
"for path in modulations_paths:\n",
" image_datasets += [LazyImageDirectoryDataset(path, 0, read_black_hot=True, transforms = [normalize_image, lambda x: x**30, normalize_image, lambda x: x[0,(x.sum(axis=2)>1)[0]][None,:,:]**(1/10), normalize_image, scale_dynamic_range])]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c62857ac-0f35-4dbb-a22b-2f8993b0c002",
"metadata": {},
"outputs": [],
"source": [
"spectrogram_size = (1,2048,2048)\n",
"\n",
"ripple_transform = RandomRippleNoiseTransform((0.1,0.3), num_emitors=8, base_freq=3)\n",
"\n",
"tone_transforms = []\n",
"tone_transforms += [BlurTransform(strength=0.3, blur_shape=(2,1))]\n",
"tone_transforms += [ripple_transform]\n",
"tone_dataset = GeneratorFunctionDataset(tone_generator_function(spectrogram_size[-1], max_height=40), 0, transforms=tone_transforms)\n",
"\n",
"signal_transforms = [normalize_image]\n",
"signal_transforms += [RandomImageResizeTransform([0.95,3])]\n",
"signal_transforms += [BlurTransform(strength=1, blur_shape=(5,1))]\n",
"signal_transforms += [ripple_transform]\n",
"image_ds = ConcatDataset(image_datasets, transforms = signal_transforms)\n",
"\n",
"\n",
"soi_transforms = [normalize_image]\n",
"soi_transforms += [RandomGaussianNoiseTransform(mean=0, range=(0.001,0.002))]\n",
"soi_transforms += [RandomRippleNoiseTransform((0.01,0.03), num_emitors=8, base_freq=3)]\n",
"soi_dataset = SOIExtractorDataset(soi_path,1, transforms=soi_transforms,read_black_hot=True, filter_strength=50)\n",
"\n",
"\n",
"\n",
"repeat_image_ds = ConcatDataset(image_datasets, transforms = [RandomImageResizeTransform([0.3,1.5])])\n",
"repeater_transforms = [BlurTransform(strength=1, blur_shape=(5,1)), ripple_transform]\n",
"repeater_signal_dataset = GeneratorFunctionDataset(repeated_signal_generator_function(lambda: repeat_image_ds[0][0], min_gap=10, max_gap=40, min_repeats=3, max_repeats=10), 0, transforms=repeater_transforms)\n",
"\n",
"composite_transforms = []\n",
"composite_transforms += [normalize_image] # inf norm\n",
"composite_transforms += [RandomRippleNoiseTransform((0.3,0.5), num_emitors=8, base_freq=1)] #light but noticeable ripple in the full image;\n",
"composite_transforms += [RandomGaussianNoiseTransform(mean=0, range=(0.3,0.5))]\n",
"composite_transforms += [scale_dynamic_range] #adjusts the noise floor in each vertical column\n",
"composite_transforms += [normalize_image] # inf norm\n",
"\n",
"composite_spectrogram_dataset = YOLOImageCompositeDataset(spectrogram_size, transforms=composite_transforms, dataset_size=250000)\n",
"composite_spectrogram_dataset.add_component(tone_dataset, min_to_add=0, max_to_add=2)\n",
"composite_spectrogram_dataset.add_component(image_ds, min_to_add=0, max_to_add=3)\n",
"composite_spectrogram_dataset.add_component(soi_dataset, min_to_add=0, max_to_add=2)\n",
"composite_spectrogram_dataset.add_component(repeater_signal_dataset, min_to_add=0, max_to_add=2)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6f6f004d-cfa4-4130-b497-7d92d4dd0bdb",
"metadata": {},
"outputs": [],
"source": [
"image, labels = composite_spectrogram_dataset[0]\n",
"plt.rcParams[\"figure.figsize\"] = (4,4)\n",
"plt.rcParams[\"figure.figsize\"] = (10,10)\n",
"plot_boxes_on_image(1 - image, labels)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c5d7c80d-bf90-4cab-b51f-cf031b5a3d8b",
"metadata": {},
"outputs": [],
"source": [
"plt.rcParams[\"figure.figsize\"] = (10,10)\n",
"plt.imshow(image[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d506c801-bf8d-45ed-bddb-b7edf808ce96",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "35d4b962-ac26-44b0-8292-806abbce4997",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

0 comments on commit b57789d

Please sign in to comment.