-
Notifications
You must be signed in to change notification settings - Fork 2
/
Normalize_Data.py
72 lines (63 loc) · 2.25 KB
/
Normalize_Data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import unpickle as up
#import Spect_Generate_Resample_File_Time as gen
import numpy as np
from dat_extract.extract.Ship_Variable_Extraction import Ship
from tqdm import tqdm
from colorama import init, Fore
init()
rootdir = "D:\ShippingCINMS_data\COP" #set dir for the extraction
destination_folder = 'D:\Pickled_Data_2\\'
f_80 = 'D:\CUGN_line_80.nc'
f_90 = 'D:\CUGN_line_90.nc'
def get_sizes(ships):
#result = [np.size(ship.spect,1) for ship in ships]
result = []
for ship in ships:
result.append(np.size(ship.spect,1))
return result
def find_done(folder):
i=0
for ships in up.unpickle_batch(folder,100,6500,9000):
for ship in ships:
if ship.spect is None:
i+=1
up.one_jar(folder,ship,True)
tqdm.write(str(i))
pass
def calc_pad(des_size,size):
sub = des_size - size
if (sub % 2) == 0:
pre_pad = sub//2
post_pad = sub//2
else:
pre_pad = sub//2
post_pad = ((sub//2) + 1)
return pre_pad, post_pad
def normalize(folder):
x=0
y=0
for ships in up.unpickle_batch(folder,50,4200,9000):
#print(np.size(ships))
#print(ships[0].spect.shape)
sizes = get_sizes(ships)
#avg_size = int(np.mean(sizes))
square_size = np.size(ships[0].spect,0)
for i in tqdm(range(len(ships)),"This Batch: ",bar_format="{l_bar}%s{bar}%s{r_bar}" % (Fore.YELLOW, Fore.RESET)):
try:
pad_length = (square_size - sizes[i])
beg_pad,end_pad = calc_pad(square_size,sizes[i])
if(pad_length>=0):
ships[i].spect = np.pad(ships[i].spect,[(0,0),(beg_pad,end_pad)],mode='constant')
else:
ships[i].spect = ships[i].spect[:,abs(beg_pad):sizes[i]+end_pad]
# print(ships[i].spect.shape)
# print(x)
x+=1
up.one_jar(folder,ships[i],False)
except:
y+=1
up.one_jar(folder,ships[i],True)
tqdm.write(str(y))
pass
# find_done(destination_folder)
normalize(destination_folder)