Merge pull request #113 from pycroscopy/use_dict_2

EELS auto id and quantification

pyTEMlib/eels_dialog.py

@@ -59,20 +59,18 @@ def __init__(self, dataset=None):
 
             self.edges = {}
 
+
             self.show_regions = False
             self.show()
 
             self.set_dataset(dataset)
-            # TODO: set elements does not work correctly for periodic table
-            # selected_edges = eels.find_edges(dataset, sensitivity=3)
-            selected_edges = []
             initial_elements = []
 
-            for edge in selected_edges:
-                initial_elements.append(edge.split('-')[0])
-            if len(initial_elements) > 0:
-                self.set_elements(initial_elements)
-
+            for key in self.edges:
+                if key.isdigit():
+                    if 'element' in self.edges[key]:
+                        initial_elements.append(self.edges[key]['element'])
+           
             self.pt_dialog = eels_dialog_utilities.PeriodicTableDialog(energy_scale=self.energy_scale,
                                                                        initial_elements=initial_elements)
             self.pt_dialog.signal_selected[list].connect(self.set_elements)
@@ -233,8 +231,8 @@ def on_enter(self):
             elif sender.objectName() == 'element_edit':
                 if str(sender.displayText()).strip() == '0':
                     sender.setText('PT')
-                    self.pt_dialog.energy_scale = self.energy_scale
-                    self.pt_dialog.show()
+                    #self.pt_dialog.energy_scale = self.energy_scale
+                    #self.pt_dialog.show()
                 else:
                     self.update_element(str(sender.displayText()).strip())
                 self.update()
@@ -621,6 +619,22 @@ def do_fit_button_click(self):
             self.update()
             self.plot()
 
+        def do_auto_id_button_click(self):
+            found_edges = eels.auto_id_edges(self.dataset)
+            selected_elements = []
+            for key in found_edges:
+                self.set_elements([key])
+                selected_elements.append(key)
+            for button in self.pt_dialog.button:
+                if button.text() in selected_elements:
+                    button.setChecked(True)
+            self.update()
+
+        def do_select_button_click(self):
+            self.pt_dialog.energy_scale = self.energy_scale
+            self.pt_dialog.show()
+            self.update()
+
         def set_action(self):
             self.ui.edit1.editingFinished.connect(self.on_enter)
             self.ui.edit2.editingFinished.connect(self.on_enter)
@@ -642,6 +656,8 @@ def set_action(self):
 
             self.ui.do_all_button.clicked.connect(self.do_all_button_click)
             self.ui.do_fit_button.clicked.connect(self.do_fit_button_click)
+            self.ui.auto_id_button.clicked.connect(self.do_auto_id_button_click)
+            self.ui.select_button.clicked.connect(self.do_select_button_click)
 
 
     class CurveVisualizer(object):

pyTEMlib/eels_dialog_utilities.py

@@ -65,6 +65,7 @@ def __init__(self, initial_elements=None, energy_scale=None, parent=None):
 
             if initial_elements is None:
                 initial_elements = [' ']
+            self.initial_elements = initial_elements
             if energy_scale is None:
                 energy_scale = [100., 150., 200.]
             self.parent = parent

pyTEMlib/eels_dlg.py

@@ -85,6 +85,14 @@ def __init__(self, dialog):
             self.separator2.setText("Elements")
             self.layout.addWidget(self.separator2, row, 0, 1, 4)
             ######################################################################
+            row += 1
+            self.auto_id_button = QtWidgets.QPushButton('AUto ID', dialog)
+            self.select_button = QtWidgets.QPushButton('Select', dialog)
+
+            self.layout.addWidget(self.auto_id_button, row, 0)
+            self.layout.addWidget(self.select_button, row, 2)
+
+            row += 1
 
             name = 'edge'
             self.label3 = QtWidgets.QLabel('Edge:')

pyTEMlib/eels_tools.py

@@ -254,7 +254,7 @@ def get_z(z):
     return z_out
 
 
-def list_all_edges(z):
+def list_all_edges(z, verbose=False):
     """List all ionization edges of an element with atomic number z
 
     Parameters
@@ -271,14 +271,19 @@ def list_all_edges(z):
     element = str(z)
     x_sections = get_x_sections()
     out_string = ''
-    print('Major edges')
+    if verbose:
+        print('Major edges')
+    edge_list = {x_sections[element]['name']: {}}
+
     for key in all_edges:
         if key in x_sections[element]:
             if 'onset' in x_sections[element][key]:
-                print(f" {x_sections[element]['name']}-{key}: {x_sections[element][key]['onset']:8.1f} eV ")
+                if verbose:
+                    print(f" {x_sections[element]['name']}-{key}: {x_sections[element][key]['onset']:8.1f} eV ")
                 out_string = out_string + f" {x_sections[element]['name']}-{key}: " \
                                           f"{x_sections[element][key]['onset']:8.1f} eV /n"
-    return out_string
+                edge_list[x_sections[element]['name']][key] =  x_sections[element][key]['onset']
+    return out_string, edge_list
 
 
 def find_major_edges(edge_onset, maximal_chemical_shift=5.):
@@ -432,6 +437,135 @@ def find_edges(dataset, sensitivity=2.5):
     return selected_edges
 
 
+def assign_likely_edges(edge_channels, energy_scale): 
+    edges_in_list = []
+    result = {}
+    for channel in edge_channels: 
+        if channel not in edge_channels[edges_in_list]:
+            shift = 5
+            element_list = find_major_edges(energy_scale[channel], maximal_chemical_shift=shift)
+            while len(element_list) < 1:
+                shift+=1
+                element_list = find_major_edges(energy_scale[channel], maximal_chemical_shift=shift)
+
+            if len(element_list) > 1:
+                while len(element_list) > 0:
+                    shift-=1
+                    element_list = find_major_edges(energy_scale[channel], maximal_chemical_shift=shift)
+                element_list = find_major_edges(energy_scale[channel], maximal_chemical_shift=shift+1)
+            element = (element_list[:4]).strip()
+            z = get_z(element)
+            result[element] =[]
+            _, edge_list = list_all_edges(z)
+
+            for peak in edge_list:
+                for edge in edge_list[peak]:
+                    possible_minor_edge = np.argmin(np.abs(energy_scale[edge_channels]-edge_list[peak][edge]))
+                    if np.abs(energy_scale[edge_channels[possible_minor_edge]]-edge_list[peak][edge]) < 3:
+                        #print('nex', next_e)
+                        edges_in_list.append(possible_minor_edge)
+
+                        result[element].append(edge)
+
+    return result
+
+
+def auto_id_edges(dataset):
+    edge_channels = identify_edges(dataset)
+    dim = dataset.get_spectrum_dims()
+    energy_scale = np.array(dataset._axes[dim[0]])
+    found_edges = assign_likely_edges(edge_channels, energy_scale)
+    return found_edges
+
+
+def identify_edges(dataset, noise_level=2.0):
+    """
+    Using first derivative to determine edge onsets
+    Any peak in first derivative higher than noise_level times standard deviation will be considered
+    
+    Parameters
+    ----------
+    dataset: sidpy.Dataset
+        the spectrum
+    noise_level: float
+        ths number times standard deviation in first derivative decides on whether an edge onset is significant
+        
+    Return
+    ------
+    edge_channel: numpy.ndarray
+    
+    """
+    dim = dataset.get_spectrum_dims()
+    energy_scale = np.array(dataset._axes[dim[0]])
+    dispersion = get_slope(energy_scale)
+    spec = scipy.ndimage.gaussian_filter(dataset, 3/dispersion)  # smooth with 3eV wideGaussian
+
+    first_derivative = spec - np.roll(spec, +2) 
+    first_derivative[:3] = 0
+    first_derivative[-3:] = 0
+
+    # find if there is a strong edge at high energy_scale
+    noise_level = noise_level*np.std(first_derivative[3:50])
+    [edge_channels, _] = scipy.signal.find_peaks(first_derivative, noise_level)
+
+    return edge_channels
+
+
+def add_element_to_dataset(dataset, z):
+    """
+    """
+    # We check whether this element is already in the
+    energy_scale = dataset.energy_loss
+    zz = get_z(z)
+    if 'edges' not in dataset.metadata:
+         dataset.metadata['edges'] = {'model': {}, 'use_low_loss': False}
+    index = 0
+    for key, edge in dataset.metadata['edges'].items():
+        if key.isdigit():
+            index += 1
+            if 'z' in edge:
+                if zz == edge['z']:
+                    index = int(key)
+                    break
+
+    major_edge = ''
+    minor_edge = ''
+    all_edges = {}
+    x_section = get_x_sections(zz)
+    edge_start = 10  # int(15./ft.get_slope(self.energy_scale)+0.5)
+    for key in x_section:
+        if len(key) == 2 and key[0] in ['K', 'L', 'M', 'N', 'O'] and key[1].isdigit():
+            if energy_scale[edge_start] < x_section[key]['onset'] < energy_scale[-edge_start]:
+                if key in ['K1', 'L3', 'M5', 'M3']:
+                    major_edge = key
+
+                all_edges[key] = {'onset': x_section[key]['onset']}
+
+    if major_edge != '':
+        key = major_edge
+    elif minor_edge != '':
+        key = minor_edge
+    else:
+        print(f'Could not find no edge of {zz} in spectrum')
+        return False
+
+
+    if str(index) not in dataset.metadata['edges']:
+        dataset.metadata['edges'][str(index)] = {}
+
+    start_exclude = x_section[key]['onset'] - x_section[key]['excl before']
+    end_exclude = x_section[key]['onset'] + x_section[key]['excl after']
+
+    dataset.metadata['edges'][str(index)] = {'z': zz, 'symmetry': key, 'element': elements[zz],
+                              'onset': x_section[key]['onset'], 'end_exclude': end_exclude,
+                              'start_exclude': start_exclude}
+    dataset.metadata['edges'][str(index)]['all_edges'] = all_edges
+    dataset.metadata['edges'][str(index)]['chemical_shift'] = 0.0
+    dataset.metadata['edges'][str(index)]['areal_density'] = 0.0
+    dataset.metadata['edges'][str(index)]['original_onset'] = dataset.metadata['edges'][str(index)]['onset']
+    return True
+
+
 def make_edges(edges_present, energy_scale, e_0, coll_angle, low_loss=None):
     """Makes the edges dictionary for quantification
 
@@ -490,6 +624,48 @@ def make_edges(edges_present, energy_scale, e_0, coll_angle, low_loss=None):
 
     return edges
 
+def fit_dataset(dataset):
+    energy_scale = dataset.energy_loss
+    if 'fit_area' not in dataset.metadata['edges']:
+        dataset.metadata['edges']['fit_area'] = {}
+    if 'fit_start' not in dataset.metadata['edges']['fit_area']:
+        dataset.metadata['edges']['fit_area']['fit_start'] = energy_scale[50]
+    if 'fit_end' not in dataset.metadata['edges']['fit_area']:
+        dataset.metadata['edges']['fit_area']['fit_end'] = energy_scale[-2]
+    dataset.metadata['edges']['use_low_loss'] = False
+
+    if 'experiment' in dataset.metadata:
+        exp = dataset.metadata['experiment']
+        if 'convergence_angle' not in exp:
+            raise ValueError('need a convergence_angle in experiment of metadata dictionary ')
+        alpha = exp['convergence_angle']
+        beta = exp['collection_angle']
+        beam_kv = exp['acceleration_voltage']
+        energy_scale = dataset.energy_loss
+        eff_beta = effective_collection_angle(energy_scale, alpha, beta, beam_kv)
+        edges = make_cross_sections(dataset.metadata['edges'], np.array(energy_scale), beam_kv, eff_beta)
+        dataset.metadata['edges'] = fit_edges2(dataset, energy_scale, edges)
+        areal_density = []
+        elements = []
+        for key in edges:
+            if key.isdigit():  # only edges have numbers in that dictionary
+                elements.append(edges[key]['element'])
+                areal_density.append(edges[key]['areal_density'])
+        areal_density = np.array(areal_density)
+        out_string = '\nRelative composition: \n'
+        for i, element in enumerate(elements):
+            out_string += f'{element}: {areal_density[i] / areal_density.sum() * 100:.1f}%  '
+
+        print(out_string)
+
+
+def auto_chemical_composition(dataset):
+
+    found_edges = auto_id_edges(dataset)
+    for key in found_edges:
+        add_element_to_dataset(dataset, key)
+    fit_dataset(dataset)
+
 
 def make_cross_sections(edges, energy_scale, e_0, coll_angle, low_loss=None):
     """Updates the edges dictionary with collection angle-integrated X-ray photo-absorption cross-sections

pyTEMlib/image_tools.py

@@ -19,7 +19,6 @@
 
 from tqdm.auto import trange, tqdm
 
-
 # import itertools
 from itertools import product
 
@@ -202,6 +201,7 @@ def power_spectrum(dset, smoothing=3):
 
     return power_spec
 
+
 def diffractogram_spots(dset, spot_threshold):
     """Find spots in diffractogram and sort them by distance from center
 
@@ -448,7 +448,7 @@ def rigid_registration(dataset):
     """
     Rigid registration of image stack with sub-pixel accuracy
 
-    Uses phase_cross_correlation from skimage.registration
+    Uses phase_cross_correlation from 'skimage.registration'
     (we determine drift from one image to next)
 
     Parameters
@@ -533,7 +533,25 @@ def rig_reg_drift(dset, rel_drift):
     drift: list of drift in pixel
     """
 
-    rig_reg = np.zeros(dset.shape)
+    frame_dim = []
+    spatial_dim = []
+    selection = []
+
+    for i, axis in dset._axes.items():
+        if axis.dimension_type.name == 'SPATIAL':
+            spatial_dim.append(i)
+            selection.append(slice(None))
+        else:
+            frame_dim.append(i)
+            selection.append(slice(0, 1))
+
+    if len(spatial_dim) != 2:
+        print('need two spatial dimensions')
+    if len(frame_dim) != 1:
+        print('need one frame dimensions')
+
+    rig_reg = np.zeros([dset.shape[frame_dim[0]], dset.shape[spatial_dim[0]], dset.shape[spatial_dim[1]]])
+
     # absolute drift
     drift = np.array(rel_drift).copy()
 
@@ -544,8 +562,9 @@ def rig_reg_drift(dset, rel_drift):
     drift = drift - center_drift
     # Shift images
     for i in range(rig_reg.shape[0]):
+        selection[frame_dim[0]] = slice(i, i+1)
         # Now we shift
-        rig_reg[i, :, :] = ndimage.shift(dset[i], [drift[i, 0], drift[i, 1]], order=3)
+        rig_reg[i, :, :] = ndimage.shift(dset[tuple(selection)].squeeze(), [drift[i, 0], drift[i, 1]], order=3)
     return rig_reg, drift
 
 

pyTEMlib/version.py

@@ -1,6 +1,6 @@
 """
 version
 """
-_version = '0.2023.2.0'
+_version = '0.2023.3.0'
 __version__ = _version
 _time = '2023-02-04 19:58:26'

setup.py

@@ -35,6 +35,7 @@
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: Scientific/Engineering :: Information Analysis'],
     keywords=['imaging', 'spectra', 'transmission', 'electron', 'microscopy',

tests/test_eels_tools.py

@@ -54,7 +54,7 @@ def test_get_x_sections(self):
         self.assertEqual(x['name'], 'Si')
 
     def test_list_all_edges(self):
-        z = eels.list_all_edges(14)
+        z, _ = eels.list_all_edges(14)
         self.assertEqual(z[:6], ' Si-K1')
 
     def test_find_major_edge(self):