Merge pull request #328 from JoQCcoz/adding_tags

Change to the multishot per hole calculator

Smartscope/core/data_manipulations.py

@@ -1,12 +1,11 @@
 
-from typing import List, Optional, Dict
+from typing import List, Optional
 import logging
 import random
-from copy import copy
 from functools import partial
 from Smartscope.lib.image.target import Target
 from smartscope_connector.Datatypes.querylist import QueryList
-from Smartscope.core.selector_sorter import SelectorSorter, SelectorValueParser, initialize_selector
+from Smartscope.core.selector_sorter import initialize_selector
 from Smartscope.core.settings.worker import PLUGINS_FACTORY
 import numpy as np
 

Smartscope/core/db_manipulations.py

@@ -129,7 +129,7 @@ def viewer_only(user):
     return False
 
 
-def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_group_size=1, iterations=500, score_weight=2):
+def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_group_size=1, iterations=500, stop_iter=100, score_weight=2):
     if len(hole_models) == 0:
         return  hole_models
     logger.debug(
@@ -185,8 +185,8 @@ def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_gr
             score_no_change = 0
         else:
             score_no_change += 1
-            if score_no_change == 250:
-                logger.debug('No changes for 250 iterations, stopping')
+            if score_no_change == stop_iter:
+                logger.debug(f'No changes for {stop_iter} iterations, stopping')
                 break
 
     logger.info(f'Best hole grouping: Coverage= {best[1]}, num_groups={best[2]}, score= {best[0]}')
@@ -293,92 +293,92 @@ def add_high_mag(grid, parent):
     return hm, created
 
 
-def select_n_squares(parent, n):
-    squares = np.array(parent.squaremodel_set.all().filter(selected=False, status=None).order_by('area'))
-    squares = [s for s in squares if s.is_good() and not s.is_out_of_range()]
-    if len(squares) == 0:
-        return
-    split_squares = np.array_split(squares, n)
-    selection = []
-    with transaction.atomic():
-        for bucket in split_squares:
-            if len(bucket) == 0:
-                continue
-            selection = random.choice(bucket)
-            update(selection, selected=True, status='queued')
-
-
-def select_n_holes(parent, n, is_bis=False):
-    filter_fields = dict(selected=False, status=None) 
-    if is_bis:
-        filter_fields['bis_type'] = 'center'
-    holes = list(parent.holemodel_set.filter(
-        **filter_fields).order_by('dist_from_center'))
-
-    holes = [h for h in holes if h.is_good() and not h.is_out_of_range()]
-
-    if n <= 0:
-        with transaction.atomic():
-            for h in holes:
-                update(h, selected=True, status='queued')
-        return
-    if len(holes) == 0:
-        return 
-    n += 1
-    minimum, maximum = holes[0].dist_from_center, holes[-1].dist_from_center
-    dist_range = maximum - minimum
-    group_dist = dist_range / (n)
-    groups = [[] for x in range(n)]
-    try:
-        for h in holes:
-            group = min([int((h.dist_from_center - minimum) // group_dist), n - 1])
-            groups[group].append(h)
-    except:
-        groups = np.array_split(np.array(holes), n)
-
-    with transaction.atomic():
-        for bucket in groups[:-1]:
-            if len(bucket) == 0:
-                continue
-            selection = random.choice(bucket)
-            update(selection, selected=True, status='queued')
-
-
-def select_n_areas(parent, n, is_bis=False):
-    filter_fields = dict(selected=False, status=None)
-    if is_bis:
-        filter_fields['bis_type'] = 'center'
-    targets = parent.targets.filter(**filter_fields)
-
-    if n <= 0:
-        with transaction.atomic():
-            for t in targets:
-                if t.is_good() and not t.is_excluded()[0] and not t.is_out_of_range():
-                    update(t, selected=True, status='queued')
-        return
+# def select_n_squares(parent, n):
+#     squares = np.array(parent.squaremodel_set.all().filter(selected=False, status=None).order_by('area'))
+#     squares = [s for s in squares if s.is_good() and not s.is_out_of_range()]
+#     if len(squares) == 0:
+#         return
+#     split_squares = np.array_split(squares, n)
+#     selection = []
+#     with transaction.atomic():
+#         for bucket in split_squares:
+#             if len(bucket) == 0:
+#                 continue
+#             selection = random.choice(bucket)
+#             update(selection, selected=True, status='queued')
+
+
+# def select_n_holes(parent, n, is_bis=False):
+#     filter_fields = dict(selected=False, status=None) 
+#     if is_bis:
+#         filter_fields['bis_type'] = 'center'
+#     holes = list(parent.holemodel_set.filter(
+#         **filter_fields).order_by('dist_from_center'))
+
+#     holes = [h for h in holes if h.is_good() and not h.is_out_of_range()]
+
+#     if n <= 0:
+#         with transaction.atomic():
+#             for h in holes:
+#                 update(h, selected=True, status='queued')
+#         return
+#     if len(holes) == 0:
+#         return 
+#     n += 1
+#     minimum, maximum = holes[0].dist_from_center, holes[-1].dist_from_center
+#     dist_range = maximum - minimum
+#     group_dist = dist_range / (n)
+#     groups = [[] for x in range(n)]
+#     try:
+#         for h in holes:
+#             group = min([int((h.dist_from_center - minimum) // group_dist), n - 1])
+#             groups[group].append(h)
+#     except:
+#         groups = np.array_split(np.array(holes), n)
 
-    clusters = dict()
-    for t in targets:
-        if not t.is_good() or t.is_out_of_range():
-            continue
-        excluded, label = t.is_excluded()
-        if excluded:
-            continue
-        try:
-            clusters[label].append(t)
-        except:
-            clusters[label] = [t]
-
-    if len(clusters) > 0:
-        randomized_sample = clusters if n == len(clusters) else random.sample(list(clusters), n) if n < len(clusters) else [
-            random.choice(list(clusters)) for i in range(n)]
-        with transaction.atomic():
-            for choice in randomized_sample:
-                sele = random.choice(clusters[choice])
-                logger.debug(f'Selecting {sele.name} from cluster {choice}')
-                update(sele, selected=True, status='queued')
-    else:
-        logger.info('All targets are rejected, skipping')
+#     with transaction.atomic():
+#         for bucket in groups[:-1]:
+#             if len(bucket) == 0:
+#                 continue
+#             selection = random.choice(bucket)
+#             update(selection, selected=True, status='queued')
+
+
+# def select_n_areas(parent, n, is_bis=False):
+#     filter_fields = dict(selected=False, status=None)
+#     if is_bis:
+#         filter_fields['bis_type'] = 'center'
+#     targets = parent.targets.filter(**filter_fields)
+
+#     if n <= 0:
+#         with transaction.atomic():
+#             for t in targets:
+#                 if t.is_good() and not t.is_excluded()[0] and not t.is_out_of_range():
+#                     update(t, selected=True, status='queued')
+#         return
+
+#     clusters = dict()
+#     for t in targets:
+#         if not t.is_good() or t.is_out_of_range():
+#             continue
+#         excluded, label = t.is_excluded()
+#         if excluded:
+#             continue
+#         try:
+#             clusters[label].append(t)
+#         except:
+#             clusters[label] = [t]
+
+#     if len(clusters) > 0:
+#         randomized_sample = clusters if n == len(clusters) else random.sample(list(clusters), n) if n < len(clusters) else [
+#             random.choice(list(clusters)) for i in range(n)]
+#         with transaction.atomic():
+#             for choice in randomized_sample:
+#                 sele = random.choice(clusters[choice])
+#                 logger.debug(f'Selecting {sele.name} from cluster {choice}')
+#                 update(sele, selected=True, status='queued')
+#     else:
+#         logger.info('All targets are rejected, skipping')
 
 
 # def get_center_hole(instance:HoleModel):

Smartscope/core/main_commands.py

@@ -162,34 +162,12 @@ def regroup_bis(grid_id, square_id, reset_groups=True):
         HoleModel.objects.filter(**queryparams,status='queued',)\
             .update(selected=False,status=status.NULL,bis_group=None,bis_type=None)
 
-    # filtered_holes = HoleModel.display.filter(**queryparams,status__isnull=True)
-    holes_for_grouping = []
-    # other_holes = []
-    # for h in filtered_holes:
-    #     if h.is_good() and not h.is_excluded()[0] and not h.is_out_of_range():
-    #         holes_for_grouping.append(h)
     squares = SquareModel.display.filter(status=status.COMPLETED,**queryparams)
     for square in squares:
         group_holes_from_square_for_BIS(square, 
                                         max_radius=collection_params.bis_max_distance,
                                         min_group_size=collection_params.min_bis_group_size)
-        # logger.debug(f"Filtering square {square}, {square.pk}")
-        # targets = square.targets.filter(status__isnull=True)
-        # filtered = filter_targets(square, targets)
-        # holes_for_grouping += apply_filter(targets, filtered)
-
-        # logger.info(f'Holes for grouping = {len(holes_for_grouping)}')
-
-        # holes = group_holes_for_BIS(
-        #     holes_for_grouping,
-        #     max_radius=collection_params.bis_max_distance,
-        #     min_group_size=collection_params.min_bis_group_size,
-        # )
-
-        # with transaction.atomic():
-        #     for hole in holes:
-        #         hole.save()
-
+
     logger.info('Regrouping BIS done.')
     return squares
 

Smartscope/lib/multishot.py

@@ -115,21 +115,22 @@ def set_shots_per_hole(
         hole_size:float,
         beam_size:float,
         image_size:np.ndarray,
-        radius_step:int=0.02,
         starting_angle:float=0,
-        consider_aspect=True,
+        # consider_aspect=True,
         min_efficiency=0.85
     ):
     hole_area = np.pi*(hole_size/2)**2
     min_allowed_coverage = np.prod(image_size)/hole_area
     angle_between_shots = 2*np.pi / number_of_shots
-    aspect= 1
+    aspect= image_size[0]/image_size[1] 
     aspect_step = 1
     steps=1
-    if consider_aspect:
-        aspect = image_size[0]/image_size[1] 
+    if aspect > 1:
+
+        # aspect = image_size[0]/image_size[1] 
         aspect_step = 0.1
-        steps=20
+        steps=int((aspect-1)//aspect_step)
+        logger.info(f'Not a square detector. Will run {steps} step of aspect ratio optimization.')
 
     # start_radius = (hole_size/2 - np.sqrt(np.sum(image_size**2))/2)
     #  if number_of_shots != 1 else 0
@@ -140,17 +141,46 @@ def set_shots_per_hole(
     best_fraction_in_hole = 0
     best_hole_coverage = 0
     best_aspect_val = 0
+    best_extra_rotation = 0
+    best_radius = start_radius
+    coarse_search = True
+
+    initial_radius_step  = hole_size/10
+
 
     while True:
-        for extra_rotation in range(int(np.degrees(angle_between_shots/2))):
+        if coarse_search:
+            radius_step = initial_radius_step
+
+            num_aspect_steps = 1
+            radius=start_radius
+            end_radius = max_radius
+            angle_range = int(np.degrees(angle_between_shots/2)*1.2)
+            initial_angle_step = int(angle_range/10)
+            rotation_range = range(0,angle_range,initial_angle_step)
+            logger.info(f'Coarse search with radius step: {radius_step:.2f} and angle step: {initial_angle_step:.2f}')
+        else:
+            radius_step = 0.02
+            num_aspect_steps = steps
+
+            radius = best_radius - initial_radius_step*2
+            start_radius = radius
+            end_radius = best_radius + initial_radius_step*2
+            rotation_range = range(best_extra_rotation-initial_angle_step*2, best_extra_rotation+initial_angle_step*2+1)
+            logger.info(f'Fine search with radius: {best_radius} +/- {initial_radius_step*2} and angle step: {best_extra_rotation} +/- {initial_angle_step*2}')
+
+        for extra_rotation in rotation_range:
             center_shot = False
             hole_coverage=1
             in_hole=1
-            radius=start_radius
+
             running=True
             while running:
-                for i in range(steps):
+
+                for i in range(num_aspect_steps):
+
                     aspect_val = 1+i*aspect_step
+                    # logger.debug(f'Radius: {radius:.2f}. Rotation: {extra_rotation}. Aspect: {aspect_val}')
                     shots = []
                     remaining_number_of_shots = number_of_shots
                     new_angle_between_shots = angle_between_shots
@@ -176,14 +206,19 @@ def set_shots_per_hole(
                         best_hole_coverage = hole_coverage
                         best_shots = shots.copy()
                         best_aspect_val=aspect_val
-                        # print(f'New Best Shot! Shots: {number_of_shots}; Rotation: {extra_rotation}; Radius: {radius}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage} %; In Hole: {in_hole:.2f}; Init in Hole: {init_in_hole}; Sum in Hole: {sum_in_hole} ;Center hole: {center_shot}; Aspect val: {best_aspect_val}')
+                        best_extra_rotation = extra_rotation
+                        best_radius = radius
+                        logger.debug(f'New Best Shot! Shots: {number_of_shots}; Rotation: {extra_rotation}; Radius: {radius}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage} %; In Hole: {in_hole:.2f}; Init in Hole: {init_in_hole}; Sum in Hole: {sum_in_hole} ;Center hole: {center_shot}; Aspect val: {best_aspect_val}')
+
                     # time.sleep(0.2)
-                if radius >= max_radius:
+                if radius >= end_radius:
+                    radius = start_radius
                     break
                 radius += radius_step
                 # print('Breaking out of loop', end='\r')
-
-        if best_shots is not None:
+        if coarse_search:
+            coarse_search=False
+        elif best_shots is not None:
             logger.info(f'Shots: {number_of_shots}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage*100:.1f} %; Aspect val: {best_aspect_val}')
             return MultiShot(n_shots=number_of_shots,
                         shots=[s.tolist() for s in best_shots],

Smartscope/server/frontend/forms.py

@@ -221,6 +221,7 @@ class SetMultiShotForm(forms.Form):
     pixel_size= forms.FloatField(label='Pixel Size (A/pix)',min_value=0,required=True,help_text='Pixel size of the Record preset')
     beam_size = forms.IntegerField(label='Beam size (nm)', min_value=0, required=True, help_text='Beam diameter in nm')
     hole_size = forms.FloatField(label='Hole Size (um)', min_value=0, required=True, help_text='Grid hole size in micrometers')
+    min_number_of_shots = forms.IntegerField(label='Minimum shots', min_value=2,initial=2,help_text='Minimum number of shots per hole to try.')
     max_number_of_shots = forms.IntegerField(label='Maximum shots', min_value=2,initial=2,help_text='Maxmimum number of shots per hole to try.')
     max_efficiency = forms.FloatField(label='Mininum field of view in hole (%)',initial=85, min_value=0, max_value=100,help_text="Minimum percentage of the total field of view accross all shots to fall within the hole.")
 

Smartscope/server/frontend/views/views.py

@@ -326,12 +326,13 @@ def post(self, request, *args, **kwargs):
             if form.is_valid(): 
                 logger.debug(form.cleaned_data)
                 data=form.cleaned_data
+                min_shots = data.pop('min_number_of_shots')
                 max_shots = data.pop('max_number_of_shots')
                 max_efficiency = data.pop('max_efficiency') / 100
                 params = RecordParams(**data)
                 results = []
-                for n_shots in range(1,max_shots):
-                    shot = set_shots_per_hole(number_of_shots=n_shots+1,
+                for n_shots in range(min_shots,max_shots+1):
+                    shot = set_shots_per_hole(number_of_shots=n_shots,
                                                 hole_size=params.hole_size,
                                                 beam_size=params.beam_size_um,
                                                 image_size=params.detector_size_um,