From 4e7b33aedaccc24bd8112107334e31993dfc87c9 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 29 Aug 2024 19:45:20 +0000 Subject: [PATCH] Change to the multishot per hole calculator * Can now define a minimum number of shots to avoid unessary calculations * Accelerated the calculations with an initial coarse search followed by a fine search --- Smartscope/core/data_manipulations.py | 5 +- Smartscope/core/db_manipulations.py | 176 +++++++++++----------- Smartscope/core/main_commands.py | 24 +-- Smartscope/lib/multishot.py | 61 ++++++-- Smartscope/server/frontend/forms.py | 1 + Smartscope/server/frontend/views/views.py | 5 +- 6 files changed, 143 insertions(+), 129 deletions(-) diff --git a/Smartscope/core/data_manipulations.py b/Smartscope/core/data_manipulations.py index ab13b52a..45a98be0 100644 --- a/Smartscope/core/data_manipulations.py +++ b/Smartscope/core/data_manipulations.py @@ -1,12 +1,11 @@ -from typing import List, Optional, Dict +from typing import List, Optional import logging import random -from copy import copy from functools import partial from Smartscope.lib.image.target import Target from smartscope_connector.Datatypes.querylist import QueryList -from Smartscope.core.selector_sorter import SelectorSorter, SelectorValueParser, initialize_selector +from Smartscope.core.selector_sorter import initialize_selector from Smartscope.core.settings.worker import PLUGINS_FACTORY import numpy as np diff --git a/Smartscope/core/db_manipulations.py b/Smartscope/core/db_manipulations.py index 8fb6ceda..7a09fe01 100755 --- a/Smartscope/core/db_manipulations.py +++ b/Smartscope/core/db_manipulations.py @@ -129,7 +129,7 @@ def viewer_only(user): return False -def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_group_size=1, iterations=500, score_weight=2): +def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_group_size=1, iterations=500, stop_iter=100, score_weight=2): if len(hole_models) == 0: return hole_models logger.debug( @@ -185,8 +185,8 @@ def group_holes_for_BIS(hole_models:List[models.HoleModel], max_radius=4, min_gr score_no_change = 0 else: score_no_change += 1 - if score_no_change == 250: - logger.debug('No changes for 250 iterations, stopping') + if score_no_change == stop_iter: + logger.debug(f'No changes for {stop_iter} iterations, stopping') break logger.info(f'Best hole grouping: Coverage= {best[1]}, num_groups={best[2]}, score= {best[0]}') @@ -293,92 +293,92 @@ def add_high_mag(grid, parent): return hm, created -def select_n_squares(parent, n): - squares = np.array(parent.squaremodel_set.all().filter(selected=False, status=None).order_by('area')) - squares = [s for s in squares if s.is_good() and not s.is_out_of_range()] - if len(squares) == 0: - return - split_squares = np.array_split(squares, n) - selection = [] - with transaction.atomic(): - for bucket in split_squares: - if len(bucket) == 0: - continue - selection = random.choice(bucket) - update(selection, selected=True, status='queued') - - -def select_n_holes(parent, n, is_bis=False): - filter_fields = dict(selected=False, status=None) - if is_bis: - filter_fields['bis_type'] = 'center' - holes = list(parent.holemodel_set.filter( - **filter_fields).order_by('dist_from_center')) - - holes = [h for h in holes if h.is_good() and not h.is_out_of_range()] - - if n <= 0: - with transaction.atomic(): - for h in holes: - update(h, selected=True, status='queued') - return - if len(holes) == 0: - return - n += 1 - minimum, maximum = holes[0].dist_from_center, holes[-1].dist_from_center - dist_range = maximum - minimum - group_dist = dist_range / (n) - groups = [[] for x in range(n)] - try: - for h in holes: - group = min([int((h.dist_from_center - minimum) // group_dist), n - 1]) - groups[group].append(h) - except: - groups = np.array_split(np.array(holes), n) - - with transaction.atomic(): - for bucket in groups[:-1]: - if len(bucket) == 0: - continue - selection = random.choice(bucket) - update(selection, selected=True, status='queued') - - -def select_n_areas(parent, n, is_bis=False): - filter_fields = dict(selected=False, status=None) - if is_bis: - filter_fields['bis_type'] = 'center' - targets = parent.targets.filter(**filter_fields) - - if n <= 0: - with transaction.atomic(): - for t in targets: - if t.is_good() and not t.is_excluded()[0] and not t.is_out_of_range(): - update(t, selected=True, status='queued') - return +# def select_n_squares(parent, n): +# squares = np.array(parent.squaremodel_set.all().filter(selected=False, status=None).order_by('area')) +# squares = [s for s in squares if s.is_good() and not s.is_out_of_range()] +# if len(squares) == 0: +# return +# split_squares = np.array_split(squares, n) +# selection = [] +# with transaction.atomic(): +# for bucket in split_squares: +# if len(bucket) == 0: +# continue +# selection = random.choice(bucket) +# update(selection, selected=True, status='queued') + + +# def select_n_holes(parent, n, is_bis=False): +# filter_fields = dict(selected=False, status=None) +# if is_bis: +# filter_fields['bis_type'] = 'center' +# holes = list(parent.holemodel_set.filter( +# **filter_fields).order_by('dist_from_center')) + +# holes = [h for h in holes if h.is_good() and not h.is_out_of_range()] + +# if n <= 0: +# with transaction.atomic(): +# for h in holes: +# update(h, selected=True, status='queued') +# return +# if len(holes) == 0: +# return +# n += 1 +# minimum, maximum = holes[0].dist_from_center, holes[-1].dist_from_center +# dist_range = maximum - minimum +# group_dist = dist_range / (n) +# groups = [[] for x in range(n)] +# try: +# for h in holes: +# group = min([int((h.dist_from_center - minimum) // group_dist), n - 1]) +# groups[group].append(h) +# except: +# groups = np.array_split(np.array(holes), n) - clusters = dict() - for t in targets: - if not t.is_good() or t.is_out_of_range(): - continue - excluded, label = t.is_excluded() - if excluded: - continue - try: - clusters[label].append(t) - except: - clusters[label] = [t] - - if len(clusters) > 0: - randomized_sample = clusters if n == len(clusters) else random.sample(list(clusters), n) if n < len(clusters) else [ - random.choice(list(clusters)) for i in range(n)] - with transaction.atomic(): - for choice in randomized_sample: - sele = random.choice(clusters[choice]) - logger.debug(f'Selecting {sele.name} from cluster {choice}') - update(sele, selected=True, status='queued') - else: - logger.info('All targets are rejected, skipping') +# with transaction.atomic(): +# for bucket in groups[:-1]: +# if len(bucket) == 0: +# continue +# selection = random.choice(bucket) +# update(selection, selected=True, status='queued') + + +# def select_n_areas(parent, n, is_bis=False): +# filter_fields = dict(selected=False, status=None) +# if is_bis: +# filter_fields['bis_type'] = 'center' +# targets = parent.targets.filter(**filter_fields) + +# if n <= 0: +# with transaction.atomic(): +# for t in targets: +# if t.is_good() and not t.is_excluded()[0] and not t.is_out_of_range(): +# update(t, selected=True, status='queued') +# return + +# clusters = dict() +# for t in targets: +# if not t.is_good() or t.is_out_of_range(): +# continue +# excluded, label = t.is_excluded() +# if excluded: +# continue +# try: +# clusters[label].append(t) +# except: +# clusters[label] = [t] + +# if len(clusters) > 0: +# randomized_sample = clusters if n == len(clusters) else random.sample(list(clusters), n) if n < len(clusters) else [ +# random.choice(list(clusters)) for i in range(n)] +# with transaction.atomic(): +# for choice in randomized_sample: +# sele = random.choice(clusters[choice]) +# logger.debug(f'Selecting {sele.name} from cluster {choice}') +# update(sele, selected=True, status='queued') +# else: +# logger.info('All targets are rejected, skipping') # def get_center_hole(instance:HoleModel): diff --git a/Smartscope/core/main_commands.py b/Smartscope/core/main_commands.py index 1e640323..a4ae48e5 100755 --- a/Smartscope/core/main_commands.py +++ b/Smartscope/core/main_commands.py @@ -162,34 +162,12 @@ def regroup_bis(grid_id, square_id, reset_groups=True): HoleModel.objects.filter(**queryparams,status='queued',)\ .update(selected=False,status=status.NULL,bis_group=None,bis_type=None) - # filtered_holes = HoleModel.display.filter(**queryparams,status__isnull=True) - holes_for_grouping = [] - # other_holes = [] - # for h in filtered_holes: - # if h.is_good() and not h.is_excluded()[0] and not h.is_out_of_range(): - # holes_for_grouping.append(h) squares = SquareModel.display.filter(status=status.COMPLETED,**queryparams) for square in squares: group_holes_from_square_for_BIS(square, max_radius=collection_params.bis_max_distance, min_group_size=collection_params.min_bis_group_size) - # logger.debug(f"Filtering square {square}, {square.pk}") - # targets = square.targets.filter(status__isnull=True) - # filtered = filter_targets(square, targets) - # holes_for_grouping += apply_filter(targets, filtered) - - # logger.info(f'Holes for grouping = {len(holes_for_grouping)}') - - # holes = group_holes_for_BIS( - # holes_for_grouping, - # max_radius=collection_params.bis_max_distance, - # min_group_size=collection_params.min_bis_group_size, - # ) - - # with transaction.atomic(): - # for hole in holes: - # hole.save() - + logger.info('Regrouping BIS done.') return squares diff --git a/Smartscope/lib/multishot.py b/Smartscope/lib/multishot.py index b661f8df..2c5ddb59 100644 --- a/Smartscope/lib/multishot.py +++ b/Smartscope/lib/multishot.py @@ -115,21 +115,22 @@ def set_shots_per_hole( hole_size:float, beam_size:float, image_size:np.ndarray, - radius_step:int=0.02, starting_angle:float=0, - consider_aspect=True, + # consider_aspect=True, min_efficiency=0.85 ): hole_area = np.pi*(hole_size/2)**2 min_allowed_coverage = np.prod(image_size)/hole_area angle_between_shots = 2*np.pi / number_of_shots - aspect= 1 + aspect= image_size[0]/image_size[1] aspect_step = 1 steps=1 - if consider_aspect: - aspect = image_size[0]/image_size[1] + if aspect > 1: + + # aspect = image_size[0]/image_size[1] aspect_step = 0.1 - steps=20 + steps=int((aspect-1)//aspect_step) + logger.info(f'Not a square detector. Will run {steps} step of aspect ratio optimization.') # start_radius = (hole_size/2 - np.sqrt(np.sum(image_size**2))/2) # if number_of_shots != 1 else 0 @@ -140,17 +141,46 @@ def set_shots_per_hole( best_fraction_in_hole = 0 best_hole_coverage = 0 best_aspect_val = 0 + best_extra_rotation = 0 + best_radius = start_radius + coarse_search = True + + initial_radius_step = hole_size/10 + while True: - for extra_rotation in range(int(np.degrees(angle_between_shots/2))): + if coarse_search: + radius_step = initial_radius_step + + num_aspect_steps = 1 + radius=start_radius + end_radius = max_radius + angle_range = int(np.degrees(angle_between_shots/2)*1.2) + initial_angle_step = int(angle_range/10) + rotation_range = range(0,angle_range,initial_angle_step) + logger.info(f'Coarse search with radius step: {radius_step:.2f} and angle step: {initial_angle_step:.2f}') + else: + radius_step = 0.02 + num_aspect_steps = steps + + radius = best_radius - initial_radius_step*2 + start_radius = radius + end_radius = best_radius + initial_radius_step*2 + rotation_range = range(best_extra_rotation-initial_angle_step*2, best_extra_rotation+initial_angle_step*2+1) + logger.info(f'Fine search with radius: {best_radius} +/- {initial_radius_step*2} and angle step: {best_extra_rotation} +/- {initial_angle_step*2}') + + for extra_rotation in rotation_range: center_shot = False hole_coverage=1 in_hole=1 - radius=start_radius + running=True while running: - for i in range(steps): + + for i in range(num_aspect_steps): + aspect_val = 1+i*aspect_step + # logger.debug(f'Radius: {radius:.2f}. Rotation: {extra_rotation}. Aspect: {aspect_val}') shots = [] remaining_number_of_shots = number_of_shots new_angle_between_shots = angle_between_shots @@ -176,14 +206,19 @@ def set_shots_per_hole( best_hole_coverage = hole_coverage best_shots = shots.copy() best_aspect_val=aspect_val - # print(f'New Best Shot! Shots: {number_of_shots}; Rotation: {extra_rotation}; Radius: {radius}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage} %; In Hole: {in_hole:.2f}; Init in Hole: {init_in_hole}; Sum in Hole: {sum_in_hole} ;Center hole: {center_shot}; Aspect val: {best_aspect_val}') + best_extra_rotation = extra_rotation + best_radius = radius + logger.debug(f'New Best Shot! Shots: {number_of_shots}; Rotation: {extra_rotation}; Radius: {radius}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage} %; In Hole: {in_hole:.2f}; Init in Hole: {init_in_hole}; Sum in Hole: {sum_in_hole} ;Center hole: {center_shot}; Aspect val: {best_aspect_val}') + # time.sleep(0.2) - if radius >= max_radius: + if radius >= end_radius: + radius = start_radius break radius += radius_step # print('Breaking out of loop', end='\r') - - if best_shots is not None: + if coarse_search: + coarse_search=False + elif best_shots is not None: logger.info(f'Shots: {number_of_shots}; Efficiency: {best_fraction_in_hole*100:.1f} %; Hole coverage: {best_hole_coverage*100:.1f} %; Aspect val: {best_aspect_val}') return MultiShot(n_shots=number_of_shots, shots=[s.tolist() for s in best_shots], diff --git a/Smartscope/server/frontend/forms.py b/Smartscope/server/frontend/forms.py index db3ea815..a1de72e5 100755 --- a/Smartscope/server/frontend/forms.py +++ b/Smartscope/server/frontend/forms.py @@ -221,6 +221,7 @@ class SetMultiShotForm(forms.Form): pixel_size= forms.FloatField(label='Pixel Size (A/pix)',min_value=0,required=True,help_text='Pixel size of the Record preset') beam_size = forms.IntegerField(label='Beam size (nm)', min_value=0, required=True, help_text='Beam diameter in nm') hole_size = forms.FloatField(label='Hole Size (um)', min_value=0, required=True, help_text='Grid hole size in micrometers') + min_number_of_shots = forms.IntegerField(label='Minimum shots', min_value=2,initial=2,help_text='Minimum number of shots per hole to try.') max_number_of_shots = forms.IntegerField(label='Maximum shots', min_value=2,initial=2,help_text='Maxmimum number of shots per hole to try.') max_efficiency = forms.FloatField(label='Mininum field of view in hole (%)',initial=85, min_value=0, max_value=100,help_text="Minimum percentage of the total field of view accross all shots to fall within the hole.") diff --git a/Smartscope/server/frontend/views/views.py b/Smartscope/server/frontend/views/views.py index 94927b5d..ed6fc9ce 100755 --- a/Smartscope/server/frontend/views/views.py +++ b/Smartscope/server/frontend/views/views.py @@ -326,12 +326,13 @@ def post(self, request, *args, **kwargs): if form.is_valid(): logger.debug(form.cleaned_data) data=form.cleaned_data + min_shots = data.pop('min_number_of_shots') max_shots = data.pop('max_number_of_shots') max_efficiency = data.pop('max_efficiency') / 100 params = RecordParams(**data) results = [] - for n_shots in range(1,max_shots): - shot = set_shots_per_hole(number_of_shots=n_shots+1, + for n_shots in range(min_shots,max_shots+1): + shot = set_shots_per_hole(number_of_shots=n_shots, hole_size=params.hole_size, beam_size=params.beam_size_um, image_size=params.detector_size_um,