Dev

.gitignore

@@ -31,4 +31,7 @@ docker-compose.yml.bak
 .config
 .python_history
 MyDocker/*
-
+.ipython
+.mysql_history
+.gitignore
+.ipython/profile_default/history.sqlite

.ipython/profile_default/startup/README

@@ -0,0 +1,11 @@
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+    00-first.py
+    50-middle.py
+    99-last.ipy

Docker/Dockerfile-base

@@ -1,50 +1,34 @@
-FROM nvidia/cuda:10.1-base-ubuntu18.04
-
+FROM nvidia/cuda:12.1.0-base-ubuntu18.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y \
 	python3-dev default-libmysqlclient-dev build-essential wget libglib2.0-0 ffmpeg libsm6 libxext6 curl mariadb-server && \
 	apt-get clean && rm -rf /var/cache/apt/lists
 
-# RUN service mariadb stop && service mariadb disable
-
 RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
 	bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 && \
 	rm Miniconda3-latest-Linux-x86_64.sh
 
-RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_4.11.15_RHEL7-64_CUDA10.1.sh && \
-	yes | bash imod_4.11.15_RHEL7-64_CUDA10.1.sh -name IMOD && \
-	rm imod_4.11.15_RHEL7-64_CUDA10.1.sh
+RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_4.11.24_RHEL7-64_CUDA10.1.sh && ls && \
+	yes | bash imod_4.11.24_RHEL7-64_CUDA10.1.sh -name IMOD && \
+	rm imod_4.11.24_RHEL7-64_CUDA10.1.sh
 
 ADD config/singularity/ctffind /usr/local/
 
 ADD config/docker/requirements.txt .
 
-# RUN	mkdir /opt/logs/ /mnt/fake_scope/ /mnt/fake_scope/raw/ /opt/nginx-mount/
-
-
-
-ENV PATH=$PATH:/opt/smartscope/Smartscope/bin:/opt/miniconda3/bin:$IMOD_DIR/bin
+ENV PATH=$PATH:/opt/miniconda3/bin
 
 RUN conda update -y conda && \
-	yes | conda install python=3.9 cudatoolkit=10.2 cudnn=7.6 && \
+	yes | conda install python=3.9 cudatoolkit=10.2 cudnn=7.6 git && \
 	yes | pip install numpy==1.21.0 && \
 	yes | pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu102 && \
 	yes | pip install -r requirements.txt && \
 	yes | pip install ipython && \
-	yes | pip install sphinx-multiversion && \
 	conda clean --all
 
-# RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate &&\
-# 	tar -xvf Smartscope0.6.tar.gz -C /opt/ &&\
-# 	rm Smartscope0.6.tar.gz
-
 # create a non-root user
 ARG USER_ID=9999
 
-RUN	useradd -m --no-log-init --system  --uid $USER_ID smartscope_user
-
-# RUN chmod 777 -R /mnt/ && \
-# 	chmod 777 /opt/logs/ && \
-# 	chmod 777 /opt/nginx-mount/
+RUN	useradd -m --no-log-init --system  --uid $USER_ID smartscope_user

Docker/Dockerfile-smartscope

@@ -35,13 +35,15 @@ ENV	ALLOWED_HOSTS=localhost \
 	MYSQL_HOST=db \
 	MYSQL_PORT=3306 \
 	MYSQL_USERNAME=root \
-	MYSQL_ROOT_PASSWORD=pass \ 
+	MYSQL_PASSWORD=pass \ 
 	DB_NAME=smartscope \
 	REDIS_HOST=cache \
 	REDIS_PORT=6379 \
 	USE_AWS=False \
 	FORCE_CPU=False
 
+ENV PATH=$PATH:/opt/smartscope/Smartscope/bin:/usr/local/IMOD/bin
+
 ADD . /opt/smartscope/
 
 RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate && \

Docker/SmartScope/database.conf

@@ -1,3 +1,6 @@
 MYSQL_USER=root
-MYSQL_ROOT_PASSWORD=pass
-MYSQL_DATABASE=smartscope
+MYSQL_PASSWORD=pass
+MYSQL_DATABASE=smartscope
+MYSQL_HOST=db
+MYSQL_PORT=3306
+MYSQL_SSL=False

Docker/SmartScope/smartscope.conf

@@ -3,12 +3,10 @@ ALLOWED_HOSTS=localhost
 AUTO_MIGRATION=true
 USE_SSL=false
 USE_LONGTERMSTORAGE=False
+TIMEZONE=America/New_York
 
 ### EXPERT SETTINGS ###
 ### EVERYTHING BELOW SHOULD NOT NEED TO BE CHANGED###
-#Other database settings
-# MYSQL_HOST=db
-# MYSQL_PORT=3306
 
 # #Executables
 # IMOD_DIR=/usr/local/IMOD	

Smartscope/bin/smartscope.py

@@ -10,6 +10,7 @@
 import logging
 from Smartscope.core.main_commands import main
 
+
 logger = logging.getLogger(__name__)
 
 

Smartscope/bin/smartscope.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+smartscope.py $@ &

Smartscope/core/autoscreen.py

@@ -2,7 +2,6 @@
 import os
 import sys
 import time
-import shlex
 from Smartscope.core.microscope_interfaces import FakeScopeInterface, TFSSerialemInterface, JEOLSerialemInterface
 from Smartscope.core.selectors import selector_wrapper
 from Smartscope.core.models import ScreeningSession, HoleModel, SquareModel, Process, HighMagModel
@@ -13,6 +12,7 @@
 from Smartscope.core.protocols import get_or_set_protocol
 from Smartscope.lib.Datatypes.microscope import Microscope,Detector,AtlasSettings
 from Smartscope.lib.preprocessing_methods import processing_worker_wrapper
+from Smartscope.core.preprocessing_pipelines import load_preprocessing_pipeline
 from Smartscope.lib.file_manipulations import get_file_and_process, create_grid_directories
 from Smartscope.lib.transformations import register_to_other_montage, register_targets_by_proximity
 from Smartscope.core.db_manipulations import update, select_n_areas, queue_atlas, add_targets, group_holes_for_BIS
@@ -23,7 +23,6 @@
 from django.utils import timezone
 import multiprocessing
 import logging
-import subprocess
 import numpy as np
 from pathlib import Path
 
@@ -104,18 +103,20 @@ def run_grid(grid, session, processing_queue, scope):
         session (ScreeningSession): ScreeningSession object from Smartscope.server.models
     """
 
-    if grid.status == 'complete':
-        logger.info(f'{grid.name} already complete')
-        return
-    if grid.status == 'aborting':
-        logger.info(f'Aborting {grid.name}')
-        return
+
 
     session_id = session.pk
     microscope = session.microscope_id
 
     # Set the Websocket_update_decorator grid property
     update.grid = grid
+    if grid.status == 'complete':
+        logger.info(f'{grid.name} already complete')
+        return
+    if grid.status == 'aborting':
+        logger.info(f'Aborting {grid.name}')
+        update(grid, status='complete')
+        return
 
     logger.info(f'Starting {grid.name}') 
 
@@ -131,7 +132,8 @@ def run_grid(grid, session, processing_queue, scope):
     # ADD the new protocol loader
     protocol = get_or_set_protocol(grid)
     resume_incomplete_processes(processing_queue, grid, session.microscope_id)
-    subprocess.Popen(shlex.split(f'smartscope.py highmag_processing smartscopePipeline {grid.grid_id} 1'))
+    preprocessing = load_preprocessing_pipeline(Path('preprocessing.json'))
+    preprocessing.start(grid)
     is_stop_file(session_id)
     atlas = queue_atlas(grid)
     scope.loadGrid(grid.position)
@@ -141,7 +143,6 @@ def run_grid(grid, session, processing_queue, scope):
     grid_type = grid.holeType
     grid_mesh = grid.meshMaterial
     if atlas.status == 'queued' or atlas.status == 'started':
-
         atlas = update(atlas, status='started')
         logger.info('Waiting on atlas file')
         runAcquisition(scope,protocol.atlas.acquisition,params,atlas)
@@ -168,6 +169,7 @@ def run_grid(grid, session, processing_queue, scope):
         grid = update(grid, refresh_from_db=True, last_update=None)
         params = grid.params_id
         if grid.status == 'aborting':
+            preprocessing.stop(grid)
             break
         else:
             squares, holes = get_queue(grid)
@@ -370,6 +372,9 @@ def autoscreen(session_id):
     except Exception as e:
         logger.exception(e)
         status = 'error'
+        if grid in locals():
+            update.grid = grid
+            update(grid, status='error')
     except KeyboardInterrupt:
         logger.info('Stopping Smartscope.py autoscreen')
         status = 'killed'

Smartscope/core/cache.py

@@ -6,10 +6,8 @@
 
 logger = logging.getLogger(__name__)
 
-
-def save_multishot_from_cache(multishot_id, directory):
-    multishot = cache.get(multishot_id)
-    logger.debug(f'Getting multishot from cache at id {multishot_id}:\n{multishot}')
-    with open(Path(directory,'multishot.json'),'w') as f:
-        f.write(multishot)
-
+def save_json_from_cache(item_id, directory, file_root_name):
+    cached_item = cache.get(item_id)
+    logger.debug(f'Getting {file_root_name} from cache at id {item_id}:\n{cached_item}')
+    with open(Path(directory,f'{file_root_name}.json'),'w') as f:
+        f.write(cached_item)

Smartscope/core/finders.py

@@ -20,3 +20,5 @@ def find_targets(montage: Montage, methods: list):
             logger.debug(f"{method} was successful: {success}, Is Classifier: {method.target_class is TargetClass.CLASSIFIER}")
 
             return targets, method.name, method.name if method.target_class is TargetClass.CLASSIFIER else None, additional_outputs
+
+        return [], '', None, dict()

Smartscope/core/microscope_interfaces.py

@@ -246,7 +246,7 @@ def reset_image_shift(self):
         return sem.ResetImageShift()
 
     def image_shift_by_microns(self,isX,isY,tiltAngle):
-        sem.ImageShiftByMicrons(isX - self.state.imageShiftX, isY - self.state.imageShiftY, 0)
+        sem.ImageShiftByMicrons(isX - self.state.imageShiftX, isY - self.state.imageShiftY, 0, 1)
         self.state.imageShiftX = isX
         self.state.imageShiftY = isY
         sem.SetDefocus(self.state.currentDefocus - isY * math.sin(math.radians(tiltAngle)))     
@@ -390,6 +390,9 @@ def medium_mag_hole(self, file=''):
     def focusDrift(self, def1, def2, step, drifTarget):
         pass
 
+    def tiltTo(self,tiltAngle):
+        pass
+
     def highmag(self, file='', frames=True, earlyReturn=False):
         if not frames:
             generate_fake_file(file, 'highmag', sleeptime=7, destination_dir=self.microscope.scopePath)

Smartscope/core/models/models_actions.py

@@ -14,9 +14,12 @@ def targets_methods(instance):
     if (output:=cache.get(cache_key)) is not None:
         logger.debug(f'Loading targets_method for {instance} from cache.')
         return output
+    default_output = dict(finders=[], classifiers=[], selectors=[])
+    if not hasattr(instance,'targets'):
+        return default_output
     targets = instance.targets.values_list('pk', flat=True)
     if len(targets) == 0:
-        return dict(finders=[], classifiers=[], selectors=[])
+        return default_output
     contenttype = ContentType.objects.get_for_model(instance.targets.first())
 
     finders = list(Finder.objects.filter(content_type=contenttype, object_id__in=targets).values_list('method_name', flat=True).distinct())

Smartscope/core/models/session.py

@@ -601,7 +601,7 @@ def targets(self):
         return self.squaremodel_set.all()
 
     # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600)
-    def toSVG(self, display_type, method):
+    def svg(self, display_type, method):
         targets = list(SquareModel.display.filter(atlas_id=self.atlas_id))
         return drawAtlas(self,targets , display_type, method)
 
@@ -782,7 +782,7 @@ def targets(self):
 
 
     # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600)
-    def toSVG(self, display_type, method):
+    def svg(self, display_type, method):
         holes = list(HoleModel.display.filter(square_id=self.square_id))
         sq = drawSquare(self, holes, display_type, method)
 
@@ -883,7 +883,7 @@ def id(self):
         return self.hole_id
 
     # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600)
-    def toSVG(self, display_type, method):
+    def svg(self, display_type, method):
         holes = list(self.targets)
         if self.shape_x is None:  # There was an error in previous version where shape wasn't set.
             set_shape_values(self)
@@ -981,9 +981,8 @@ def parent(self):
     def set_parent(self, parent):
         self.hole_id = parent
     # endaliases
-
-    @property
-    def SVG(self):
+
+    def svg(self, *args, **kwargs):
         return drawHighMag(self)
 
     @property