Merge pull request #105 from spacetelescope/fix-git-sync

git-sync ignores whitespace diffs.
spacetelescope · Jul 12, 2021 · aa8b5f3 · aa8b5f3
2 parents 2e3cfdf + 8e28c50
commit aa8b5f3
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 81 deletions.
diff --git a/deployments/common/image/base.yml b/deployments/common/image/base.yml
@@ -4,8 +4,8 @@ channels:
   - http://ssb.stsci.edu/astroconda
   - defaults
 dependencies:
-  #- bqplot-image-gl==1.4.1
-  #- bqplot
+  - bqplot-image-gl
+  - bqplot
   - jupyterlab_widgets
   - ipywidgets
   - ipydatawidgets
@@ -27,7 +27,8 @@ dependencies:
     - nbgitpuller
     - ipyvuetify>=1.4.1
     - git+https://github.com/yuvipanda/jupyter-desktop-server
-     #- git+https://github.com/mariobuikhuizen/ipysplitpanes --trusted-host github.com
-     #- git+https://github.com/mariobuikhuizen/ipygoldenlayout --trusted-host github.com
-     #- ipysplitpanes==0.1.0
-     #- ipygoldenlayout==0.3.0
+    #- git+https://github.com/mariobuikhuizen/ipysplitpanes --trusted-host github.com
+    #- git+https://github.com/mariobuikhuizen/ipygoldenlayout --trusted-host github.com
+    # BELOW: these shouldn't need to be here...
+    - ipysplitpanes==0.1.0
+    - ipygoldenlayout==0.3.0
diff --git a/deployments/common/image/common-scripts/git-sync b/deployments/common/image/common-scripts/git-sync
@@ -45,12 +45,18 @@ class GitSync(object):
                 logging.info('Moved {} to {} to avoid conflict with upstream'.format(f, new_file_name))
 
     def find_untracked_local_files(self):
+        ignore = ['.gitignore', 'README.md']
+
         proc = subprocess.Popen(
-            ['git ls-files --others --exclude-standard'],
+            ['git ls-files'],
             stdout=subprocess.PIPE, shell=True
         )
         (output, err) = proc.communicate()
-        return [f for f in output.decode('utf-8').split('\n') if len(f) > 0]
+        untracked = [f for f in output.decode('utf-8').split('\n') if len(f) > 0 and f not in ignore]
+        for f in untracked:
+            logging.debug('Found untracked local file: {}'.format(f))
+
+        return untracked
 
     def find_modified_local_files(self):
         proc = subprocess.Popen(
@@ -68,48 +74,45 @@ class GitSync(object):
 
         return modified
 
-    def find_upstream_updates(self, mode):
-        logging.info('Get list of files that have been added or modified upstream...')
-
-        def check_upstream(m):
-            output = subprocess.check_output([
-                'git', 'log', '..origin/{}'.format(self.branch_name),
-                '--oneline', '--name-status'
-            ], cwd=self.repo_dir).decode()
-            files = []
-            for line in output.split('\n'):
+    def find_upstream_updates(self):
+        logging.info('Get list of files that have been modified or added upstream...')
+
+        modes = ['A', 'M']
+        output = subprocess.check_output([
+            'git', 'log', '..origin/{}'.format(self.branch_name),
+            '--oneline', '--name-status'
+        ], cwd=self.repo_dir).decode()
+        files = []
+        for line in output.split('\n'):
+            for m in modes:
                 if line.startswith(m):
                     f = os.path.relpath(line.split('\t', 1)[1], self.repo_dir)
-                    logging.debug('New or modified upstream file: [{}] {}'.format(m, f))
+                    logging.debug('Upstream file [{}]: {}'.format(m, f))
                     files.append(f)
 
-            return files
-
-        if mode in ['A', 'M']:
-            return check_upstream(mode)
-        else:
-            raise Exception('mode must be either A or M')
+        return files
 
     def merge(self):
         logging.info('Merging {} into local clone...'.format(self.branch_name))
         user_email = 'archive@stsci.edu'
         user_name = 'git-sync'
-        # BUG: is the commit part working?
         os.system('git -c user.email={} -c user.name={} merge -Xours --no-edit origin/{}'.format(
             user_email, user_name, self.branch_name
         ))
 
     def prepare_clone(self):
-        new_upstream_files = self.find_upstream_updates('A')
-        modified_upstream_files = self.find_upstream_updates('M')
+        modified_upstream_files = self.find_upstream_updates()
+        modified_local_files = self.find_modified_local_files()
         untracked_local_files = self.find_untracked_local_files()
 
         # move certain files to avoid conflicts with upstream
         # - tracked local files that have been modified
-        # - untracked local files that have been created,
-        #   and upstream files of the same names have also been created
-        files_to_move = self.find_modified_local_files()
-        files_to_move.extend([f for f in untracked_local_files if f in new_upstream_files])
+        # - untracked local files that have been created, where upstream
+        #   files of the same names have also been created or modified
+        files_to_move = modified_local_files + modified_upstream_files
+        files_to_move.extend([f for f in untracked_local_files if modified_upstream_files])
+
+        # BUG/TODO: if a local file has been removed via "git rm", check it out before moving files
         self.move_files(files_to_move)
 
         # if local files have been removed, but still exist upstream, restore them

diff --git a/deployments/jwebbinar/image/environments/frozen/jdaviz-frozen.yml b/deployments/jwebbinar/image/environments/frozen/jdaviz-frozen.yml
@@ -41,7 +41,7 @@ dependencies:
   - idna=2.10
   - imagecodecs=2021.6.8
   - imageio=2.9.0
-  - importlib-metadata=4.6.0
+  - importlib-metadata=4.6.1
   - ipydatawidgets=4.2.0
   - ipykernel=6.0.1
   - ipympl=0.7.0
@@ -102,7 +102,6 @@ dependencies:
   - olefile=0.46
   - openjpeg=2.4.0
   - openssl=1.1.1k
-  - packaging=20.9
   - pandoc=2.14.0.3
   - partd=1.2.0
   - pexpect=4.8.0
@@ -117,7 +116,7 @@ dependencies:
   - pyparsing=2.4.7
   - pyrsistent=0.17.3
   - pysocks=1.7.1
-  - python=3.9.5
+  - python=3.9.6
   - python-dateutil=2.8.1
   - python_abi=3.9
   - pythreejs=2.3.0
@@ -146,7 +145,7 @@ dependencies:
   - yaml=0.2.5
   - zeromq=4.3.4
   - zfp=0.5.5
-  - zipp=3.4.1
+  - zipp=3.5.0
   - zlib=1.2.11
   - zstd=1.5.0
   - pip:
@@ -184,7 +183,7 @@ dependencies:
     - filelock==3.0.12
     - flake8==3.9.2
     - freetype-py==2.2.0
-    - glue-astronomy==0.1
+    - glue-astronomy==0.2
     - glue-core==1.0.1
     - glue-jupyter==0.7
     - glue-vispy-viewers==1.0.2
@@ -205,7 +204,7 @@ dependencies:
     - jupyter-bokeh==3.0.2
     - jupyter-packaging==0.7.12
     - jupyter-server==1.6.1
-    - jupyter-server-proxy==3.0.2
+    - jupyter-server-proxy==3.1.0
     - jupyterlab==3.0.12
     - jupyterlab-server==2.6.0
     - jwst==0.17.1
@@ -222,6 +221,7 @@ dependencies:
     - notebook==6.3.0
     - numpy==1.20.1
     - numpydoc==1.1.0
+    - packaging==20.9
     - pandas==1.2.3
     - pandocfilters==1.4.3
     - papermill==2.3.3
@@ -249,7 +249,7 @@ dependencies:
     - qtconsole==5.0.3
     - qtpy==1.9.0
     - radio-beam==0.3.3
-    - regex==2021.7.1
+    - regex==2021.7.6
     - regions==0.4
     - requests-mock==1.9.3
     - requests-unixsocket==0.2.0

diff --git a/deployments/jwebbinar/image/environments/frozen/jwebbinar-frozen.yml b/deployments/jwebbinar/image/environments/frozen/jwebbinar-frozen.yml
@@ -67,7 +67,7 @@ dependencies:
     - asdf==2.8.1
     - astropy==4.2.1
     - astropy-sphinx-theme==1.1
-    - astroquery==0.4.3.dev6862
+    - astroquery==0.4.3.dev6900
     - astroscrappy==1.0.8
     - async-generator==1.10
     - async-timeout==3.0.1
@@ -115,7 +115,7 @@ dependencies:
     - html5lib==1.1
     - imageio==2.9.0
     - imagesize==1.2.0
-    - importlib-metadata==4.6.0
+    - importlib-metadata==4.6.1
     - importlib-resources==5.2.0
     - iniconfig==1.1.1
     - ipydatawidgets==4.2.0
@@ -137,15 +137,15 @@ dependencies:
     - jinja2==3.0.1
     - jmespath==0.10.0
     - joblib==1.0.1
-    - jplephem==2.15
+    - jplephem==2.16
     - json5==0.9.6
     - jsonschema==3.2.0
     - jupyter-bokeh==3.0.2
     - jupyter-client==6.1.12
     - jupyter-core==4.7.1
     - jupyter-packaging==0.7.12
     - jupyter-server==1.9.0
-    - jupyter-server-proxy==3.0.2
+    - jupyter-server-proxy==3.1.0
     - jupyterlab==3.0.12
     - jupyterlab-pygments==0.1.2
     - jupyterlab-server==2.6.0
@@ -176,8 +176,8 @@ dependencies:
     - notebook==6.4.0
     - numpydoc==1.1.0
     - openpyxl==3.0.7
-    - packaging==20.9
-    - pandas==1.2.5
+    - packaging==21.0
+    - pandas==1.3.0
     - pandeia-engine==1.6.1
     - pandocfilters==1.4.3
     - pandokia==2.3.0
@@ -188,7 +188,7 @@ dependencies:
     - pexpect==4.8.0
     - photutils==1.1.0
     - pickleshare==0.7.5
-    - pillow==8.3.0
+    - pillow==8.3.1
     - pipdeptree==2.0.0
     - pluggy==0.13.1
     - poppy==0.9.2
@@ -221,7 +221,7 @@ dependencies:
     - pyzmq==22.1.0
     - qtconsole==5.1.1
     - qtpy==1.9.0
-    - regex==2021.7.1
+    - regex==2021.7.6
     - requests==2.25.1
     - requests-mock==1.9.3
     - requests-unixsocket==0.2.0
@@ -264,7 +264,7 @@ dependencies:
     - testpath==0.5.0
     - textwrap3==0.9.2
     - threadpoolctl==2.1.0
-    - tifffile==2021.6.14
+    - tifffile==2021.7.2
     - toml==0.10.2
     - toolz==0.11.1
     - tornado==6.1
@@ -280,5 +280,5 @@ dependencies:
     - widgetsnbextension==3.5.1
     - xlrd==2.0.1
     - yarl==1.6.3
-    - zipp==3.4.1
+    - zipp==3.5.0
 prefix: /opt/conda/envs/jwebbinar
diff --git a/deployments/jwebbinar/image/environments/jdaviz/jdaviz.yml b/deployments/jwebbinar/image/environments/jdaviz/jdaviz.yml
@@ -1,19 +1,19 @@
 name: jdaviz
 channels:
-        - conda-forge
-        - https://ssb.stsci.edu/astroconda
-        - defaults
+    - conda-forge
+    - https://ssb.stsci.edu/astroconda
+    - defaults
 dependencies:
-        - pip
-        - bqplot-image-gl
-        - bqplot
-        - jupyterlab_widgets
-        - ipydatawidgets
-        - ipykernel
-        - ipympl
-        - ipython
-        - ipyvolume
-        - ipyvue
-        - ipyvuetify
-        - ipywebrtc
-        - ipywidgets
+    - pip
+    - bqplot-image-gl
+    - bqplot
+    - jupyterlab_widgets
+    - ipydatawidgets
+    - ipykernel
+    - ipympl
+    - ipython
+    - ipyvolume
+    - ipyvue
+    - ipyvuetify
+    - ipywebrtc
+    - ipywidgets
diff --git a/deployments/jwebbinar/image/environments/jwebbinar/jwebbinar.yml b/deployments/jwebbinar/image/environments/jwebbinar/jwebbinar.yml
@@ -1,12 +1,12 @@
 name: jwebbinar
 channels:
-        - conda-forge
-        - manics
-        - https://ssb.stsci.edu/astroconda
-        - defaults
+    - conda-forge
+    - manics
+    - https://ssb.stsci.edu/astroconda
+    - defaults
 dependencies:
-        - pip
-        - freetds
-        - fftw
-        - pyfftw
-        - python==3.8
+    - pip
+    - freetds
+    - fftw
+    - pyfftw
+    - python==3.8
diff --git a/deployments/jwebbinar/image/environments/setup-notebooks b/deployments/jwebbinar/image/environments/setup-notebooks
@@ -5,12 +5,10 @@
 
 /opt/common-scripts/git-sync https://github.com/spacetelescope/jwebbinar_prep current_webbinar /home/jovyan/jwebbinar_prep
 
-/opt/common-scripts/set-notebook-kernel jwebbinar JWebbinar jwebbinar_prep/ifu_session/MRS_Notebook2.ipynb
-/opt/common-scripts/set-notebook-kernel jwebbinar JWebbinar jwebbinar_prep/ifu_session/jwebbinar5_nirspecifu.ipynb
-/opt/common-scripts/set-notebook-kernel jdaviz Jdaviz jwebbinar_prep/ifu_session/Cube_Analysis_Webbinar_solutions.ipynb
+/opt/common-scripts/set-notebook-kernel jwebbinar JWebbinar jwebbinar_prep/pointsource_imaging/MIRI_Aperture_Photometry_*.ipynb
+/opt/common-scripts/set-notebook-kernel jwebbinar JWebbinar jwebbinar_prep/pointsource_imaging/NIRCam_Basic_PSF_Photometry_*.ipynb
 
 mkdir -p /opt/environments/jwebbinar/tests
 mkdir -p /opt/environments/jdaviz/tests
-ls -1 /home/jovyan/jwebbinar_prep/ifu_session/MRS_Notebook2.ipynb  | grep -v __ | grep -v _live  >/opt/environments/jwebbinar/tests/notebooks
-ls -1 /home/jovyan/jwebbinar_prep/ifu_session/jwebbinar5_nirspecifu.ipynb  | grep -v __ | grep -v _live  >/opt/environments/jwebbinar/tests/notebooks
-ls -1 /home/jovyan/jwebbinar_prep/ifu_session/Cube_Analysis_Webbinar_solutions.ipynb  | grep -v __ | grep -v _live  >/opt/environments/jdaviz/tests/notebooks
+ls -1 /home/jovyan/jwebbinar_prep/pointsource_imaging/MIRI_Aperture_Photometry_*.ipynb | grep -v __ | grep -v _live  >/opt/environments/jwebbinar/tests/notebooks
+ls -1 /home/jovyan/jwebbinar_prep/pointsource_imaging/NIRCam_Basic_PSF_Photometry_*.ipynb | grep -v __ | grep -v _live  >/opt/environments/jwebbinar/tests/notebooks
diff --git a/derived-env b/derived-env
@@ -7,7 +7,7 @@
 export ACCOUNT=${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com
 export ECR_ACCOUNT=${CENTRAL_ECR_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com
 
-if $USE_CENTRAL_ECR
+if [ "$USE_CENTRAL_ECR" = true ]
 then
     export ECR_ACCOUNT_TO_USE=$ECR_ACCOUNT
     export IMAGE_REPO=${DEPLOYMENT_NAME}