Merge pull request #45 from LewisResearchGroup/develop

v0.2.3

InnoSetup/InnoSetupScript.iss

@@ -37,8 +37,8 @@ Name: "english"; MessagesFile: "compiler:Default.isl"
 Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
 
 [Files]
-Source: "C:\Users\soere\workspace\ms-mint-app\specfiles\dist\Mint\{#MyAppExeName}"; DestDir: "{app}"; Flags: ignoreversion
-Source: "C:\Users\soere\workspace\ms-mint-app\specfiles\dist\Mint\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "C:\Users\soere\workspace\ms-mint-app\pyinstaller\dist\Mint\{#MyAppExeName}"; DestDir: "{app}"; Flags: ignoreversion
+Source: "C:\Users\soere\workspace\ms-mint-app\pyinstaller\dist\Mint\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
 ; NOTE: Don't use "Flags: ignoreversion" on any shared system files
 
 [Icons]

Makefile

@@ -2,7 +2,7 @@ lint:
 	flake8
 
 pyinstaller:
-	cd specfiles && pyinstaller --onedir --noconfirm Mint__onedir__.spec --additional-hooks-dir=hooks
+	cd specfiles && pyinstaller --noconfirm Mint.spec ../scripts/Mint.py
 
 docs:
 	mkdocs build && mkdocs gh-deploy

README.md

@@ -16,6 +16,22 @@ More information on how to install and run the program can be found in the [Docu
 
 ## Release notes
 
+### 0.3.0 Milestones
+- Be able to share a workspace with another user
+- Be able to rename a workspace
+- Explanation of expected metadata in app
+  - explain what can be put as colors
+- describe accepted file types in ms_files tab
+- target-list add columns database_ref, formula, polarity
+- add descriptions to the optimization tab
+- total ion chromatogram (TIC)
+- full extracted ion chromatogram (EIC or XIC)
+- feature to convert intensities to concentrations
+
+### 0.2.2
+- version strings in GUI
+- fixed issues with QC figures
+
 ### 0.2.1
 - uses ms-mint 0.2.1 with new implementation of `peak_area_top3`
 - new quality control tab

ms_mint_app/__init__.py

@@ -0,0 +1,3 @@
+from ._version import get_versions
+__version__ = get_versions()["version"]
+del get_versions

ms_mint_app/app.py

@@ -18,13 +18,17 @@
 from dash.exceptions import PreventUpdate
 from dash.dcc import Download
 from dash_extensions.enrich import FileSystemCache
+from dash.long_callback import DiskcacheLongCallbackManager
+
+
 
 import dash_bootstrap_components as dbc
 
 from flask_caching import Cache
 from flask_login import current_user
 
 import ms_mint
+import ms_mint_app
 
 from . import tools as T
 
@@ -61,6 +65,18 @@ def make_dirs():
     "CACHE_DEFAULT_TIMEOUT": 300,
 }
 
+logging.info(f'CACHEDIR: {CACHEDIR}')
+logging.info(f'TMPDIR: {TMPDIR}')
+
+## Diskcache
+from uuid import uuid4
+import diskcache
+launch_uid = uuid4()
+cache = diskcache.Cache(CACHEDIR)
+long_callback_manager = DiskcacheLongCallbackManager(
+    cache, cache_by=[lambda: launch_uid], expire=60,
+)
+
 pd.options.display.max_colwidth = 1000
 
 _modules = [
@@ -166,8 +182,11 @@ def make_dirs():
         html.Div(id="tab-content"),
         html.Div(id="viewport-container", style={"visibility": "hidden"}),
         _outputs,
+        html.Div(f'ms-mint: {ms_mint.__version__}'),
+        html.Div(f'ms-mint-app: {ms_mint_app.__version__}'),
     ],
     style={"margin": "2%"},
+
 )
 
 
@@ -242,6 +261,7 @@ def create_app(**kwargs):
 
     app = dash.Dash(
         __name__,
+        long_callback_manager=long_callback_manager,
         external_stylesheets=[
             dbc.themes.MINTY,
             "https://codepen.io/chriddyp/pen/bWLwgP.css",

ms_mint_app/ms_files.py

@@ -233,6 +233,7 @@ def ms_delete(n_clicks, rows, wdir):
         id='ms-uploader',
     )
     def upload_completed(status):
+        logging.warning(f'Upload status: {status} ({type(status)})')
         return [str(fn) for fn in status.uploaded_files]
 
     @app.callback(

ms_mint_app/plotting.py

@@ -316,7 +316,7 @@ def fill_options(tab, wdir):
         options = [{"value": x, "label": x} for x in cols]
         return [options] * 7
 
-    @app.callback(
+    @app.long_callback(
         Output("plot-figures", "children"),
         Input("plot-update", "n_clicks"),
         State("plot-kind", "value"),
@@ -338,6 +338,7 @@ def fill_options(tab, wdir):
         State("plot-palette", "value"),
         State("plot-options", "value"),
         State("wdir", "children"),
+        cancel=[Input("plot-update", "n_clicks")],
     )
     def create_figure(
         n_clicks,

ms_mint_app/tools.py

@@ -354,7 +354,16 @@ def get_metadata(wdir):
 
     df = df[df["MS-file"] != ""]
 
-    df = df.groupby("MS-file").first().reindex(ms_files).reset_index()
+    new_files = [e for e in ms_files if e not in df['MS-file'].values]
+
+    df = df.groupby("MS-file").first().reindex(ms_files, ).reset_index()
+
+    print('NEW FILES:', new_files)
+    if new_files :
+        # Default for PeakOpt for new files should be False
+        ndx = df[df['MS-file'].isin(new_files)].index
+        print('INDEX:', ndx)
+        df.loc[ndx, 'PeakOpt'] = False
 
     if "PeakOpt" not in df.columns:
         df["PeakOpt"] = False
@@ -386,7 +395,7 @@ def init_metadata(ms_files):
     df["InAnalysis"] = True
     df["Label"] = ""
     df["Color"] = None
-    df["Type"] = "Biological Sample"
+    df["Type"] = "Unknown"
     df["RunOrder"] = ""
     df["Batch"] = ""
     df["Row"] = ""
@@ -605,10 +614,24 @@ def fig_to_src(dpi=100):
     return "data:image/png;base64,{}".format(encoded)
 
 
-def merge_metadata(old, new):
-    old = old.set_index("MS-file")
+def merge_metadata(old: pd.DataFrame, new: pd.DataFrame, index_col='MS-file') -> pd.DataFrame:
+    """
+    This function updates one existing dataframe 
+    with information from a second dataframe.
+    If a column of the new dataframe does not 
+    exist it will be created.
+
+    Parameters:
+    old (pd.DataFrame): The DataFrame to merge new data into.
+    new (pd.DataFrame): The DataFrame containing the new data to merge.
+
+    Returns:
+    pd.DataFrame: The merged DataFrame.
+
+    """    
+    old = old.set_index(index_col)
 
-    new = new.groupby("MS-file").first().replace("null", None)
+    new = new.groupby(index_col).first().replace("null", None)
 
     for col in new.columns:
         if col == "" or col.startswith("Unnamed"):
@@ -621,6 +644,7 @@ def merge_metadata(old, new):
                 continue
             if ndx in old.index:
                 old.loc[ndx, col] = value
+
     return old.reset_index()
 
 

pyinstaller/Mint.spec

@@ -3,25 +3,18 @@ import os
 from PyInstaller.utils.hooks import collect_submodules
 
 src_dir = os.path.abspath(os.path.join(SPECPATH, os.pardir))
-hooks_dir = os.path.join(src_dir, 'specfiles', 'hooks')
+hooks_dir = os.path.join(src_dir, 'pyinstaller', 'hooks')
 script = os.path.join(src_dir, 'scripts', 'Mint.py')
 
 
-hiddenimports_sklearn = collect_submodules('sklearn')
-hiddenimports_bs4 = collect_submodules('bs4')
-hiddenimports_scipy = collect_submodules('scipy')
-hiddenimports_pyarrow = collect_submodules('pyarrow')
-hiddenimports_ms_mint_app = collect_submodules('ms_mint_app')
-hiddenimports_packaging = collect_submodules('packaging')
-
-
 all_hidden_imports = (
     collect_submodules('sklearn')
     + collect_submodules('bs4')
     + collect_submodules('scipy')
     + collect_submodules('pyarrow')
     + collect_submodules('ms_mint_app')
     + collect_submodules('packaging')
+    + collect_submodules('brotli')
 )
 
 

pyinstaller/hooks/hook-pymzml.py

@@ -1,3 +1,4 @@
 from PyInstaller.utils.hooks import collect_data_files
 
 datas = collect_data_files("pymzml")
+hiddenimports = ["pymzml.obo"]

requirements.txt

@@ -1,7 +1,8 @@
-ms-mint==0.2.1
+ms-mint==0.2.2
 xlsxwriter
 waitress
 dash
+dash[diskcache]
 dash_extensions
 dash_bootstrap_components
 flask_login
@@ -16,4 +17,6 @@ Flask-Caching==1.10.1
 beautifulsoup4
 pandas>=1.4,<1.5
 packaging==21.3.0
-beautifulsoup4
+beautifulsoup4
+brotlipy
+dash[diskcache]

setup.py

@@ -8,7 +8,7 @@
 
 
 install_requires = [
-    "ms-mint==0.2.1",
+    "ms-mint",
     "xlsxwriter",
     "waitress",
     "dash",

tests/test__tools.py

@@ -0,0 +1,115 @@
+import pandas as pd
+from pathlib import Path as P
+
+from ms_mint_app import tools as T
+
+def test__merge_metadata():
+    # Define the old DataFrame
+    old_data = {
+      "MS-file": ["file1", "file2", "file3"],
+      "col1": [1, 2, 3],
+      "col2": ["a", "b", "c"],
+      "col3": ["x", "y", "z"]
+    }
+    old_df = pd.DataFrame(data=old_data)
+
+    # Define the new DataFrame
+    new_data = {
+      "MS-file": ["file1", "file2", "file3", "not-in-old"],
+      "col1": [1, 2, 6, 7],
+      "col2": ["d", "b", "f", "g"],
+      "new_col": ['i', 'j', 'k', 'l']
+    }
+    new_df = pd.DataFrame(data=new_data)
+
+    # Define the new DataFrame
+    expected = {
+      "MS-file": ["file1", "file2", "file3"],
+      "col1": [1, 2, 6],
+      "col2": ["d", "b", "f"],
+      "col3": ["x", "y", "z"],
+      "new_col": ['i', 'j', 'k']
+    }
+    expected_df = pd.DataFrame(data=expected)
+
+    actual_df = T.merge_metadata(old_df, new_df)
+
+    print('Old datafame:')
+    print(old_df)
+    print('New datafame:')
+    print(new_df)
+    print('Expected result:')
+    print(expected_df)
+    print('Actual result:')
+    print(actual_df)
+
+    assert actual_df.equals(expected_df), actual_df
+
+
+
+def test__get_metadata(tmp_path):
+
+    T.create_workspace(tmp_path, 'test')
+
+    # Create working directory
+    wdir = P(tmp_path/'workspaces', 'test')
+
+    ms_files_path = P(wdir/'ms_files')
+
+    # Create files
+    open(ms_files_path/'F1.mzXML', 'w').close()
+    open(ms_files_path/'F2.mzXML', 'w').close()
+
+    for subpath in P(tmp_path).rglob('*'):
+        print(subpath)
+
+    metadata = T.get_metadata(wdir)
+
+    print(metadata)
+
+    for e in metadata['MS-file']:
+        print(f'"{e}"')
+
+    assert 'F1' in list(metadata['MS-file']), metadata
+    assert 'F2' in list(metadata['MS-file']), metadata
+
+
+
+def test__get_metadata_after_more_files_added(tmp_path):
+
+    T.create_workspace(tmp_path, 'test')
+
+    # Create working directory
+    wdir = P(tmp_path/'workspaces', 'test')
+
+    ms_files_path = P(wdir/'ms_files')
+
+    # Create files
+    open(ms_files_path/'F1.mzXML', 'w').close()
+    open(ms_files_path/'F2.mzXML', 'w').close()
+
+    for subpath in P(tmp_path).rglob('*'):
+        print(subpath)
+
+    metadata = T.get_metadata(wdir)
+
+    metadata.loc[0, 'PeakOpt'] = True
+
+    print(metadata)
+
+    T.write_metadata(metadata, wdir)
+
+    # Create more files
+    open(ms_files_path/'F3.mzXML', 'w').close()
+    open(ms_files_path/'F4.mzXML', 'w').close()    
+
+    for subpath in P(tmp_path).rglob('*'):
+        print(subpath)
+
+    metadata = T.get_metadata(wdir)
+
+    print(metadata)
+
+    assert all(metadata.PeakOpt == [True, False, False, False])
+
+