REF: remove biom script; make commands importable and tab-completable

Fixes biocore#283. Changes also necessary for biocore#656.

.travis.yml

@@ -24,7 +24,7 @@ install:
   - pip install -e . --no-deps
 script:
   - nosetests --with-doctest --with-coverage
-  - flake8 biom setup.py scripts
+  - flake8 biom setup.py
   - biom show-install-info
   - make -C doc html
   # we can only validate the tables if we have H5PY

MANIFEST.in

@@ -5,7 +5,6 @@ include ChangeLog.md
 graft biom
 graft support_files
 graft examples
-graft scripts
 graft doc
 
 prune docs/_build

biom/cli/__init__.py

@@ -6,15 +6,25 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 # ----------------------------------------------------------------------------
 
-from .installation_informer import show_install_info
-from .table_subsetter import subset_table
-from .table_summarizer import summarize_table
-from .table_normalizer import normalize_table
-from .metadata_adder import add_metadata
-from .table_validator import validate_table
-from .table_converter import convert
-from .util import write_biom_table
+from __future__ import division
 
-__all__ = ['validate_table', 'summarize_table', 'add_metadata',
-           'show_install_info', 'normalize_table', 'subset_table',
-           'convert', 'write_biom_table']
+from importlib import import_module
+
+import click
+import biom
+
+
+@click.group()
+@click.version_option(version=biom.__version__)
+def cli():
+    pass
+
+
+import_module('biom.cli.table_summarizer')
+import_module('biom.cli.metadata_adder')
+import_module('biom.cli.table_converter')
+import_module('biom.cli.installation_informer')
+import_module('biom.cli.table_subsetter')
+import_module('biom.cli.table_normalizer')
+import_module('biom.cli.table_head')
+import_module('biom.cli.table_validator')

biom/cli/installation_informer.py

@@ -10,8 +10,30 @@
 
 import sys
 
+import click
 
+from biom.cli import cli
+
+
+@cli.command(name='show-install-info')
 def show_install_info():
+    """Provide information about the biom-format installation.
+
+    Provide information about the biom-format installation, including settings
+    pulled from the configuration file. For more details, see
+    http://biom-format.org
+
+    Example usage:
+
+    Display biom-format installation information:
+
+    $ biom show-install-info
+
+    """
+    click.echo(_show_install_info())
+
+
+def _show_install_info():
     lines = []
     lines.extend(_get_formatted_system_info())
     lines.extend(_get_formatted_dependency_version_info())

biom/cli/metadata_adder.py

@@ -8,7 +8,110 @@
 
 from __future__ import division
 
+import click
+
+from biom import load_table
+from biom.cli import cli
+from biom.cli.util import write_biom_table
 from biom.parse import MetadataMap
+from biom.util import HAVE_H5PY
+
+
+@cli.command(name='add-metadata')
+@click.option('-i', '--input-fp', required=True,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The input BIOM table')
+@click.option('-o', '--output-fp', required=True,
+              type=click.Path(exists=False, dir_okay=False),
+              help='The output BIOM table')
+@click.option('-m', '--sample-metadata-fp', required=False,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The sample metadata mapping file (will add sample '
+                   'metadata to the input BIOM table, if provided).')
+@click.option('--observation-metadata-fp', required=False,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The observation metadata mapping file (will add '
+                   'observation metadata to the input BIOM table, if '
+                   'provided).')
+@click.option('--sc-separated', required=False, type=click.STRING,
+              help='Comma-separated list of the metadata fields to split '
+                   'on semicolons. This is useful for hierarchical data such '
+                   'as taxonomy or functional categories.')
+@click.option('--sc-pipe-separated', required=False, type=click.STRING,
+              help='Comma-separated list of the metadata fields to split '
+                   'on semicolons and pipes ("|"). This is useful for '
+                   'hierarchical data such as functional categories with '
+                   'one-to-many mappings (e.g. x;y;z|x;y;w)).')
+@click.option('--int-fields', required=False, type=click.STRING,
+              help='Comma-separated list of the metadata fields to cast '
+                   'to integers. This is useful for integer data such as '
+                   '"DaysSinceStart".')
+@click.option('--float-fields', required=False, type=click.STRING,
+              help='Comma-separated list of the metadata fields to cast '
+                   'to floating point numbers. This is useful for real number '
+                   'data such as "pH".')
+@click.option('--sample-header', required=False, type=click.STRING,
+              help='Comma-separated list of the sample metadata field '
+                   'names. This is useful if a header line is not provided '
+                   'with the metadata, if you want to rename the fields, or '
+                   'if you want to include only the first n fields where n is '
+                   'the number of entries provided here.')
+@click.option('--observation-header', required=False, type=click.STRING,
+              help='Comma-separated list of the observation metadata '
+                   'field names. This is useful if a header line is not '
+                   'provided with the metadata, if you want to rename the '
+                   'fields, or if you want to include only the first n fields '
+                   'where n is the number of entries provided here.')
+@click.option('--output-as-json', default=not HAVE_H5PY, is_flag=True,
+              help='Write the output file in JSON format.')
+def add_metadata(input_fp, output_fp, sample_metadata_fp,
+                 observation_metadata_fp, sc_separated, sc_pipe_separated,
+                 int_fields, float_fields, sample_header, observation_header,
+                 output_as_json):
+    """Add metadata to a BIOM table.
+
+    Add sample and/or observation metadata to BIOM-formatted files. See
+    examples here: http://biom-format.org/documentation/adding_metadata.html
+
+    Example usage:
+
+    Add sample metadata to a BIOM table:
+
+    $ biom add-metadata -i otu_table.biom -o table_with_sample_metadata.biom
+      -m sample_metadata.txt
+    """
+    table = load_table(input_fp)
+    if sample_metadata_fp is not None:
+        sample_metadata_f = open(sample_metadata_fp, 'U')
+    else:
+        sample_metadata_f = None
+    if observation_metadata_fp is not None:
+        observation_metadata_f = open(observation_metadata_fp, 'U')
+    else:
+        observation_metadata_f = None
+    if sc_separated is not None:
+        sc_separated = sc_separated.split(',')
+    if sc_pipe_separated is not None:
+        sc_pipe_separated = sc_pipe_separated.split(',')
+    if int_fields is not None:
+        int_fields = int_fields.split(',')
+    if float_fields is not None:
+        float_fields = float_fields.split(',')
+    if sample_header is not None:
+        sample_header = sample_header.split(',')
+    if observation_header is not None:
+        observation_header = observation_header.split(',')
+
+    result = _add_metadata(table, sample_metadata_f, observation_metadata_f,
+                           sc_separated, sc_pipe_separated, int_fields,
+                           float_fields, sample_header, observation_header)
+
+    if output_as_json:
+        fmt = 'json'
+    else:
+        fmt = 'hdf5'
+
+    write_biom_table(result, fmt, output_fp)
 
 
 def _split_on_semicolons(x):
@@ -33,10 +136,10 @@ def _float(x):
         return x
 
 
-def add_metadata(table, sample_metadata=None, observation_metadata=None,
-                 sc_separated=None, sc_pipe_separated=None, int_fields=None,
-                 float_fields=None, sample_header=None,
-                 observation_header=None):
+def _add_metadata(table, sample_metadata=None, observation_metadata=None,
+                  sc_separated=None, sc_pipe_separated=None, int_fields=None,
+                  float_fields=None, sample_header=None,
+                  observation_header=None):
 
     if sample_metadata is None and observation_metadata is None:
         raise ValueError('Must specify sample_metadata and/or '

biom/cli/table_converter.py

@@ -8,7 +8,13 @@
 
 from __future__ import division
 
+import click
+
+from biom import load_table
+from biom.cli import cli
 from biom.cli.util import write_biom_table
+from biom.parse import MetadataMap
+
 
 table_types = ["OTU table",
                "Pathway table",
@@ -32,12 +38,103 @@
 }
 
 
-def convert(table, output_filepath, sample_metadata=None,
-            observation_metadata=None,
-            to_json=False, to_hdf5=False, to_tsv=False,
-            collapsed_samples=False, collapsed_observations=False,
-            header_key=None, output_metadata_id=None, table_type=None,
-            process_obs_metadata=None, tsv_metadata_formatter='sc_separated'):
+@cli.command(name='convert')
+@click.option('-i', '--input-fp', required=True,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The input BIOM table')
+@click.option('-o', '--output-fp', required=True,
+              type=click.Path(exists=False, dir_okay=False),
+              help='The output BIOM table')
+@click.option('-m', '--sample-metadata-fp', required=False,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The sample metadata mapping file (will add sample '
+                   'metadata to the input BIOM table, if provided).')
+@click.option('--observation-metadata-fp', required=False,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The observation metadata mapping file (will add '
+                   'observation metadata to the input BIOM table, if '
+                   'provided).')
+@click.option('--to-json', default=False, is_flag=True,
+              help='Output as JSON-formatted table.')
+@click.option('--to-hdf5', default=False, is_flag=True,
+              help='Output as HDF5-formatted table.')
+@click.option('--to-tsv', default=False, is_flag=True,
+              help='Output as TSV-formatted (classic) table.')
+@click.option('--collapsed-samples', default=False, is_flag=True,
+              help='If --to_hdf5 is passed and the original table is a '
+                   'BIOM table with collapsed samples, this will '
+                   'update the sample metadata of the table to '
+                   'the supported HDF5 collapsed format.')
+@click.option('--collapsed-observations', default=False, is_flag=True,
+              help='If --to_hdf5 is passed and the original table is a '
+                   'BIOM table with collapsed observations, this will '
+                   'update the observation metadata of the table '
+                   'to the supported HDF5 collapsed format.')
+@click.option('--header-key', required=False, type=click.STRING,
+              help='The observation metadata to include from the input '
+                   'BIOM table file when creating a tsv table file. '
+                   'By default no observation metadata will be included.')
+@click.option('--output-metadata-id', required=False, type=click.STRING,
+              help='The name to be given to the observation metadata '
+                   'column when creating a tsv table file if the column '
+                   'should be renamed.')
+@click.option('--table-type', required=False,
+              type=click.Choice(table_types),
+              help='The type of the table.')
+@click.option('--process-obs-metadata', required=False,
+              type=click.Choice(
+                observation_metadata_types),
+              help='Process metadata associated with observations when '
+              'converting from a classic table.')
+@click.option('--tsv-metadata-formatter', required=False,
+              default='sc_separated',
+              type=click.Choice(
+                observation_metadata_formatters),
+              help='Method for formatting the observation metadata.')
+def convert(input_fp, output_fp, sample_metadata_fp, observation_metadata_fp,
+            to_json, to_hdf5, to_tsv, collapsed_samples,
+            collapsed_observations, header_key, output_metadata_id, table_type,
+            process_obs_metadata, tsv_metadata_formatter):
+    """Convert to/from the BIOM table format.
+
+    Convert between BIOM table formats. See examples here:
+    http://biom-format.org/documentation/biom_conversion.html
+
+    Example usage:
+
+    Convert a "classic" BIOM file (tab-separated text) to an HDF5 BIOM
+    formatted OTU table:
+
+    $ biom convert -i table.txt -o table.biom --to-hdf5
+    """
+    if sum([to_tsv, to_hdf5, to_json]) > 1:
+        raise ValueError("--to-tsv, --to-json, and --to-hdf5 are mutually "
+                         "exclusive. You can only pass one of these options.")
+
+    table = load_table(input_fp)
+    if sample_metadata_fp is not None:
+        with open(sample_metadata_fp, 'U') as f:
+            sample_metadata_f = MetadataMap.from_file(f)
+    else:
+        sample_metadata_f = None
+    if observation_metadata_fp is not None:
+        with open(observation_metadata_fp, 'U') as f:
+            observation_metadata_f = MetadataMap.from_file(f)
+    else:
+        observation_metadata_f = None
+
+    _convert(table, output_fp, sample_metadata_f, observation_metadata_f,
+             to_json, to_hdf5, to_tsv, collapsed_samples,
+             collapsed_observations, header_key, output_metadata_id,
+             table_type, process_obs_metadata, tsv_metadata_formatter)
+
+
+def _convert(table, output_filepath, sample_metadata=None,
+             observation_metadata=None, to_json=False, to_hdf5=False,
+             to_tsv=False, collapsed_samples=False,
+             collapsed_observations=False, header_key=None,
+             output_metadata_id=None, table_type=None,
+             process_obs_metadata=None, tsv_metadata_formatter='sc_separated'):
 
     if sum([to_tsv, to_hdf5, to_json]) == 0:
         raise ValueError("Must specify an output format")

biom/cli/table_head.py

@@ -0,0 +1,46 @@
+# ----------------------------------------------------------------------------
+# Copyright (c) 2011-2013, The BIOM Format Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# ----------------------------------------------------------------------------
+
+from __future__ import division
+
+import click
+
+from biom import load_table
+from biom.cli import cli
+
+
+@cli.command()
+@click.option('-i', '--input-fp', required=True,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The input BIOM table')
+@click.option('-o', '--output-fp', default=None,
+              type=click.Path(writable=True),
+              help='An output file-path', required=False)
+@click.option('-n', '--n-obs', default=5, type=int,
+              help="The number of observations to show",
+              required=False)
+@click.option('-m', '--n-samp', default=5, type=int,
+              help="The number of samples to show",
+              required=False)
+def head(input_fp, output_fp, n_obs, n_samp):
+    """Dump the first bit of a table.
+
+    Example usage:
+
+    Print out the upper left corner of a BIOM table to standard out:
+
+    $ biom head -i table.biom
+
+    """
+    table = load_table(input_fp).head(n=n_obs, m=n_samp)
+
+    if output_fp is None:
+        click.echo(str(table))
+    else:
+        with open(output_fp, 'w') as fp:
+            fp.write(str(table))