Conda support

.travis.yml

@@ -6,6 +6,14 @@ cache:
   - pip
 install:
   - "pip install setuptools --upgrade; pip install -r test_requirements.txt; pip install -e git+https://github.com/django/django-contrib-comments.git#egg=django-contrib-comments; python setup.py install" 
+  - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - "$HOME/miniconda/bin/pip install -r test_requirements.txt; $HOME/miniconda/bin/pip install -e git+https://github.com/django/django-contrib-comments.git#egg=django-contrib-comments; $HOME/miniconda/bin/python setup.py install"
+  - hash -r
+  - $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no
+  - $HOME/miniconda/bin/conda update -q conda
+  # Useful for debugging any issues with conda
+  - $HOME/miniconda/bin/conda info -a
 # command to run tests
 env:
   - TESTCASE=tests/test_handler.py
@@ -14,6 +22,7 @@ env:
   - TESTCASE=tests/tests.py
 script: 
   - nosetests $TESTCASE --with-coverage --cover-package=zappa --with-timer
+  - unset VIRTUAL_ENV; PATH="$HOME/miniconda/bin:$PATH" CONDA_PREFIX=$HOME/miniconda nosetests $TESTCASE --with-coverage --cover-package=zappa --with-timer
     #  - coverage combine --append
 after_success:
   coveralls

README.md

@@ -478,6 +478,7 @@ to change Zappa's behavior. Use these at your own risk!
         ],
         "exception_handler": "your_module.report_exception", // function that will be invoked in case Zappa sees an unhandled exception raised from your code
         "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. To exclude boto3 and botocore (available in an older version on Lambda), add "boto3*" and "botocore*".
+        "exclude_conda_packages": ["boto3","botocore","pip","python","readline","sqlite","tk","wheel"] // When using conda, a list of conda packages to remove before zipping.
         "extends": "stage_name", // Duplicate and extend another stage's settings. For example, `dev-asia` could extend from `dev-common` with a different `s3_bucket` value.
         "extra_permissions": [{ // Attach any extra permissions to this policy. Default None
             "Effect": "Allow",
@@ -516,7 +517,7 @@ to change Zappa's behavior. Use these at your own risk!
         "vpc_config": { // Optional VPC configuration for Lambda function
             "SubnetIds": [ "subnet-12345678" ], // Note: not all availability zones support Lambda!
             "SecurityGroupIds": [ "sg-12345678" ]
-        }
+        },
     }
 }
 ```

zappa/cli.py

@@ -1743,26 +1743,33 @@ def create_package(self):
             inspect.getfile(inspect.currentframe())))
         handler_file = os.sep.join(current_file.split(os.sep)[0:]) + os.sep + 'handler.py'
 
+        conda_mode = self.stage_config.get('conda_mode', False)
+        exclude_conda_packages = self.stage_config.get('exclude_conda_packages',
+                    ['pip','python','readline','sqlite','wheel', 'boto3', 'botocore', 'tk'])  
         # Create the zip file(s)
         if self.stage_config.get('slim_handler', False):
             # Create two zips. One with the application and the other with just the handler.
             # https://github.com/Miserlou/Zappa/issues/510
             self.zip_path = self.zappa.create_lambda_zip(
                 prefix=self.lambda_name,
                 use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True),
-                exclude=self.stage_config.get('exclude', [])
+                exclude=self.stage_config.get('exclude', []),
+                conda_mode=conda_mode,
+                exclude_conda_packages=exclude_conda_packages
             )
 
             # Make sure the normal venv is not included in the handler's zip
             exclude = self.stage_config.get('exclude', [])
-            cur_venv = self.zappa.get_current_venv()
-            exclude.append(cur_venv.split('/')[-1])
+            if not conda_mode:
+                cur_venv = self.zappa.get_current_venv()
+                exclude.append(cur_venv.split('/')[-1])
             self.handler_path = self.zappa.create_lambda_zip(
                 prefix='handler_{0!s}'.format(self.lambda_name),
                 venv=self.zappa.create_handler_venv(),
                 handler_file=handler_file,
                 slim_handler=True,
-                exclude=exclude
+                exclude=exclude,
+                conda_mode=False
             )
         else:
             # Create a single zip that has the handler and application
@@ -1774,7 +1781,9 @@ def create_package(self):
                     'exclude',
                     # Exclude packages already builtin to the python lambda environment
                     # https://github.com/Miserlou/Zappa/issues/556
-                    ["boto3", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"])
+                    ["boto3", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"]),
+                conda_mode=conda_mode,
+                exclude_conda_packages=exclude_conda_packages
             )
             # Warn if this is too large for Lambda.
             file_stats = os.stat(self.zip_path)
@@ -1924,7 +1933,8 @@ def remove_uploaded_zip(self):
             self.zappa.remove_from_s3(self.zip_path, self.s3_bucket_name)
             if self.stage_config.get('slim_handler', False):
                 # Need to keep the project zip as the slim handler uses it.
-                self.zappa.remove_from_s3(self.handler_path, self.s3_bucket_name)
+                if self.handler_path is not None:
+                    self.zappa.remove_from_s3(self.handler_path, self.s3_bucket_name)
 
 
     def on_exit(self):
@@ -2127,17 +2137,28 @@ def deploy_api_gateway(self, api_id):
         return endpoint_url
 
     def check_venv(self):
-        """ Ensure we're inside a virtualenv. """
+        """ Ensure we're inside a virtualenv or conda environment. """
         if self.zappa:
             venv = self.zappa.get_current_venv()
         else:
             # Just for `init`, when we don't have settings yet.
             temp_zappa = Zappa()
             venv = temp_zappa.get_current_venv()
         if not venv:
-            raise ClickException(
-                click.style("Zappa", bold=True) + " requires an " + click.style("active virtual environment", bold=True, fg="red") + "!\n" +
-                "Learn more about virtual environments here: " + click.style("http://docs.python-guide.org/en/latest/dev/virtualenvs/", bold=False, fg="cyan"))
+            if os.getenv('CONDA_PREFIX', os.getenv('CONDA_ENV_PATH')) is not None:
+                if not self.stage_config.get('conda_mode', False):
+                    raise ClickException(
+                        "You seem to be using a conda environment. Set " + click.style("conda_mode=true", bold=True) + " when using conda.")
+            else:
+                if not self.stage_config.get('conda_mode', False):
+                    raise ClickException(
+                        click.style("Zappa", bold=True) + " requires an " + click.style("active virtual or conda environment", bold=True, fg="red") + "!\n" +
+                        "Learn more about virtual environments here: " + click.style("http://docs.python-guide.org/en/latest/dev/virtualenvs/", bold=False, fg="cyan"))
+                else:
+                    raise ClickException(
+                        click.style("Zappa", bold=True) + "  in conda mode requires an " + click.style("active conda environment", bold=True, fg="red") + "!\n" +
+                        "Learn more about conda environments here: " + click.style("https://conda.io/docs/using/envs.html", bold=False, fg="cyan"))
+
 
 ####################################################################
 # Main

zappa/util.py

@@ -1,19 +1,26 @@
 import calendar
+from collections import namedtuple
 import datetime
 import durationpy
 import fnmatch
-import json
+from logging import getLogger
 import os
 import re
 import requests
+import shlex
 import shutil
 import stat
+from subprocess import CalledProcessError, PIPE, Popen
+import sys
 import urlparse
 
+logger = getLogger(__name__)
+
 ##
 # Settings / Packaging
 ##
 
+
 def copytree(src, dst, symlinks=False, ignore=None):
     """
     This is a contributed re-implementation of 'copytree' that
@@ -167,15 +174,11 @@ def get_event_source(event_source, lambda_arn, target_function, boto_session, dr
     to schedule this event, and return the event source.
 
     """
-    import kappa.function
-    import kappa.restapi
     import kappa.event_source.dynamodb_stream
     import kappa.event_source.kinesis
     import kappa.event_source.s3
     import kappa.event_source.sns
     import kappa.event_source.cloudwatch
-    import kappa.policy
-    import kappa.role
     import kappa.awsclient
 
     class PseudoContext(object):
@@ -300,6 +303,30 @@ def check_new_version_available(this_version):
         return False
 
 
+##
+# Subprocess
+##
+
+def call(command, path=None, env_vars=None, raise_on_error=True):
+    if not env_vars:
+        env_vars = {}
+    #Make sure PATH is forwarded to env_vars
+    if 'PATH' not in env_vars:
+        env_vars['PATH'] = os.getenv('PATH')
+    path = sys.prefix if path is None else os.path.abspath(path)
+    p = Popen(shlex.split(command), cwd=path, stdout=PIPE, stderr=PIPE, env=env_vars)
+    stdout, stderr = p.communicate()
+    rc = p.returncode
+    logger.debug("{0} $  {1}\n"
+                 "  stdout: {2}\n"
+                 "  stderr: {3}\n"
+                 "  rc: {4}"
+                 .format(path, command, stdout, stderr, rc))
+    if raise_on_error and rc != 0:
+        raise CalledProcessError(rc, command + "\nstdout: {0}\nstderr: {1}".format(stdout, stderr))
+    Response = namedtuple('Response', ['stdout', 'stderr', 'rc'])
+    return Response(stdout.decode('utf-8'), stderr.decode('utf-8'), int(rc))
+
 class InvalidAwsLambdaName(Exception):
     """Exception: proposed AWS Lambda name is invalid"""
     pass