Created deployment option for sqlite-backed DB

README.md

@@ -200,10 +200,12 @@ Topics in this section:
 - [How to use Depicted effectively](#how-to-use-depicted-effectively)
 - [Example tools](#example-tools)
 - [The API documentation](#api)
-- [Deployment to App Engine](#deployment)
+- [Deployment to production (Sqlite, App Engine, etc)](#deployment)
 
 ## Running the server locally
 
+**WARNING LABEL**: This is *only* for local development. If you actually want a reliable server that will run for days, see [the section on deployment](#deployment) below.
+
 1. Have a version of [Python 2.7](http://www.python.org/download/releases/2.7/) installed.
 1. Download [PhantomJS](http://phantomjs.org/) for your machine.
 1. Download [ImageMagick](http://www.imagemagick.org/script/binary-releases.php) for your machine.
@@ -655,6 +657,38 @@ Marks a release candidate as having all runs reported.
 
 ## Deployment
 
+### Sqlite instance
+
+Here's how to run a production-grade version of the server on your a machine using sqlite as the database. This will also work on VMs if you set the database location to a persistent filesystem.
+
+1. `cd` into the `deployment` directory:
+1. Run this command:
+
+        make sqlite_deploy
+
+1. Copy the `sqlite_deploy` directory to wherever you want to run the server
+1. `cd` into the `sqlite_deploy` directory
+1. Create a new python virtual environment and activate it:
+
+        virtualenv .
+        source bin/activate
+
+1. Install all dependencies into the environment:
+
+        pip install -r requirements.txt
+        pip install -e .
+
+1. Run the server
+
+        ./run.sh
+
+1. Note that all of the data for the server will live in the `sqlite_deploy` directory in a file named `data.db`.
+1. When you quit the server, don't forget to deactivate your virtual environment:
+
+        deactivate
+
+### App Engine managed VMs
+
 Here's how to deploy to Google App Engine / CloudSQL / Google Compute Engine. This guide is still a little rough.
 
 1. `cd` into the `deployment` directory:

deployment/.gitignore

@@ -1,2 +1,2 @@
 appengine_deploy
-templates_compiled.zip
+sqlite_deploy

deployment/Makefile

@@ -1,7 +1,8 @@
 all:
+	# Do nothing. You probably don't want both.
 
 clean:
-	rm -Rf appengine_deploy
+	rm -Rf appengine_deploy sqlite_deploy
 
 appengine_deploy:
 	rm -Rf appengine_deploy
@@ -10,3 +11,8 @@ appengine_deploy:
 	    "from dpxdt.server import app; \
 	     app.jinja_env.compile_templates( \
 	         './appengine_deploy/templates_compiled.zip')"
+
+sqlite_deploy:
+	rm -Rf sqlite_deploy
+	cp -R -L sqlite sqlite_deploy
+	cd sqlite_deploy && ./bootstrap.py

deployment/sqlite/bootstrap.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# Copyright 2015 Brett Slatkin
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Bootstraps a new installation by setting up the production environment."""
+
+import os
+os.environ['YOURAPPLICATION_SETTINGS'] = '../../settings.cfg'
+os.environ['SQLITE_PRODUCTION'] = 'Yes'
+
+from dpxdt.server import db
+from dpxdt.server import models
+from dpxdt.server import utils
+
+db.create_all()
+
+build = models.Build(name='Primary build')
+db.session.add(build)
+db.session.commit()
+
+api_key = models.ApiKey(
+    id=utils.human_uuid(),
+    secret=utils.password_uuid(),
+    purpose='Local workers',
+    superuser=True,
+    build_id=build.id)
+db.session.add(api_key)
+db.session.commit()
+
+db.session.flush()
+
+with open('flags.cfg', 'a') as f:
+    f.write('--release_client_id=%s\n' % api_key.id)
+    f.write('--release_client_secret=%s\n' % api_key.secret)

deployment/sqlite/dpxdt

@@ -0,0 +1 @@
+../../dpxdt

deployment/sqlite/flags.cfg

@@ -0,0 +1,17 @@
+--port=5000
+--enable_api_server
+--enable_queue_workers
+--fetch_use_internal_redirects
+--release_server_prefix=http://localhost:5000/api
+--queue_server_prefix=http://localhost:5000/api/work_queue
+--fetch_frequency=10
+--fetch_threads=10
+--capture_threads=10
+--capture_wait_seconds=2
+--phantomjs_timeout=20
+--queue_idle_poll_seconds=30
+--queue_busy_poll_seconds=1
+--pdiff_threads=10
+--pdiff_wait_seconds=2
+--pdiff_timeout=20
+--polltime=1

deployment/sqlite/requirements.txt

@@ -0,0 +1 @@
+../../requirements.txt

deployment/sqlite/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+source bin/activate
+pip install -r requirements.txt
+pip install -e .
+trap "deactivate" 0
+
+export YOURAPPLICATION_SETTINGS=../../settings.cfg
+export SQLITE_PRODUCTION=Yes
+
+dpxdt_server --flagfile=flags.cfg

deployment/sqlite/settings.cfg

@@ -0,0 +1 @@
+SQLALCHEMY_DATABASE_URI = 'sqlite:///../../data.db'

deployment/sqlite/setup.py

@@ -0,0 +1 @@
+../../setup.py

dpxdt/server/utils.py

@@ -80,7 +80,7 @@ def jsonify_error(message_or_exception, status_code=400):
     else:
         message = message_or_exception
 
-    logging.debug('Returning status=%d, error message: %s',
+    logging.debug('Returning status=%s, error message: %s',
                   status_code, message)
     response = jsonify(error=message)
     response.status_code = status_code
@@ -153,7 +153,9 @@ def is_production():
     """Returns True if this is the production environment."""
     # TODO: Support other deployment situations.
     return (
-        os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Engine'))
+        os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Engine')
+        or
+        'SQLITE_PRODUCTION' in os.environ)
 
 
 def get_deployment_timestamp():

requirements.txt

@@ -19,4 +19,4 @@ pyimgur==0.5.2
 python-gflags==2.0
 requests==2.5.1
 watchdog==0.8.3
-wsgiref==0.1.
+wsgiref==0.1.2