Merge branch 'rebrickable-data'

.gitignore

@@ -72,3 +72,5 @@ pyrebrickable-api
 
 #pytest
 .pytest_cache/
+
+pyrebrickable-data/rebrickable\.db

.travis.yml

@@ -11,8 +11,7 @@ python:
 install:
 - python patch_swagger.py
 - "./generate_api.sh"
-- pip install -U tox-travis
-- pip install twine
+- pip install -r dev-requirements.txt
 script:
 - "./validate_spec.sh swagger.json"
 - tox
@@ -31,7 +30,6 @@ deploy:
   file_glob: true
   file:
     - dist/*
-    - rebrickable-api.tar.gz
   on:
     repo: rienafairefr/pyrebrickable
     tags: true

README.md

@@ -3,39 +3,51 @@
 
 # pyrebrickable
 
-This is pyrebrickable, containing two packages:
+This is pyrebrickable, containing three packages.
+
+Combined documentation can be found in the accompanying Github Pages [here](https://rienafairefr.github.io/pyrebrickable)
 
 ## rebrickable_api
 
-The rebrickable_api Python package is automatically generated by the [Swagger Codegen](https://github.com/swagger-api/swagger-codegen) project.
+The rebrickable_api Python package is automatically generated by the [Swagger Codegen](https://github.com/swagger-api/swagger-codegen) project,
+you can install it 'standalone' with `pip install pyrebrickable-api`
 
 ## rebrickable_cli
 
 This Python package provides a CLI wrapper around the Rebrickable API<br> <br> It supports the v3 API through it's openAPI specification.<br> https://rebrickable.com/api/v3/swagger/?format=openapi<br> Models for Part, Set, etc. have been manually added to provide meaningful results from HTTP responses<br> <br> Some endpoints might not work, don't hesitate to file an issue<br>
 
 Once the pyrebrickable package is installed through `pip install pyrebrickable` you should have access to a `rebrickable` command.
+You can also install it 'standalone' with `pip install pyrebrickable-cli`
+
+## rebrickable_data
 
-Combined documentation can be found in a Github Pages repo [here](https://rienafairefr.github.io/pyrebrickable)
+This package uses the monthly CSV database dumps, import them into a
+local database and provide you a way through SQLAlchemy to query that
+database
 
-## Requirements.
+You can also install it 'standalone' with `pip install pyrebrickable-data`
+
+# Requirements.
 
 Python 2.7 and 3.4+
 
-## Installation & Usage
+# Installation & Usage
 
 ```
 pip install pyrebrickable
 ```
 
-## Getting Started
+# Getting Started
 
-With the package installed, run
+For the CLI: with the package installed, run
 
 `rebrickable --help`
 
 You can also read the [CLI docs](https://rienafairefr.github.io/pyrebrickable/cli.html), auto-generated documentation from the [Click](http//click.pocoo.org) definitions in [cli.py](pyrebrickable-cli/rebrickable_cli/cli.py)
 
-Or you can directly use the API through `from rebrickable_api import [...]`, see the [API docs](https://rienafairefr.github.io/pyrebrickable/api.html) for how to use it
+To use the API, use `from rebrickable_api import [...]`, see the [API docs](https://rienafairefr.github.io/pyrebrickable/api.html) for how to use it
+
+To use the rebrickable_data package, see [Data docs](https://rienafairefr.github.io/pyrebrickable/data.html)
 
 ## Author
 

dev-requirements.txt

@@ -6,5 +6,9 @@ pytest
 mock
 -e pyrebrickable-api
 -e pyrebrickable-cli
+-e pyrebrickable-data
 pytest-cov
 coveralls
+tox
+tox-travis
+twine

docs/api.rst

@@ -1,10 +1,20 @@
-API Documentation
+API documentation
 =================
 
 
+There are two sub-APIs, the Lego and Users API, the documentation auto-generated from the swagger spec is here:
+
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :caption: Contents:
 
-   Lego API <../pyrebrickable-api/docs/LegoApi.md>
-   Users API <../pyrebrickable-api/docs/UsersApi.md>
+     Lego API <../pyrebrickable-api/docs/LegoApi>
+     Users API <../pyrebrickable-api/docs/UsersApi>
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/conf.py

@@ -16,9 +16,9 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+#import os
+#import sys
+#sys.path.insert(0, os.path.abspath('.'))
 
 
 # -- General configuration ------------------------------------------------
@@ -96,7 +96,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -162,3 +162,30 @@
      author, 'pyrebrickable', 'One line description of project.',
      'Miscellaneous'),
 ]
+
+
+def run_apidoc(_):
+    from sphinx.ext.apidoc import main
+    import os
+    import sys
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+    cur_dir = os.path.abspath(os.path.dirname(__file__))
+    modules = [
+        ('pyrebrickable-api','api'),
+        ('pyrebrickable-cli','cli')
+    ]
+
+    def rel(pth):
+        return os.path.abspath(os.path.join(cur_dir, "..", pth))
+
+    excludes = ['../**setup.py']
+
+    for module, dir_module in modules:
+        output_dir = rel(os.path.join('docs', 'reference', dir_module))
+        cmd = ['--force', '--separate', '-o', output_dir, rel(module)] + excludes
+        print('Calling sphinx-apidoc: ['+ ' '.join(cmd) + ']')
+        main(cmd)
+
+
+def setup(app):
+    app.connect('builder-inited', run_apidoc)

docs/data.rst

@@ -0,0 +1,48 @@
+pyrebrickable_data documentation
+================================
+
+Installation
+------------
+
+As part of pyrebrickable
+
+    pip install pyrebrickable
+
+Or standalone
+
+    pip install pyrebrickable-data
+
+
+Download and Import
+-------------------
+
+Before anything, you need to download and import the database CSV dumps into your own local database
+The local database will be created in your user data directory (somewhere in ~/.local/share, or %APPDATA% for Windows)
+The database dumps will be saved there as well. Use:
+
+    python -m rebrickable_data.download
+    python -m rebrickable_data.import
+
+This can take some time (a few minutes)
+
+Usage
+-----
+
+After you have downloaded and imported the data, you can use the data through the SQLAlchemy interface:
+
+    from rebrickable_data.database import Session
+    from rebrickable_data.models import Part
+
+    # all the parts
+    Session.query(Part).all()
+
+etc.
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/index.rst

@@ -10,8 +10,11 @@ Welcome to pyrebrickable's documentation!
    :maxdepth: 1
    :caption: Contents:
 
-   CLI <cli.rst>
-   API <api.rst>
+     CLI <cli.rst>
+     API <api.rst>
+     CLI Reference <reference/cli/modules.rst>
+     Auto-generated API Client Reference <reference/api/modules.rst>
+     Database <data.rst>
 
 
 Indices and tables

generate_api.sh

@@ -3,12 +3,11 @@ echo -- `cat swagger.json`
 
 VERSION=${TRAVIS_TAG:-${TAG_NAME:-dev}}
 
-docker run --rm --user `id -u`:`id -g` -v ${PWD}:/local swaggerapi/swagger-codegen-cli:v2.3.0\
-           generate -i /local/swagger.json\
-           --git-user-id rienafairefr\
-           --git-repo-id pyrebrickable\
-           -l python -o /local/pyrebrickable-api -DprojectName=pyrebrickable_api -DpackageName=rebrickable_api\
-           -DpackageVersion="$VERSION" -DappDescription="This is rebrickable_api, an autogenerated package to access\
+docker run --rm --user `id -u`:`id -g` -v ${PWD}:/local swaggerapi/swagger-codegen-cli:v2.3.0 \
+           generate -i /local/swagger.json \
+           --git-user-id rienafairefr \
+           --git-repo-id pyrebrickable \
+           -l python -o /local/pyrebrickable-api -DprojectName=pyrebrickable_api -DpackageName=rebrickable_api \
+           -DpackageVersion="$VERSION" -DappDescription="This is rebrickable_api, an autogenerated package to access \
+           -DappName="pyrebrickable-api" -DinfoEmail="rienafairefr@gmail.com" \
            www.rebrickable.com"
-
-tar -zcvf rebrickable-api.tar.gz pyrebrickable-api

pyrebrickable-cli/setup.py

@@ -41,7 +41,7 @@
 Some endpoints might not work, don't hesitate to file an issue""",
     author_email="",
     url="",
-    keywords=["Swagger", ""],
+    keywords=["LEGO", "Rebrickable", "CLI"],
     entry_points={
         'console_scripts': [
             'rebrickable=rebrickable_cli.cli:main'

pyrebrickable-data/rebrickable_data/database.py

@@ -0,0 +1,11 @@
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from rebrickable_data.utils import data_dir
+
+db_path = os.path.join(data_dir, 'rebrickable.db')
+db_url = 'sqlite:///%s' % db_path
+engine = create_engine(db_url)
+Session = sessionmaker(bind=engine)

pyrebrickable-data/rebrickable_data/download.py

@@ -0,0 +1,25 @@
+import os
+
+import requests
+
+from rebrickable_data.utils import data_dir, data_files
+
+url_format = 'https://m.rebrickable.com/media/downloads/%s.csv'
+
+
+def download_data_files():
+    for data_file in data_files:
+        csv_path = os.path.join(data_dir, data_file + '.csv')
+        url = url_format % data_file
+        if not os.path.exists(csv_path):
+            response = requests.get(url)
+            if response.ok:
+                with open(csv_path, 'wb') as csv_file:
+                    csv_file.write(response.content)
+                print('OK, downloaded %s' % data_file)
+            else:
+                print('could not get remote data at %s' % url)
+
+
+if __name__ == '__main__':
+    download_data_files()

pyrebrickable-data/rebrickable_data/import.py

@@ -0,0 +1,68 @@
+import csv
+import os
+
+from sqlalchemy import inspect, Table
+
+from rebrickable_data.models import Base
+from rebrickable_data.utils import data_dir, data_files
+from rebrickable_data.database import db_url, engine, Session
+
+
+def get_class_by_tablename(tablename):
+    """Return class reference mapped to table.
+
+    :param tablename: String with name of table.
+    :return: Class reference or None.
+    """
+    for c in Base._decl_class_registry.values():
+        if hasattr(c, '__tablename__') and c.__tablename__ == tablename:
+            return c
+    return Base.metadata.tables.get(tablename)
+
+
+def import_main():
+    print('')
+    metadata = Base.metadata
+
+    metadata.create_all(engine)
+
+    # create a Session
+    session = Session()
+
+    for data_file in data_files:
+        print('Importing %s ... ' % data_file, end='')
+
+        model_type = get_class_by_tablename(data_file)
+
+        assert model_type is not None
+
+        if session.query(model_type).first():
+            print('There is already data in %s, not importing.' % data_file)
+            continue
+
+        csv_path = os.path.join(data_dir, data_file+'.csv')
+        with open(csv_path, 'r') as csv_file:
+            reader = csv.DictReader(csv_file)
+
+            objects = list(reader)
+
+            print(' %i rows ...' % len(objects), end='')
+
+            for k in reader.fieldnames:
+                if k.startswith('is_'):
+                    for row in objects:
+                        row[k] = row[k] == 't'
+
+        if isinstance(model_type, Table):
+            ins = model_type.insert(objects)
+            conn = engine.connect()
+            conn.execute(ins)
+        else:
+            session.bulk_insert_mappings(model_type, objects)
+
+        session.commit()
+        print('ok')
+
+
+if __name__ == '__main__':
+    import_main()

pyrebrickable-data/rebrickable_data/models/__init__.py

@@ -0,0 +1,9 @@
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+from rebrickable_data.models.colors import *
+from rebrickable_data.models.inventories import *
+from rebrickable_data.models.parts import *
+from rebrickable_data.models.sets import *
+from rebrickable_data.models.themes import *

pyrebrickable-data/rebrickable_data/models/colors.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Integer, Column, String, Boolean
+
+from rebrickable_data.models import Base
+
+
+class Color(Base):
+    __tablename__ = 'colors'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    rgb = Column(String)
+    is_trans = Column(Boolean)