Move examples to query docstring.

Pull examples with doctest and exit less early for Method parsing.

Move example docstring from connection.py->client.py

Change clas to klass.

gcloud/datastore/client.py

@@ -443,6 +443,37 @@ def query(self, **kwargs):
         """Proxy to :class:`gcloud.datastore.query.Query`.
 
         Passes our ``project``.
+
+        Using query to search a datastore::
+
+        >>> from gcloud import datastore
+        >>> client = datastore.Client()
+        >>> query = client.query(kind='MyKind')
+        >>> query.add_filter('property', '=', 'val')
+
+        Using the query iterator's
+        :meth:`next_page() <gcloud.datastore.query.Iterator.next_page>` method:
+
+        >>> query_iter = query.fetch()
+        >>> entities, more_results, cursor = query_iter.next_page()
+        >>> entities
+        [<list of Entity unmarshalled from protobuf>]
+        >>> more_results
+        <boolean of more results>
+        >>> cursor
+        <string containing cursor where fetch stopped>
+
+        Under the hood this is doing:
+
+        >>> connection.run_query('project', query.to_protobuf())
+        [<list of Entity Protobufs>], cursor, more_results, skipped_results
+
+        :type **kwargs: dict
+        :param **kwargs: Parameters for initializing and instance of
+                         :class:`gcloud.datastore.query.Query`.
+
+        :rtype: :class:`gcloud.datastore.query.Query`
+        :returns: An instance of :class:`gcloud.datastore.query.Query`
         """
         if 'client' in kwargs:
             raise TypeError('Cannot pass client')

gcloud/datastore/connection.py

@@ -227,29 +227,7 @@ def run_query(self, project, query_pb, namespace=None,
         :meth:`gcloud.datastore.query.Query.fetch` method.
 
         Under the hood, the :class:`gcloud.datastore.query.Query` class
-        uses this method to fetch data:
-
-        >>> from gcloud import datastore
-        >>> client = datastore.Client()
-        >>> query = client.query(kind='MyKind')
-        >>> query.add_filter('property', '=', 'val')
-
-        Using the query iterator's
-        :meth:`next_page() <.datastore.query.Iterator.next_page>` method:
-
-        >>> query_iter = query.fetch()
-        >>> entities, more_results, cursor = query_iter.next_page()
-        >>> entities
-        [<list of Entity unmarshalled from protobuf>]
-        >>> more_results
-        <boolean of more results>
-        >>> cursor
-        <string containing cursor where fetch stopped>
-
-        Under the hood this is doing:
-
-        >>> connection.run_query('project', query.to_protobuf())
-        [<list of Entity Protobufs>], cursor, more_results, skipped_results
+        uses this method to fetch data.
 
         :type project: string
         :param project: The project over which to run the query.

scripts/generate_json_docs.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import argparse
+import cgi
+import doctest
 import inspect
 import json
 import os
@@ -27,6 +29,9 @@
 from verify_included_modules import get_public_modules
 
 
+_DOCSTRING_TEST_PARSER = doctest.DocTestParser()
+
+
 class Module(object):
 
     def __init__(self, module_id, name, description=None,
@@ -148,36 +153,38 @@ def from_pdoc(cls, element):
         components = element.refname.split('.')
 
         mod = __import__(components[0])
-
         for comp in components[1:]:
             mod = getattr(mod, comp)
 
-        build_source(mod, method)
+        # Get method line number.
+        method.add_source_line(get_source_line_number(mod))
+
+        # Get method Examples.
+        examples = get_examples_from_docstring(element.docstring)
+        if examples:
+            method.add_example(examples)
 
         if element.docstring:
             if not isinstance(element, pdoc.Class) and element.cls:
-                cls = element.cls.cls
+                klass = element.cls.cls
             elif element.cls:
-                cls = element.cls
+                klass = element.cls
             else:
-                cls = None
+                klass = None
 
             # Hack for old-style classes
-            if str(cls)[0] != '<':
-                cls = '<class \'' + str(cls) + '\'>'
+            if not str(klass).startswith('<'):
+                klass = '<class \'%s\'>' % (klass,)
 
             try:
-                method_info = parse_docstring(element.docstring, cls)
+                method_info = parse_docstring(element.docstring, klass)
             except (MethodParsingException, IndexError):
                 return method
 
             for name, data in method_info['arguments'].items():
                 param = Param.from_docstring_section(name, data)
                 method.add_param(param)
 
-            if method_info.get('example'):
-                method.add_example(method_info['example'])
-
             if method_info.get('return'):
                 if len(method_info['return']['type_name']) > 0:
                     type_name = method_info.get('return').get('type_name')
@@ -296,9 +303,35 @@ def clean_source_path(module):
     return '%s.py' % (source_id.replace('.', '/'),)
 
 
+def get_examples_from_docstring(doc_str):
+    """Parse doctest style code examples from a docstring."""
+    examples = _DOCSTRING_TEST_PARSER.get_examples(doc_str)
+    example_str = ''
+    for example in examples:
+        example_str += '%s' % (example.source,)
+        example_str += '%s' % (example.want,)
+
+    return cgi.escape(example_str)
+
+
+def get_source_line_number(module):
+    if isinstance(module, (types.ModuleType, types.ClassType,
+                           types.MethodType, types.FunctionType,
+                           types.TracebackType, types.FrameType,
+                           types.CodeType, types.TypeType)):
+
+        _, line = inspect.getsourcelines(module)
+        source_path = clean_source_path(module)
+
+        if line:
+            source_path = source_path + '#L' + str(line)
+        return source_path
+
+
 def process_code_blocks(doc):
     blocks = []
     index = 0
+
     for line in doc.splitlines(True):
         if len(blocks) - 1 < index:
             blocks.append('')