new documentation system [closes #720, closes #768]

base/util.jl

@@ -235,58 +235,27 @@ end
 
 # help
 
-function parse_help(stream)
-    helpdb = Dict()
-    for l = each_line(stream)
-        if isempty(l)
-            continue
-        end
-        if length(l) >= 3 && l[1:3]=="## "
-            heading = l[4:end-1]
-            category = Dict()
-            helpdb[heading] = category
-            continue
-        end
-        if l[1]=='`'
-            parts = split(l, '—')
-            sig = parts[1][2:end-2]
-            if length(parts) > 1
-                desc = parts[2]
-            else
-                desc = ""
-            end
-            m = match(r"(\w+!?)\(", sig)
-            if m != nothing
-                # found something of the form "f("
-                funcname = m.captures[1]
-            else
-                # otherwise use whatever's between the ``
-                funcname = sig
+_jl_help_categories = nothing
+_jl_help_functions = nothing
+
+function _jl_init_help()
+    global _jl_help_categories, _jl_help_functions
+    if _jl_help_categories == nothing
+        println("Loading help data...")
+        load("$JULIA_HOME/doc/helpdb.jl")
+        _jl_help_categories = Dict()
+        _jl_help_functions = Dict()
+        for (cat,func,desc) in _jl_help_db()
+            if !has(_jl_help_categories, cat)
+                _jl_help_categories[cat] = {}
             end
-            entry = (sig, desc)
-            if has(category,funcname)
-                push(category[funcname], entry)
-            else
-                category[funcname] = {entry}
+            push(_jl_help_categories[cat], func)
+            if !has(_jl_help_functions, func)
+                _jl_help_functions[func] = {}
             end
+            push(_jl_help_functions[func], desc)
         end
     end
-    helpdb
-end
-
-_jl_helpdb = nothing
-
-const _jl_help_url = "https://raw.github.com/JuliaLang/julialang.github.com/master/manual/standard-library-reference/index.md"
-
-function _jl_init_help()
-    global _jl_helpdb
-    if _jl_helpdb == nothing
-        println("Downloading help data...")
-        cmd = `curl -s $_jl_help_url`
-        stream = fdio(read_from(cmd).fd, true)
-        spawn(cmd)
-        _jl_helpdb = parse_help(stream)
-    end
 end
 
 function help()
@@ -301,7 +270,7 @@ function help()
  for one of the following categories:
 
 ")
-    for (cat, tabl) = _jl_helpdb
+    for (cat, tabl) = _jl_help_categories
         if !isempty(tabl)
             print("  ")
             show(cat); println()
@@ -311,12 +280,12 @@ end
 
 function help(cat::String)
     _jl_init_help()
-    if !has(_jl_helpdb, cat)
+    if !has(_jl_help_categories, cat)
         # if it's not a category, try another named thing
         return help_for(cat)
     end
     println("Help is available for the following items:")
-    for (func, _) = _jl_helpdb[cat]
+    for func = _jl_help_categories[cat]
         print(func, " ")
     end
     println()
@@ -328,25 +297,17 @@ function _jl_print_help_entries(entries)
         if !first
             println()
         end
-        print(desc[1], "\n ", desc[2])
+        println(strip(desc))
         first = false
     end
 end
 
 help_for(s::String) = help_for(s, 0)
 function help_for(fname::String, obj)
     _jl_init_help()
-    n = 0
-    for (cat, tabl) = _jl_helpdb
-        for (func, entries) = tabl
-            if func == fname
-                _jl_print_help_entries(entries)
-                n+=1
-                break
-            end
-        end
-    end
-    if n == 0
+    if has(_jl_help_functions, fname)
+        _jl_print_help_entries(_jl_help_functions[fname])
+    else
         if isgeneric(obj)
             repl_show(obj); println()
         else
@@ -360,23 +321,20 @@ function apropos(txt::String)
     n = 0
     r = Regex("\\Q$txt", PCRE_CASELESS)
     first = true
-    for (cat, tabl) = _jl_helpdb
+    for (cat, _) in _jl_help_categories
         if matches(r, cat)
             println("Category: \"$cat\"")
             first = false
         end
     end
-    for (cat, tabl) = _jl_helpdb
-        for (func, entries) = tabl
-            if matches(r, func) || anyp(e->(matches(r,e[1]) || matches(r,e[2])),
-                                        entries)
-                if !first
-                    println()
-                end
-                _jl_print_help_entries(entries)
-                first = false
-                n+=1
+    for (func, entries) in _jl_help_functions
+        if matches(r, func) || anyp(e->matches(r,e), entries)
+            if !first
+                println()
             end
+            _jl_print_help_entries(entries)
+            first = false
+            n+=1
         end
     end
     if n == 0

doc/.gitignore

@@ -0,0 +1 @@
+_build/

doc/Makefile

@@ -0,0 +1,158 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  helpdb.jl  to make the REPL help db"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/JuliaLanguage.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/JuliaLanguage.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/JuliaLanguage"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/JuliaLanguage"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+helpdb.jl: stdlib/base.rst sphinx/jlhelp.py
+	$(SPHINXBUILD) -b jlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/jlhelp
+	mv $(BUILDDIR)/jlhelp/stdlib/base.jl helpdb.jl

doc/README.md

@@ -0,0 +1,34 @@
+Julia Documentation README
+==========================
+
+Julia's documentation is written in reStructuredText, a good reference for which
+is the [Documenting Python](http://docs.python.org/devguide/documenting.html)
+chapter of the Python Developer's Guide.
+
+
+Building the documentation
+--------------------------
+
+The documentation is built using [Sphinx](http://sphinx.pocoo.org/) and LaTeX.
+On ubuntu, you'll need the following packages installed:
+
+    python-sphinx
+    texlive
+    texlive-latex-extra
+
+Then run
+
+    $ make helpdb.jl
+    $ make html
+    $ make latexpdf
+
+
+File layout
+-----------
+
+    conf.py             Sphinx configuration
+    helpdb.jl           REPL help database
+    sphinx/             Sphinx extensions and plugins
+    sphinx/jlhelp.py    Sphinx plugin to build helpdb.jl
+
+