DOCS-127 - Add KSQL doc to CP

WIP

WIP

WIP

WIP

docs/Makefile

@@ -0,0 +1,181 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext livehtml
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  livehtml   to make standalone HTML files automatically watching for changes"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+livehtml:
+	python autoreload.py
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/KafkaRESTProxy.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/KafkaRESTProxy.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/KafkaRESTProxy"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/KafkaRESTProxy"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

docs/README.md

@@ -1,7 +1,6 @@
 # KSQL Documentation
 
-| Overview |[Quick Start](/docs/quickstart#quick-start) | [Concepts](/docs/concepts.md#concepts) | [Syntax Reference](/docs/syntax-reference.md#syntax-reference) |[Demo](/ksql-clickstream-demo#clickstream-analysis) | [Examples](/docs/examples.md#examples) | [FAQ](/docs/faq.md#frequently-asked-questions)  | [Roadmap](/docs/roadmap.md#roadmap) | 
-|---|----|-----|----|----|----|----|----|
+The full Confluent Platform documentation is available at [docs.confluent.io](https://docs.confluent.io/current/ksql/docs/index.html).
 
 > *Important: This release is a **developer preview** and is free and open-source from Confluent under the Apache 2.0 license. Do not run KSQL against a production cluster.*
 
@@ -26,7 +25,41 @@ You can use KSQL in standalone, client-server, application, and embedded modes.
 
 ## Getting Started
 
-* Beginners: Try the [interactive quick start](/docs/quickstart#quick-start). The quick start configures a single instance in a lightweight Docker container or in a Kafka cluster. It demonstrates a simple workflow using KSQL to write streaming queries against data in Kafka.
-* Advanced users: Try the [end-to-end Clickstream Analysis demo](/ksql-clickstream-demo#clickstream-analysis).
+* Beginners: Try the [interactive quick start](quickstart/index.rst). The quick start configures a single instance in a lightweight Docker container or in a Kafka cluster. It demonstrates a simple workflow using KSQL to write streaming queries against data in Kafka.
+* Advanced users: Try the [end-to-end Clickstream Analysis demo](ksql-clickstream-demo/index.rst).
 
+# Table of contents
 
+- [Quick Start](quickstart/index.rst)
+- [Configuring KSQL](config-ksql.rst)
+- [Concepts](concepts.rst)
+- [Syntax Reference](syntax-reference.rst)
+- [Clickstream Analysis Demo](ksql-clickstream-demo/index.rst)
+- [KSQL Examples](examples.rst)
+- [Frequently Asked Questions](faq.rst)
+
+# Building the documentation
+
+This documentation is built using [Sphinx](http://sphinx-doc.org). It also uses some extensions for theming and REST API
+documentation support.
+
+Start by installing the requirements:
+
+    pip install -r requirements.txt
+
+Then you can generate the HTML version of the docs:
+
+    make html
+
+The root of the documentation will be at `_build/html/index.html`
+
+While editing the documentation, you can get a live preview using python-livepreview. Install the Python library:
+
+    pip install livereload
+
+Then run the monitoring script in the background:
+
+    python autoreload.py &
+
+If you install the [browser extensions](http://livereload.com/) then everything should update every time any files are
+saved without any manual steps on your part.

docs/concepts.rst

@@ -0,0 +1,153 @@
+.. _ksql_concepts:
+
+Concepts
+========
+
+==========
+Components
+==========
+
+The main components of KSQL are the KSQL CLI and the KSQL server.
+
+KSQL CLI
+--------
+
+The KSQL CLI allows you to interactively write KSQL queries. Its
+interface should be familiar to users of MySQL, Postgres, Oracle, Hive,
+Presto, etc.
+
+The KSQL CLI acts as a client to the KSQL server.
+
+KSQL Server
+-----------
+
+The KSQL server runs the engine that executes KSQL queries, which
+includes the data processing as well as reading data from and writing
+data to the target Kafka cluster.
+
+===========
+Terminology
+===========
+
+When using KSQL, the following terminology is used.
+
+Stream
+------
+
+A stream is an unbounded sequence of structured data (“facts”). For
+example, we could have a stream of financial transactions such as “Alice
+sent $100 to Bob, then Charlie sent $50 to Bob”. Facts in a stream are
+immutable, which means new facts can be inserted to a stream, but
+existing facts can never be updated or deleted. Streams can be created
+from a Kafka topic or derived from existing streams and tables. In both
+cases, a stream’s underlying data is durably stored (persisted) within a
+Kafka topic on the Kafka brokers.
+
+Table
+-----
+
+A table is a view of a stream, or another table, and represents a
+collection of evolving facts. For example, we could have a table that
+contains the latest financial information such as “Bob’s current account
+balance is $150”. It is the equivalent of a traditional database table
+but enriched by streaming semantics such as windowing. Facts in a table
+are mutable, which means new facts can be inserted to the table, and
+existing facts can be updated or deleted. Tables can be created from a
+Kafka topic or derived from existing streams and tables. In both cases,
+a table’s underlying data is durably stored (persisted) within a Kafka
+topic on the Kafka brokers.
+
+.. _modes-of-operation:
+
+==================
+Modes of operation
+==================
+
+Standalone mode
+---------------
+
+In stand-alone mode, both the KSQL client and server components are
+co-located on the same machine, in the same JVM, and are started
+together. This makes standalone mode very convenient for local
+development and testing.
+
+.. image:: img/standalone-mode.png
+
+To run KSQL in standalone mode:
+
+-  Start the KSQL CLI and the server components all in the same JVM:
+
+   -  Start with default settings:
+
+      .. code:: bash
+
+		$ ./bin/ksql-cli local
+
+   -  Start with :ref:`custom
+      settings <configuring-ksql>`, pointing
+      KSQL at a specific Kafka cluster (see the Streams
+      `bootstrap.servers <streams_developer-guide_required-configs>`
+      setting):
+
+      .. code:: bash
+
+		$ ./bin/ksql-cli local --bootstrap-server kafka-broker-1:9092 \
+                       --properties-file path/to/ksql-cli.properties
+
+Client-server mode
+------------------
+
+In client-server mode, you can run a pool of KSQL servers on remote
+machines, VMs, or containers. The CLI then connects to these remote KSQL
+servers over HTTP.
+
+.. image:: img/client-server.png
+
+To run KSQL in client-server mode:
+
+-  Start any number of server nodes:
+
+   -  Start with default settings:
+
+      .. code:: bash
+
+		$ ./bin/ksql-server-start
+
+   -  Start with :ref:`custom
+      settings <configuring-ksql>`, pointing
+      KSQL at a specific Kafka cluster (see Streams :ref:`bootstrap servers <streams_developer-guide_required-configs>` setting):
+
+      .. code:: bash
+
+         $ hostname
+         my-ksql-server
+
+         $ cat ksql-server.properties
+         # You must set at least the following two properties
+         bootstrap.servers=kafka-broker-1:9092
+         # Note: `application.id` is not really needed but you must set it
+         #       because of a known issue in the KSQL Developer Preview
+         application.id=app-id-setting-is-ignored
+         
+         # Optional settings below, only for illustration purposes
+         # The hostname/port on which the server node will listen for client connections
+         listeners=http://0.0.0.0:8090
+
+      To start the server node with the settings above:
+
+      .. code:: bash
+
+		$ ./bin/ksql-server-start ksql-server.properties
+
+-  Start any number of CLIs, specifying the desired KSQL server address
+   as the ``remote`` endpoint:
+
+   .. code:: bash
+
+       $ ./bin/ksql-cli remote http://my-ksql-server:8090
+
+All KSQL servers (and their engines) share the work of processing KSQL
+queries that are submitted to them:
+
+- To add processing capacity, start more KSQL servers (scale out). You can do this during live operations.
+- To remove processing capacity, stop some of the running KSQL servers. You can do this during live operations. The remaining KSQL servers will automatically take over the processing work of the stopped servers. Make sure that at least one KSQL server is running, otherwise your queries will not be executed any longer.