RFC: new approach for snippet injection

google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java

@@ -753,13 +753,19 @@ public int hashCode() {
    * Updates dataset information.
    *
    * <p>Example of updating a dataset by changing its description.
-   * <pre> {@code
-   * String datasetName = "my_dataset_name";
-   * String newDescription = "some_new_description";
-   * Dataset oldDataset = bigquery.getDataset(datasetName);
-   * DatasetInfo datasetInfo = oldDataset.toBuilder().setDescription(newDescription).build();
-   * Dataset newDataset = bigquery.update(datasetInfo);
-   * }</pre>
+   * <!--SNIPPET bigquery_update_table_description-->
+   * <pre>{@code
+   *    // String datasetName = "my_dataset_name";
+   *    // String tableName = "my_table_name";
+   *    // String newDescription = "new_description";
+   *
+   *    Table beforeTable = bigquery.getTable(datasetName, tableName);
+   *    TableInfo tableInfo = beforeTable.toBuilder()
+   *        .setDescription(newDescription)
+   *        .build();
+   *    Table afterTable = bigquery.update(tableInfo);
+   *     }</pre>
+   * <!--SNIPPET bigquery_update_table_description-->
    *
    * @throws BigQueryException upon failure
    */
@@ -785,7 +791,7 @@ public int hashCode() {
    * String datasetName = "my_dataset_name";
    * String tableName = "my_table_name";
    * Table beforeTable = bigquery.getTable(datasetName, tableName);
-   * 
+   *
    * // Set table to expire 5 days from now.
    * long expirationMillis = DateTime.now().plusDays(5).getMillis();
    * TableInfo tableInfo = beforeTable.toBuilder()
@@ -1104,7 +1110,7 @@ TableResult listTableData(
    * // BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
    * String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
    * QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query).build();
-   * 
+   *
    * // Print the results.
    * for (FieldValueList row : bigquery.query(queryConfig).iterateAll()) {
    *   for (FieldValue val : row) {

google-cloud-clients/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/MessageReceiver.java

@@ -22,27 +22,31 @@
 public interface MessageReceiver {
   /**
    * Called when a message is received by the subscriber. The implementation must arrange for {@link
-   * AckReplyConsumer#ack()} or {@link
-   * AckReplyConsumer#nack()} to be called after processing the {@code message}.
+   * AckReplyConsumer#ack()} or {@link AckReplyConsumer#nack()} to be called after processing the
+   * {@code message}.
+   * <!--SNIPPET receiveMessage-->
    *
-   * <p>This {@code MessageReceiver} passes all messages to a {@code BlockingQueue}.
-   * This method can be called concurrently from multiple threads,
-   * so it is important that the queue be thread-safe.
+   * <pre>{@code
+   * // This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}. This method can
+   * // be called concurrently from multiple threads, so it is important that the queue be
+   * // thread-safe.
+   * //
+   * // This example is for illustration. Implementations may directly process messages instead of
+   * // sending them to queues.
+   * MessageReceiver receiver =
+   *     new MessageReceiver() {
+   *       public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
+   *         if (blockingQueue.offer(message)) {
+   *           consumer.ack();
+   *         } else {
+   *           consumer.nack();
+   *         }
+   *       }
+   *     };
    *
-   * This example is for illustration. Implementations may directly process messages
-   * instead of sending them to queues.
-   * <pre> {@code
-   * MessageReceiver receiver = new MessageReceiver() {
-   *   public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
-   *     if (blockingQueue.offer(message)) {
-   *       consumer.ack();
-   *     } else {
-   *       consumer.nack();
-   *     }
-   *   }
-   * };
    * }</pre>
    *
+   * <!--SNIPPET receiveMessage-->
    */
   void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer);
 }

google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java

@@ -121,12 +121,12 @@ public Dataset updateDataset(String datasetName, String newDescription) {
   /**
    * Example of updating a table by changing its description.
    */
-  // [TARGET update(TableInfo, TableOption...)]
-  // [VARIABLE "my_dataset_name"]
-  // [VARIABLE "my_table_name"]
-  // [VARIABLE "new_description"]
   public Table updateTableDescription(String datasetName, String tableName, String newDescription) {
     // [START bigquery_update_table_description]
+    // String datasetName = "my_dataset_name";
+    // String tableName = "my_table_name";
+    // String newDescription = "new_description";
+
     Table beforeTable = bigquery.getTable(datasetName, tableName);
     TableInfo tableInfo = beforeTable.toBuilder()
         .setDescription(newDescription)

google-cloud-examples/src/main/java/com/google/cloud/examples/pubsub/snippets/MessageReceiverSnippets.java

@@ -36,25 +36,25 @@ public MessageReceiverSnippets(BlockingQueue<PubsubMessage> blockingQueue) {
     this.blockingQueue = blockingQueue;
   }
 
-  /**
-   * This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}.
-   * This method can be called concurrently from multiple threads,
-   * so it is important that the queue be thread-safe.
-   *
-   * This example is for illustration. Implementations may directly process messages
-   * instead of sending them to queues.
-   */
-  // [TARGET receiveMessage(PubsubMessage, AckReplyConsumer)]
   public MessageReceiver messageReceiver() {
-    MessageReceiver receiver = new MessageReceiver() {
-      public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
-        if (blockingQueue.offer(message)) {
-          consumer.ack();
-        } else {
-          consumer.nack();
-        }
-      }
-    };
+    // SNIPPET receiveMessage
+    // This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}. This method can
+    // be called concurrently from multiple threads, so it is important that the queue be
+    // thread-safe.
+    //
+    // This example is for illustration. Implementations may directly process messages instead of
+    // sending them to queues.
+    MessageReceiver receiver =
+        new MessageReceiver() {
+          public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
+            if (blockingQueue.offer(message)) {
+              consumer.ack();
+            } else {
+              consumer.nack();
+            }
+          }
+        };
+    // SNIPPET receiveMessage
     return receiver;
   }
 }

utilities/snippets.go

@@ -0,0 +1,230 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime/pprof"
+	"strings"
+)
+
+func init() {
+	log.SetFlags(0)
+	log.SetPrefix("snippet: ")
+}
+
+func main() {
+	cpuprof := flag.String("cpuprofile", "", "write CPU profile to this file")
+	flag.Parse()
+
+	if cp := *cpuprof; cp != "" {
+		f, err := os.Create(cp)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	files := map[string]string{}
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.Mode().IsRegular() || filepath.Ext(path) != ".java" {
+			return nil
+		}
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		files[path] = string(b)
+		return nil
+	}
+	for _, dir := range flag.Args() {
+		if err := filepath.Walk(dir, walkFn); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	snip := map[string]string{}
+	for file, txt := range files {
+		if err := getSnip(file, txt, snip); err != nil {
+			log.Fatal(err)
+		}
+		if err := getCloud(file, txt, snip); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	rd := rewriteData{
+		rewrite: map[string]string{},
+		used:    map[string]bool{},
+	}
+	for file, txt := range files {
+		if err := writeSnip(file, txt, snip, rd); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for file, txt := range rd.rewrite {
+		if err := ioutil.WriteFile(file, []byte(txt), 0644); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for key := range snip {
+		if !rd.used[key] {
+			log.Printf("unused snippet: %q", key)
+		}
+	}
+}
+
+func getCloud(file, txt string, snip map[string]string) error {
+	const cloudPrefix = "// [START "
+	const cloudSuffix = "// [END %s]"
+
+	ftxt := txt
+	for {
+		if p := strings.Index(txt, cloudPrefix); p >= 0 {
+			txt = txt[p:]
+		} else {
+			return nil
+		}
+
+		var tag string
+		if p := strings.Index(txt, "]\n"); p >= 0 {
+			// "// [START foo]" -> "foo"
+			tag = txt[10:p]
+			txt = txt[p+1:]
+		} else {
+			tag = txt
+			txt = ""
+		}
+
+		endTag := fmt.Sprintf(cloudSuffix, tag)
+		if p := strings.Index(txt, endTag); p >= 0 {
+			key := fmt.Sprintf("<!--SNIPPET %s-->", tag)
+			snipTxt := strings.Trim(txt[:p], "\n\r")
+			if _, exist := snip[key]; exist {
+				snip[key] = strings.Join([]string{snip[key], snipTxt}, "")
+			}
+
+			snip[key] = snipTxt
+			txt = txt[p+len(endTag):]
+		} else {
+			return fmt.Errorf("[START %s]:%d snippet %q not closed", file, lineNum(ftxt, txt), tag)
+		}
+	}
+}
+
+func getSnip(file, txt string, snip map[string]string) error {
+	const snipPrefix = "// SNIPPET "
+
+	ftxt := txt
+	for {
+		if p := strings.Index(txt, snipPrefix); p >= 0 {
+			txt = txt[p:]
+		} else {
+			return nil
+		}
+
+		var key string
+		if p := strings.IndexByte(txt, '\n'); p >= 0 {
+			key = txt[:p]
+			txt = txt[p:]
+		} else {
+			key = txt
+			txt = ""
+		}
+
+		if p := strings.Index(txt, key); p >= 0 {
+			// "// SNIPPET foo" -> "<!--SNIPPET foo-->"
+			key = fmt.Sprintf("<!--%s-->", strings.TrimSpace(key[3:]))
+
+			if _, exist := snip[key]; exist {
+				return fmt.Errorf("%s:%d snippet %q has already been defined", file, lineNum(ftxt, txt), key)
+			}
+
+			snip[key] = strings.Trim(txt[:p], "\n\r")
+			txt = txt[p+len(snipPrefix):]
+		} else {
+			return fmt.Errorf("%s:%d snippet %q not closed", file, lineNum(ftxt, txt), key)
+		}
+	}
+}
+
+type rewriteData struct {
+	rewrite map[string]string
+	used    map[string]bool
+}
+
+func writeSnip(file, txt string, snip map[string]string, rd rewriteData) error {
+	const snipPrefix = "<!--SNIPPET "
+
+	ftxt := txt
+	var buf bytes.Buffer
+	for {
+		if p := strings.Index(txt, snipPrefix); p >= 0 {
+			buf.WriteString(txt[:p])
+			txt = txt[p:]
+		} else if buf.Len() == 0 {
+			return nil
+		} else {
+			buf.WriteString(txt)
+			rd.rewrite[file] = buf.String()
+			return nil
+		}
+
+		var key string
+		if p := strings.Index(txt, "-->"); p >= 0 {
+			key = txt[:p+3]
+			txt = txt[p+3:]
+		}
+
+		rep, ok := snip[key]
+		if ok {
+			rd.used[key] = true
+		} else {
+			return fmt.Errorf("%s:%d snippet target %q undefined", file, lineNum(ftxt, txt), key)
+		}
+
+		if p := strings.Index(txt, key); p >= 0 {
+			buf.WriteString(key)
+			buf.WriteString("\n   * <pre>{@code\n   *")
+			buf.WriteString(strings.Replace(rep, "\n", "\n   *", -1))
+			buf.WriteString(" }</pre>\n   * ")
+			buf.WriteString(key)
+			txt = txt[p+len(key):]
+		} else {
+			return fmt.Errorf("%s:%d snippet target %q not closed", file, lineNum(ftxt, txt), key)
+		}
+	}
+}
+
+// Give strings s and suf where suf is a suffix of s, lineNum reports
+// the line number on which suf starts.
+func lineNum(s, suf string) int {
+	pre := s[:len(s)-len(suf)]
+	return strings.Count(pre, "\n") + 1
+}