From 83ccf082ceb129f536e8a01b755c15ec0f5c2495 Mon Sep 17 00:00:00 2001
From: Pei He <peihe@google.com>
Date: Wed, 24 Aug 2016 17:26:50 -0700
Subject: [PATCH 1/8] Move the samples data to gs://apache-beam-samples/

---
 .../java/org/apache/beam/examples/DebuggingWordCount.java    | 4 ++--
 .../main/java/org/apache/beam/examples/MinimalWordCount.java | 2 +-
 .../java/org/apache/beam/examples/WindowedWordCount.java     | 2 +-
 .../src/main/java/org/apache/beam/examples/WordCount.java    | 4 ++--
 .../apache/beam/examples/complete/StreamingWordExtract.java  | 2 +-
 .../main/java/org/apache/beam/examples/complete/TfIdf.java   | 4 ++--
 .../apache/beam/examples/complete/TopWikipediaSessions.java  | 5 +++--
 .../apache/beam/examples/complete/TrafficMaxLaneFlow.java    | 2 +-
 .../org/apache/beam/examples/complete/TrafficRoutes.java     | 2 +-
 .../apache/beam/examples/cookbook/DatastoreWordCount.java    | 2 +-
 .../java/org/apache/beam/examples/cookbook/DeDupExample.java | 4 ++--
 .../org/apache/beam/examples/cookbook/TriggerExample.java    | 2 +-
 .../java/org/apache/beam/examples/MinimalWordCountJava8.java | 2 +-
 .../apache/beam/examples/complete/game/HourlyTeamScore.java  | 2 +-
 .../org/apache/beam/examples/complete/game/UserScore.java    | 2 +-
 .../org/apache/beam/examples/MinimalWordCountJava8Test.java  | 2 +-
 16 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
index 5a0930cd35f5b..be3aa419b5379 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
@@ -100,8 +100,8 @@
  * that changing the default worker log level to TRACE or DEBUG will significantly increase
  * the amount of logs output.
  *
- * <p>The input file defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt} and can be
- * overridden with {@code --inputFile}.
+ * <p>The input file defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}
+ * and can be overridden with {@code --inputFile}.
  */
 public class DebuggingWordCount {
   /** A DoFn that filters for a specific key based upon a regular expression. */
diff --git a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
index df725e3f15858..f28a20cf94491 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
@@ -87,7 +87,7 @@ public static void main(String[] args) {
     // Concept #1: Apply a root transform to the pipeline; in this case, TextIO.Read to read a set
     // of input text files. TextIO.Read returns a PCollection where each element is one line from
     // the input text (a set of Shakespeare's texts).
-    p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+    p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
      // Concept #2: Apply a ParDo transform to our PCollection of text lines. This ParDo invokes a
      // DoFn (defined in-line) on each element that tokenizes the text line into individual words.
      // The ParDo returns a PCollection<String>, where each element is an individual word in
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
index 5f60524209c19..7af354cee0fab 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
@@ -82,7 +82,7 @@
  *
  * <p>Optionally specify the input file path via:
  * {@code --inputFile=gs://INPUT_PATH},
- * which defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt}.
+ * which defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}.
  *
  * <p>Specify an output BigQuery dataset and optionally, a table for the output. If you don't
  * specify the table, one will be created for you using the job name. If you don't specify the
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index d42d6214973d3..0275651bf288b 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -89,8 +89,8 @@
  *   --output=gs://YOUR_OUTPUT_PREFIX
  * }</pre>
  *
- * <p>The input file defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt} and can be
- * overridden with {@code --inputFile}.
+ * <p>The input file defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}
+ * and can be overridden with {@code --inputFile}.
  */
 public class WordCount {
 
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java b/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
index 348bab84b996d..869ea69b8ae47 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
@@ -104,7 +104,7 @@ static TableSchema getSchema() {
   private interface StreamingWordExtractOptions
       extends ExampleOptions, ExampleBigQueryTableOptions, StreamingOptions {
     @Description("Path of the file to read from")
-    @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+    @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
     String getInputFile();
     void setInputFile(String value);
   }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
index a5a939263ee4e..87023edac29ac 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
@@ -83,7 +83,7 @@
  *   --output=gs://YOUR_OUTPUT_PREFIX
  * }</pre>
  *
- * <p>The default input is {@code gs://dataflow-samples/shakespeare/} and can be overridden with
+ * <p>The default input is {@code gs://apache-beam-samples/shakespeare/} and can be overridden with
  * {@code --input}.
  */
 public class TfIdf {
@@ -94,7 +94,7 @@ public class TfIdf {
    */
   private static interface Options extends PipelineOptions {
     @Description("Path to the directory or GCS prefix containing files to read from")
-    @Default.String("gs://dataflow-samples/shakespeare/")
+    @Default.String("gs://apache-beam-samples/shakespeare/")
     String getInput();
     void setInput(String value);
 
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
index 1b2064ad068ee..d597258d3b923 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
@@ -64,7 +64,7 @@
  *   --output=gs://YOUR_OUTPUT_PREFIX
  * }</pre>
  *
- * <p>The default input is {@code gs://dataflow-samples/wikipedia_edits/*.json} and can be
+ * <p>The default input is {@code gs://apache-beam-samples/wikipedia_edits/*.json} and can be
  * overridden with {@code --input}.
  *
  * <p>The input for this example is large enough that it's a good place to enable (experimental)
@@ -77,7 +77,8 @@
  * This will automatically scale the number of workers up over time until the job completes.
  */
 public class TopWikipediaSessions {
-  private static final String EXPORTED_WIKI_TABLE = "gs://dataflow-samples/wikipedia_edits/*.json";
+  private static final String EXPORTED_WIKI_TABLE =
+      "gs://apache-beam-samples/wikipedia_edits/*.json";
 
   /**
    * Extracts user and timestamp from a TableRow representing a Wikipedia edit.
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
index 1b27e650f03e9..e4569600c05c6 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
@@ -303,7 +303,7 @@ public PCollection<String> apply(PBegin begin) {
     */
   private interface TrafficMaxLaneFlowOptions extends ExampleOptions, ExampleBigQueryTableOptions {
     @Description("Path of the file to read from")
-    @Default.String("gs://dataflow-samples/traffic_sensor/"
+    @Default.String("gs://apache-beam-samples/traffic_sensor/"
         + "Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv")
     String getInputFile();
     void setInputFile(String value);
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
index f3c2d3936ee74..95336c644a79f 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
@@ -313,7 +313,7 @@ public PCollection<String> apply(PBegin begin) {
   */
   private interface TrafficRoutesOptions extends ExampleOptions, ExampleBigQueryTableOptions {
     @Description("Path of the file to read from")
-    @Default.String("gs://dataflow-samples/traffic_sensor/"
+    @Default.String("gs://apache-beam-samples/traffic_sensor/"
         + "Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv")
     String getInputFile();
     void setInputFile(String value);
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
index 9a9e79968670e..eb2165f91e19b 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
@@ -151,7 +151,7 @@ public void processElement(ProcessContext c) {
    */
   public static interface Options extends PipelineOptions {
     @Description("Path of the file to read from and store to Datastore")
-    @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+    @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
     String getInput();
     void setInput(String value);
 
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
index d573bcd9d295e..57917109addd1 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
@@ -47,7 +47,7 @@
  * and an output prefix on GCS:
  *   --output=gs://YOUR_OUTPUT_PREFIX
  *
- * <p>The input defaults to {@code gs://dataflow-samples/shakespeare/*} and can be
+ * <p>The input defaults to {@code gs://apache-beam-samples/shakespeare/*} and can be
  * overridden with {@code --input}.
  */
 public class DeDupExample {
@@ -59,7 +59,7 @@ public class DeDupExample {
    */
   private static interface Options extends PipelineOptions {
     @Description("Path to the directory or GCS prefix containing files to read from")
-    @Default.String("gs://dataflow-samples/shakespeare/*")
+    @Default.String("gs://apache-beam-samples/shakespeare/*")
     String getInput();
     void setInput(String value);
 
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
index db59435555458..263054138dd64 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
@@ -420,7 +420,7 @@ public interface TrafficFlowOptions
       extends ExampleOptions, ExampleBigQueryTableOptions, StreamingOptions {
 
     @Description("Input file to read from")
-    @Default.String("gs://dataflow-samples/traffic_sensor/"
+    @Default.String("gs://apache-beam-samples/traffic_sensor/"
         + "Freeways-5Minaa2010-01-01_to_2010-02-15.csv")
     String getInput();
     void setInput(String value);
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
index a49da7bdfbb65..24dd6f9b2c483 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
@@ -55,7 +55,7 @@ public static void main(String[] args) {
 
     Pipeline p = Pipeline.create(options);
 
-    p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+    p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
      .apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
          .withOutputType(TypeDescriptors.strings()))
      .apply(Filter.by((String word) -> !word.isEmpty()))
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index d408e2132dabf..cf1389981fc5b 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -70,7 +70,7 @@
  * timestamped after 23:59 PST on 2015-10-18 should not be included in the analysis.
  * To indicate a time before which data should be filtered out, include the {@code --startMin} arg.
  * If you're using the default input specified in {@link UserScore},
- * "gs://dataflow-samples/game/gaming_data*.csv", then
+ * "gs://apache-beam-samples/game/gaming_data*.csv", then
  * {@code --startMin=2015-11-16-16-10 --stopMin=2015-11-17-16-10} are good values.
  */
 public class HourlyTeamScore extends UserScore {
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 65036cee6b922..f05879f93106f 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -182,7 +182,7 @@ public static interface Options extends PipelineOptions {
     @Description("Path to the data file(s) containing game data.")
     // The default maps to two large Google Cloud Storage files (each ~12GB) holding two subsequent
     // day's worth (roughly) of data.
-    @Default.String("gs://dataflow-samples/game/gaming_data*.csv")
+    @Default.String("gs://apache-beam-samples/game/gaming_data*.csv")
     String getInput();
     void setInput(String value);
 
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
index 85841a7801474..181921920b002 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
@@ -60,7 +60,7 @@ public void testMinimalWordCountJava8() throws Exception {
     Pipeline p = TestPipeline.create();
     p.getOptions().as(GcsOptions.class).setGcsUtil(buildMockGcsUtil());
 
-    p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+    p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
      .apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
          .withOutputType(TypeDescriptors.strings()))
      .apply(Filter.by((String word) -> !word.isEmpty()))

From 4e6230cc734ab3dba081e04d135a285b73008270 Mon Sep 17 00:00:00 2001
From: Scott Wegner <swegner@google.com>
Date: Wed, 17 Aug 2016 14:38:36 -0700
Subject: [PATCH 2/8] Update DoFn javadocs to remove references to OldDoFn and
 Dataflow

---
 .../examples/common/PubsubFileInjector.java   |   2 +-
 .../apache/beam/sdk/util/DoFnRunnerBase.java  |  16 +-
 .../beam/sdk/util/GroupAlsoByWindowsDoFn.java |   2 +-
 .../apache/beam/sdk/util/ReduceFnRunner.java  |   5 +-
 .../beam/sdk/util/SimpleDoFnRunner.java       |   4 +-
 .../ImmutabilityCheckingBundleFactory.java    |   4 +-
 .../direct/TransformEvaluatorFactory.java     |   3 +-
 .../beam/runners/dataflow/util/DoFnInfo.java  |   7 +-
 .../translation/MultiOutputWordCountTest.java |   2 +-
 .../spark/translation/SerializationTest.java  |   4 +-
 .../org/apache/beam/sdk/AggregatorValues.java |   4 +-
 .../beam/sdk/transforms/Aggregator.java       |  14 +-
 .../beam/sdk/transforms/CombineFns.java       |  18 +-
 .../org/apache/beam/sdk/transforms/DoFn.java  |  23 +-
 .../beam/sdk/transforms/DoFnTester.java       |  62 ++--
 .../beam/sdk/transforms/GroupByKey.java       |   7 +-
 .../beam/sdk/transforms/PTransform.java       |   2 +-
 .../org/apache/beam/sdk/transforms/ParDo.java | 306 +++++++++---------
 .../beam/sdk/transforms/SimpleFunction.java   |   6 +-
 .../sdk/transforms/windowing/PaneInfo.java    |  10 +-
 .../beam/sdk/util/BaseExecutionContext.java   |   4 +-
 .../util/ReifyTimestampAndWindowsDoFn.java    |   4 +-
 .../beam/sdk/util/SerializableUtils.java      |   2 +-
 .../beam/sdk/util/SystemDoFnInternal.java     |   7 +-
 .../beam/sdk/util/WindowingInternals.java     |   3 +-
 .../DoFnDelegatingAggregatorTest.java         |   2 +-
 .../beam/sdk/transforms/DoFnTesterTest.java   |   3 +-
 .../beam/sdk/transforms/NoOpOldDoFn.java      |   2 +-
 28 files changed, 263 insertions(+), 265 deletions(-)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/PubsubFileInjector.java b/examples/java/src/main/java/org/apache/beam/examples/common/PubsubFileInjector.java
index e6a1495e545d5..4634159826d35 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/PubsubFileInjector.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/PubsubFileInjector.java
@@ -69,7 +69,7 @@ public Bound publish(String outputTopic) {
     }
   }
 
-  /** A OldDoFn that publishes non-empty lines to Google Cloud PubSub. */
+  /** A {@link OldDoFn} that publishes non-empty lines to Google Cloud PubSub. */
   public static class Bound extends OldDoFn<String, Void> {
     private final String outputTopic;
     private final String timestampLabelKey;
diff --git a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
index 8a0f6bf868d91..04a0978b60301 100644
--- a/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
+++ b/runners/core-java/src/main/java/org/apache/beam/sdk/util/DoFnRunnerBase.java
@@ -58,10 +58,10 @@
  */
 public abstract class DoFnRunnerBase<InputT, OutputT> implements DoFnRunner<InputT, OutputT> {
 
-  /** The OldDoFn being run. */
+  /** The {@link OldDoFn} being run. */
   public final OldDoFn<InputT, OutputT> fn;
 
-  /** The context used for running the OldDoFn. */
+  /** The context used for running the {@link OldDoFn}. */
   public final DoFnContext<InputT, OutputT> context;
 
   protected DoFnRunnerBase(
@@ -164,8 +164,8 @@ public void finishBundle() {
   /**
    * A concrete implementation of {@code OldDoFn.Context} used for running a {@link OldDoFn}.
    *
-   * @param <InputT> the type of the OldDoFn's (main) input elements
-   * @param <OutputT> the type of the OldDoFn's (main) output elements
+   * @param <InputT> the type of the {@link OldDoFn} (main) input elements
+   * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
    */
   private static class DoFnContext<InputT, OutputT>
       extends OldDoFn<InputT, OutputT>.Context {
@@ -350,7 +350,7 @@ protected <AggInputT, AggOutputT> Aggregator<AggInputT, AggOutputT> createAggreg
   }
 
   /**
-   * Returns a new {@code OldDoFn.ProcessContext} for the given element.
+   * Returns a new {@link OldDoFn.ProcessContext} for the given element.
    */
   protected OldDoFn<InputT, OutputT>.ProcessContext createProcessContext(
       WindowedValue<InputT> elem) {
@@ -366,11 +366,11 @@ private boolean isSystemDoFn() {
   }
 
   /**
-   * A concrete implementation of {@code OldDoFn.ProcessContext} used for
+   * A concrete implementation of {@link OldDoFn.ProcessContext} used for
    * running a {@link OldDoFn} over a single element.
    *
-   * @param <InputT> the type of the OldDoFn's (main) input elements
-   * @param <OutputT> the type of the OldDoFn's (main) output elements
+   * @param <InputT> the type of the {@link OldDoFn} (main) input elements
+   * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
    */
   static class DoFnProcessContext<InputT, OutputT>
       extends OldDoFn<InputT, OutputT>.ProcessContext {
diff --git a/runners/core-java/src/main/java/org/apache/beam/sdk/util/GroupAlsoByWindowsDoFn.java b/runners/core-java/src/main/java/org/apache/beam/sdk/util/GroupAlsoByWindowsDoFn.java
index f82e5dfe32e7b..f386dfba12201 100644
--- a/runners/core-java/src/main/java/org/apache/beam/sdk/util/GroupAlsoByWindowsDoFn.java
+++ b/runners/core-java/src/main/java/org/apache/beam/sdk/util/GroupAlsoByWindowsDoFn.java
@@ -26,7 +26,7 @@
 import org.apache.beam.sdk.values.KV;
 
 /**
- * OldDoFn that merges windows and groups elements in those windows, optionally
+ * {@link OldDoFn} that merges windows and groups elements in those windows, optionally
  * combining values.
  *
  * @param <K> key type
diff --git a/runners/core-java/src/main/java/org/apache/beam/sdk/util/ReduceFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/sdk/util/ReduceFnRunner.java
index 61e5b21ebfd93..7c3e4d749a1f3 100644
--- a/runners/core-java/src/main/java/org/apache/beam/sdk/util/ReduceFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/sdk/util/ReduceFnRunner.java
@@ -33,7 +33,6 @@
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.beam.sdk.transforms.OldDoFn;
 import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -177,8 +176,8 @@ public class ReduceFnRunner<K, InputT, OutputT, W extends BoundedWindow> {
    * Store the previously emitted pane (if any) for each window.
    *
    * <ul>
-   * <li>State: The previous {@link PaneInfo} passed to the user's {@link OldDoFn#processElement},
-   * if any.
+   * <li>State: The previous {@link PaneInfo} passed to the user's {@code DoFn.ProcessElement}
+   * method, if any.
    * <li>Style style: DIRECT
    * <li>Merging: Always keyed by actual window, so does not depend on {@link #activeWindows}.
    * Cleared when window is merged away.
diff --git a/runners/core-java/src/main/java/org/apache/beam/sdk/util/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/sdk/util/SimpleDoFnRunner.java
index 6c1cf451d61a6..1ebe5a874e74a 100644
--- a/runners/core-java/src/main/java/org/apache/beam/sdk/util/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/sdk/util/SimpleDoFnRunner.java
@@ -28,8 +28,8 @@
 /**
  * Runs a {@link OldDoFn} by constructing the appropriate contexts and passing them in.
  *
- * @param <InputT> the type of the OldDoFn's (main) input elements
- * @param <OutputT> the type of the OldDoFn's (main) output elements
+ * @param <InputT> the type of the {@link OldDoFn} (main) input elements
+ * @param <OutputT> the type of the {@link OldDoFn} (main) output elements
  */
 public class SimpleDoFnRunner<InputT, OutputT> extends DoFnRunnerBase<InputT, OutputT>{
 
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
index d5c0f0c354e60..71bd8b4e50053 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutabilityCheckingBundleFactory.java
@@ -25,7 +25,7 @@
 import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.util.IllegalMutationException;
 import org.apache.beam.sdk.util.MutationDetector;
 import org.apache.beam.sdk.util.MutationDetectors;
@@ -40,7 +40,7 @@
  * elements added to the bundle will be encoded by the {@link Coder} of the underlying
  * {@link PCollection}.
  *
- * <p>This catches errors during the execution of a {@link OldDoFn} caused by modifying an element
+ * <p>This catches errors during the execution of a {@link DoFn} caused by modifying an element
  * after it is added to an output {@link PCollection}.
  */
 class ImmutabilityCheckingBundleFactory implements BundleFactory {
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
index e9fa06b107f70..ecf2da8f67a85 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/TransformEvaluatorFactory.java
@@ -22,7 +22,6 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.OldDoFn;
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
@@ -37,7 +36,7 @@ public interface TransformEvaluatorFactory {
    * Create a new {@link TransformEvaluator} for the application of the {@link PTransform}.
    *
    * <p>Any work that must be done before input elements are processed (such as calling
-   * {@link OldDoFn#startBundle(OldDoFn.Context)}) must be done before the
+   * {@code DoFn.StartBundle}) must be done before the
    * {@link TransformEvaluator} is made available to the caller.
    *
    * <p>May return null if the application cannot produce an evaluator (for example, it is a
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
index 139db9da0ab98..949c381489936 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DoFnInfo.java
@@ -24,10 +24,10 @@
 import org.apache.beam.sdk.values.PCollectionView;
 
 /**
- * Wrapper class holding the necessary information to serialize a OldDoFn.
+ * Wrapper class holding the necessary information to serialize a {@link OldDoFn}.
  *
- * @param <InputT> the type of the (main) input elements of the OldDoFn
- * @param <OutputT> the type of the (main) output elements of the OldDoFn
+ * @param <InputT> the type of the (main) input elements of the {@link OldDoFn}
+ * @param <OutputT> the type of the (main) output elements of the {@link OldDoFn}
  */
 public class DoFnInfo<InputT, OutputT> implements Serializable {
   private final OldDoFn<InputT, OutputT> doFn;
@@ -66,3 +66,4 @@ public Coder<InputT> getInputCoder() {
     return inputCoder;
   }
 }
+
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
index 517596aab7822..acfa3dfae2b80 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/MultiOutputWordCountTest.java
@@ -100,7 +100,7 @@ public void testRun() throws Exception {
   }
 
   /**
-   * A OldDoFn that tokenizes lines of text into individual words.
+   * A {@link DoFn} that tokenizes lines of text into individual words.
    */
   static class ExtractWordsFn extends DoFn<String, String> {
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
index 0e9121ca87f97..22a40cd29e2ca 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java
@@ -142,7 +142,7 @@ public void testRun() throws Exception {
   }
 
   /**
-   * A OldDoFn that tokenizes lines of text into individual words.
+   * A {@link DoFn} that tokenizes lines of text into individual words.
    */
   static class ExtractWordsFn extends DoFn<StringHolder, StringHolder> {
     private static final Pattern WORD_BOUNDARY = Pattern.compile("[^a-zA-Z']+");
@@ -170,7 +170,7 @@ public void processElement(ProcessContext c) {
   }
 
   /**
-   * A OldDoFn that converts a Word and Count into a printable string.
+   * A {@link DoFn} that converts a Word and Count into a printable string.
    */
   private static class FormatCountsFn extends DoFn<KV<StringHolder, Long>, StringHolder> {
     @ProcessElement
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
index 6297085319e2a..1fd034a02cc01 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/AggregatorValues.java
@@ -21,11 +21,11 @@
 import java.util.Map;
 import org.apache.beam.sdk.transforms.Aggregator;
 import org.apache.beam.sdk.transforms.Combine.CombineFn;
-import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.DoFn;
 
 /**
  * A collection of values associated with an {@link Aggregator}. Aggregators declared in a
- * {@link OldDoFn} are emitted on a per-{@code OldDoFn}-application basis.
+ * {@link DoFn} are emitted on a per-{@link DoFn}-application basis.
  *
  * @param <T> the output type of the aggregator
  */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
index db4ab33ff1b0e..67d399fbc8207 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
@@ -25,8 +25,8 @@
  * to be combined across all bundles.
  *
  * <p>Aggregators are created by calling
- * {@link OldDoFn#createAggregator OldDoFn.createAggregatorForDoFn},
- * typically from the {@link OldDoFn} constructor. Elements can be added to the
+ * {@link DoFn#createAggregator DoFn.createAggregatorForDoFn},
+ * typically from the {@link DoFn} constructor. Elements can be added to the
  * {@code Aggregator} by calling {@link Aggregator#addValue}.
  *
  * <p>Aggregators are visible in the monitoring UI, when the pipeline is run
@@ -37,14 +37,14 @@
  *
  * <p>Example:
  * <pre> {@code
- * class MyDoFn extends OldDoFn<String, String> {
+ * class MyDoFn extends DoFn<String, String> {
  *   private Aggregator<Integer, Integer> myAggregator;
  *
  *   public MyDoFn() {
  *     myAggregator = createAggregatorForDoFn("myAggregator", new Sum.SumIntegerFn());
  *   }
  *
- *   @Override
+ *   @ProcessElement
  *   public void processElement(ProcessContext c) {
  *     myAggregator.addValue(1);
  *   }
@@ -79,8 +79,8 @@ interface AggregatorFactory {
     /**
      * Create an aggregator with the given {@code name} and {@link CombineFn}.
      *
-     *  <p>This method is called to create an aggregator for a {@link OldDoFn}. It receives the
-     *  class of the {@link OldDoFn} being executed and the context of the step it is being
+     *  <p>This method is called to create an aggregator for a {@link DoFn}. It receives the
+     *  class of the {@link DoFn} being executed and the context of the step it is being
      *  executed in.
      */
     <InputT, AccumT, OutputT> Aggregator<InputT, OutputT> createAggregatorForDoFn(
@@ -90,7 +90,7 @@ <InputT, AccumT, OutputT> Aggregator<InputT, OutputT> createAggregatorForDoFn(
 
   // TODO: Consider the following additional API conveniences:
   // - In addition to createAggregatorForDoFn(), consider adding getAggregator() to
-  //   avoid the need to store the aggregator locally in a OldDoFn, i.e., create
+  //   avoid the need to store the aggregator locally in a DoFn, i.e., create
   //   if not already present.
   // - Add a shortcut for the most common aggregator:
   //   c.createAggregatorForDoFn("name", new Sum.SumIntegerFn()).
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index 9fa8ded6b2409..6f059932f5f47 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -67,7 +67,7 @@ public class CombineFns {
    * <p>The same {@link TupleTag} cannot be used in a composition multiple times.
    *
    * <p>Example:
-   * <pre>{ @code
+   * <pre><code>
    * PCollection<KV<K, Integer>> latencies = ...;
    *
    * TupleTag<Integer> maxLatencyTag = new TupleTag<Integer>();
@@ -75,7 +75,7 @@ public class CombineFns {
    *
    * SimpleFunction<Integer, Integer> identityFn =
    *     new SimpleFunction<Integer, Integer>() {
-   *       @Override
+   *      {@literal @}Override
    *       public Integer apply(Integer input) {
    *           return input;
    *       }};
@@ -87,8 +87,8 @@ public class CombineFns {
    *
    * PCollection<T> finalResultCollection = maxAndMean
    *     .apply(ParDo.of(
-   *         new OldDoFn<KV<K, CoCombineResult>, T>() {
-   *           @Override
+   *         new DoFn<KV<K, CoCombineResult>, T>() {
+   *          {@literal @}ProcessElement
    *           public void processElement(ProcessContext c) throws Exception {
    *             KV<K, CoCombineResult> e = c.element();
    *             Integer maxLatency = e.getValue().get(maxLatencyTag);
@@ -97,7 +97,7 @@ public class CombineFns {
    *             c.output(...some T...);
    *           }
    *         }));
-   * } </pre>
+   * </code></pre>
    */
   public static ComposeKeyedCombineFnBuilder composeKeyed() {
     return new ComposeKeyedCombineFnBuilder();
@@ -110,7 +110,7 @@ public static ComposeKeyedCombineFnBuilder composeKeyed() {
    * <p>The same {@link TupleTag} cannot be used in a composition multiple times.
    *
    * <p>Example:
-   * <pre>{ @code
+   * <pre><code>
    * PCollection<Integer> globalLatencies = ...;
    *
    * TupleTag<Integer> maxLatencyTag = new TupleTag<Integer>();
@@ -130,8 +130,8 @@ public static ComposeKeyedCombineFnBuilder composeKeyed() {
    *
    * PCollection<T> finalResultCollection = maxAndMean
    *     .apply(ParDo.of(
-   *         new OldDoFn<CoCombineResult, T>() {
-   *           @Override
+   *         new DoFn<CoCombineResult, T>() {
+   *          {@literal @}ProcessElement
    *           public void processElement(ProcessContext c) throws Exception {
    *             CoCombineResult e = c.element();
    *             Integer maxLatency = e.get(maxLatencyTag);
@@ -140,7 +140,7 @@ public static ComposeKeyedCombineFnBuilder composeKeyed() {
    *             c.output(...some T...);
    *           }
    *         }));
-   * } </pre>
+   * </code></pre>
    */
   public static ComposeCombineFnBuilder compose() {
     return new ComposeCombineFnBuilder();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 9f898261c7bb3..59c832323686f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -63,8 +63,6 @@
  * that satisfies the requirements described there. See the {@link ProcessElement}
  * for details.
  *
- * <p>This functionality is experimental and likely to change.
- *
  * <p>Example usage:
  *
  * <pre> {@code
@@ -123,7 +121,7 @@ public abstract class Context {
      *
      * <p>If invoked from {@link ProcessElement}), the timestamp
      * must not be older than the input element's timestamp minus
-     * {@link OldDoFn#getAllowedTimestampSkew}.  The output element will
+     * {@link DoFn#getAllowedTimestampSkew}.  The output element will
      * be in the same windows as the input element.
      *
      * <p>If invoked from {@link StartBundle} or {@link FinishBundle},
@@ -172,7 +170,7 @@ public abstract class Context {
      *
      * <p>If invoked from {@link ProcessElement}), the timestamp
      * must not be older than the input element's timestamp minus
-     * {@link OldDoFn#getAllowedTimestampSkew}.  The output element will
+     * {@link DoFn#getAllowedTimestampSkew}.  The output element will
      * be in the same windows as the input element.
      *
      * <p>If invoked from {@link StartBundle} or {@link FinishBundle},
@@ -190,7 +188,7 @@ public abstract <T> void sideOutputWithTimestamp(
   }
 
   /**
-   * Information accessible when running {@link OldDoFn#processElement}.
+   * Information accessible when running a {@link DoFn.ProcessElement} method.
    */
   public abstract class ProcessContext extends Context {
 
@@ -359,9 +357,14 @@ public OutputReceiver<OutputT> outputReceiver() {
    * Annotation for the method to use to prepare an instance for processing a batch of elements.
    * The method annotated with this must satisfy the following constraints:
    * <ul>
-   *   <li>It must have at least one argument.
+   *   <li>It must have exactly one argument.
    *   <li>Its first (and only) argument must be a {@link DoFn.Context}.
    * </ul>
+   *
+   * <p>A simple method declaration would look like:
+   * <code>
+   *   public void setup(DoFn.Context c) { .. }
+   * </code>
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
@@ -414,13 +417,13 @@ public OutputReceiver<OutputT> outputReceiver() {
   /**
    * Returns an {@link Aggregator} with aggregation logic specified by the
    * {@link CombineFn} argument. The name provided must be unique across
-   * {@link Aggregator}s created within the OldDoFn. Aggregators can only be created
+   * {@link Aggregator}s created within the {@link DoFn}. Aggregators can only be created
    * during pipeline construction.
    *
    * @param name the name of the aggregator
    * @param combiner the {@link CombineFn} to use in the aggregator
    * @return an aggregator for the provided name and combiner in the scope of
-   *         this OldDoFn
+   *         this {@link DoFn}
    * @throws NullPointerException if the name or combiner is null
    * @throws IllegalArgumentException if the given name collides with another
    *         aggregator in this scope
@@ -447,13 +450,13 @@ public OutputReceiver<OutputT> outputReceiver() {
   /**
    * Returns an {@link Aggregator} with the aggregation logic specified by the
    * {@link SerializableFunction} argument. The name provided must be unique
-   * across {@link Aggregator}s created within the OldDoFn. Aggregators can only be
+   * across {@link Aggregator}s created within the {@link DoFn}. Aggregators can only be
    * created during pipeline construction.
    *
    * @param name the name of the aggregator
    * @param combiner the {@link SerializableFunction} to use in the aggregator
    * @return an aggregator for the provided name and combiner in the scope of
-   *         this OldDoFn
+   *         this {@link DoFn}
    * @throws NullPointerException if the name or combiner is null
    * @throws IllegalArgumentException if the given name collides with another
    *         aggregator in this scope
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index 82c12938fa684..6801768c10967 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -50,12 +50,12 @@
 import org.joda.time.Instant;
 
 /**
- * A harness for unit-testing a {@link OldDoFn}.
+ * A harness for unit-testing a {@link DoFn}.
  *
  * <p>For example:
  *
  * <pre> {@code
- * OldDoFn<InputT, OutputT> fn = ...;
+ * DoFn<InputT, OutputT> fn = ...;
  *
  * DoFnTester<InputT, OutputT> fnTester = DoFnTester.of(fn);
  *
@@ -72,17 +72,17 @@
  * Assert.assertThat(fnTester.processBundle(i1, i2, ...), Matchers.hasItems(...));
  * } </pre>
  *
- * @param <InputT> the type of the {@code OldDoFn}'s (main) input elements
- * @param <OutputT> the type of the {@code OldDoFn}'s (main) output elements
+ * @param <InputT> the type of the {@link DoFn}'s (main) input elements
+ * @param <OutputT> the type of the {@link DoFn}'s (main) output elements
  */
 public class DoFnTester<InputT, OutputT> {
   /**
    * Returns a {@code DoFnTester} supporting unit-testing of the given
-   * {@link OldDoFn}.
+   * {@link DoFn}.
    */
   @SuppressWarnings("unchecked")
-  public static <InputT, OutputT> DoFnTester<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn) {
-    return new DoFnTester<InputT, OutputT>(fn);
+  public static <InputT, OutputT> DoFnTester<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
+    return new DoFnTester<>(DoFnAdapters.toOldDoFn(fn));
   }
 
   /**
@@ -90,19 +90,19 @@ public static <InputT, OutputT> DoFnTester<InputT, OutputT> of(OldDoFn<InputT, O
    * {@link OldDoFn}.
    */
   @SuppressWarnings("unchecked")
-  public static <InputT, OutputT> DoFnTester<InputT, OutputT>
-      of(DoFn<InputT, OutputT> fn) {
-    return new DoFnTester<InputT, OutputT>(DoFnAdapters.toOldDoFn(fn));
+   public static <InputT, OutputT> DoFnTester<InputT, OutputT>
+      of(OldDoFn<InputT, OutputT> fn) {
+    return new DoFnTester<>(fn);
   }
 
   /**
    * Registers the tuple of values of the side input {@link PCollectionView}s to
-   * pass to the {@link OldDoFn} under test.
+   * pass to the {@link DoFn} under test.
    *
    * <p>Resets the state of this {@link DoFnTester}.
    *
    * <p>If this isn't called, {@code DoFnTester} assumes the
-   * {@link OldDoFn} takes no side inputs.
+   * {@link DoFn} takes no side inputs.
    */
   public void setSideInputs(Map<PCollectionView<?>, Map<BoundedWindow, ?>> sideInputs) {
     this.sideInputs = sideInputs;
@@ -110,7 +110,7 @@ public void setSideInputs(Map<PCollectionView<?>, Map<BoundedWindow, ?>> sideInp
   }
 
   /**
-   * Registers the values of a side input {@link PCollectionView} to pass to the {@link OldDoFn}
+   * Registers the values of a side input {@link PCollectionView} to pass to the {@link DoFn}
    * under test.
    *
    * <p>The provided value is the final value of the side input in the specified window, not
@@ -129,7 +129,7 @@ public <T> void setSideInput(PCollectionView<T> sideInput, BoundedWindow window,
   }
 
   /**
-   * Whether or not a {@link DoFnTester} should clone the {@link OldDoFn} under test.
+   * Whether or not a {@link DoFnTester} should clone the {@link DoFn} under test.
    */
   public enum CloningBehavior {
     CLONE,
@@ -137,14 +137,14 @@ public enum CloningBehavior {
   }
 
   /**
-   * Instruct this {@link DoFnTester} whether or not to clone the {@link OldDoFn} under test.
+   * Instruct this {@link DoFnTester} whether or not to clone the {@link DoFn} under test.
    */
   public void setCloningBehavior(CloningBehavior newValue) {
     this.cloningBehavior = newValue;
   }
 
   /**
-   *  Indicates whether this {@link DoFnTester} will clone the {@link OldDoFn} under test.
+   *  Indicates whether this {@link DoFnTester} will clone the {@link DoFn} under test.
    */
   public CloningBehavior getCloningBehavior() {
     return cloningBehavior;
@@ -166,7 +166,7 @@ public List<OutputT> processBundle(Iterable <? extends InputT> inputElements) th
   }
 
   /**
-   * A convenience method for testing {@link OldDoFn DoFns} with bundles of elements.
+   * A convenience method for testing {@link DoFn DoFns} with bundles of elements.
    * Logic proceeds as follows:
    *
    * <ol>
@@ -182,9 +182,9 @@ public final List<OutputT> processBundle(InputT... inputElements) throws Excepti
   }
 
   /**
-   * Calls {@link OldDoFn#startBundle} on the {@code OldDoFn} under test.
+   * Calls the {@link DoFn.StartBundle} method on the {@link DoFn} under test.
    *
-   * <p>If needed, first creates a fresh instance of the OldDoFn under test.
+   * <p>If needed, first creates a fresh instance of the {@link DoFn} under test.
    */
   public void startBundle() throws Exception {
     resetState();
@@ -210,14 +210,14 @@ private static void unwrapUserCodeException(UserCodeException e) throws Exceptio
   }
 
   /**
-   * Calls {@link OldDoFn#processElement} on the {@code OldDoFn} under test, in a
-   * context where {@link OldDoFn.ProcessContext#element} returns the
+   * Calls the {@link DoFn.ProcessElement} method on the {@link DoFn} under test, in a
+   * context where {@link DoFn.ProcessContext#element} returns the
    * given element.
    *
    * <p>Will call {@link #startBundle} automatically, if it hasn't
    * already been called.
    *
-   * @throws IllegalStateException if the {@code OldDoFn} under test has already
+   * @throws IllegalStateException if the {@code DoFn} under test has already
    * been finished
    */
   public void processElement(InputT element) throws Exception {
@@ -235,12 +235,12 @@ public void processElement(InputT element) throws Exception {
   }
 
   /**
-   * Calls {@link OldDoFn#finishBundle} of the {@code OldDoFn} under test.
+   * Calls the {@link DoFn.FinishBundle} method of the {@link DoFn} under test.
    *
    * <p>Will call {@link #startBundle} automatically, if it hasn't
    * already been called.
    *
-   * @throws IllegalStateException if the {@code OldDoFn} under test has already
+   * @throws IllegalStateException if the {@link DoFn} under test has already
    * been finished
    */
   public void finishBundle() throws Exception {
@@ -674,7 +674,7 @@ protected <AggInputT, AggOutputT> Aggregator<AggInputT, AggOutputT> createAggreg
 
   /////////////////////////////////////////////////////////////////////////////
 
-  /** The possible states of processing a OldDoFn. */
+  /** The possible states of processing a {@link DoFn}. */
   enum State {
     UNSTARTED,
     STARTED,
@@ -683,23 +683,23 @@ enum State {
 
   private final PipelineOptions options = PipelineOptionsFactory.create();
 
-  /** The original OldDoFn under test. */
+  /** The original {@link OldDoFn} under test. */
   private final OldDoFn<InputT, OutputT> origFn;
 
   /**
-   * Whether to clone the original {@link OldDoFn} or just use it as-is.
+   * Whether to clone the original {@link DoFn} or just use it as-is.
    *
-   * <p></p>Worker-side {@link OldDoFn DoFns} may not be serializable, and are not required to be.
+   * <p>Worker-side {@link DoFn DoFns} may not be serializable, and are not required to be.
    */
   private CloningBehavior cloningBehavior = CloningBehavior.CLONE;
 
-  /** The side input values to provide to the OldDoFn under test. */
+  /** The side input values to provide to the {@link DoFn} under test. */
   private Map<PCollectionView<?>, Map<BoundedWindow, ?>> sideInputs =
       new HashMap<>();
 
   private Map<String, Object> accumulators;
 
-  /** The output tags used by the OldDoFn under test. */
+  /** The output tags used by the {@link DoFn} under test. */
   private TupleTag<OutputT> mainOutputTag = new TupleTag<>();
 
   /** The original OldDoFn under test, if started. */
@@ -708,7 +708,7 @@ enum State {
   /** The ListOutputManager to examine the outputs. */
   private Map<TupleTag<?>, List<WindowedValue<?>>> outputs;
 
-  /** The state of processing of the OldDoFn under test. */
+  /** The state of processing of the {@link DoFn} under test. */
   private State state;
 
   private DoFnTester(OldDoFn<InputT, OutputT> origFn) {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
index ed7f411aec11c..3a3da65e077d6 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/GroupByKey.java
@@ -63,18 +63,19 @@
  * {@code Coder} of the values of the input.
  *
  * <p>Example of use:
- * <pre> {@code
+ * <pre><code>
  * PCollection<KV<String, Doc>> urlDocPairs = ...;
  * PCollection<KV<String, Iterable<Doc>>> urlToDocs =
  *     urlDocPairs.apply(GroupByKey.<String, Doc>create());
  * PCollection<R> results =
- *     urlToDocs.apply(ParDo.of(new OldDoFn<KV<String, Iterable<Doc>>, R>() {
+ *     urlToDocs.apply(ParDo.of(new DoFn<KV<String, Iterable<Doc>>, R>() {
+ *      {@literal @}ProcessElement
  *       public void processElement(ProcessContext c) {
  *         String url = c.element().getKey();
  *         Iterable<Doc> docsWithThatUrl = c.element().getValue();
  *         ... process all docs having that url ...
  *       }}));
- * } </pre>
+ * </code></pre>
  *
  * <p>{@code GroupByKey} is a key primitive in data-parallel
  * processing, since it is the main way to efficiently bring
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
index 19abef90cea1b..4a58141ab503f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
@@ -146,7 +146,7 @@
  * implementing {@code Serializable}.
  *
  * <p>{@code PTransform} is marked {@code Serializable} solely
- * because it is common for an anonymous {@code OldDoFn},
+ * because it is common for an anonymous {@link DoFn},
  * instance to be created within an
  * {@code apply()} method of a composite {@code PTransform}.
  *
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 5efbe9f190240..f9cb557c1e5df 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -22,7 +22,6 @@
 import java.util.Arrays;
 import java.util.List;
 import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.runners.PipelineRunner;
@@ -50,13 +49,12 @@
  * <p>The {@link ParDo} processing style is similar to what happens inside
  * the "Mapper" or "Reducer" class of a MapReduce-style algorithm.
  *
- * <h2>{@link OldDoFn DoFns}</h2>
+ * <h2>{@link DoFn DoFns}</h2>
  *
  * <p>The function to use to process each element is specified by a
- * {@link OldDoFn OldDoFn&lt;InputT, OutputT&gt;}, primarily via its
- * {@link OldDoFn#processElement processElement} method. The {@link OldDoFn} may also
- * override the default implementations of {@link OldDoFn#startBundle startBundle}
- * and {@link OldDoFn#finishBundle finishBundle}.
+ * {@link DoFn DoFn&lt;InputT, OutputT&gt;}, primarily via its
+ * {@link DoFn.ProcessElement ProcessElement} method. The {@link DoFn} may also
+ * provide a {@link DoFn.StartBundle StartBundle} and {@link DoFn.FinishBundle finishBundle} method.
  *
  * <p>Conceptually, when a {@link ParDo} transform is executed, the
  * elements of the input {@link PCollection} are first divided up
@@ -66,39 +64,38 @@
  *
  * <ol>
  *   <li>If required, a fresh instance of the argument {@link DoFn} is created
- *     on a worker, and {@link DoFn#setup()} is called on this instance. This may be through
- *     deserialization or other means. A {@link PipelineRunner} may reuse {@link DoFn} instances for
- *     multiple bundles. A {@link DoFn} that has terminated abnormally (by throwing an
+ *     on a worker, and the {@link DoFn.Setup} method is called on this instance. This may be
+ *     through deserialization or other means. A {@link PipelineRunner} may reuse {@link DoFn}
+ *     instances for multiple bundles. A {@link DoFn} that has terminated abnormally (by throwing an
  *     {@link Exception}) will never be reused.</li>
- *   <li>The {@link OldDoFn OldDoFn's} {@link OldDoFn#startBundle} method is called to
- *     initialize it. If this method is not overridden, the call may be optimized
- *     away.</li>
- *   <li>The {@link OldDoFn OldDoFn's} {@link OldDoFn#processElement} method
+ *   <li>The {@link DoFn DoFn's} {@link DoFn.StartBundle} method, if provided, is called to
+ *     initialize it.</li>
+ *   <li>The {@link DoFn DoFn's} {@link DoFn.ProcessElement} method
  *     is called on each of the input elements in the bundle.</li>
- *   <li>The {@link OldDoFn OldDoFn's} {@link OldDoFn#finishBundle} method is called
- *     to complete its work. After {@link OldDoFn#finishBundle} is called, the
- *     framework will not again invoke {@link OldDoFn#processElement} or
- *     {@link OldDoFn#finishBundle}
- *     until a new call to {@link OldDoFn#startBundle} has occurred.
- *     If this method is not overridden, this call may be optimized away.</li>
- *   <li>If any of {@link DoFn#setup}, {@link DoFn#startBundle}, {@link DoFn#processElement} or
- *     {@link DoFn#finishBundle} throw an exception, {@link DoFn#teardown} will be called on the
- *     {@link DoFn} instance.</li>
- *   <li>If a runner will no longer use a {@link DoFn}, {@link DoFn#teardown()} will be called on
- *     the discarded instance.</li>
+ *   <li>The {@link DoFn DoFn's} {@link DoFn.FinishBundle} method, if provided, is called
+ *     to complete its work. After {@link DoFn.FinishBundle} is called, the
+ *     framework will not again invoke {@link DoFn.ProcessElement} or
+ *     {@link DoFn.FinishBundle}
+ *     until a new call to {@link DoFn.StartBundle} has occurred.</li>
+ *   <li>If any of {@link DoFn.Setup}, {@link DoFn.StartBundle}, {@link DoFn.ProcessElement} or
+ *     {@link DoFn.FinishBundle} methods throw an exception, the {@link DoFn.Teardown} method, if
+ *     provided, will be called on the {@link DoFn} instance.</li>
+ *   <li>If a runner will no longer use a {@link DoFn}, the {@link DoFn.Teardown} method, if
+ *     provided, will be called on the discarded instance.</li>
  * </ol>
  *
- * Each of the calls to any of the {@link OldDoFn OldDoFn's} processing
+ * Each of the calls to any of the {@link DoFn DoFn's} processing
  * methods can produce zero or more output elements. All of the
- * of output elements from all of the {@link OldDoFn} instances
+ * of output elements from all of the {@link DoFn} instances
  * are included in the output {@link PCollection}.
  *
  * <p>For example:
  *
- * <pre> {@code
+ * <pre><code>
  * PCollection<String> lines = ...;
  * PCollection<String> words =
- *     lines.apply(ParDo.of(new OldDoFn<String, String>() {
+ *     lines.apply(ParDo.of(new DoFn<String, String>() {
+ *        {@literal @}ProcessElement
  *         public void processElement(ProcessContext c) {
  *           String line = c.element();
  *           for (String word : line.split("[^a-zA-Z']+")) {
@@ -106,13 +103,14 @@
  *           }
  *         }}));
  * PCollection<Integer> wordLengths =
- *     words.apply(ParDo.of(new OldDoFn<String, Integer>() {
+ *     words.apply(ParDo.of(new DoFn<String, Integer>() {
+ *        {@literal @}ProcessElement
  *         public void processElement(ProcessContext c) {
  *           String word = c.element();
  *           Integer length = word.length();
  *           c.output(length);
  *         }}));
- * } </pre>
+ * </code></pre>
  *
  * <p>Each output element has the same timestamp and is in the same windows
  * as its corresponding input element, and the output {@code PCollection}
@@ -131,9 +129,9 @@
  *
  * <pre> {@code
  * PCollection<String> words =
- *     lines.apply("ExtractWords", ParDo.of(new OldDoFn<String, String>() { ... }));
+ *     lines.apply("ExtractWords", ParDo.of(new DoFn<String, String>() { ... }));
  * PCollection<Integer> wordLengths =
- *     words.apply("ComputeWordLengths", ParDo.of(new OldDoFn<String, Integer>() { ... }));
+ *     words.apply("ComputeWordLengths", ParDo.of(new DoFn<String, Integer>() { ... }));
  * } </pre>
  *
  * <h2>Side Inputs</h2>
@@ -145,17 +143,18 @@
  * {@link PCollection PCollections} computed by earlier pipeline operations,
  * passed in to the {@link ParDo} transform using
  * {@link #withSideInputs}, and their contents accessible to each of
- * the {@link OldDoFn} operations via {@link OldDoFn.ProcessContext#sideInput sideInput}.
+ * the {@link DoFn} operations via {@link DoFn.ProcessContext#sideInput sideInput}.
  * For example:
  *
- * <pre> {@code
+ * <pre><code>
  * PCollection<String> words = ...;
  * PCollection<Integer> maxWordLengthCutOff = ...; // Singleton PCollection
  * final PCollectionView<Integer> maxWordLengthCutOffView =
  *     maxWordLengthCutOff.apply(View.<Integer>asSingleton());
  * PCollection<String> wordsBelowCutOff =
  *     words.apply(ParDo.withSideInputs(maxWordLengthCutOffView)
- *                      .of(new OldDoFn<String, String>() {
+ *                      .of(new DoFn<String, String>() {
+ *        {@literal @}ProcessElement
  *         public void processElement(ProcessContext c) {
  *           String word = c.element();
  *           int lengthCutOff = c.sideInput(maxWordLengthCutOffView);
@@ -163,7 +162,7 @@
  *             c.output(word);
  *           }
  *         }}));
- * } </pre>
+ * </code></pre>
  *
  * <h2>Side Outputs</h2>
  *
@@ -174,13 +173,13 @@
  * and bundled in a {@link PCollectionTuple}. The {@link TupleTag TupleTags}
  * to be used for the output {@link PCollectionTuple} are specified by
  * invoking {@link #withOutputTags}. Unconsumed side outputs do not
- * necessarily need to be explicitly specified, even if the {@link OldDoFn}
- * generates them. Within the {@link OldDoFn}, an element is added to the
+ * necessarily need to be explicitly specified, even if the {@link DoFn}
+ * generates them. Within the {@link DoFn}, an element is added to the
  * main output {@link PCollection} as normal, using
- * {@link OldDoFn.Context#output}, while an element is added to a side output
- * {@link PCollection} using {@link OldDoFn.Context#sideOutput}. For example:
+ * {@link DoFn.Context#output}, while an element is added to a side output
+ * {@link PCollection} using {@link DoFn.Context#sideOutput}. For example:
  *
- * <pre> {@code
+ * <pre><code>
  * PCollection<String> words = ...;
  * // Select words whose length is below a cut off,
  * // plus the lengths of words that are above the cut off.
@@ -201,10 +200,11 @@
  *         .withOutputTags(wordsBelowCutOffTag,
  *                         TupleTagList.of(wordLengthsAboveCutOffTag)
  *                                     .and(markedWordsTag))
- *         .of(new OldDoFn<String, String>() {
+ *         .of(new DoFn<String, String>() {
  *             // Create a tag for the unconsumed side output.
  *             final TupleTag<String> specialWordsTag =
  *                 new TupleTag<String>(){};
+ *            {@literal @}ProcessElement
  *             public void processElement(ProcessContext c) {
  *               String word = c.element();
  *               if (word.length() <= wordLengthCutOff) {
@@ -230,14 +230,13 @@
  *     results.get(wordLengthsAboveCutOffTag);
  * PCollection<String> markedWords =
  *     results.get(markedWordsTag);
- * } </pre>
+ * </code></pre>
  *
  * <h2>Properties May Be Specified In Any Order</h2>
  *
  * <p>Several properties can be specified for a {@link ParDo}
- * {@link PTransform}, including name, side inputs, side output tags,
- * and {@link OldDoFn} to invoke. Only the {@link OldDoFn} is required; the
- * name is encouraged but not required, and side inputs and side
+ * {@link PTransform}, including side inputs, side output tags,
+ * and {@link DoFn} to invoke. Only the {@link DoFn} is required; side inputs and side
  * output tags are only specified when they're needed. These
  * properties can be specified in any order, as long as they're
  * specified before the {@link ParDo} {@link PTransform} is applied.
@@ -250,23 +249,23 @@
  * {@link ParDo.Bound} nested classes, each of which offer
  * property setter instance methods to enable setting additional
  * properties. {@link ParDo.Bound} is used for {@link ParDo}
- * transforms whose {@link OldDoFn} is specified and whose input and
+ * transforms whose {@link DoFn} is specified and whose input and
  * output static types have been bound. {@link ParDo.Unbound ParDo.Unbound} is used
  * for {@link ParDo} transforms that have not yet had their
- * {@link OldDoFn} specified. Only {@link ParDo.Bound} instances can be
+ * {@link DoFn} specified. Only {@link ParDo.Bound} instances can be
  * applied.
  *
  * <p>Another benefit of this approach is that it reduces the number
  * of type parameters that need to be specified manually. In
  * particular, the input and output types of the {@link ParDo}
  * {@link PTransform} are inferred automatically from the type
- * parameters of the {@link OldDoFn} argument passed to {@link ParDo#of}.
+ * parameters of the {@link DoFn} argument passed to {@link ParDo#of}.
  *
  * <h2>Output Coders</h2>
  *
  * <p>By default, the {@link Coder Coder&lt;OutputT&gt;} for the
  * elements of the main output {@link PCollection PCollection&lt;OutputT&gt;} is
- * inferred from the concrete type of the {@link OldDoFn OldDoFn&lt;InputT, OutputT&gt;}.
+ * inferred from the concrete type of the {@link DoFn DoFn&lt;InputT, OutputT&gt;}.
  *
  * <p>By default, the {@link Coder Coder&lt;SideOutputT&gt;} for the elements of
  * a side output {@link PCollection PCollection&lt;SideOutputT&gt;} is inferred
@@ -286,74 +285,74 @@
  * This style of {@code TupleTag} instantiation is used in the example of
  * multiple side outputs, above.
  *
- * <h2>Serializability of {@link OldDoFn DoFns}</h2>
+ * <h2>Serializability of {@link DoFn DoFns}</h2>
  *
- * <p>A {@link OldDoFn} passed to a {@link ParDo} transform must be
- * {@link Serializable}. This allows the {@link OldDoFn} instance
+ * <p>A {@link DoFn} passed to a {@link ParDo} transform must be
+ * {@link Serializable}. This allows the {@link DoFn} instance
  * created in this "main program" to be sent (in serialized form) to
  * remote worker machines and reconstituted for bundles of elements
- * of the input {@link PCollection} being processed. A {@link OldDoFn}
+ * of the input {@link PCollection} being processed. A {@link DoFn}
  * can have instance variable state, and non-transient instance
  * variable state will be serialized in the main program and then
  * deserialized on remote worker machines for some number of bundles
  * of elements to process.
  *
- * <p>{@link OldDoFn DoFns} expressed as anonymous inner classes can be
+ * <p>{@link DoFn DoFns} expressed as anonymous inner classes can be
  * convenient, but due to a quirk in Java's rules for serializability,
  * non-static inner or nested classes (including anonymous inner
  * classes) automatically capture their enclosing class's instance in
  * their serialized state. This can lead to including much more than
- * intended in the serialized state of a {@link OldDoFn}, or even things
+ * intended in the serialized state of a {@link DoFn}, or even things
  * that aren't {@link Serializable}.
  *
  * <p>There are two ways to avoid unintended serialized state in a
- * {@link OldDoFn}:
+ * {@link DoFn}:
  *
  * <ul>
  *
- * <li>Define the {@link OldDoFn} as a named, static class.
+ * <li>Define the {@link DoFn} as a named, static class.
  *
- * <li>Define the {@link OldDoFn} as an anonymous inner class inside of
+ * <li>Define the {@link DoFn} as an anonymous inner class inside of
  * a static method.
  *
  * </ul>
  *
  * <p>Both of these approaches ensure that there is no implicit enclosing
- * instance serialized along with the {@link OldDoFn} instance.
+ * instance serialized along with the {@link DoFn} instance.
  *
  * <p>Prior to Java 8, any local variables of the enclosing
  * method referenced from within an anonymous inner class need to be
- * marked as {@code final}. If defining the {@link OldDoFn} as a named
+ * marked as {@code final}. If defining the {@link DoFn} as a named
  * static class, such variables would be passed as explicit
  * constructor arguments and stored in explicit instance variables.
  *
  * <p>There are three main ways to initialize the state of a
- * {@link OldDoFn} instance processing a bundle:
+ * {@link DoFn} instance processing a bundle:
  *
  * <ul>
  *
  * <li>Define instance variable state (including implicit instance
  * variables holding final variables captured by an anonymous inner
- * class), initialized by the {@link OldDoFn}'s constructor (which is
+ * class), initialized by the {@link DoFn}'s constructor (which is
  * implicit for an anonymous inner class). This state will be
- * automatically serialized and then deserialized in the {@code OldDoFn}
+ * automatically serialized and then deserialized in the {@link DoFn}
  * instances created for bundles. This method is good for state
- * known when the original {@code OldDoFn} is created in the main
+ * known when the original {@link DoFn} is created in the main
  * program, if it's not overly large. This is not suitable for any
- * state which must only be used for a single bundle, as {@link OldDoFn OldDoFn's}
+ * state which must only be used for a single bundle, as {@link DoFn DoFn's}
  * may be used to process multiple bundles.
  *
  * <li>Compute the state as a singleton {@link PCollection} and pass it
- * in as a side input to the {@link OldDoFn}. This is good if the state
+ * in as a side input to the {@link DoFn}. This is good if the state
  * needs to be computed by the pipeline, or if the state is very large
  * and so is best read from file(s) rather than sent as part of the
- * {@code OldDoFn}'s serialized state.
+ * {@link DoFn DoFn's} serialized state.
  *
- * <li>Initialize the state in each {@link OldDoFn} instance, in
- * {@link OldDoFn#startBundle}. This is good if the initialization
+ * <li>Initialize the state in each {@link DoFn} instance, in a
+ * {@link DoFn.StartBundle} method. This is good if the initialization
  * doesn't depend on any information known only by the main program or
  * computed by earlier pipeline operations, but is the same for all
- * instances of this {@link OldDoFn} for all program executions, say
+ * instances of this {@link DoFn} for all program executions, say
  * setting up empty caches or initializing constant data.
  *
  * </ul>
@@ -363,16 +362,16 @@
  * <p>{@link ParDo} operations are intended to be able to run in
  * parallel across multiple worker machines. This precludes easy
  * sharing and updating mutable state across those machines. There is
- * no support in the Google Cloud Dataflow system for communicating
+ * no support in the Beam model for communicating
  * and synchronizing updates to shared state across worker machines,
  * so programs should not access any mutable static variable state in
- * their {@link OldDoFn}, without understanding that the Java processes
+ * their {@link DoFn}, without understanding that the Java processes
  * for the main program and workers will each have its own independent
  * copy of such state, and there won't be any automatic copying of
  * that state across Java processes. All information should be
- * communicated to {@link OldDoFn} instances via main and side inputs and
+ * communicated to {@link DoFn} instances via main and side inputs and
  * serialized state, and all output should be communicated from a
- * {@link OldDoFn} instance via main and side outputs, in the absence of
+ * {@link DoFn} instance via main and side outputs, in the absence of
  * external communication mechanisms written by user code.
  *
  * <h2>Fault Tolerance</h2>
@@ -380,29 +379,28 @@
  * <p>In a distributed system, things can fail: machines can crash,
  * machines can be unable to communicate across the network, etc.
  * While individual failures are rare, the larger the job, the greater
- * the chance that something, somewhere, will fail. The Google Cloud
- * Dataflow service strives to mask such failures automatically,
- * principally by retrying failed {@link OldDoFn} bundle. This means
- * that a {@code OldDoFn} instance might process a bundle partially, then
- * crash for some reason, then be rerun (often on a different worker
- * machine) on that same bundle and on the same elements as before.
- * Sometimes two or more {@link OldDoFn} instances will be running on the
+ * the chance that something, somewhere, will fail. Beam runners may strive
+ * to mask such failures by retrying failed {@link DoFn} bundle. This means
+ * that a {@link DoFn} instance might process a bundle partially, then
+ * crash for some reason, then be rerun (often in a new JVM) on that
+ * same bundle and on the same elements as before.
+ * Sometimes two or more {@link DoFn} instances will be running on the
  * same bundle simultaneously, with the system taking the results of
  * the first instance to complete successfully. Consequently, the
- * code in a {@link OldDoFn} needs to be written such that these
+ * code in a {@link DoFn} needs to be written such that these
  * duplicate (sequential or concurrent) executions do not cause
- * problems. If the outputs of a {@link OldDoFn} are a pure function of
+ * problems. If the outputs of a {@link DoFn} are a pure function of
  * its inputs, then this requirement is satisfied. However, if a
- * {@link OldDoFn OldDoFn's} execution has external side-effects, such as performing
- * updates to external HTTP services, then the {@link OldDoFn OldDoFn's} code
+ * {@link DoFn DoFn's} execution has external side-effects, such as performing
+ * updates to external HTTP services, then the {@link DoFn DoFn's} code
  * needs to take care to ensure that those updates are idempotent and
  * that concurrent updates are acceptable. This property can be
  * difficult to achieve, so it is advisable to strive to keep
- * {@link OldDoFn DoFns} as pure functions as much as possible.
+ * {@link DoFn DoFns} as pure functions as much as possible.
  *
  * <h2>Optimization</h2>
  *
- * <p>The Google Cloud Dataflow service automatically optimizes a
+ * <p>Beam runners may choose to apply optimizations to a
  * pipeline before it is executed. A key optimization, <i>fusion</i>,
  * relates to {@link ParDo} operations. If one {@link ParDo} operation produces a
  * {@link PCollection} that is then consumed as the main input of another
@@ -419,18 +417,16 @@
  * written to disk, saving all the I/O and space expense of
  * constructing it.
  *
- * <p>The Google Cloud Dataflow service applies fusion as much as
- * possible, greatly reducing the cost of executing pipelines. As a
- * result, it is essentially "free" to write {@link ParDo} operations in a
+ * <p>When Beam runners apply fusion optimization, it is essentially "free"
+ * to write {@link ParDo} operations in a
  * very modular, composable style, each {@link ParDo} operation doing one
  * clear task, and stringing together sequences of {@link ParDo} operations to
  * get the desired overall effect. Such programs can be easier to
  * understand, easier to unit-test, easier to extend and evolve, and
  * easier to reuse in new programs. The predefined library of
- * PTransforms that come with Google Cloud Dataflow makes heavy use of
- * this modular, composable style, trusting to the Google Cloud
- * Dataflow service's optimizer to "flatten out" all the compositions
- * into highly optimized stages.
+ * PTransforms that come with Beam makes heavy use of
+ * this modular, composable style, trusting to the runner to
+ * "flatten out" all the compositions into highly optimized stages.
  *
  * @see <a href="https://cloud.google.com/dataflow/model/par-do">the web
  * documentation for ParDo</a>
@@ -443,15 +439,15 @@ public class ParDo {
    *
    * <p>Side inputs are {@link PCollectionView PCollectionViews}, whose contents are
    * computed during pipeline execution and then made accessible to
-   * {@link OldDoFn} code via {@link OldDoFn.ProcessContext#sideInput sideInput}. Each
-   * invocation of the {@link OldDoFn} receives the same values for these
+   * {@link DoFn} code via {@link DoFn.ProcessContext#sideInput sideInput}. Each
+   * invocation of the {@link DoFn} receives the same values for these
    * side inputs.
    *
    * <p>See the discussion of Side Inputs above for more explanation.
    *
    * <p>The resulting {@link PTransform} is incomplete, and its
    * input/output types are not yet bound. Use
-   * {@link ParDo.Unbound#of} to specify the {@link OldDoFn} to
+   * {@link ParDo.Unbound#of} to specify the {@link DoFn} to
    * invoke, which will also bind the input/output types of this
    * {@link PTransform}.
    */
@@ -464,13 +460,13 @@ public static Unbound withSideInputs(PCollectionView<?>... sideInputs) {
     *
    * <p>Side inputs are {@link PCollectionView}s, whose contents are
    * computed during pipeline execution and then made accessible to
-   * {@code OldDoFn} code via {@link OldDoFn.ProcessContext#sideInput sideInput}.
+   * {@link DoFn} code via {@link DoFn.ProcessContext#sideInput sideInput}.
    *
    * <p>See the discussion of Side Inputs above for more explanation.
    *
    * <p>The resulting {@link PTransform} is incomplete, and its
    * input/output types are not yet bound. Use
-   * {@link ParDo.Unbound#of} to specify the {@link OldDoFn} to
+   * {@link ParDo.Unbound#of} to specify the {@link DoFn} to
    * invoke, which will also bind the input/output types of this
    * {@link PTransform}.
    */
@@ -486,11 +482,11 @@ public static Unbound withSideInputs(
    *
    * <p>{@link TupleTag TupleTags} are used to name (with its static element
    * type {@code T}) each main and side output {@code PCollection<T>}.
-   * This {@link PTransform PTransform's} {@link OldDoFn} emits elements to the main
+   * This {@link PTransform PTransform's} {@link DoFn} emits elements to the main
    * output {@link PCollection} as normal, using
-   * {@link OldDoFn.Context#output}. The {@link OldDoFn} emits elements to
+   * {@link DoFn.Context#output}. The {@link DoFn} emits elements to
    * a side output {@code PCollection} using
-   * {@link OldDoFn.Context#sideOutput}, passing that side output's tag
+   * {@link DoFn.Context#sideOutput}, passing that side output's tag
    * as an argument. The result of invoking this {@link PTransform}
    * will be a {@link PCollectionTuple}, and any of the the main and
    * side output {@code PCollection}s can be retrieved from it via
@@ -501,7 +497,7 @@ public static Unbound withSideInputs(
    *
    * <p>The resulting {@link PTransform} is incomplete, and its input
    * type is not yet bound. Use {@link ParDo.UnboundMulti#of}
-   * to specify the {@link OldDoFn} to invoke, which will also bind the
+   * to specify the {@link DoFn} to invoke, which will also bind the
    * input type of this {@link PTransform}.
    */
   public static <OutputT> UnboundMulti<OutputT> withOutputTags(
@@ -510,6 +506,20 @@ public static <OutputT> UnboundMulti<OutputT> withOutputTags(
     return new Unbound().withOutputTags(mainOutputTag, sideOutputTags);
   }
 
+  /**
+   * Creates a {@link ParDo} {@link PTransform} that will invoke the
+   * given {@link DoFn} function.
+   *
+   * <p>The resulting {@link PTransform PTransform's} types have been bound, with the
+   * input being a {@code PCollection<InputT>} and the output a
+   * {@code PCollection<OutputT>}, inferred from the types of the argument
+   * {@code DoFn<InputT, OutputT>}. It is ready to be applied, or further
+   * properties can be set on it first.
+   */
+  public static <InputT, OutputT> Bound<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
+    return of(adapt(fn), fn.getClass());
+  }
+
   /**
    * Creates a {@link ParDo} {@link PTransform} that will invoke the
    * given {@link OldDoFn} function.
@@ -537,29 +547,11 @@ private static <InputT, OutputT> Bound<InputT, OutputT> of(
     return DoFnAdapters.toOldDoFn(fn);
   }
 
-  /**
-   * Creates a {@link ParDo} {@link PTransform} that will invoke the
-   * given {@link DoFn} function.
-   *
-   * <p>The resulting {@link PTransform PTransform's} types have been bound, with the
-   * input being a {@code PCollection<InputT>} and the output a
-   * {@code PCollection<OutputT>}, inferred from the types of the argument
-   * {@code OldDoFn<InputT, OutputT>}. It is ready to be applied, or further
-   * properties can be set on it first.
-   *
-   * <p>{@link DoFn} is an experimental alternative to
-   * {@link OldDoFn} which simplifies accessing the window of the element.
-   */
-  @Experimental
-  public static <InputT, OutputT> Bound<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
-    return of(adapt(fn), fn.getClass());
-  }
-
   /**
    * An incomplete {@link ParDo} transform, with unbound input/output types.
    *
    * <p>Before being applied, {@link ParDo.Unbound#of} must be
-   * invoked to specify the {@link OldDoFn} to invoke, which will also
+   * invoked to specify the {@link DoFn} to invoke, which will also
    * bind the input/output types of this {@link PTransform}.
    */
   public static class Unbound {
@@ -619,6 +611,18 @@ public <OutputT> UnboundMulti<OutputT> withOutputTags(TupleTag<OutputT> mainOutp
           name, sideInputs, mainOutputTag, sideOutputTags);
     }
 
+    /**
+     * Returns a new {@link ParDo} {@link PTransform} that's like this
+     * transform but which will invoke the given {@link DoFn}
+     * function, and which has its input and output types bound. Does
+     * not modify this transform. The resulting {@link PTransform} is
+     * sufficiently specified to be applied, but more properties can
+     * still be specified.
+     */
+    public <InputT, OutputT> Bound<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
+      return of(adapt(fn), fn.getClass());
+    }
+
     /**
      * Returns a new {@link ParDo} {@link PTransform} that's like this
      * transform but that will invoke the given {@link OldDoFn}
@@ -638,24 +642,11 @@ private <InputT, OutputT> Bound<InputT, OutputT> of(
         OldDoFn<InputT, OutputT> fn, Class<?> fnClass) {
       return new Bound<>(name, sideInputs, fn, fnClass);
     }
-
-
-    /**
-     * Returns a new {@link ParDo} {@link PTransform} that's like this
-     * transform but which will invoke the given {@link DoFn}
-     * function, and which has its input and output types bound. Does
-     * not modify this transform. The resulting {@link PTransform} is
-     * sufficiently specified to be applied, but more properties can
-     * still be specified.
-     */
-    public <InputT, OutputT> Bound<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
-      return of(adapt(fn), fn.getClass());
-    }
   }
 
   /**
    * A {@link PTransform} that, when applied to a {@code PCollection<InputT>},
-   * invokes a user-specified {@code OldDoFn<InputT, OutputT>} on all its elements,
+   * invokes a user-specified {@code DoFn<InputT, OutputT>} on all its elements,
    * with all its outputs collected into an output
    * {@code PCollection<OutputT>}.
    *
@@ -756,9 +747,9 @@ protected String getKindString() {
     /**
      * {@inheritDoc}
      *
-     * <p>{@link ParDo} registers its internal {@link OldDoFn} as a subcomponent for display data.
-     * {@link OldDoFn} implementations can register display data by overriding
-     * {@link OldDoFn#populateDisplayData}.
+     * <p>{@link ParDo} registers its internal {@link DoFn} as a subcomponent for display data.
+     * {@link DoFn} implementations can register display data by overriding
+     * {@link DoFn#populateDisplayData}.
      */
     @Override
     public void populateDisplayData(Builder builder) {
@@ -780,7 +771,7 @@ public List<PCollectionView<?>> getSideInputs() {
    * input type.
    *
    * <p>Before being applied, {@link ParDo.UnboundMulti#of} must be
-   * invoked to specify the {@link OldDoFn} to invoke, which will also
+   * invoked to specify the {@link DoFn} to invoke, which will also
    * bind the input type of this {@link PTransform}.
    *
    * @param <OutputT> the type of the main output {@code PCollection} elements
@@ -836,38 +827,41 @@ public UnboundMulti<OutputT> withSideInputs(
 
     /**
      * Returns a new multi-output {@link ParDo} {@link PTransform}
-     * that's like this transform but that will invoke the given
-     * {@link OldDoFn} function, and that has its input type bound.
+     * that's like this transform but which will invoke the given
+     * {@link DoFn} function, and which has its input type bound.
      * Does not modify this transform. The resulting
      * {@link PTransform} is sufficiently specified to be applied, but
      * more properties can still be specified.
      */
-    public <InputT> BoundMulti<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn) {
-      return of(fn, fn.getClass());
-    }
-
-    public <InputT> BoundMulti<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn, Class<?> fnClass) {
-      return new BoundMulti<>(
-              name, sideInputs, mainOutputTag, sideOutputTags, fn, fnClass);
+    public <InputT> BoundMulti<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
+      return of(adapt(fn), fn.getClass());
     }
 
     /**
      * Returns a new multi-output {@link ParDo} {@link PTransform}
-     * that's like this transform but which will invoke the given
-     * {@link DoFn} function, and which has its input type bound.
+     * that's like this transform but that will invoke the given
+     * {@link OldDoFn} function, and that has its input type bound.
      * Does not modify this transform. The resulting
      * {@link PTransform} is sufficiently specified to be applied, but
      * more properties can still be specified.
+     *
+     * @deprecated please port your {@link OldDoFn} to a {@link DoFn}
      */
-    public <InputT> BoundMulti<InputT, OutputT> of(DoFn<InputT, OutputT> fn) {
-      return of(adapt(fn), fn.getClass());
+    @Deprecated
+    public <InputT> BoundMulti<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn) {
+      return of(fn, fn.getClass());
+    }
+
+    private <InputT> BoundMulti<InputT, OutputT> of(OldDoFn<InputT, OutputT> fn, Class<?> fnClass) {
+      return new BoundMulti<>(
+              name, sideInputs, mainOutputTag, sideOutputTags, fn, fnClass);
     }
   }
 
   /**
    * A {@link PTransform} that, when applied to a
    * {@code PCollection<InputT>}, invokes a user-specified
-   * {@code OldDoFn<InputT, OutputT>} on all its elements, which can emit elements
+   * {@code DoFn<InputT, OutputT>} on all its elements, which can emit elements
    * to any of the {@link PTransform}'s main and side output
    * {@code PCollection}s, which are bundled into a result
    * {@code PCollectionTuple}.
@@ -939,7 +933,7 @@ public PCollectionTuple apply(PCollection<? extends InputT> input) {
           input.isBounded());
 
       // The fn will likely be an instance of an anonymous subclass
-      // such as OldDoFn<Integer, String> { }, thus will have a high-fidelity
+      // such as DoFn<Integer, String> { }, thus will have a high-fidelity
       // TypeDescriptor for the output type.
       outputs.get(mainOutputTag).setTypeDescriptorInternal(fn.getOutputTypeDescriptor());
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SimpleFunction.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SimpleFunction.java
index bf075f8dcbd29..86046598fb4c4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SimpleFunction.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/SimpleFunction.java
@@ -37,7 +37,7 @@ SimpleFunction<InputT, OutputT> fromSerializableFunctionWithOutputType(
 
   /**
    * Returns a {@link TypeDescriptor} capturing what is known statically
-   * about the input type of this {@code OldDoFn} instance's most-derived
+   * about the input type of this {@link SimpleFunction} instance's most-derived
    * class.
    *
    * <p>See {@link #getOutputTypeDescriptor} for more discussion.
@@ -48,10 +48,10 @@ public TypeDescriptor<InputT> getInputTypeDescriptor() {
 
   /**
    * Returns a {@link TypeDescriptor} capturing what is known statically
-   * about the output type of this {@code OldDoFn} instance's
+   * about the output type of this {@link SimpleFunction} instance's
    * most-derived class.
    *
-   * <p>In the normal case of a concrete {@code OldDoFn} subclass with
+   * <p>In the normal case of a concrete {@link SimpleFunction} subclass with
    * no generic type parameters of its own (including anonymous inner
    * classes), this will be a complete non-generic type, which is good
    * for choosing a default output {@code Coder<OutputT>} for the output
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
index 0c87e2271ec92..727a4925cb31a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/PaneInfo.java
@@ -29,15 +29,15 @@
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.OldDoFn;
 import org.apache.beam.sdk.util.VarInt;
 
 /**
  * Provides information about the pane an element belongs to. Every pane is implicitly associated
  * with a window. Panes are observable only via the
- * {@link OldDoFn.ProcessContext#pane} method of the context
- * passed to a {@link OldDoFn#processElement} overridden method.
+ * {@link DoFn.ProcessContext#pane} method of the context
+ * passed to a {@link DoFn.ProcessElement} method.
  *
  * <p>Note: This does not uniquely identify a pane, and should not be used for comparisons.
  */
@@ -72,8 +72,8 @@ public final class PaneInfo {
    * definitions:
    * <ol>
    * <li>We'll call a pipeline 'simple' if it does not use
-   * {@link OldDoFn.Context#outputWithTimestamp} in
-   * any {@code OldDoFn}, and it uses the same
+   * {@link DoFn.Context#outputWithTimestamp} in
+   * any {@link DoFn}, and it uses the same
    * {@link org.apache.beam.sdk.transforms.windowing.Window.Bound#withAllowedLateness}
    * argument value on all windows (or uses the default of {@link org.joda.time.Duration#ZERO}).
    * <li>We'll call an element 'locally late', from the point of view of a computation on a
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
index dead76eb3d001..9ee55ad3a4f0b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
@@ -106,7 +106,7 @@ public Collection<? extends T> getAllStepContexts() {
 
   /**
    * Hook for subclasses to implement that will be called whenever
-   * {@link OldDoFn.Context#output}
+   * {@code DoFn.Context#output}
    * is called.
    */
   @Override
@@ -114,7 +114,7 @@ public void noteOutput(WindowedValue<?> output) {}
 
   /**
    * Hook for subclasses to implement that will be called whenever
-   * {@link OldDoFn.Context#sideOutput}
+   * {@code DoFn.Context#sideOutput}
    * is called.
    */
   @Override
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestampAndWindowsDoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestampAndWindowsDoFn.java
index 2808ca90568e3..8f3f540e66ae4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestampAndWindowsDoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ReifyTimestampAndWindowsDoFn.java
@@ -21,8 +21,8 @@
 import org.apache.beam.sdk.values.KV;
 
 /**
- * OldDoFn that makes timestamps and window assignments explicit in the value part of each key/value
- * pair.
+ * {@link OldDoFn} that makes timestamps and window assignments explicit in the value part of each
+ * key/value pair.
  *
  * @param <K> the type of the keys of the input and output {@code PCollection}s
  * @param <V> the type of the values of the input {@code PCollection}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
index 354aa5d91182d..6b3218ecf199c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableUtils.java
@@ -102,7 +102,7 @@ public static <T extends Serializable> T clone(T value) {
    */
   public static CloudObject ensureSerializable(Coder<?> coder) {
     // Make sure that Coders are java serializable as well since
-    // they are regularly captured within OldDoFn's.
+    // they are regularly captured within DoFn's.
     Coder<?> copy = (Coder<?>) ensureSerializable((Serializable) coder);
 
     CloudObject cloudObject = copy.asCloudObject();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SystemDoFnInternal.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SystemDoFnInternal.java
index e9904b2d7660a..004496b8319b3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SystemDoFnInternal.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SystemDoFnInternal.java
@@ -22,15 +22,14 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
-import org.apache.beam.sdk.transforms.OldDoFn;
 
 /**
- * Annotation to mark {@link OldDoFn DoFns} as an internal component of the Dataflow SDK.
+ * Annotation to mark {@code DoFns} as an internal component of the Beam SDK.
  *
  * <p>Currently, the only effect of this is to mark any aggregators reported by an annotated
- * {@code OldDoFn} as a system counter (as opposed to a user counter).
+ * {@code DoFn} as a system counter (as opposed to a user counter).
  *
- * <p>This is internal to the Dataflow SDK.
+ * <p>This is internal to the Beam SDK.
  */
 @Documented
 @Retention(RetentionPolicy.RUNTIME)
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingInternals.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingInternals.java
index 54158d24e5208..016276cb4bf34 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingInternals.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/WindowingInternals.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.util.Collection;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.state.StateInternals;
@@ -28,7 +29,7 @@
 import org.joda.time.Instant;
 
 /**
- * Interface that may be required by some (internal) {@code OldDoFn}s to implement windowing. It
+ * Interface that may be required by some (internal) {@link DoFn}s to implement windowing. It
  * should not be necessary for general user code to interact with this at all.
  *
  * <p>This interface should be provided by runner implementors to support windowing on their runner.
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnDelegatingAggregatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnDelegatingAggregatorTest.java
index 25b909aabbaf3..c072fd7c47db4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnDelegatingAggregatorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnDelegatingAggregatorTest.java
@@ -35,7 +35,7 @@
 import org.mockito.MockitoAnnotations;
 
 /**
- * Tests for OldDoFn.DelegatingAggregator.
+ * Tests for {@link OldDoFn.DelegatingAggregator}.
  */
 @RunWith(JUnit4.class)
 public class DoFnDelegatingAggregatorTest {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
index 2f1519cc42764..2649be5aadf30 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/DoFnTesterTest.java
@@ -276,7 +276,8 @@ public void processElement(ProcessContext c) throws Exception {
   }
 
   /**
-   * A OldDoFn that adds values to an aggregator and converts input to String in processElement.
+   * An {@link OldDoFn} that adds values to an aggregator and converts input to String in
+   * {@link OldDoFn#processElement).
    */
   private static class CounterDoFn extends OldDoFn<Long, String> {
     Aggregator<Long, Long> agg = createAggregator("ctr", new Sum.SumLongFn());
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/NoOpOldDoFn.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/NoOpOldDoFn.java
index c73251000650a..302b66aa7c4eb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/NoOpOldDoFn.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/NoOpOldDoFn.java
@@ -28,7 +28,7 @@
 
 /**
  * A {@link OldDoFn} that does nothing with provided elements. Used for testing
- * methods provided by the OldDoFn abstract class.
+ * methods provided by the {@link OldDoFn} abstract class.
  *
  * @param <InputT> unused.
  * @param <OutputT> unused.

From b04776d4d5235340148233f5f9e3f26f4c01076b Mon Sep 17 00:00:00 2001
From: Pei He <peihe@google.com>
Date: Wed, 10 Aug 2016 15:21:41 -0700
Subject: [PATCH 3/8] [BEAM-545] Promote JobName to PipelineOptions

---
 .../common/ExampleBigQueryTableOptions.java   |  3 +-
 .../beam/examples/common/ExampleOptions.java  | 43 -----------------
 ...mplePubsubTopicAndSubscriptionOptions.java |  2 +-
 .../common/ExamplePubsubTopicOptions.java     |  2 +-
 .../runners/flink/FlinkPipelineOptions.java   | 34 -------------
 .../options/DataflowPipelineOptions.java      | 48 -------------------
 .../options/DataflowPipelineOptionsTest.java  | 42 +++++++++++++---
 .../beam/sdk/options/PipelineOptions.java     | 42 ++++++++++++++++
 8 files changed, 80 insertions(+), 136 deletions(-)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleBigQueryTableOptions.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleBigQueryTableOptions.java
index 2eef525faed0b..5d815c72cc09d 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleBigQueryTableOptions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleBigQueryTableOptions.java
@@ -49,8 +49,7 @@ public interface ExampleBigQueryTableOptions extends GcpOptions {
   static class BigQueryTableFactory implements DefaultValueFactory<String> {
     @Override
     public String create(PipelineOptions options) {
-      return options.as(ExampleOptions.class).getNormalizedUniqueName()
-          .replace('-', '_');
+      return options.getJobName().replace('-', '_');
     }
   }
 }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleOptions.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleOptions.java
index 8b7ed073f3593..85643e41f352f 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/ExampleOptions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExampleOptions.java
@@ -17,17 +17,9 @@
  */
 package org.apache.beam.examples.common;
 
-import com.google.common.base.MoreObjects;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptions;
-import org.joda.time.DateTimeUtils;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
 /**
  * Options that can be used to configure the Beam examples.
@@ -42,39 +34,4 @@ public interface ExampleOptions extends PipelineOptions {
   @Default.Integer(1)
   int getInjectorNumWorkers();
   void setInjectorNumWorkers(int numWorkers);
-
-  @Description("A normalized unique name that is used to name anything related to the pipeline."
-      + "It defaults to ApplicationName-UserName-Date-RandomInteger")
-  @Default.InstanceFactory(NormalizedUniqueNameFactory.class)
-  String getNormalizedUniqueName();
-  void setNormalizedUniqueName(String numWorkers);
-
-  /**
-   * Returns a normalized unique name constructed from {@link ApplicationNameOptions#getAppName()},
-   * the local system user name (if available), the current time, and a random integer.
-   *
-   * <p>The normalization makes sure that the name matches the pattern of
-   * [a-z]([-a-z0-9]*[a-z0-9])?.
-   */
-  public static class NormalizedUniqueNameFactory implements DefaultValueFactory<String> {
-    private static final DateTimeFormatter FORMATTER =
-        DateTimeFormat.forPattern("MMddHHmmss").withZone(DateTimeZone.UTC);
-
-    @Override
-    public String create(PipelineOptions options) {
-      String appName = options.as(ApplicationNameOptions.class).getAppName();
-      String normalizedAppName = appName == null || appName.length() == 0 ? "BeamApp"
-          : appName.toLowerCase()
-                   .replaceAll("[^a-z0-9]", "0")
-                   .replaceAll("^[^a-z]", "a");
-      String userName = MoreObjects.firstNonNull(System.getProperty("user.name"), "");
-      String normalizedUserName = userName.toLowerCase()
-                                          .replaceAll("[^a-z0-9]", "0");
-      String datePart = FORMATTER.print(DateTimeUtils.currentTimeMillis());
-
-      String randomPart = Integer.toHexString(ThreadLocalRandom.current().nextInt());
-      return String.format("%s-%s-%s-%s",
-          normalizedAppName, normalizedUserName, datePart, randomPart);
-    }
-  }
 }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicAndSubscriptionOptions.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicAndSubscriptionOptions.java
index 36893a3a69909..7f954a1c18509 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicAndSubscriptionOptions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicAndSubscriptionOptions.java
@@ -39,7 +39,7 @@ static class PubsubSubscriptionFactory implements DefaultValueFactory<String> {
     @Override
     public String create(PipelineOptions options) {
       return "projects/" + options.as(GcpOptions.class).getProject()
-          + "/subscriptions/" + options.as(ExampleOptions.class).getNormalizedUniqueName();
+          + "/subscriptions/" + options.getJobName();
     }
   }
 }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicOptions.java b/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicOptions.java
index 1c9270b8f0656..71879b7242765 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicOptions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/common/ExamplePubsubTopicOptions.java
@@ -39,7 +39,7 @@ static class PubsubTopicFactory implements DefaultValueFactory<String> {
     @Override
     public String create(PipelineOptions options) {
       return "projects/" + options.as(GcpOptions.class).getProject()
-          + "/topics/" + options.as(ExampleOptions.class).getNormalizedUniqueName();
+          + "/topics/" + options.getJobName();
     }
   }
 }
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
index 6d1a8d040f1a5..6561fa5ef996c 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java
@@ -22,14 +22,9 @@
 import java.util.List;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
-import org.joda.time.DateTimeUtils;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
 /**
  * Options which can be used to configure a Flink PipelineRunner.
@@ -49,15 +44,6 @@ public interface FlinkPipelineOptions extends PipelineOptions, ApplicationNameOp
   List<String> getFilesToStage();
   void setFilesToStage(List<String> value);
 
-  /**
-   * The job name is used to identify jobs running on a Flink cluster.
-   */
-  @Description("Flink job name, to uniquely identify active jobs. "
-      + "Defaults to using the ApplicationName-UserName-Date.")
-  @Default.InstanceFactory(JobNameFactory.class)
-  String getJobName();
-  void setJobName(String value);
-
   /**
    * The url of the Flink JobManager on which to execute pipelines. This can either be
    * the the address of a cluster JobManager, in the form "host:port" or one of the special
@@ -93,24 +79,4 @@ public interface FlinkPipelineOptions extends PipelineOptions, ApplicationNameOp
   @Default.Long(-1L)
   Long getExecutionRetryDelay();
   void setExecutionRetryDelay(Long delay);
-
-
-  class JobNameFactory implements DefaultValueFactory<String> {
-    private static final DateTimeFormatter FORMATTER =
-        DateTimeFormat.forPattern("MMddHHmmss").withZone(DateTimeZone.UTC);
-
-    @Override
-    public String create(PipelineOptions options) {
-      String appName = options.as(ApplicationNameOptions.class).getAppName();
-      String normalizedAppName = appName == null || appName.length() == 0 ? "FlinkRunner"
-          : appName.toLowerCase()
-          .replaceAll("[^a-z0-9]", "0")
-          .replaceAll("^[^a-z]", "a");
-      String userName = System.getProperty("user.name", "");
-      String normalizedUserName = userName.toLowerCase()
-          .replaceAll("[^a-z0-9]", "0");
-      String datePart = FORMATTER.print(DateTimeUtils.currentTimeMillis());
-      return normalizedAppName + "-" + normalizedUserName + "-" + datePart;
-    }
-  }
 }
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 8ef43c5efaf66..9f58f93c9ecc3 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -20,7 +20,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.isNullOrEmpty;
 
-import com.google.common.base.MoreObjects;
 import java.io.IOException;
 import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.sdk.options.ApplicationNameOptions;
@@ -35,10 +34,6 @@
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
 import org.apache.beam.sdk.util.IOChannelUtils;
-import org.joda.time.DateTimeUtils;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
 /**
  * Options that can be used to configure the {@link DataflowRunner}.
@@ -75,21 +70,6 @@ public interface DataflowPipelineOptions
   String getStagingLocation();
   void setStagingLocation(String value);
 
-  /**
-   * The Dataflow job name is used as an idempotence key within the Dataflow service.
-   * If there is an existing job that is currently active, another active job with the same
-   * name will not be able to be created. Defaults to using the ApplicationName-UserName-Date.
-   */
-  @Description("The Dataflow job name is used as an idempotence key within the Dataflow service. "
-      + "For each running job in the same GCP project, jobs with the same name cannot be created "
-      + "unless the new job is an explicit update of the previous one. Defaults to using "
-      + "ApplicationName-UserName-Date. The job name must match the regular expression "
-      + "'[a-z]([-a-z0-9]{0,38}[a-z0-9])?'. The runner will automatically truncate the name of the "
-      + "job and convert to lower case.")
-  @Default.InstanceFactory(JobNameFactory.class)
-  String getJobName();
-  void setJobName(String value);
-
   /**
    * Whether to update the currently running pipeline with the same name as this one.
    */
@@ -99,34 +79,6 @@ public interface DataflowPipelineOptions
   boolean isUpdate();
   void setUpdate(boolean value);
 
-  /**
-   * Returns a normalized job name constructed from {@link ApplicationNameOptions#getAppName()}, the
-   * local system user name (if available), and the current time. The normalization makes sure that
-   * the job name matches the required pattern of [a-z]([-a-z0-9]*[a-z0-9])? and length limit of 40
-   * characters.
-   *
-   * <p>This job name factory is only able to generate one unique name per second per application
-   * and user combination.
-   */
-  public static class JobNameFactory implements DefaultValueFactory<String> {
-    private static final DateTimeFormatter FORMATTER =
-        DateTimeFormat.forPattern("MMddHHmmss").withZone(DateTimeZone.UTC);
-
-    @Override
-    public String create(PipelineOptions options) {
-      String appName = options.as(ApplicationNameOptions.class).getAppName();
-      String normalizedAppName = appName == null || appName.length() == 0 ? "dataflow"
-          : appName.toLowerCase()
-                   .replaceAll("[^a-z0-9]", "0")
-                   .replaceAll("^[^a-z]", "a");
-      String userName = MoreObjects.firstNonNull(System.getProperty("user.name"), "");
-      String normalizedUserName = userName.toLowerCase()
-                                          .replaceAll("[^a-z0-9]", "0");
-      String datePart = FORMATTER.print(DateTimeUtils.currentTimeMillis());
-      return normalizedAppName + "-" + normalizedUserName + "-" + datePart;
-    }
-  }
-
   /**
    * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}.
    */
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
index 61d39923ae5dd..202d04b1e9a85 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
@@ -53,7 +53,13 @@ public void testUserNameIsNotSet() {
     System.getProperties().remove("user.name");
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
     options.setAppName("TestApplication");
-    assertEquals("testapplication--1208190706", options.getJobName());
+    String[] nameComponents = options.getJobName().split("-");
+    assertEquals(4, nameComponents.length);
+    assertEquals("testapplication", nameComponents[0]);
+    assertEquals("", nameComponents[1]);
+    assertEquals("1208190706", nameComponents[2]);
+    // Verify the last component is a hex integer (unsigned).
+    Long.parseLong(nameComponents[3], 16);
     assertTrue(options.getJobName().length() <= 40);
   }
 
@@ -63,9 +69,13 @@ public void testAppNameAndUserNameAreLong() {
     System.getProperties().put("user.name", "abcdeabcdeabcdeabcdeabcdeabcde");
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
     options.setAppName("1234567890123456789012345678901234567890");
-    assertEquals(
-        "a234567890123456789012345678901234567890-abcdeabcdeabcdeabcdeabcdeabcde-1208190706",
-        options.getJobName());
+    String[] nameComponents = options.getJobName().split("-");
+    assertEquals(4, nameComponents.length);
+    assertEquals("a234567890123456789012345678901234567890", nameComponents[0]);
+    assertEquals("abcdeabcdeabcdeabcdeabcdeabcde", nameComponents[1]);
+    assertEquals("1208190706", nameComponents[2]);
+    // Verify the last component is a hex integer (unsigned).
+    Long.parseLong(nameComponents[3], 16);
   }
 
   @Test
@@ -74,7 +84,13 @@ public void testAppNameIsLong() {
     System.getProperties().put("user.name", "abcde");
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
     options.setAppName("1234567890123456789012345678901234567890");
-    assertEquals("a234567890123456789012345678901234567890-abcde-1208190706", options.getJobName());
+    String[] nameComponents = options.getJobName().split("-");
+    assertEquals(4, nameComponents.length);
+    assertEquals("a234567890123456789012345678901234567890", nameComponents[0]);
+    assertEquals("abcde", nameComponents[1]);
+    assertEquals("1208190706", nameComponents[2]);
+    // Verify the last component is a hex integer (unsigned).
+    Long.parseLong(nameComponents[3], 16);
   }
 
   @Test
@@ -83,7 +99,13 @@ public void testUserNameIsLong() {
     System.getProperties().put("user.name", "abcdeabcdeabcdeabcdeabcdeabcde");
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
     options.setAppName("1234567890");
-    assertEquals("a234567890-abcdeabcdeabcdeabcdeabcdeabcde-1208190706", options.getJobName());
+    String[] nameComponents = options.getJobName().split("-");
+    assertEquals(4, nameComponents.length);
+    assertEquals("a234567890", nameComponents[0]);
+    assertEquals("abcdeabcdeabcdeabcdeabcdeabcde", nameComponents[1]);
+    assertEquals("1208190706", nameComponents[2]);
+    // Verify the last component is a hex integer (unsigned).
+    Long.parseLong(nameComponents[3], 16);
   }
 
   @Test
@@ -92,7 +114,13 @@ public void testUtf8UserNameAndApplicationNameIsNormalized() {
     System.getProperties().put("user.name", "ði ıntəˈnæʃənəl ");
     DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
     options.setAppName("fəˈnɛtık əsoʊsiˈeıʃn");
-    assertEquals("f00n0t0k00so0si0e00n-0i00nt00n000n0l0-1208190706", options.getJobName());
+    String[] nameComponents = options.getJobName().split("-");
+    assertEquals(4, nameComponents.length);
+    assertEquals("f00n0t0k00so0si0e00n", nameComponents[0]);
+    assertEquals("0i00nt00n000n0l0", nameComponents[1]);
+    assertEquals("1208190706", nameComponents[2]);
+    // Verify the last component is a hex integer (unsigned).
+    Long.parseLong(nameComponents[3], 16);
   }
 
   @Test
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
index db54d0ac76b67..701ae70eafde4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PipelineOptions.java
@@ -22,8 +22,10 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.google.auto.service.AutoService;
+import com.google.common.base.MoreObjects;
 import java.lang.reflect.Proxy;
 import java.util.ServiceLoader;
+import java.util.concurrent.ThreadLocalRandom;
 import javax.annotation.concurrent.ThreadSafe;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.options.GoogleApiDebugOptions.GoogleApiTracer;
@@ -33,6 +35,10 @@
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.Context;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.joda.time.DateTimeUtils;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 /**
  * PipelineOptions are used to configure Pipelines. You can extend {@link PipelineOptions}
@@ -258,6 +264,13 @@ public static enum CheckEnabled {
   String getTempLocation();
   void setTempLocation(String value);
 
+  @Description("Name of the pipeline execution."
+      + "It must match the regular expression '[a-z]([-a-z0-9]{0,38}[a-z0-9])?'."
+      + "It defaults to ApplicationName-UserName-Date-RandomInteger")
+  @Default.InstanceFactory(JobNameFactory.class)
+  String getJobName();
+  void setJobName(String numWorkers);
+
   /**
    * A {@link DefaultValueFactory} that obtains the class of the {@code DirectRunner} if it exists
    * on the classpath, and throws an exception otherwise.
@@ -284,4 +297,33 @@ public Class<? extends PipelineRunner> create(PipelineOptions options) {
       }
     }
   }
+
+  /**
+   * Returns a normalized job name constructed from {@link ApplicationNameOptions#getAppName()},
+   * the local system user name (if available), the current time, and a random integer.
+   *
+   * <p>The normalization makes sure that the name matches the pattern of
+   * [a-z]([-a-z0-9]*[a-z0-9])?.
+   */
+  static class JobNameFactory implements DefaultValueFactory<String> {
+    private static final DateTimeFormatter FORMATTER =
+        DateTimeFormat.forPattern("MMddHHmmss").withZone(DateTimeZone.UTC);
+
+    @Override
+    public String create(PipelineOptions options) {
+      String appName = options.as(ApplicationNameOptions.class).getAppName();
+      String normalizedAppName = appName == null || appName.length() == 0 ? "BeamApp"
+          : appName.toLowerCase()
+                   .replaceAll("[^a-z0-9]", "0")
+                   .replaceAll("^[^a-z]", "a");
+      String userName = MoreObjects.firstNonNull(System.getProperty("user.name"), "");
+      String normalizedUserName = userName.toLowerCase()
+                                          .replaceAll("[^a-z0-9]", "0");
+      String datePart = FORMATTER.print(DateTimeUtils.currentTimeMillis());
+
+      String randomPart = Integer.toHexString(ThreadLocalRandom.current().nextInt());
+      return String.format("%s-%s-%s-%s",
+          normalizedAppName, normalizedUserName, datePart, randomPart);
+    }
+  }
 }

From 81352b4264999a4a26da8be0bb93bf3a218354d4 Mon Sep 17 00:00:00 2001
From: Thomas Groh <tgroh@google.com>
Date: Thu, 25 Aug 2016 14:58:56 -0700
Subject: [PATCH 4/8] Make WriteTest more resilient to Randomness

In the worst case scenario for random key assignment in
Write.ApplyShardingKey, the chance of the number of records per output
shard was too high. This makes the test significantly less likely to
flake.
---
 .../src/test/java/org/apache/beam/sdk/io/WriteTest.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteTest.java
index 28651884d2982..997566ae619b8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteTest.java
@@ -192,11 +192,12 @@ public void testShardedWriteBalanced() {
       inputs.add(String.format("elt%04d", i));
     }
 
+    int numShards = 10;
     runShardedWrite(
         inputs,
         new WindowAndReshuffle<>(
             Window.<String>into(Sessions.withGapDuration(Duration.millis(1)))),
-        Optional.of(10));
+        Optional.of(numShards));
 
     // Check that both the min and max number of results per shard are close to the expected.
     int min = Integer.MAX_VALUE;
@@ -205,7 +206,9 @@ public void testShardedWriteBalanced() {
       min = Math.min(min, i);
       max = Math.max(max, i);
     }
-    assertThat((double) min, Matchers.greaterThanOrEqualTo(max * 0.9));
+    double expected = numElements / (double) numShards;
+    assertThat((double) min, Matchers.greaterThanOrEqualTo(expected * 0.6));
+    assertThat((double) max, Matchers.lessThanOrEqualTo(expected * 1.4));
   }
 
   /**

From 89cf4613465647e2711983674879afd5f67c519d Mon Sep 17 00:00:00 2001
From: Dan Halperin <dhalperi@google.com>
Date: Tue, 23 Aug 2016 18:54:23 -0700
Subject: [PATCH 5/8] checkstyle: prohibit API client repackaged Guava

Apparently the IllegalImport check only blocks packages,
so we had to move to Regexp to get individual classes.

As a bonus, this enforcement let us remove two bogus dependencies.
Smaller JARs for the win!
---
 runners/spark/pom.xml                                  |  4 ----
 .../streaming/StreamingTransformTranslator.java        |  6 +++---
 .../build-tools/src/main/resources/beam/checkstyle.xml | 10 +++++++++-
 .../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java       |  2 +-
 sdks/java/io/hdfs/pom.xml                              |  5 -----
 .../java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java |  2 +-
 6 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index a6311b5dcf1f6..a5e99a06e80b9 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -154,10 +154,6 @@
       <artifactId>auto-service</artifactId>
       <optional>true</optional>
     </dependency>
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 43160f5500669..5f35ebb201862 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -17,9 +17,9 @@
  */
 package org.apache.beam.runners.spark.translation.streaming;
 
-import com.google.api.client.util.Lists;
-import com.google.api.client.util.Maps;
-import com.google.api.client.util.Sets;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.common.reflect.TypeToken;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
diff --git a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
index 47ddc5b2e37ad..c7d9b2c2a8132 100644
--- a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
@@ -100,7 +100,15 @@ page at http://checkstyle.sourceforge.net/config.html -->
     </module>
 
     <module name="IllegalImport">
-      <property name="illegalPkgs" value="com.google.api.client.repackaged, com.google.api.client.util.util.ByteStreams, com.google.api.client.util.Charsets, com.google.api.client.util.Collections2, com.google.api.client.util.Joiner, com.google.api.client.util.Lists, com.google.api.client.util.Maps, com.google.api.client.util.Objects, com.google.api.client.util.Preconditions, com.google.api.client.util.Sets, com.google.api.client.util.Strings, com.google.api.client.util.Throwables, io.netty.util.internal"/>
+      <property name="illegalPkgs" value="com.google.api.client.repackaged, io.netty.util.internal"/>
+    </module>
+
+    <!--
+        IllegalImport cannot blacklist classes, and c.g.api.client.util is used for some shaded
+        code and some useful code. So we need to fall back to Regexp.
+    -->
+    <module name="RegexpSinglelineJava">
+      <property name="format" value="com\.google\.api\.client\.util\.(ByteStreams|Charsets|Collections2|Joiner|Lists|Maps|Objects|Preconditions|Sets|Strings|Throwables)"/>
     </module>
 
     <module name="UnusedImports">
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index a6d7e2f5e957c..7a7575bfdd24c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -33,7 +33,6 @@
 import static org.mockito.Mockito.when;
 
 import com.google.api.client.util.Data;
-import com.google.api.client.util.Strings;
 import com.google.api.services.bigquery.model.ErrorProto;
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobConfigurationExtract;
@@ -50,6 +49,7 @@
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
diff --git a/sdks/java/io/hdfs/pom.xml b/sdks/java/io/hdfs/pom.xml
index 07b9eb6b4bfe7..0ec542ccf9d80 100644
--- a/sdks/java/io/hdfs/pom.xml
+++ b/sdks/java/io/hdfs/pom.xml
@@ -81,11 +81,6 @@
       <artifactId>annotations</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
diff --git a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
index 0b538b355bc18..6d30307f2e07f 100644
--- a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
+++ b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HDFSFileSink.java
@@ -19,8 +19,8 @@
 
 import static com.google.common.base.Preconditions.checkState;
 
-import com.google.api.client.util.Maps;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import java.io.IOException;
 import java.util.Map;

From e233e5f64d8bfeb5b4da7d96515e939c4bfd8b0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jean-Baptiste=20Onofr=C3=A9?= <jbonofre@apache.org>
Date: Thu, 25 Aug 2016 14:32:20 +0200
Subject: [PATCH 6/8] [BEAM-294] Rename dataflow references to beam

---
 runners/spark/pom.xml                                     | 2 +-
 .../java/org/apache/beam/runners/spark/SparkRunner.java   | 2 +-
 .../org/apache/beam/runners/spark/TestSparkRunner.java    | 2 +-
 .../beam/runners/spark/aggregators/NamedAggregators.java  | 2 +-
 .../org/apache/beam/runners/spark/examples/WordCount.java | 2 +-
 .../runners/spark/io/hadoop/ShardNameTemplateHelper.java  | 6 +++---
 .../beam/runners/spark/translation/DoFnFunction.java      | 2 +-
 .../spark/translation/SparkPipelineTranslator.java        | 2 +-
 .../runners/spark/translation/SparkRuntimeContext.java    | 2 +-
 .../runners/spark/translation/TransformTranslator.java    | 4 ++--
 .../streaming/StreamingTransformTranslator.java           | 8 ++++----
 .../apache/beam/runners/spark/util/BroadcastHelper.java   | 4 ++--
 .../spark/translation/TransformTranslatorTest.java        | 2 +-
 .../spark/translation/streaming/KafkaStreamingTest.java   | 2 +-
 14 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index a5e99a06e80b9..b924cb8e3ea88 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -330,7 +330,7 @@
               </goals>
               <configuration>
                 <relocations>
-                  <!-- relocate Guava used by Dataflow (v18) since it conflicts with 
+                  <!-- relocate Guava used by Beam (v18) since it conflicts with
                     version used by Hadoop (v11) -->
                   <relocation>
                     <pattern>com.google.common</pattern>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
index 2ce1ff697e42a..fa85a2e25e262 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkRunner.java
@@ -50,7 +50,7 @@
 /**
  * The SparkRunner translate operations defined on a pipeline to a representation
  * executable by Spark, and then submitting the job to Spark to be executed. If we wanted to run
- * a dataflow pipeline with the default options of a single threaded spark instance in local mode,
+ * a Beam pipeline with the default options of a single threaded spark instance in local mode,
  * we would do the following:
  *
  * {@code
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
index 50ed5f3f5ce8c..376b80ff953b2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/TestSparkRunner.java
@@ -28,7 +28,7 @@
 
 /**
  * The SparkRunner translate operations defined on a pipeline to a representation executable
- * by Spark, and then submitting the job to Spark to be executed. If we wanted to run a dataflow
+ * by Spark, and then submitting the job to Spark to be executed. If we wanted to run a Beam
  * pipeline with the default options of a single threaded spark instance in local mode, we would do
  * the following:
  *
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
index c15e276c43dbb..e2cd9632e37b7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
@@ -32,7 +32,7 @@
 
 /**
  * This class wraps a map of named aggregators. Spark expects that all accumulators be declared
- * before a job is launched. Dataflow allows aggregators to be used and incremented on the fly.
+ * before a job is launched. Beam allows aggregators to be used and incremented on the fly.
  * We create a map of named aggregators and instantiate in the the spark context before the job
  * is launched. We can then add aggregators on the fly in Spark.
  */
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
index 06770305f2c14..1af84add67ecb 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/examples/WordCount.java
@@ -110,7 +110,7 @@ public PCollection<KV<String, Long>> apply(PCollection<String> lines) {
    */
   public static interface WordCountOptions extends PipelineOptions {
     @Description("Path of the file to read from")
-    @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+    @Default.String("gs://beam-samples/shakespeare/kinglear.txt")
     String getInputFile();
     void setInputFile(String value);
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/ShardNameTemplateHelper.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/ShardNameTemplateHelper.java
index 7f8e2978c439d..4a7058bfd521d 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/ShardNameTemplateHelper.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/ShardNameTemplateHelper.java
@@ -36,9 +36,9 @@ public final class ShardNameTemplateHelper {
 
   private static final Logger LOG = LoggerFactory.getLogger(ShardNameTemplateHelper.class);
 
-  public static final String OUTPUT_FILE_PREFIX = "spark.dataflow.fileoutputformat.prefix";
-  public static final String OUTPUT_FILE_TEMPLATE = "spark.dataflow.fileoutputformat.template";
-  public static final String OUTPUT_FILE_SUFFIX = "spark.dataflow.fileoutputformat.suffix";
+  public static final String OUTPUT_FILE_PREFIX = "spark.beam.fileoutputformat.prefix";
+  public static final String OUTPUT_FILE_TEMPLATE = "spark.beam.fileoutputformat.template";
+  public static final String OUTPUT_FILE_SUFFIX = "spark.beam.fileoutputformat.suffix";
 
   private ShardNameTemplateHelper() {
   }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
index 800d614e5e964..454b7607e9df2 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/DoFnFunction.java
@@ -31,7 +31,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Dataflow's Do functions correspond to Spark's FlatMap functions.
+ * Beam's Do functions correspond to Spark's FlatMap functions.
  *
  * @param <InputT> Input element type.
  * @param <OutputT> Output element type.
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkPipelineTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkPipelineTranslator.java
index 997940bb99de6..1f7ccf1e005b7 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkPipelineTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkPipelineTranslator.java
@@ -20,7 +20,7 @@
 import org.apache.beam.sdk.transforms.PTransform;
 
 /**
- * Translator to support translation between Dataflow transformations and Spark transformations.
+ * Translator to support translation between Beam transformations and Spark transformations.
  */
 public interface SparkPipelineTranslator {
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 4bc0c00973f8b..2634c65b24484 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -57,7 +57,7 @@ public class SparkRuntimeContext implements Serializable {
   private final String serializedPipelineOptions;
 
   /**
-   * Map fo names to dataflow aggregators.
+   * Map fo names to Beam aggregators.
    */
   private final Map<String, Aggregator<?, ?>> aggregators = new HashMap<>();
   private transient CoderRegistry coderRegistry;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
index 08e3fda7a36b5..eaceb852ce344 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java
@@ -94,7 +94,7 @@
 import scala.Tuple2;
 
 /**
- * Supports translation between a DataFlow transform, and Spark's operations on RDDs.
+ * Supports translation between a Beam transform, and Spark's operations on RDDs.
  */
 public final class TransformTranslator {
 
@@ -895,7 +895,7 @@ private static Map<TupleTag<?>, BroadcastHelper<?>> getSideInputs(
   }
 
   /**
-   * Translator matches Dataflow transformation with the appropriate evaluator.
+   * Translator matches Beam transformation with the appropriate evaluator.
    */
   public static class Translator implements SparkPipelineTranslator {
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index 5f35ebb201862..43dcef665fb2a 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -68,7 +68,7 @@
 
 
 /**
- * Supports translation between a DataFlow transform, and Spark's operations on DStreams.
+ * Supports translation between a Beam transform, and Spark's operations on DStreams.
  */
 public final class StreamingTransformTranslator {
 
@@ -349,13 +349,13 @@ public void evaluate(Window.Bound<T> transform, EvaluationContext context) {
         (TransformEvaluator<TransformT>) EVALUATORS.get(clazz);
     if (transform == null) {
       if (UNSUPPORTED_EVALUATORS.contains(clazz)) {
-        throw new UnsupportedOperationException("Dataflow transformation " + clazz
+        throw new UnsupportedOperationException("Beam transformation " + clazz
           .getCanonicalName()
           + " is currently unsupported by the Spark streaming pipeline");
       }
       // DStream transformations will transform an RDD into another RDD
       // Actions will create output
-      // In Dataflow it depends on the PTransform's Input and Output class
+      // In Beam it depends on the PTransform's Input and Output class
       Class<?> pTOutputClazz = getPTransformOutputClazz(clazz);
       if (PDone.class.equals(pTOutputClazz)) {
         return foreachRDD(rddTranslator);
@@ -373,7 +373,7 @@ public void evaluate(Window.Bound<T> transform, EvaluationContext context) {
   }
 
   /**
-   * Translator matches Dataflow transformation with the appropriate Spark streaming evaluator.
+   * Translator matches Beam transformation with the appropriate Spark streaming evaluator.
    * rddTranslator uses Spark evaluators in transform/foreachRDD to evaluate the transformation
    */
   public static class Translator implements SparkPipelineTranslator {
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
index 29c2dd995b6e6..5f0c79576e595 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
@@ -34,12 +34,12 @@
 public abstract class BroadcastHelper<T> implements Serializable {
 
   /**
-   * If the property {@code dataflow.spark.directBroadcast} is set to
+   * If the property {@code beam.spark.directBroadcast} is set to
    * {@code true} then Spark serialization (Kryo) will be used to broadcast values
    * in View objects. By default this property is not set, and values are coded using
    * the appropriate {@link Coder}.
    */
-  public static final String DIRECT_BROADCAST = "dataflow.spark.directBroadcast";
+  public static final String DIRECT_BROADCAST = "beam.spark.directBroadcast";
 
   private static final Logger LOG = LoggerFactory.getLogger(BroadcastHelper.class);
 
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
index f61ad1c9a9fda..f72eba7ea6cac 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/TransformTranslatorTest.java
@@ -56,7 +56,7 @@ public class TransformTranslatorTest {
 
   /**
    * Builds a simple pipeline with TextIO.Read and TextIO.Write, runs the pipeline
-   * in DirectRunner and on SparkRunner, with the mapped dataflow-to-spark
+   * in DirectRunner and on SparkRunner, with the mapped beam-to-spark
    * transforms. Finally it makes sure that the results are the same for both runs.
    */
   @Test
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index 27d6f5ea60689..ac77922bc57c3 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -56,7 +56,7 @@ public class KafkaStreamingTest {
           new EmbeddedKafkaCluster.EmbeddedZookeeper();
   private static final EmbeddedKafkaCluster EMBEDDED_KAFKA_CLUSTER =
           new EmbeddedKafkaCluster(EMBEDDED_ZOOKEEPER.getConnection(), new Properties());
-  private static final String TOPIC = "kafka_dataflow_test_topic";
+  private static final String TOPIC = "kafka_beam_test_topic";
   private static final Map<String, String> KAFKA_MESSAGES = ImmutableMap.of(
       "k1", "v1", "k2", "v2", "k3", "v3", "k4", "v4"
   );

From e871f1731d849fa7eda0a508b57e8b94514eb236 Mon Sep 17 00:00:00 2001
From: Ian Zhou <ianzhou@google.com>
Date: Mon, 15 Aug 2016 15:39:56 -0700
Subject: [PATCH 7/8] Modified BigtableIO to use DoFn setup/tearDown methods
 instead of startBundle/finishBundle

---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java    | 17 +++++++++++++----
 .../sdk/io/gcp/bigtable/BigtableService.java    |  9 ++++++++-
 .../io/gcp/bigtable/BigtableServiceImpl.java    |  8 ++++++++
 .../sdk/io/gcp/bigtable/BigtableIOTest.java     |  3 +++
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 3a9ffce487f6f..67dde507313d8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -515,9 +515,13 @@ public BigtableWriterFn(String tableId, BigtableService bigtableService) {
         this.failures = new ConcurrentLinkedQueue<>();
       }
 
-      @StartBundle
-      public void startBundle(Context c) throws Exception {
+      @Setup
+      public void setup() throws Exception {
         bigtableWriter = bigtableService.openForWriting(tableId);
+      }
+
+      @StartBundle
+      public void startBundle(Context c) {
         recordsWritten = 0;
       }
 
@@ -531,12 +535,17 @@ public void processElement(ProcessContext c) throws Exception {
 
       @FinishBundle
       public void finishBundle(Context c) throws Exception {
-        bigtableWriter.close();
-        bigtableWriter = null;
+        bigtableWriter.flush();
         checkForFailures();
         logger.info("Wrote {} records", recordsWritten);
       }
 
+      @Teardown
+      public void tearDown() throws Exception {
+        bigtableWriter.close();
+        bigtableWriter = null;
+      }
+
       @Override
       public void populateDisplayData(DisplayData.Builder builder) {
         Write.this.populateDisplayData(builder);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
index ecd38a7468e4b..c656bbbf3fd99 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
@@ -50,10 +50,17 @@ ListenableFuture<MutateRowResponse> writeRecord(KV<ByteString, Iterable<Mutation
         throws IOException;
 
     /**
-     * Closes the writer.
+     * Flushes the writer.
      *
      * @throws IOException if any writes did not succeed
      */
+    void flush() throws IOException;
+
+    /**
+     * Closes the writer.
+     *
+     * @throws IOException if there is an error closing the writer
+     */
     void close() throws IOException;
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
index 07a183efc9d96..a40264329ef04 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
@@ -182,6 +182,14 @@ public BigtableWriterImpl(BigtableSession session, BigtableTableName tableName)
       partialBuilder = MutateRowRequest.newBuilder().setTableName(tableName.toString());
     }
 
+    @Override
+    public void flush() throws IOException {
+      if (bulkMutation != null) {
+        bulkMutation.flush();
+        executor.flush();
+      }
+    }
+
     @Override
     public void close() throws IOException {
       try {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
index 61b404ae441db..d60ede65aeeac 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
@@ -821,6 +821,9 @@ public ListenableFuture<MutateRowResponse> writeRecord(
       return Futures.immediateFuture(MutateRowResponse.getDefaultInstance());
     }
 
+    @Override
+    public void flush() {}
+
     @Override
     public void close() {}
   }

From 226dea2f04de2c000733f1182bdd3d18d516d4e4 Mon Sep 17 00:00:00 2001
From: staslev <staslevin@gmail.com>
Date: Fri, 26 Aug 2016 10:26:38 +0300
Subject: [PATCH 8/8] Added support for reporting aggregator values to Spark
 sinks

---
 runners/spark/pom.xml                         |   6 +
 .../runners/spark/SparkPipelineOptions.java   |   7 +-
 .../spark/aggregators/NamedAggregators.java   |  38 +++-
 .../aggregators/metrics/AggregatorMetric.java |  44 +++++
 .../metrics/AggregatorMetricSource.java       |  49 +++++
 .../metrics/WithNamedAggregatorsSupport.java  | 169 ++++++++++++++++++
 .../aggregators/metrics/package-info.java     |  22 +++
 .../aggregators/metrics/sink/CsvSink.java     |  39 ++++
 .../metrics/sink/GraphiteSink.java            |  39 ++++
 .../metrics/sink/package-info.java            |  23 +++
 .../beam/runners/spark/io/ConsoleIO.java      |   2 +-
 .../runners/spark/io/hadoop/HadoopIO.java     |   4 +-
 .../translation/SparkRuntimeContext.java      |  29 ++-
 .../runners/spark/util/BroadcastHelper.java   |   4 +-
 .../spark/InMemoryMetricsSinkRule.java        |  32 ++++
 .../runners/spark/SimpleWordCountTest.java    |  12 ++
 .../metrics/sink/InMemoryMetrics.java         |  79 ++++++++
 .../src/test/resources/metrics.properties     |  29 +++
 18 files changed, 611 insertions(+), 16 deletions(-)
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetric.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetricSource.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/WithNamedAggregatorsSupport.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/CsvSink.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/GraphiteSink.java
 create mode 100644 runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/package-info.java
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/InMemoryMetricsSinkRule.java
 create mode 100644 runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java
 create mode 100644 runners/spark/src/test/resources/metrics.properties

diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index b924cb8e3ea88..b928b4452967e 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -37,6 +37,7 @@
     <spark.version>1.6.2</spark.version>
     <hadoop.version>2.2.0</hadoop.version>
     <kafka.version>0.8.2.1</kafka.version>
+    <dropwizard.metrics.version>3.1.2</dropwizard.metrics.version>
   </properties>
 
   <profiles>
@@ -231,6 +232,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>${dropwizard.metrics.version}</version>
+    </dependency>
 
     <!-- test dependencies -->
     <dependency>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 080ff19a0c25f..be4f7f06a9bc0 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -38,10 +38,15 @@ public interface SparkPipelineOptions extends PipelineOptions, StreamingOptions,
           + "execution is stopped")
   @Default.Long(-1)
   Long getTimeout();
-  void setTimeout(Long batchInterval);
+  void setTimeout(Long timeoutMillis);
 
   @Description("Batch interval for Spark streaming in milliseconds.")
   @Default.Long(1000)
   Long getBatchIntervalMillis();
   void setBatchIntervalMillis(Long batchInterval);
+
+  @Description("Enable/disable sending aggregator values to Spark's metric sinks")
+  @Default.Boolean(true)
+  Boolean getEnableSparkSinks();
+  void setEnableSparkSinks(Boolean enableSparkSinks);
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
index e2cd9632e37b7..4e96466706dc9 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/NamedAggregators.java
@@ -18,13 +18,18 @@
 
 package org.apache.beam.runners.spark.aggregators;
 
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.TreeMap;
+
 import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
 import org.apache.beam.sdk.coders.Coder;
@@ -69,6 +74,22 @@ public <T> T getValue(String name, Class<T> typeClass) {
     return typeClass.cast(mNamedAggregators.get(name).render());
   }
 
+  /**
+   * @return a map of all the aggregator names and their <b>rendered </b>values
+   */
+  public Map<String, ?> renderAll() {
+    return
+        ImmutableMap.copyOf(
+            Maps.transformValues(mNamedAggregators,
+                new Function<State<?, ?, ?>, Object>() {
+
+                  @Override
+                  public Object apply(State<?, ?, ?> state) {
+                    return state.render();
+                  }
+                }));
+  }
+
   /**
    * Merges another NamedAggregators instance with this instance.
    *
@@ -116,6 +137,7 @@ public String toString() {
    * @param <OutputT>   Output data type
    */
   public interface State<InputT, InterT, OutputT> extends Serializable {
+
     /**
      * @param element new element to update state
      */
@@ -133,16 +155,16 @@ public interface State<InputT, InterT, OutputT> extends Serializable {
   /**
    * =&gt; combineFunction in data flow.
    */
-  public static class CombineFunctionState<InputT, InterT, OutpuT>
-      implements State<InputT, InterT, OutpuT> {
+  public static class CombineFunctionState<InputT, InterT, OutputT>
+      implements State<InputT, InterT, OutputT> {
 
-    private Combine.CombineFn<InputT, InterT, OutpuT> combineFn;
+    private Combine.CombineFn<InputT, InterT, OutputT> combineFn;
     private Coder<InputT> inCoder;
     private SparkRuntimeContext ctxt;
     private transient InterT state;
 
     public CombineFunctionState(
-        Combine.CombineFn<InputT, InterT, OutpuT> combineFn,
+        Combine.CombineFn<InputT, InterT, OutputT> combineFn,
         Coder<InputT> inCoder,
         SparkRuntimeContext ctxt) {
       this.combineFn = combineFn;
@@ -157,7 +179,7 @@ public void update(InputT element) {
     }
 
     @Override
-    public State<InputT, InterT, OutpuT> merge(State<InputT, InterT, OutpuT> other) {
+    public State<InputT, InterT, OutputT> merge(State<InputT, InterT, OutputT> other) {
       this.state = combineFn.mergeAccumulators(ImmutableList.of(current(), other.current()));
       return this;
     }
@@ -168,12 +190,12 @@ public InterT current() {
     }
 
     @Override
-    public OutpuT render() {
+    public OutputT render() {
       return combineFn.extractOutput(state);
     }
 
     @Override
-    public Combine.CombineFn<InputT, InterT, OutpuT> getCombineFn() {
+    public Combine.CombineFn<InputT, InterT, OutputT> getCombineFn() {
       return combineFn;
     }
 
@@ -192,7 +214,7 @@ private void writeObject(ObjectOutputStream oos) throws IOException {
     @SuppressWarnings("unchecked")
     private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException {
       ctxt = (SparkRuntimeContext) ois.readObject();
-      combineFn = (Combine.CombineFn<InputT, InterT, OutpuT>) ois.readObject();
+      combineFn = (Combine.CombineFn<InputT, InterT, OutputT>) ois.readObject();
       inCoder = (Coder<InputT>) ois.readObject();
       try {
         state = combineFn.getAccumulatorCoder(ctxt.getCoderRegistry(), inCoder)
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetric.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetric.java
new file mode 100644
index 0000000000000..c07a0697e8f18
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetric.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics;
+
+import com.codahale.metrics.Metric;
+
+import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+
+/**
+ * An adapter between the {@link NamedAggregators} and codahale's {@link Metric}
+ * interface.
+ */
+public class AggregatorMetric implements Metric {
+
+  private final NamedAggregators namedAggregators;
+
+  private AggregatorMetric(final NamedAggregators namedAggregators) {
+    this.namedAggregators = namedAggregators;
+  }
+
+  public static AggregatorMetric of(final NamedAggregators namedAggregators) {
+    return new AggregatorMetric(namedAggregators);
+  }
+
+  NamedAggregators getNamedAggregators() {
+    return namedAggregators;
+  }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetricSource.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetricSource.java
new file mode 100644
index 0000000000000..0658e049a999e
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/AggregatorMetricSource.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics;
+
+import com.codahale.metrics.MetricRegistry;
+
+import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import org.apache.spark.metrics.source.Source;
+
+/**
+ * A Spark {@link Source} that is tailored to expose an {@link AggregatorMetric},
+ * wrapping an underlying {@link NamedAggregators} instance.
+ */
+public class AggregatorMetricSource implements Source {
+
+  private static final String SOURCE_NAME = "NamedAggregators";
+
+  private final MetricRegistry metricRegistry = new MetricRegistry();
+
+  public AggregatorMetricSource(final NamedAggregators aggregators) {
+    metricRegistry.register(SOURCE_NAME, AggregatorMetric.of(aggregators));
+  }
+
+  @Override
+  public String sourceName() {
+    return SOURCE_NAME;
+  }
+
+  @Override
+  public MetricRegistry metricRegistry() {
+    return metricRegistry;
+  }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/WithNamedAggregatorsSupport.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/WithNamedAggregatorsSupport.java
new file mode 100644
index 0000000000000..88e2211cf9e1c
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/WithNamedAggregatorsSupport.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+
+import java.util.Map;
+import java.util.SortedMap;
+
+import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link MetricRegistry} decorator-like* that supports {@link AggregatorMetric} by exposing
+ * the underlying * {@link org.apache.beam.runners.spark.aggregators.NamedAggregators}'
+ * aggregators as {@link Gauge}s.
+ * <p>
+ * *{@link MetricRegistry} is not an interface, so this is not a by-the-book decorator.
+ * That said, it delegates all metric related getters to the "decorated" instance.
+ * </p>
+ */
+public class WithNamedAggregatorsSupport extends MetricRegistry {
+
+  private static final Logger LOG = LoggerFactory.getLogger(WithNamedAggregatorsSupport.class);
+
+  private MetricRegistry internalMetricRegistry;
+
+  private WithNamedAggregatorsSupport(final MetricRegistry internalMetricRegistry) {
+    this.internalMetricRegistry = internalMetricRegistry;
+  }
+
+  public static WithNamedAggregatorsSupport forRegistry(final MetricRegistry metricRegistry) {
+    return new WithNamedAggregatorsSupport(metricRegistry);
+  }
+
+  @Override
+  public SortedMap<String, Timer> getTimers(final MetricFilter filter) {
+    return internalMetricRegistry.getTimers(filter);
+  }
+
+  @Override
+  public SortedMap<String, Meter> getMeters(final MetricFilter filter) {
+    return internalMetricRegistry.getMeters(filter);
+  }
+
+  @Override
+  public SortedMap<String, Histogram> getHistograms(final MetricFilter filter) {
+    return internalMetricRegistry.getHistograms(filter);
+  }
+
+  @Override
+  public SortedMap<String, Counter> getCounters(final MetricFilter filter) {
+    return internalMetricRegistry.getCounters(filter);
+  }
+
+  @Override
+  public SortedMap<String, Gauge> getGauges(final MetricFilter filter) {
+    return
+        new ImmutableSortedMap.Builder<String, Gauge>(
+            Ordering.from(String.CASE_INSENSITIVE_ORDER))
+            .putAll(internalMetricRegistry.getGauges(filter))
+            .putAll(extractGauges(internalMetricRegistry, filter))
+            .build();
+  }
+
+  private Map<String, Gauge> extractGauges(final MetricRegistry metricRegistry,
+                                           final MetricFilter filter) {
+
+    // find the AggregatorMetric metrics from within all currently registered metrics
+    final Optional<Map<String, Gauge>> gauges =
+        FluentIterable
+            .from(metricRegistry.getMetrics().entrySet())
+            .firstMatch(isAggregatorMetric())
+            .transform(toGauges());
+
+    return
+        gauges.isPresent()
+            ? Maps.filterEntries(gauges.get(), matches(filter))
+            : ImmutableMap.<String, Gauge>of();
+  }
+
+  private Function<Map.Entry<String, Metric>, Map<String, Gauge>> toGauges() {
+    return new Function<Map.Entry<String, Metric>, Map<String, Gauge>>() {
+      @Override
+      public Map<String, Gauge> apply(final Map.Entry<String, Metric> entry) {
+        final NamedAggregators agg = ((AggregatorMetric) entry.getValue()).getNamedAggregators();
+        final Map<String, Gauge> gaugeMap = Maps.transformEntries(agg.renderAll(), toGauge());
+        return Maps.filterValues(gaugeMap, Predicates.notNull());
+      }
+    };
+  }
+
+  private Maps.EntryTransformer<String, Object, Gauge> toGauge() {
+    return new Maps.EntryTransformer<String, Object, Gauge>() {
+
+      @Override
+      public Gauge transformEntry(final String name, final Object rawValue) {
+        return new Gauge<Double>() {
+
+          @Override
+          public Double getValue() {
+            // at the moment the metric's type is assumed to be
+            // compatible with Double. While far from perfect, it seems reasonable at
+            // this point in time
+            try {
+              return Double.parseDouble(rawValue.toString());
+            } catch (final Exception e) {
+              LOG.warn("Failed reporting metric with name [{}], of type [{}], since it could not be"
+                  + " converted to double", name, rawValue.getClass().getSimpleName(), e);
+              return null;
+            }
+          }
+        };
+      }
+    };
+  }
+
+  private Predicate<Map.Entry<String, Gauge>> matches(final MetricFilter filter) {
+    return new Predicate<Map.Entry<String, Gauge>>() {
+      @Override
+      public boolean apply(final Map.Entry<String, Gauge> entry) {
+        return filter.matches(entry.getKey(), entry.getValue());
+      }
+    };
+  }
+
+  private Predicate<Map.Entry<String, Metric>> isAggregatorMetric() {
+    return new Predicate<Map.Entry<String, Metric>>() {
+      @Override
+      public boolean apply(final Map.Entry<String, Metric> metricEntry) {
+        return (metricEntry.getValue() instanceof AggregatorMetric);
+      }
+    };
+  }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java
new file mode 100644
index 0000000000000..f19f63536001e
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Defines classes for integrating with Spark's metrics mechanism (Sinks, Sources, etc.).
+ */
+package org.apache.beam.runners.spark.aggregators.metrics;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/CsvSink.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/CsvSink.java
new file mode 100644
index 0000000000000..af1601aea8be8
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/CsvSink.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics.sink;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Properties;
+
+import org.apache.beam.runners.spark.aggregators.metrics.AggregatorMetric;
+import org.apache.beam.runners.spark.aggregators.metrics.WithNamedAggregatorsSupport;
+import org.apache.spark.metrics.sink.Sink;
+
+/**
+ * A Spark {@link Sink} that is tailored to report {@link AggregatorMetric} metrics
+ * to a CSV file.
+ */
+public class CsvSink extends org.apache.spark.metrics.sink.CsvSink {
+  public CsvSink(final Properties properties,
+                 final MetricRegistry metricRegistry,
+                 final org.apache.spark.SecurityManager securityMgr) {
+    super(properties, WithNamedAggregatorsSupport.forRegistry(metricRegistry), securityMgr);
+  }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/GraphiteSink.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/GraphiteSink.java
new file mode 100644
index 0000000000000..7a45ef7f48822
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/GraphiteSink.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics.sink;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Properties;
+
+import org.apache.beam.runners.spark.aggregators.metrics.AggregatorMetric;
+import org.apache.beam.runners.spark.aggregators.metrics.WithNamedAggregatorsSupport;
+import org.apache.spark.metrics.sink.Sink;
+
+/**
+ * A Spark {@link Sink} that is tailored to report {@link AggregatorMetric} metrics
+ * to Graphite.
+ */
+public class GraphiteSink extends org.apache.spark.metrics.sink.GraphiteSink {
+  public GraphiteSink(final Properties properties,
+                      final MetricRegistry metricRegistry,
+                      final org.apache.spark.SecurityManager securityMgr) {
+    super(properties, WithNamedAggregatorsSupport.forRegistry(metricRegistry), securityMgr);
+  }
+}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/package-info.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/package-info.java
new file mode 100644
index 0000000000000..2e6dd0d3a9dbc
--- /dev/null
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/sink/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Spark sinks that support
+ * the {@link org.apache.beam.runners.spark.aggregators.metrics.AggregatorMetric}.
+ */
+package org.apache.beam.runners.spark.aggregators.metrics.sink;
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
index eefea77de5d2f..b1c567c757d75 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/ConsoleIO.java
@@ -47,7 +47,7 @@ public static <T> Unbound<T> from(int num) {
 
     /**
      * {@link PTransform} writing {@link PCollection} on the console.
-     * @param <T>
+     * @param <T> the type of the elements in the {@link PCollection}
      */
     public static class Unbound<T> extends PTransform<PCollection<T>, PDone> {
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/HadoopIO.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/HadoopIO.java
index 7b106105ee190..70bec78e51f41 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/HadoopIO.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/io/hadoop/HadoopIO.java
@@ -55,8 +55,8 @@ public static <K, V> Bound<K, V> from(String filepattern,
 
     /**
      * A {@link PTransform} reading bounded collection of data from HDFS.
-     * @param <K>
-     * @param <V>
+     * @param <K> the type of the keys
+     * @param <V> the type of the values
      */
     public static class Bound<K, V> extends PTransform<PInput, PCollection<KV<K, V>>> {
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
index 2634c65b24484..4e4cd1a170ce3 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkRuntimeContext.java
@@ -20,14 +20,19 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+
 import com.google.common.collect.ImmutableList;
+
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+
+import org.apache.beam.runners.spark.SparkPipelineOptions;
 import org.apache.beam.runners.spark.aggregators.AggAccumParam;
 import org.apache.beam.runners.spark.aggregators.NamedAggregators;
+import org.apache.beam.runners.spark.aggregators.metrics.AggregatorMetricSource;
 import org.apache.beam.sdk.AggregatorValues;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
@@ -41,7 +46,9 @@
 import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.spark.Accumulator;
+import org.apache.spark.SparkEnv$;
 import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.metrics.MetricsSystem;
 
 
 /**
@@ -63,8 +70,9 @@ public class SparkRuntimeContext implements Serializable {
   private transient CoderRegistry coderRegistry;
 
   SparkRuntimeContext(JavaSparkContext jsc, Pipeline pipeline) {
-    this.accum = jsc.accumulator(new NamedAggregators(), new AggAccumParam());
-    this.serializedPipelineOptions = serializePipelineOptions(pipeline.getOptions());
+    final SparkPipelineOptions opts = pipeline.getOptions().as(SparkPipelineOptions.class);
+    accum = registerMetrics(jsc, opts);
+    serializedPipelineOptions = serializePipelineOptions(opts);
   }
 
   private static String serializePipelineOptions(PipelineOptions pipelineOptions) {
@@ -83,6 +91,23 @@ private static PipelineOptions deserializePipelineOptions(String serializedPipel
     }
   }
 
+  private Accumulator<NamedAggregators> registerMetrics(final JavaSparkContext jsc,
+                                                        final SparkPipelineOptions opts) {
+    final NamedAggregators initialValue = new NamedAggregators();
+    final Accumulator<NamedAggregators> accum = jsc.accumulator(initialValue, new AggAccumParam());
+
+    if (opts.getEnableSparkSinks()) {
+      final MetricsSystem metricsSystem = SparkEnv$.MODULE$.get().metricsSystem();
+      final AggregatorMetricSource aggregatorMetricSource =
+          new AggregatorMetricSource(initialValue);
+      // in case the context was not cleared
+      metricsSystem.removeSource(aggregatorMetricSource);
+      metricsSystem.registerSource(aggregatorMetricSource);
+    }
+
+    return accum;
+  }
+
   /**
    * Retrieves corresponding value of an aggregator.
    *
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
index 5f0c79576e595..5c13b80058943 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/util/BroadcastHelper.java
@@ -58,7 +58,7 @@ public static <T> BroadcastHelper<T> create(T value, Coder<T> coder) {
    * A {@link BroadcastHelper} that relies on the underlying
    * Spark serialization (Kryo) to broadcast values. This is appropriate when
    * broadcasting very large values, since no copy of the object is made.
-   * @param <T>
+   * @param <T> the type of the value stored in the broadcast variable
    */
   static class DirectBroadcastHelper<T> extends BroadcastHelper<T> {
     private Broadcast<T> bcast;
@@ -86,7 +86,7 @@ public void broadcast(JavaSparkContext jsc) {
    * A {@link BroadcastHelper} that uses a
    * {@link Coder} to encode values as byte arrays
    * before broadcasting.
-   * @param <T>
+   * @param <T> the type of the value stored in the broadcast variable
    */
   static class CodedBroadcastHelper<T> extends BroadcastHelper<T> {
     private Broadcast<byte[]> bcast;
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/InMemoryMetricsSinkRule.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/InMemoryMetricsSinkRule.java
new file mode 100644
index 0000000000000..506dbbdeed7d0
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/InMemoryMetricsSinkRule.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark;
+
+import org.apache.beam.runners.spark.aggregators.metrics.sink.InMemoryMetrics;
+import org.junit.rules.ExternalResource;
+
+/**
+ * A rule that cleans the {@link InMemoryMetrics} after the tests has finished.
+ */
+class InMemoryMetricsSinkRule extends ExternalResource {
+  @Override
+  protected void before() throws Throwable {
+    InMemoryMetrics.clearAll();
+  }
+}
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java
index f644765682f18..8b7762fd73364 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java
@@ -19,6 +19,8 @@
 package org.apache.beam.runners.spark;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertThat;
 
 import com.google.common.collect.ImmutableSet;
@@ -27,6 +29,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
+import org.apache.beam.runners.spark.aggregators.metrics.sink.InMemoryMetrics;
 import org.apache.beam.runners.spark.examples.WordCount;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -39,12 +42,17 @@
 import org.apache.commons.io.FileUtils;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExternalResource;
 import org.junit.rules.TemporaryFolder;
 
 /**
  * Simple word count test.
  */
 public class SimpleWordCountTest {
+
+  @Rule
+  public ExternalResource inMemoryMetricsSink = new InMemoryMetricsSinkRule();
+
   private static final String[] WORDS_ARRAY = {
       "hi there", "hi", "hi sue bob",
       "hi sue", "", "bob hi"};
@@ -54,6 +62,8 @@ public class SimpleWordCountTest {
 
   @Test
   public void testInMem() throws Exception {
+    assertThat(InMemoryMetrics.valueOf("emptyLines"), is(nullValue()));
+
     SparkPipelineOptions options = PipelineOptionsFactory.as(SparkPipelineOptions.class);
     options.setRunner(SparkRunner.class);
     Pipeline p = Pipeline.create(options);
@@ -66,6 +76,8 @@ public void testInMem() throws Exception {
 
     EvaluationResult res = (EvaluationResult) p.run();
     res.close();
+
+    assertThat(InMemoryMetrics.<Double>valueOf("emptyLines"), is(1d));
   }
 
   @Rule
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java
new file mode 100644
index 0000000000000..35e67173cba8a
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/aggregators/metrics/sink/InMemoryMetrics.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.runners.spark.aggregators.metrics.sink;
+
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Properties;
+
+import org.apache.beam.runners.spark.aggregators.metrics.WithNamedAggregatorsSupport;
+import org.apache.spark.metrics.sink.Sink;
+
+/**
+ * An in-memory {@link Sink} implementation for tests.
+ */
+public class InMemoryMetrics implements Sink {
+
+  private static WithNamedAggregatorsSupport extendedMetricsRegistry;
+  private static MetricRegistry internalMetricRegistry;
+
+  public InMemoryMetrics(final Properties properties,
+                         final MetricRegistry metricRegistry,
+                         final org.apache.spark.SecurityManager securityMgr) {
+    extendedMetricsRegistry = WithNamedAggregatorsSupport.forRegistry(metricRegistry);
+    internalMetricRegistry = metricRegistry;
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T> T valueOf(final String name) {
+    final T retVal;
+
+    if (extendedMetricsRegistry != null
+        && extendedMetricsRegistry.getGauges().containsKey(name)) {
+      retVal = (T) extendedMetricsRegistry.getGauges().get(name).getValue();
+    } else {
+      retVal = null;
+    }
+
+    return retVal;
+  }
+
+  public static void clearAll() {
+    if (internalMetricRegistry != null) {
+      internalMetricRegistry.removeMatching(MetricFilter.ALL);
+    }
+  }
+
+  @Override
+  public void start() {
+
+  }
+
+  @Override
+  public void stop() {
+
+  }
+
+  @Override
+  public void report() {
+
+  }
+
+}
diff --git a/runners/spark/src/test/resources/metrics.properties b/runners/spark/src/test/resources/metrics.properties
new file mode 100644
index 0000000000000..4aa01d2dfd4fe
--- /dev/null
+++ b/runners/spark/src/test/resources/metrics.properties
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*.sink.memory.class=org.apache.beam.runners.spark.aggregators.metrics.sink.InMemoryMetrics
+
+#*.sink.csv.class=org.apache.beam.runners.spark.aggregators.metrics.sink.CsvSink
+#*.sink.csv.directory=/tmp/spark-metrics
+#*.sink.csv.period=1
+#*.sink.graphite.unit=SECONDS
+
+#*.sink.graphite.class=org.apache.beam.runners.spark.aggregators.metrics.sink.GraphiteSink
+#*.sink.graphite.host=YOUR_HOST
+#*.sink.graphite.port=2003
+#*.sink.graphite.prefix=spark
+#*.sink.graphite.period=1
+#*.sink.graphite.unit=SECONDS