From b8ada86cfdad716c15058257cb66da8a87bc4cca Mon Sep 17 00:00:00 2001
From: Kris Wilson <kwilson@twopensource.com>
Date: Thu, 18 Jan 2018 18:27:32 -0800
Subject: [PATCH 1/5] [pantsd] Add RunTracker stats.

---
 src/python/pants/bin/daemon_pants_runner.py   |  9 ++++-
 src/python/pants/bin/goal_runner.py           |  1 +
 src/python/pants/bin/local_pants_runner.py    |  7 ++++
 src/python/pants/goal/BUILD                   |  5 +++
 src/python/pants/goal/context.py              |  7 +++-
 src/python/pants/goal/pantsd_stats.py         | 28 ++++++++++++++
 src/python/pants/goal/run_tracker.py          | 16 ++++++--
 .../pants/pantsd/service/pailgun_service.py   | 38 ++++++++++---------
 .../pants/pantsd/service/scheduler_service.py |  7 ++++
 9 files changed, 93 insertions(+), 25 deletions(-)
 create mode 100644 src/python/pants/goal/pantsd_stats.py

diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py
index 22c6f636cee..6293d5f1653 100644
--- a/src/python/pants/bin/daemon_pants_runner.py
+++ b/src/python/pants/bin/daemon_pants_runner.py
@@ -72,7 +72,8 @@ class DaemonPantsRunner(ProcessManager):
   N.B. this class is primarily used by the PailgunService in pantsd.
   """
 
-  def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_exception=None):
+  def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, preceding_graph_size,
+               deferred_exception=None):
     """
     :param socket socket: A connected socket capable of speaking the nailgun protocol.
     :param Exiter exiter: The Exiter instance for this run.
@@ -82,6 +83,7 @@ def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_
                                            construction. In the event of an exception, this will be
                                            None.
     :param threading.RLock fork_lock: A lock to use during forking for thread safety.
+    :param int preceding_graph_size: The size of the graph pre-warming, for stats.
     :param Exception deferred_exception: A deferred exception from the daemon's graph construction.
                                          If present, this will be re-raised in the client context.
     """
@@ -92,6 +94,7 @@ def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_
     self._env = env
     self._graph_helper = graph_helper
     self._fork_lock = fork_lock
+    self._preceding_graph_size = preceding_graph_size
     self._deferred_exception = deferred_exception
 
   def _make_identity(self):
@@ -201,7 +204,9 @@ def post_fork_child(self):
         self._raise_deferred_exc()
 
         # Otherwise, conduct a normal run.
-        LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper).run()
+        runner = LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper)
+        runner.set_preceding_graph_size(self._preceding_graph_size)
+        runner.run()
       except KeyboardInterrupt:
         self._exiter.exit(1, msg='Interrupted by user.\n')
       except Exception:
diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py
index 880cb45eb0a..d89887f5955 100644
--- a/src/python/pants/bin/goal_runner.py
+++ b/src/python/pants/bin/goal_runner.py
@@ -250,6 +250,7 @@ def run(self):
 
     try:
       result = self._execute_engine()
+      self._run_tracker.pantsd_stats.set_resulting_graph_size(self._context.scheduler.graph_len())
       if result:
         self._run_tracker.set_root_outcome(WorkUnit.FAILURE)
     except KeyboardInterrupt:
diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py
index af90baa0830..1fb753615e1 100644
--- a/src/python/pants/bin/local_pants_runner.py
+++ b/src/python/pants/bin/local_pants_runner.py
@@ -31,6 +31,10 @@ def __init__(self, exiter, args, env, daemon_build_graph=None, options_bootstrap
     self._env = env
     self._daemon_build_graph = daemon_build_graph
     self._options_bootstrapper = options_bootstrapper
+    self._preceding_graph_size = -1
+
+  def set_preceding_graph_size(self, size):
+    self._preceding_graph_size = size
 
   def run(self):
     profile_path = self._env.get('PANTS_PROFILE')
@@ -70,6 +74,9 @@ def _run(self):
       if repro:
         repro.capture(run_tracker.run_info.get_as_dict())
 
+      # Record the preceding product graph size.
+      run_tracker.pantsd_stats.set_resident_graph_size(self._preceding_graph_size)
+
       # Setup and run GoalRunner.
       goal_runner = GoalRunner.Factory(root_dir,
                                        options,
diff --git a/src/python/pants/goal/BUILD b/src/python/pants/goal/BUILD
index 8004507ced7..878f1cd9790 100644
--- a/src/python/pants/goal/BUILD
+++ b/src/python/pants/goal/BUILD
@@ -61,6 +61,11 @@ python_library(
   ],
 )
 
+python_library(
+  name = 'pantsd_stats',
+  sources = ['pantsd_stats.py'],
+)
+
 python_library(
   name = 'products',
   sources = ['products.py'],
diff --git a/src/python/pants/goal/context.py b/src/python/pants/goal/context.py
index d65ee749859..4b70396ec70 100644
--- a/src/python/pants/goal/context.py
+++ b/src/python/pants/goal/context.py
@@ -80,8 +80,6 @@ def __init__(self, options, run_tracker, target_roots,
     self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
     self._replace_targets(target_roots)
     self._invalidation_report = invalidation_report
-    # TODO(#4769): This should not be exposed to anyone.
-    # Note that the Context created in unit tests by BaseTest uses a different codepath.
     self._scheduler = scheduler
 
   @property
@@ -144,6 +142,11 @@ def scm(self):
     """
     return self._scm
 
+  @property
+  def scheduler(self):
+    """Returns the scheduler instance used for this run."""
+    return self._scheduler
+
   @property
   def workspace(self):
     """Returns the current workspace, if any."""
diff --git a/src/python/pants/goal/pantsd_stats.py b/src/python/pants/goal/pantsd_stats.py
new file mode 100644
index 00000000000..0b30c736214
--- /dev/null
+++ b/src/python/pants/goal/pantsd_stats.py
@@ -0,0 +1,28 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+
+class PantsDaemonStats(object):
+  """Tracks various stats about the daemon."""
+
+  def __init__(self, dir):
+    self.resident_graph_size = None
+    self.resulting_graph_size = None
+    # TODO: Dir is currently unused, but plumbed for later use.
+    self._dir = dir
+
+  def set_resident_graph_size(self, size):
+    self.resident_graph_size = size
+
+  def set_resulting_graph_size(self, size):
+    self.resulting_graph_size = size
+
+  def get_all(self):
+    return {
+      'resident_graph_size': self.resident_graph_size,
+      'resulting_graph_size': self.resulting_graph_size,
+    }
diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py
index fc6a7e16340..0c90683dbbc 100644
--- a/src/python/pants/goal/run_tracker.py
+++ b/src/python/pants/goal/run_tracker.py
@@ -25,6 +25,7 @@
 from pants.build_graph.target import Target
 from pants.goal.aggregated_timings import AggregatedTimings
 from pants.goal.artifact_cache_stats import ArtifactCacheStats
+from pants.goal.pantsd_stats import PantsDaemonStats
 from pants.reporting.report import Report
 from pants.stats.statsdb import StatsDBFactory
 from pants.subsystem.subsystem import Subsystem
@@ -92,6 +93,7 @@ def __init__(self, *args, **kwargs):
     self.cumulative_timings = None
     self.self_timings = None
     self.artifact_cache_stats = None
+    self.pantsd_stats = None
 
     # Initialized in `start()`.
     self.report = None
@@ -160,8 +162,10 @@ def initialize(self):
     # Initialize the run.
     millis = int((self._run_timestamp * 1000) % 1000)
     run_id = 'pants_run_{}_{}_{}'.format(
-      time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)), millis,
-      uuid.uuid4().hex)
+      time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)),
+      millis,
+      uuid.uuid4().hex
+    )
 
     info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope)
     self.run_info_dir = os.path.join(info_dir, run_id)
@@ -182,8 +186,11 @@ def initialize(self):
     self.self_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'self_timings'))
 
     # Hit/miss stats for the artifact cache.
-    self.artifact_cache_stats = \
-      ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats'))
+    self.artifact_cache_stats = ArtifactCacheStats(os.path.join(self.run_info_dir,
+                                                                'artifact_cache_stats'))
+
+    # Hit/miss stats for the artifact cache.
+    self.pantsd_stats = PantsDaemonStats(os.path.join(self.run_info_dir, 'pantsd_stats'))
 
     return run_id
 
@@ -342,6 +349,7 @@ def store_stats(self):
       'cumulative_timings': self.cumulative_timings.get_all(),
       'self_timings': self.self_timings.get_all(),
       'artifact_cache_stats': self.artifact_cache_stats.get_all(),
+      'pantsd_stats': self.pantsd_stats.get_all(),
       'outcomes': self.outcomes
     }
     # Dump individual stat file.
diff --git a/src/python/pants/pantsd/service/pailgun_service.py b/src/python/pants/pantsd/service/pailgun_service.py
index e83febab38e..358e994dee9 100644
--- a/src/python/pants/pantsd/service/pailgun_service.py
+++ b/src/python/pants/pantsd/service/pailgun_service.py
@@ -57,25 +57,28 @@ def runner_factory(sock, arguments, environment):
       graph_helper = None
       deferred_exc = None
 
+      # Capture the size of the graph prior to any warming, for stats.
+      preceding_graph_size = self._scheduler_service.product_graph_len()
+      self._logger.debug('resident graph size: %s', preceding_graph_size)
+
       self._logger.debug('execution commandline: %s', arguments)
-      if self._scheduler_service:
-        self._logger.debug('args are: %s', arguments)
-        options, _ = OptionsInitializer(OptionsBootstrapper(args=arguments)).setup(init_logging=False)
-        target_roots = self._target_roots_class.create(
-          options,
-          change_calculator=self._scheduler_service.change_calculator
+      options, _ = OptionsInitializer(OptionsBootstrapper(args=arguments)).setup(init_logging=False)
+      target_roots = self._target_roots_class.create(
+        options,
+        change_calculator=self._scheduler_service.change_calculator
+      )
+
+      try:
+        self._logger.debug('warming the product graph via %s', self._scheduler_service)
+        # N.B. This call is made in the pre-fork daemon context for reach and reuse of the
+        # resident scheduler.
+        graph_helper = self._scheduler_service.warm_product_graph(target_roots)
+      except Exception:
+        deferred_exc = sys.exc_info()
+        self._logger.warning(
+          'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
+          ''.join(traceback.format_exception(*deferred_exc))
         )
-        try:
-          self._logger.debug('warming the product graph via %s', self._scheduler_service)
-          # N.B. This call is made in the pre-fork daemon context for reach and reuse of the
-          # resident scheduler.
-          graph_helper = self._scheduler_service.warm_product_graph(target_roots)
-        except Exception:
-          deferred_exc = sys.exc_info()
-          self._logger.warning(
-            'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
-            ''.join(traceback.format_exception(*deferred_exc))
-          )
 
       return self._runner_class(
         sock,
@@ -84,6 +87,7 @@ def runner_factory(sock, arguments, environment):
         environment,
         graph_helper,
         self.fork_lock,
+        preceding_graph_size,
         deferred_exc
       )
 
diff --git a/src/python/pants/pantsd/service/scheduler_service.py b/src/python/pants/pantsd/service/scheduler_service.py
index 15965a25874..7cabc8190b8 100644
--- a/src/python/pants/pantsd/service/scheduler_service.py
+++ b/src/python/pants/pantsd/service/scheduler_service.py
@@ -87,6 +87,13 @@ def _process_event_queue(self):
 
     self._event_queue.task_done()
 
+  def product_graph_len(self):
+    """Provides the size of the captive product graph.
+
+    :returns: The node count for the captive product graph.
+    """
+    return self._scheduler.graph_len()
+
   def warm_product_graph(self, spec_roots):
     """Runs an execution request against the captive scheduler given a set of input specs to warm.
 

From c0ed579dd18c3fd2b24fb3267d580b0257057202 Mon Sep 17 00:00:00 2001
From: Kris Wilson <kwilson@twopensource.com>
Date: Mon, 22 Jan 2018 18:50:48 -0800
Subject: [PATCH 2/5] Cleanups.

---
 src/python/pants/goal/pantsd_stats.py | 4 +---
 src/python/pants/goal/run_tracker.py  | 4 ++--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/python/pants/goal/pantsd_stats.py b/src/python/pants/goal/pantsd_stats.py
index 0b30c736214..09b7e31c716 100644
--- a/src/python/pants/goal/pantsd_stats.py
+++ b/src/python/pants/goal/pantsd_stats.py
@@ -9,11 +9,9 @@
 class PantsDaemonStats(object):
   """Tracks various stats about the daemon."""
 
-  def __init__(self, dir):
+  def __init__(self):
     self.resident_graph_size = None
     self.resulting_graph_size = None
-    # TODO: Dir is currently unused, but plumbed for later use.
-    self._dir = dir
 
   def set_resident_graph_size(self, size):
     self.resident_graph_size = size
diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py
index 0c90683dbbc..16a32133905 100644
--- a/src/python/pants/goal/run_tracker.py
+++ b/src/python/pants/goal/run_tracker.py
@@ -189,8 +189,8 @@ def initialize(self):
     self.artifact_cache_stats = ArtifactCacheStats(os.path.join(self.run_info_dir,
                                                                 'artifact_cache_stats'))
 
-    # Hit/miss stats for the artifact cache.
-    self.pantsd_stats = PantsDaemonStats(os.path.join(self.run_info_dir, 'pantsd_stats'))
+    # Daemon stats.
+    self.pantsd_stats = PantsDaemonStats()
 
     return run_id
 

From 9c1cd161e57a117a8b16ddf8220d52d5dbc341fa Mon Sep 17 00:00:00 2001
From: Kris Wilson <kwilson@twopensource.com>
Date: Mon, 22 Jan 2018 23:04:24 -0800
Subject: [PATCH 3/5] Quick test.

---
 tests/python/pants_test/goal/test_run_tracker_integration.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/python/pants_test/goal/test_run_tracker_integration.py b/tests/python/pants_test/goal/test_run_tracker_integration.py
index 10712c92ee8..7e96afd6afa 100644
--- a/tests/python/pants_test/goal/test_run_tracker_integration.py
+++ b/tests/python/pants_test/goal/test_run_tracker_integration.py
@@ -28,6 +28,7 @@ def test_stats_local_json_file(self):
         self.assertIn('run_info', stats_json)
         self.assertIn('self_timings', stats_json)
         self.assertIn('cumulative_timings', stats_json)
+        self.assertIn('pantsd_stats', stats_json)
 
   def test_workunit_failure(self):
     pants_run = self.run_pants([

From dc39ac0847b59955186c136b745227cbb3a25397 Mon Sep 17 00:00:00 2001
From: Kris Wilson <kwilson@twopensource.com>
Date: Tue, 23 Jan 2018 09:58:24 -0800
Subject: [PATCH 4/5] Review feedback and fixups.

---
 src/python/pants/bin/local_pants_runner.py | 2 +-
 src/python/pants/goal/BUILD                | 1 +
 src/python/pants/goal/pantsd_stats.py      | 8 ++++----
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py
index 1fb753615e1..62aa52a23f6 100644
--- a/src/python/pants/bin/local_pants_runner.py
+++ b/src/python/pants/bin/local_pants_runner.py
@@ -75,7 +75,7 @@ def _run(self):
         repro.capture(run_tracker.run_info.get_as_dict())
 
       # Record the preceding product graph size.
-      run_tracker.pantsd_stats.set_resident_graph_size(self._preceding_graph_size)
+      run_tracker.pantsd_stats.set_preceding_graph_size(self._preceding_graph_size)
 
       # Setup and run GoalRunner.
       goal_runner = GoalRunner.Factory(root_dir,
diff --git a/src/python/pants/goal/BUILD b/src/python/pants/goal/BUILD
index 878f1cd9790..94320b42840 100644
--- a/src/python/pants/goal/BUILD
+++ b/src/python/pants/goal/BUILD
@@ -81,6 +81,7 @@ python_library(
   dependencies = [
     ':aggregated_timings',
     ':artifact_cache_stats',
+    ':pantsd_stats',
     '3rdparty/python:requests',
     '3rdparty/python:pyopenssl',
     'src/python/pants/base:build_environment',
diff --git a/src/python/pants/goal/pantsd_stats.py b/src/python/pants/goal/pantsd_stats.py
index 09b7e31c716..880940db6f8 100644
--- a/src/python/pants/goal/pantsd_stats.py
+++ b/src/python/pants/goal/pantsd_stats.py
@@ -10,17 +10,17 @@ class PantsDaemonStats(object):
   """Tracks various stats about the daemon."""
 
   def __init__(self):
-    self.resident_graph_size = None
+    self.preceding_graph_size = None
     self.resulting_graph_size = None
 
-  def set_resident_graph_size(self, size):
-    self.resident_graph_size = size
+  def set_preceding_graph_size(self, size):
+    self.preceding_graph_size = size
 
   def set_resulting_graph_size(self, size):
     self.resulting_graph_size = size
 
   def get_all(self):
     return {
-      'resident_graph_size': self.resident_graph_size,
+      'preceding_graph_size': self.preceding_graph_size,
       'resulting_graph_size': self.resulting_graph_size,
     }

From 4ed7be1995c305f6bd58f82153324fe38d5f6998 Mon Sep 17 00:00:00 2001
From: Kris Wilson <kwilson@twopensource.com>
Date: Tue, 23 Jan 2018 12:27:18 -0800
Subject: [PATCH 5/5] Stus feedback.

---
 src/python/pants/bin/goal_runner.py |  2 +-
 src/python/pants/goal/context.py    | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py
index d89887f5955..012b4996d44 100644
--- a/src/python/pants/bin/goal_runner.py
+++ b/src/python/pants/bin/goal_runner.py
@@ -250,7 +250,7 @@ def run(self):
 
     try:
       result = self._execute_engine()
-      self._run_tracker.pantsd_stats.set_resulting_graph_size(self._context.scheduler.graph_len())
+      self._context.set_resulting_graph_size_in_runtracker()
       if result:
         self._run_tracker.set_root_outcome(WorkUnit.FAILURE)
     except KeyboardInterrupt:
diff --git a/src/python/pants/goal/context.py b/src/python/pants/goal/context.py
index 4b70396ec70..f74aec0e09f 100644
--- a/src/python/pants/goal/context.py
+++ b/src/python/pants/goal/context.py
@@ -142,11 +142,6 @@ def scm(self):
     """
     return self._scm
 
-  @property
-  def scheduler(self):
-    """Returns the scheduler instance used for this run."""
-    return self._scheduler
-
   @property
   def workspace(self):
     """Returns the current workspace, if any."""
@@ -160,6 +155,12 @@ def __str__(self):
     ident = Target.identify(self.targets())
     return 'Context(id:{}, targets:{})'.format(ident, self.targets())
 
+  def set_resulting_graph_size_in_runtracker(self):
+    """Sets the resulting graph size in the run tracker's daemon stats object."""
+    node_count = self._scheduler.graph_len()
+    self.run_tracker.pantsd_stats.set_resulting_graph_size(node_count)
+    return node_count
+
   def submit_background_work_chain(self, work_chain, parent_workunit_name=None):
     """
     :API: public