[pantsd] Add RunTracker stats.

src/python/pants/bin/daemon_pants_runner.py

@@ -72,7 +72,8 @@ class DaemonPantsRunner(ProcessManager):
   N.B. this class is primarily used by the PailgunService in pantsd.
   """
 
-  def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_exception=None):
+  def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, preceding_graph_size,
+               deferred_exception=None):
     """
     :param socket socket: A connected socket capable of speaking the nailgun protocol.
     :param Exiter exiter: The Exiter instance for this run.
@@ -82,6 +83,7 @@ def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_
                                            construction. In the event of an exception, this will be
                                            None.
     :param threading.RLock fork_lock: A lock to use during forking for thread safety.
+    :param int preceding_graph_size: The size of the graph pre-warming, for stats.
     :param Exception deferred_exception: A deferred exception from the daemon's graph construction.
                                          If present, this will be re-raised in the client context.
     """
@@ -92,6 +94,7 @@ def __init__(self, socket, exiter, args, env, graph_helper, fork_lock, deferred_
     self._env = env
     self._graph_helper = graph_helper
     self._fork_lock = fork_lock
+    self._preceding_graph_size = preceding_graph_size
     self._deferred_exception = deferred_exception
 
   def _make_identity(self):
@@ -201,7 +204,9 @@ def post_fork_child(self):
         self._raise_deferred_exc()
 
         # Otherwise, conduct a normal run.
-        LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper).run()
+        runner = LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper)
+        runner.set_preceding_graph_size(self._preceding_graph_size)
+        runner.run()
       except KeyboardInterrupt:
         self._exiter.exit(1, msg='Interrupted by user.\n')
       except Exception:

src/python/pants/bin/goal_runner.py

@@ -250,6 +250,7 @@ def run(self):
 
     try:
       result = self._execute_engine()
+      self._context.set_resulting_graph_size_in_runtracker()
       if result:
         self._run_tracker.set_root_outcome(WorkUnit.FAILURE)
     except KeyboardInterrupt:

src/python/pants/bin/local_pants_runner.py

@@ -31,6 +31,10 @@ def __init__(self, exiter, args, env, daemon_build_graph=None, options_bootstrap
     self._env = env
     self._daemon_build_graph = daemon_build_graph
     self._options_bootstrapper = options_bootstrapper
+    self._preceding_graph_size = -1
+
+  def set_preceding_graph_size(self, size):
+    self._preceding_graph_size = size
 
   def run(self):
     profile_path = self._env.get('PANTS_PROFILE')
@@ -70,6 +74,9 @@ def _run(self):
       if repro:
         repro.capture(run_tracker.run_info.get_as_dict())
 
+      # Record the preceding product graph size.
+      run_tracker.pantsd_stats.set_preceding_graph_size(self._preceding_graph_size)
+
       # Setup and run GoalRunner.
       goal_runner = GoalRunner.Factory(root_dir,
                                        options,

src/python/pants/goal/BUILD

@@ -61,6 +61,11 @@ python_library(
   ],
 )
 
+python_library(
+  name = 'pantsd_stats',
+  sources = ['pantsd_stats.py'],
+)
+
 python_library(
   name = 'products',
   sources = ['products.py'],
@@ -76,6 +81,7 @@ python_library(
   dependencies = [
     ':aggregated_timings',
     ':artifact_cache_stats',
+    ':pantsd_stats',
     '3rdparty/python:requests',
     '3rdparty/python:pyopenssl',
     'src/python/pants/base:build_environment',

src/python/pants/goal/context.py

@@ -80,8 +80,6 @@ def __init__(self, options, run_tracker, target_roots,
     self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
     self._replace_targets(target_roots)
     self._invalidation_report = invalidation_report
-    # TODO(#4769): This should not be exposed to anyone.
-    # Note that the Context created in unit tests by BaseTest uses a different codepath.
     self._scheduler = scheduler
 
   @property
@@ -157,6 +155,12 @@ def __str__(self):
     ident = Target.identify(self.targets())
     return 'Context(id:{}, targets:{})'.format(ident, self.targets())
 
+  def set_resulting_graph_size_in_runtracker(self):
+    """Sets the resulting graph size in the run tracker's daemon stats object."""
+    node_count = self._scheduler.graph_len()
+    self.run_tracker.pantsd_stats.set_resulting_graph_size(node_count)
+    return node_count
+
   def submit_background_work_chain(self, work_chain, parent_workunit_name=None):
     """
     :API: public

src/python/pants/goal/pantsd_stats.py

@@ -0,0 +1,26 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+
+class PantsDaemonStats(object):
+  """Tracks various stats about the daemon."""
+
+  def __init__(self):
+    self.preceding_graph_size = None
+    self.resulting_graph_size = None
+
+  def set_preceding_graph_size(self, size):
+    self.preceding_graph_size = size
+
+  def set_resulting_graph_size(self, size):
+    self.resulting_graph_size = size
+
+  def get_all(self):
+    return {
+      'preceding_graph_size': self.preceding_graph_size,
+      'resulting_graph_size': self.resulting_graph_size,
+    }

src/python/pants/goal/run_tracker.py

@@ -25,6 +25,7 @@
 from pants.build_graph.target import Target
 from pants.goal.aggregated_timings import AggregatedTimings
 from pants.goal.artifact_cache_stats import ArtifactCacheStats
+from pants.goal.pantsd_stats import PantsDaemonStats
 from pants.reporting.report import Report
 from pants.stats.statsdb import StatsDBFactory
 from pants.subsystem.subsystem import Subsystem
@@ -92,6 +93,7 @@ def __init__(self, *args, **kwargs):
     self.cumulative_timings = None
     self.self_timings = None
     self.artifact_cache_stats = None
+    self.pantsd_stats = None
 
     # Initialized in `start()`.
     self.report = None
@@ -160,8 +162,10 @@ def initialize(self):
     # Initialize the run.
     millis = int((self._run_timestamp * 1000) % 1000)
     run_id = 'pants_run_{}_{}_{}'.format(
-      time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)), millis,
-      uuid.uuid4().hex)
+      time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)),
+      millis,
+      uuid.uuid4().hex
+    )
 
     info_dir = os.path.join(self.get_options().pants_workdir, self.options_scope)
     self.run_info_dir = os.path.join(info_dir, run_id)
@@ -182,8 +186,11 @@ def initialize(self):
     self.self_timings = AggregatedTimings(os.path.join(self.run_info_dir, 'self_timings'))
 
     # Hit/miss stats for the artifact cache.
-    self.artifact_cache_stats = \
-      ArtifactCacheStats(os.path.join(self.run_info_dir, 'artifact_cache_stats'))
+    self.artifact_cache_stats = ArtifactCacheStats(os.path.join(self.run_info_dir,
+                                                                'artifact_cache_stats'))
+
+    # Daemon stats.
+    self.pantsd_stats = PantsDaemonStats()
 
     return run_id
 
@@ -342,6 +349,7 @@ def store_stats(self):
       'cumulative_timings': self.cumulative_timings.get_all(),
       'self_timings': self.self_timings.get_all(),
       'artifact_cache_stats': self.artifact_cache_stats.get_all(),
+      'pantsd_stats': self.pantsd_stats.get_all(),
       'outcomes': self.outcomes
     }
     # Dump individual stat file.

src/python/pants/pantsd/service/pailgun_service.py

@@ -57,25 +57,28 @@ def runner_factory(sock, arguments, environment):
       graph_helper = None
       deferred_exc = None
 
+      # Capture the size of the graph prior to any warming, for stats.
+      preceding_graph_size = self._scheduler_service.product_graph_len()
+      self._logger.debug('resident graph size: %s', preceding_graph_size)
+
       self._logger.debug('execution commandline: %s', arguments)
-      if self._scheduler_service:
-        self._logger.debug('args are: %s', arguments)
-        options, _ = OptionsInitializer(OptionsBootstrapper(args=arguments)).setup(init_logging=False)
-        target_roots = self._target_roots_class.create(
-          options,
-          change_calculator=self._scheduler_service.change_calculator
+      options, _ = OptionsInitializer(OptionsBootstrapper(args=arguments)).setup(init_logging=False)
+      target_roots = self._target_roots_class.create(
+        options,
+        change_calculator=self._scheduler_service.change_calculator
+      )
+
+      try:
+        self._logger.debug('warming the product graph via %s', self._scheduler_service)
+        # N.B. This call is made in the pre-fork daemon context for reach and reuse of the
+        # resident scheduler.
+        graph_helper = self._scheduler_service.warm_product_graph(target_roots)
+      except Exception:
+        deferred_exc = sys.exc_info()
+        self._logger.warning(
+          'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
+          ''.join(traceback.format_exception(*deferred_exc))
         )
-        try:
-          self._logger.debug('warming the product graph via %s', self._scheduler_service)
-          # N.B. This call is made in the pre-fork daemon context for reach and reuse of the
-          # resident scheduler.
-          graph_helper = self._scheduler_service.warm_product_graph(target_roots)
-        except Exception:
-          deferred_exc = sys.exc_info()
-          self._logger.warning(
-            'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
-            ''.join(traceback.format_exception(*deferred_exc))
-          )
 
       return self._runner_class(
         sock,
@@ -84,6 +87,7 @@ def runner_factory(sock, arguments, environment):
         environment,
         graph_helper,
         self.fork_lock,
+        preceding_graph_size,
         deferred_exc
       )
 

src/python/pants/pantsd/service/scheduler_service.py

@@ -87,6 +87,13 @@ def _process_event_queue(self):
 
     self._event_queue.task_done()
 
+  def product_graph_len(self):
+    """Provides the size of the captive product graph.
+
+    :returns: The node count for the captive product graph.
+    """
+    return self._scheduler.graph_len()
+
   def warm_product_graph(self, spec_roots):
     """Runs an execution request against the captive scheduler given a set of input specs to warm.
 

tests/python/pants_test/goal/test_run_tracker_integration.py

@@ -28,6 +28,7 @@ def test_stats_local_json_file(self):
         self.assertIn('run_info', stats_json)
         self.assertIn('self_timings', stats_json)
         self.assertIn('cumulative_timings', stats_json)
+        self.assertIn('pantsd_stats', stats_json)
 
   def test_workunit_failure(self):
     pants_run = self.run_pants([