Merge pull request #3141 from dependabot/jurre/stackprof

Add option to profile dry-run using Stackprof

.github/workflows/ci.yml

@@ -93,7 +93,7 @@ jobs:
       - name: Run Python flake8 linting
         if: matrix.suite == 'python'
         run: |
-          docker run --rm "$CORE_CI_IMAGE" bash -c "pyenv exec flake8 . --count --exclude=./.*,./python/spec/fixtures --show-source --statistics"
+          docker run --rm "$CORE_CI_IMAGE" bash -c "pyenv exec flake8 python/helpers/. --count --exclude=./.*,./python/spec/fixtures --show-source --statistics"
       - name: Run Ruby Rubocop linting
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite }} && bundle exec rubocop ."

README.md

@@ -180,6 +180,13 @@ This is a "meta" gem, that simply depends on all the others. If you want to
 automatically include support for all languages, you can just include this gem
 and you'll get all you need.
 
+## Profiling
+
+You can profile a dry-run by passing the `--profile` flag when running it. This
+will generate a `stackprof-<datetime>.dump` file in the `tmp/` folder, and you
+can generate a flamegraph from this by running:
+`stackprof --d3-flamegraph tmp/stackprof-<datetime>.dump > tmp/flamegraph.html`.
+
 ## Why is this public?
 
 As the name suggests, Dependabot Core is the core of Dependabot (the rest of the

bin/docker-dev-shell

@@ -167,6 +167,7 @@ docker run --rm -ti \
   -v "$(pwd)/omnibus/dependabot-omnibus.gemspec:$CODE_DIR/omnibus/dependabot-omnibus.gemspec" \
   -v "$(pwd)/omnibus/lib:$CODE_DIR/omnibus/lib" \
   -v "$(pwd)/omnibus/spec:$CODE_DIR/omnibus/spec" \
+  -v "$(pwd)/tmp:/$CODE_DIR/tmp" \
   --name "$CONTAINER_NAME" \
   "${DOCKER_OPTS[@]}" \
   --cap-add=SYS_PTRACE \

bin/dry-run.rb

@@ -63,6 +63,7 @@
 require "byebug"
 require "logger"
 require "dependabot/logger"
+require "stackprof"
 
 Dependabot.logger = Logger.new($stdout)
 
@@ -201,6 +202,11 @@
           "Only update vulnerable dependencies") do |_value|
     $options[:security_updates_only] = true
   end
+
+  opts.on("--profile",
+          "Profile using Stackprof. Output in `tmp/stackprof-<datetime>.dump`") do
+    $options[:profile] = true
+  end
 end
 
 option_parse.parse!
@@ -272,9 +278,7 @@ def cached_dependency_files_read
   )
   FileUtils.mkdir_p(cache_dir) unless Dir.exist?(cache_dir)
 
-  if File.exist?(cache_manifest_path)
-    cached_manifest = File.read(cache_manifest_path)
-  end
+  cached_manifest = File.read(cache_manifest_path) if File.exist?(cache_manifest_path)
   cached_dependency_files = JSON.parse(cached_manifest) if cached_manifest
 
   all_files_cached = cached_dependency_files&.all? do |file|
@@ -321,9 +325,7 @@ def cached_dependency_files_read
     end
     # Initialize a git repo so that changed files can be diffed
     if $options[:write]
-      if File.exist?(".gitignore")
-        FileUtils.cp(".gitignore", File.join(cache_dir, ".gitignore"))
-      end
+      FileUtils.cp(".gitignore", File.join(cache_dir, ".gitignore")) if File.exist?(".gitignore")
       Dir.chdir(cache_dir) do
         system("git init . && git add . && git commit --allow-empty -m 'Init'")
       end
@@ -426,6 +428,8 @@ def handle_dependabot_error(error:, dependency:)
        "#{error_details.fetch(:'error-detail')}"
 end
 
+StackProf.start(raw: true) if $options[:profile]
+
 source = Dependabot::Source.new(
   provider: $options[:provider],
   repo: $repo_name,
@@ -690,6 +694,10 @@ def security_fix?(dependency)
 rescue StandardError => e
   handle_dependabot_error(error: e, dependency: dep)
 end
+
+StackProf.stop if $options[:profile]
+StackProf.results("tmp/stackprof-#{Time.now.strftime('%Y-%m-%d-%H:%M')}.dump") if $options[:profile]
+
 # rubocop:enable Metrics/BlockLength
 
 # rubocop:enable Style/GlobalVars

common/dependabot-common.gemspec

@@ -42,6 +42,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop", "~> 1.9.0"
   spec.add_development_dependency "simplecov", "~> 0.21.0"
   spec.add_development_dependency "simplecov-console", "~> 0.9.1"
+  spec.add_development_dependency "stackprof", "~> 0.2.16"
   spec.add_development_dependency "vcr", "6.0.0"
   spec.add_development_dependency "webmock", "~> 3.4"