Bundler 2: [Prerelease] Add native helpers for file parsing

.github/workflows/ci.yml

@@ -16,24 +16,25 @@ jobs:
       fail-fast: false
       matrix:
         suite:
-          - bundler
-          - cargo
-          - common
-          - composer
-          - dep
-          - docker
-          - elm
-          - git_submodules
-          - github_actions
-          - go_modules
-          - gradle
-          - hex
-          - maven
-          - npm_and_yarn
-          - nuget
-          - omnibus
-          - python
-          - terraform
+        - { path: bundler, name: bundler1 }
+        - { path: bundler, name: bundler2 }
+        - { path: cargo, name: cargo }
+        - { path: common, name: common }
+        - { path: composer, name: composer }
+        - { path: dep, name: dep }
+        - { path: docker, name: docker }
+        - { path: elm, name: elm }
+        - { path: git_submodules, name: git_submodules }
+        - { path: github_actions, name: github_actions }
+        - { path: go_modules, name: go_modules }
+        - { path: gradle, name: gradle }
+        - { path: hex, name: hex }
+        - { path: maven, name: maven }
+        - { path: npm_and_yarn, name: npm_and_yarn }
+        - { path: nuget, name: nuget }
+        - { path: omnibus, name: omnibus }
+        - { path: python, name: python }
+        - { path: terraform, name: terraform }
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -91,28 +92,29 @@ jobs:
           docker push "$CORE_CI_IMAGE:latest"
           docker push "$CORE_CI_IMAGE:ci--$BRANCH_REF"
       - name: Run Python flake8 linting
-        if: matrix.suite == 'python'
+        if: matrix.suite.name == 'python'
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c "pyenv exec flake8 python/helpers/. --count --exclude=./.*,./python/spec/fixtures --show-source --statistics"
       - name: Run Ruby Rubocop linting
         run: |
-          docker run --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite }} && bundle exec rubocop ."
+          docker run --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && bundle exec rubocop ."
       - name: Run js linting and tests
-        if: matrix.suite == 'npm_and_yarn'
+        if: matrix.suite.name == 'npm_and_yarn'
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /opt/npm_and_yarn && npm run lint"
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /opt/npm_and_yarn && npm test"
       - name: Run bundler v1 native helper specs
-        if: matrix.suite == 'bundler'
+        if: matrix.suite.name == 'bundler1'
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c \
             "cd /home/dependabot/dependabot-core/bundler/helpers/v1 && BUNDLER_VERSION=1 bundle install && BUNDLER_VERSION=1 bundle exec rspec spec"
       - name: Run bundler v2 native helper specs
-        if: matrix.suite == 'bundler'
+        if: matrix.suite.name == 'bundler2'
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c \
             "cd /home/dependabot/dependabot-core/bundler/helpers/v2 && BUNDLER_VERSION=2 bundle install && BUNDLER_VERSION=2 bundle exec rspec spec"
-      - name: Run ${{ matrix.suite }} tests with rspec
+      - name: Run ${{ matrix.suite.name }} tests with rspec
         run: |
-          docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --rm "$CORE_CI_IMAGE" bash -c \
-            "cd /home/dependabot/dependabot-core/${{ matrix.suite }} && bundle exec rspec spec"
+          echo "SUITE_NAME=${{ matrix.suite.name }}" >> $GITHUB_ENV
+          docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --env "SUITE_NAME=$SUITE_NAME" --rm "$CORE_CI_IMAGE" bash -c \
+            "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && bundle exec rspec spec"

bundler/helpers/v2/.gitignore

@@ -1,5 +1,4 @@
-/.bundle/*
-!/.bundle/config
+/.bundle
 /.env
 /tmp
 /dependabot-*.gem

bundler/helpers/v2/build

@@ -10,7 +10,6 @@ fi
 
 helpers_dir="$(dirname "${BASH_SOURCE[0]}")"
 cp -r \
-  "$helpers_dir/.bundle" \
   "$helpers_dir/lib" \
   "$helpers_dir/run.rb" \
   "$helpers_dir/Gemfile" \
@@ -20,4 +19,5 @@ cd "$install_dir"
 
 # NOTE: Sets `BUNDLED WITH` to match the installed v1 version in Gemfile.lock
 # forcing specs and native helpers to run with the same version
-BUNDLER_VERSION=2 bundle install
+BUNDLER_VERSION=2 bundle config set --local path ".bundle"
+BUNDLER_VERSION=2 bundle install --without test

bundler/helpers/v2/lib/functions.rb

@@ -1,12 +1,20 @@
+require "functions/file_parser"
+
 module Functions
   class NotImplementedError < StandardError; end
 
   def self.parsed_gemfile(lockfile_name:, gemfile_name:, dir:)
-    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+    set_bundler_flags_and_credentials(dir: dir, credentials: [],
+      using_bundler2: false)
+    FileParser.new(lockfile_name: lockfile_name).
+      parsed_gemfile(gemfile_name: gemfile_name)
   end
 
   def self.parsed_gemspec(lockfile_name:, gemspec_name:, dir:)
-    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+    set_bundler_flags_and_credentials(dir: dir, credentials: [],
+      using_bundler2: false)
+    FileParser.new(lockfile_name: lockfile_name).
+      parsed_gemspec(gemspec_name: gemspec_name)
   end
 
   def self.vendor_cache_dir(dir:)
@@ -57,7 +65,47 @@ def self.git_specs(dir:, gemfile_name:, credentials:, using_bundler2:)
 
   def self.set_bundler_flags_and_credentials(dir:, credentials:,
                                              using_bundler2:)
-    raise NotImplementedError, "Bundler 2 adapter does not yet implement #{__method__}"
+    dir = dir ? Pathname.new(dir) : dir
+    Bundler.instance_variable_set(:@root, dir)
+
+    # Remove installed gems from the default Rubygems index
+    Gem::Specification.all =
+      Gem::Specification.send(:default_stubs, "*.gemspec")
+
+    # Set auth details
+    relevant_credentials(credentials).each do |cred|
+      token = cred["token"] ||
+              "#{cred['username']}:#{cred['password']}"
+
+      Bundler.settings.set_command_option(
+        cred.fetch("host"),
+        token.gsub("@", "%40F").gsub("?", "%3F")
+      )
+    end
+
+    # NOTE: Prevent bundler from printing resolution information
+    Bundler.ui = Bundler::UI::Silent.new
+
+    # Use HTTPS for GitHub if lockfile
+    Bundler.settings.set_command_option("forget_cli_options", "true")
+    Bundler.settings.set_command_option("github.https", "true")
+  end
+
+  def self.relevant_credentials(credentials)
+    [
+      *git_source_credentials(credentials),
+      *private_registry_credentials(credentials)
+    ].select { |cred| cred["password"] || cred["token"] }
+  end
+
+  def self.private_registry_credentials(credentials)
+    credentials.
+      select { |cred| cred["type"] == "rubygems_server" }
+  end
+
+  def self.git_source_credentials(credentials)
+    credentials.
+      select { |cred| cred["type"] == "git_source" }
   end
 
   def self.conflicting_dependencies(dir:, dependency_name:, target_version:,

bundler/helpers/v2/lib/functions/file_parser.rb

@@ -0,0 +1,106 @@
+module Functions
+  class FileParser
+    def initialize(lockfile_name:)
+      @lockfile_name = lockfile_name
+    end
+
+    attr_reader :lockfile_name
+
+    def parsed_gemfile(gemfile_name:)
+      Bundler::Definition.build(gemfile_name, nil, {}).
+        dependencies.select(&:current_platform?).
+        reject { |dep| dep.source.is_a?(Bundler::Source::Gemspec) }.
+        map(&method(:serialize_bundler_dependency))
+    end
+
+    def parsed_gemspec(gemspec_name:)
+      Bundler.load_gemspec_uncached(gemspec_name).
+        dependencies.
+        map(&method(:serialize_bundler_dependency))
+    end
+
+    private
+
+    def lockfile
+      return @lockfile if defined?(@lockfile)
+
+      @lockfile =
+        begin
+          return unless lockfile_name && File.exist?(lockfile_name)
+
+          File.read(lockfile_name)
+        end
+    end
+
+    def parsed_lockfile
+      return unless lockfile
+
+      @parsed_lockfile ||= Bundler::LockfileParser.new(lockfile)
+    end
+
+    def source_from_lockfile(dependency_name)
+      parsed_lockfile&.specs.find { |s| s.name == dependency_name }&.source
+    end
+
+    def source_for(dependency)
+      source = dependency.source
+      if lockfile && default_rubygems?(source)
+        # If there's a lockfile and the Gemfile doesn't have anything
+        # interesting to say about the source, check that.
+        source = source_from_lockfile(dependency.name)
+      end
+      raise "Bad source: #{source}" unless sources.include?(source.class)
+
+      return nil if default_rubygems?(source)
+
+      details = { type: source.class.name.split("::").last.downcase }
+      if source.is_a?(Bundler::Source::Git)
+        details.merge!(git_source_details(source))
+      end
+      if source.is_a?(Bundler::Source::Rubygems)
+        details[:url] = source.remotes.first.to_s
+      end
+      details
+    end
+
+    # TODO: Remove default `master` branch
+    def git_source_details(source)
+      {
+        url: source.uri,
+        branch: source.branch || "master",
+        ref: source.ref || "master"
+      }
+    end
+
+    def default_rubygems?(source)
+      return true if source.nil?
+      return false unless source.is_a?(Bundler::Source::Rubygems)
+
+      source.remotes.any? { |r| r.to_s.include?("rubygems.org") }
+    end
+
+    def serialize_bundler_dependency(dependency)
+      {
+        name: dependency.name,
+        requirement: dependency.requirement,
+        groups: dependency.groups,
+        source: source_for(dependency),
+        type: dependency.type
+      }
+    end
+
+    # Can't be a constant because some of these don't exist in bundler
+    # 1.15, which used to cause issues on Heroku (causing exception on boot).
+    # TODO: Check if this will be an issue with multiple bundler versions
+    def sources
+      [
+        NilClass,
+        Bundler::Source::Rubygems,
+        Bundler::Source::Git,
+        Bundler::Source::Path,
+        Bundler::Source::Gemspec,
+        Bundler::Source::Metadata
+      ]
+    end
+  end
+end

bundler/helpers/v2/monkey_patches/definition_bundler_version_patch.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "bundler/definition"
+
+# Ignore the Bundler version specified in the Gemfile (since the only Bundler
+# version available to us is the one we're using).
+module BundlerDefinitionBundlerVersionPatch
+  def expanded_dependencies
+    @expanded_dependencies ||=
+      expand_dependencies(dependencies + metadata_dependencies, @remote).
+      reject { |d| d.name == "bundler" }
+  end
+end
+
+Bundler::Definition.prepend(BundlerDefinitionBundlerVersionPatch)

bundler/helpers/v2/monkey_patches/definition_ruby_version_patch.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "bundler/definition"
+
+module BundlerDefinitionRubyVersionPatch
+  def index
+    @index ||= super.tap do
+      if ruby_version
+        requested_version = ruby_version.to_gem_version_with_patchlevel
+        sources.metadata_source.specs <<
+          Gem::Specification.new("ruby\0", requested_version)
+      end
+
+      sources.metadata_source.specs <<
+        Gem::Specification.new("ruby\0", "2.5.3p105")
+    end
+  end
+end
+
+Bundler::Definition.prepend(BundlerDefinitionRubyVersionPatch)

bundler/helpers/v2/monkey_patches/git_source_patch.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "bundler/source"
+
+module Bundler
+  class Source
+    class Git
+      class GitProxy
+        private
+
+        # Bundler allows ssh authentication when talking to GitHub but there's
+        # no way for Dependabot to do so (it doesn't have any ssh keys).
+        # Instead, we convert all `git@github.com:` URLs to use HTTPS.
+        def configured_uri_for(uri)
+          uri = uri.gsub(%r{git@(.*?):/?}, 'https://\1/')
+          if /https?:/ =~ uri
+            remote = Bundler::URI(uri)
+            config_auth = Bundler.settings[remote.to_s] || Bundler.settings[remote.host]
+            remote.userinfo ||= config_auth
+            remote.to_s
+          else
+            uri
+          end
+        end
+      end
+    end
+  end
+end
+
+module Bundler
+  class Source
+    class Git < Path
+      private
+
+      def serialize_gemspecs_in(destination)
+        original_load_paths = $LOAD_PATH.dup
+        reduced_load_paths = original_load_paths.
+                             reject { |p| p.include?("/gems/") }
+
+        $LOAD_PATH.shift until $LOAD_PATH.empty?
+        reduced_load_paths.each { |p| $LOAD_PATH << p }
+
+        if destination.relative?
+          destination = destination.expand_path(Bundler.root)
+        end
+        Dir["#{destination}/#{@glob}"].each do |spec_path|
+          # Evaluate gemspecs and cache the result. Gemspecs
+          # in git might require git or other dependencies.
+          # The gemspecs we cache should already be evaluated.
+          spec = Bundler.load_gemspec(spec_path)
+          next unless spec
+
+          Bundler.rubygems.set_installed_by_version(spec)
+          Bundler.rubygems.validate(spec)
+          File.open(spec_path, "wb") { |file| file.write(spec.to_ruby) }
+        end
+        $LOAD_PATH.shift until $LOAD_PATH.empty?
+        original_load_paths.each { |p| $LOAD_PATH << p }
+      end
+    end
+  end
+end