Add content addressable gems support

Manifest.txt

@@ -357,6 +357,7 @@ lib/rubygems/compact_index_client/http_fetcher.rb
 lib/rubygems/compact_index_client/parser.rb
 lib/rubygems/compact_index_client/updater.rb
 lib/rubygems/config_file.rb
+lib/rubygems/content_address.rb
 lib/rubygems/cooldown.rb
 lib/rubygems/cooldown_option.rb
 lib/rubygems/core_ext/kernel_gem.rb

lib/bundler/endpoint_specification.rb

@@ -5,24 +5,32 @@ module Bundler
   class EndpointSpecification < Gem::Specification
     include MatchRemoteMetadata
 
-    attr_reader :name, :version, :platform, :checksum, :created_at
+    attr_reader :name, :version, :platform, :checksum, :created_at, :content_address
     attr_writer :dependencies
     attr_accessor :remote, :locked_platform
 
-    def initialize(name, version, platform, spec_fetcher, dependencies, metadata = nil)
+    def initialize(name, version, suffix, spec_fetcher, dependencies, metadata = nil)
       super()
       @name         = name
       @version      = Gem::Version.create version
-      @platform     = Gem::Platform.new(platform)
       @spec_fetcher = spec_fetcher
       @dependencies = nil
       @unbuilt_dependencies = dependencies
+      @content_address = nil
+      @required_platform = nil
 
       @loaded_from          = nil
       @remote_specification = nil
       @locked_platform = nil
 
       parse_metadata(metadata)
+
+      if Gem::ContentAddress.match?(suffix) && @required_platform
+        @content_address = suffix
+        @platform = @required_platform
+      else
+        @platform = Gem::Platform.new(suffix)
+      end
     end
 
     def insecurely_materialized?
@@ -147,7 +155,8 @@ def inspect
     private
 
     def _remote_specification
-      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @platform])
+      suffix = @content_address || @platform
+      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, suffix])
     end
 
     def local_specification_path
@@ -183,6 +192,8 @@ def parse_metadata(data)
           @required_ruby_version = Gem::Requirement.new(v)
         when "created_at"
           @created_at = parse_created_at(v.is_a?(Array) ? v.last : v)&.freeze
+        when "platform"
+          @required_platform = required_platform_from(Array(v).last)
         end
       end
     rescue StandardError => e
@@ -210,5 +221,12 @@ def parse_created_at(value)
     def build_dependency(name, requirements)
       Dependency.new(name, requirements)
     end
+
+    def required_platform_from(value)
+      op, platform = value.to_s.split(" ", 2)
+      return unless op == "=" && platform
+
+      Gem::Platform.new(platform)
+    end
   end
 end

lib/bundler/fetcher.rb

@@ -177,13 +177,13 @@ def specs_with_retry(gem_names, source)
     def specs(gem_names, source)
       index = Bundler::Index.new
 
-      fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata|
+      fetch_specs(gem_names).each do |name, version, suffix, dependencies, metadata|
         spec = if dependencies
-          EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es|
+          EndpointSpecification.new(name, version, suffix, self, dependencies, metadata).tap do |es|
             source.checksum_store.replace(es, es.checksum)
           end
         else
-          RemoteSpecification.new(name, version, platform, self)
+          RemoteSpecification.new(name, version, suffix, self)
         end
         spec.source = source
         spec.remote = @remote

lib/bundler/lazy_specification.rb

@@ -8,7 +8,7 @@ class LazySpecification
     include MatchPlatform
     include ForcePlatform
 
-    attr_reader :name, :version, :platform, :materialization
+    attr_reader :name, :version, :platform, :materialization, :content_address
     attr_accessor :source, :remote, :force_ruby_platform, :dependencies, :required_ruby_version, :required_rubygems_version
     attr_accessor :overrides
 
@@ -27,21 +27,22 @@ class LazySpecification
     alias_method :runtime_dependencies, :dependencies
 
     def self.from_spec(s)
-      lazy_spec = new(s.name, s.version, s.platform, s.source)
+      lazy_spec = new(s.name, s.version, s.platform, s.source, content_address: s.content_address)
       lazy_spec.dependencies = s.runtime_dependencies
       lazy_spec.required_ruby_version = s.required_ruby_version
       lazy_spec.required_rubygems_version = s.required_rubygems_version
       lazy_spec.overrides = s.overrides if s.is_a?(LazySpecification)
       lazy_spec
     end
 
-    def initialize(name, version, platform, source = nil, **materialization_options)
+    def initialize(name, version, platform, source = nil, content_address: nil, **materialization_options)
       @name          = name
       @version       = version
       @dependencies  = []
       @required_ruby_version = Gem::Requirement.default
       @required_rubygems_version = Gem::Requirement.default
       @platform = platform || Gem::Platform::RUBY
+      @content_address = content_address
 
       @original_source = source
       @source = source
@@ -65,7 +66,9 @@ def source_changed?
     end
 
     def full_name
-      @full_name ||= if platform == Gem::Platform::RUBY
+      @full_name ||= if Gem::ContentAddress.match?(@content_address) && platform != Gem::Platform::RUBY
+        "#{@name}-#{@version}-#{@content_address}"
+      elsif platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
         "#{@name}-#{@version}-#{platform}"
@@ -77,7 +80,7 @@ def lock_name
     end
 
     def name_tuple
-      Gem::NameTuple.new(@name, @version, @platform)
+      Gem::NameTuple.new(@name, @version, @platform, content_address: @content_address)
     end
 
     def ==(other)
@@ -114,7 +117,11 @@ def satisfies?(dependency)
 
     def to_lock
       out = String.new
-      out << "    #{lock_name}\n"
+      out << "    #{lock_name}"
+      # Append the platform additionally for content-addressable gems that contain a SHA
+      # where the platform would otherwise be
+      out << " #{platform}" if Gem::ContentAddress.match?(content_address) && platform != Gem::Platform::RUBY
+      out << "\n"
 
       dependencies.sort_by(&:to_s).uniq.each do |dep|
         next if dep.type == :development

lib/bundler/lockfile_parser.rb

@@ -264,11 +264,13 @@ def parse_checksum(line)
       checksums = $6
       name = $2
       version = $3
-      platform = $4
+      content_address = $4 if Gem::ContentAddress.match?($4)
+      platform = $4 unless content_address
 
       version = Gem::Version.new(version)
       platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
-      full_name = Gem::NameTuple.new(name, version, platform).full_name
+      name_tuple = Gem::NameTuple.new(name, version, platform, content_address: content_address)
+      full_name = name_tuple.full_name
       spec = @specs[full_name]
 
       if name == "bundler"
@@ -295,11 +297,17 @@ def parse_spec(line)
 
       if spaces.size == 4
         # only load platform for non-dependency (spec) line
-        platform = $4
+        if Gem::ContentAddress.match?($4) && $6 && $6 != Gem::Platform::RUBY.to_s
+          content_address = $4
+          platform = $6
+        else
+          platform = $4
+          content_address = $6 if Gem::ContentAddress.match?($6)
+        end
 
         version = Gem::Version.new(version)
         platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
-        @current_spec = LazySpecification.new(name, version, platform, @current_source, strict: @strict)
+        @current_spec = LazySpecification.new(name, version, platform, @current_source, content_address: content_address, strict: @strict)
         @current_source.add_dependency_names(name)
 
         @specs[@current_spec.full_name] = @current_spec

lib/bundler/match_platform.rb

@@ -2,6 +2,10 @@
 
 module Bundler
   module MatchPlatform
+    def content_address
+      nil
+    end
+
     def installable_on_platform?(target_platform) # :nodoc:
       return true if [Gem::Platform::RUBY, nil, target_platform].include?(platform)
       return true if Gem::Platform.new(platform) === target_platform
@@ -11,13 +15,24 @@ def installable_on_platform?(target_platform) # :nodoc:
 
     def self.select_best_platform_match(specs, platform, force_ruby: false, prefer_locked: false)
       matching = select_all_platform_match(specs, platform, force_ruby: force_ruby, prefer_locked: prefer_locked)
+      matching = prefer_content_addressable(matching)
 
       Gem::Platform.sort_and_filter_best_platform_match(matching, platform)
     end
 
+    def self.prefer_content_addressable(matching)
+      addressable, non_addressable = matching.partition {|s| Gem::ContentAddress.match?(s.content_address) }
+      return matching if addressable.empty?
+
+      compatible = addressable.select(&:matches_current_ruby?)
+      compatible.any? ? compatible : non_addressable
+    end
+
     def self.select_best_local_platform_match(specs, force_ruby: false, locked_platforms: nil)
       local = Bundler.local_platform
-      matching = select_all_platform_match(specs, local, force_ruby: force_ruby).filter_map {|spec| spec.materialized_for_installation(locked_platforms) }
+      matching = select_all_platform_match(specs, local, force_ruby: force_ruby)
+      matching = prefer_content_addressable(matching)
+      matching = matching.filter_map {|spec| spec.materialized_for_installation(locked_platforms) }
 
       Gem::Platform.sort_best_platform_match(matching, local)
     end
@@ -31,7 +46,6 @@ def self.select_all_platform_match(specs, platform, force_ruby: false, prefer_lo
         locked_originally = matching.select {|spec| spec.is_a?(::Bundler::LazySpecification) }
         return locked_originally if locked_originally.any?
       end
-
       matching
     end
 

lib/bundler/remote_specification.rb

@@ -10,18 +10,19 @@ class RemoteSpecification
     include MatchPlatform
     include Comparable
 
-    attr_reader :name, :version, :platform
+    attr_reader :name, :version, :platform, :content_address
     attr_writer :dependencies
     attr_accessor :source, :remote, :locked_platform, :created_at
 
-    def initialize(name, version, platform, spec_fetcher)
+    def initialize(name, version, platform, spec_fetcher, content_address: nil)
       @name         = name
       @version      = Gem::Version.create version
       @original_platform = platform || Gem::Platform::RUBY
       @platform     = Gem::Platform.new(platform)
       @spec_fetcher = spec_fetcher
       @dependencies = nil
       @locked_platform = nil
+      @content_address = content_address
     end
 
     def insecurely_materialized?
@@ -35,7 +36,9 @@ def fetch_platform
     end
 
     def full_name
-      @full_name ||= if @platform == Gem::Platform::RUBY
+      @full_name ||= if Gem::ContentAddress.match?(@content_address) && @platform != Gem::Platform::RUBY
+        "#{@name}-#{@version}-#{@content_address}"
+      elsif @platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
         "#{@name}-#{@version}-#{@platform}"

lib/bundler/resolver.rb

@@ -275,7 +275,7 @@ def incompatibilities_for(package, version)
 
     def all_versions_for(package)
       name = package.name
-      results = (@base[name] + filter_specs(@all_specs[name], package)).uniq {|spec| [spec.version.hash, spec.platform] }
+      results = (@base[name] + filter_specs(@all_specs[name], package)).uniq {|spec| [spec.version.hash, spec.platform, spec.content_address] }
 
       if name == "bundler" && !bundler_pinned_to_current_version?
         bundler_spec = Gem.loaded_specs["bundler"]

lib/bundler/rubygems_ext.rb

@@ -13,7 +13,34 @@
 # `Gem::Source` from the redefined `Gem::Specification#source`.
 require "rubygems/source"
 
+# Can be removed once RubyGems 4.0.0 support is dropped
+unless Gem::BasicSpecification.method_defined?(:content_address)
+  Gem::BasicSpecification.attr_accessor :content_address
+end
+
+# Can be removed once RubyGems 4.0.0 support is dropped
+unless Gem::NameTuple.method_defined?(:content_address)
+  Gem::NameTuple.attr_reader :content_address
+end
+
 module Gem
+  # Can be removed once RubyGems 4.0.0 support is dropped
+  unless defined?(Gem::ContentAddress)
+    module ContentAddress
+      def self.match?(token)
+        false
+      end
+
+      def self.applicable?(spec)
+        false
+      end
+
+      def self.content_addressed?(spec)
+        false
+      end
+    end
+  end
+
   # Can be removed once RubyGems 3.5.11 support is dropped
   unless Gem.respond_to?(:freebsd_platform?)
     def self.freebsd_platform?
@@ -417,7 +444,8 @@ class NameTuple
     unless Gem::NameTuple.new("a", Gem::Version.new("1"), Gem::Platform.new("x86_64-linux")).platform.is_a?(String)
       alias_method :initialize_with_platform, :initialize
 
-      def initialize(name, version, platform = Gem::Platform::RUBY)
+      def initialize(name, version, platform = Gem::Platform::RUBY, content_address = nil)
+        @content_address = content_address
         if Gem::Platform === platform
           initialize_with_platform(name, version, platform.to_s)
         else
@@ -426,7 +454,18 @@ def initialize(name, version, platform = Gem::Platform::RUBY)
       end
     end
 
+    unless instance_method(:initialize).parameters.any? {|kind, name| kind == :key && name == :content_address }
+      alias_method :initialize_without_content_address, :initialize
+
+      def initialize(name, version, platform = Gem::Platform::RUBY, content_address: nil)
+        initialize_without_content_address(name, version, platform)
+        @content_address = content_address
+      end
+    end
+
     def lock_name
+      return "#{name} (#{version}-#{content_address})" if Gem::ContentAddress.match?(content_address)
+
       if platform == Gem::Platform::RUBY
         "#{name} (#{version})"
       else

lib/bundler/rubygems_gem_installer.rb

@@ -4,6 +4,11 @@
 
 module Bundler
   class RubyGemsGemInstaller < Gem::Installer
+    # Can be removed once RubyGems 4.0.0 support is dropped
+    unless private_method_defined?(:assign_content_address)
+      private def assign_content_address; end
+    end
+
     # Cap how many jobserver slots a single gem's `make` may grab so that one
     # gem with many recipes doesn't starve the others sharing the pool. Beyond
     # a handful of jobs the extra parallelism rarely pays off in practice.
@@ -14,6 +19,8 @@ def check_executable_overwrite(filename)
     end
 
     def install
+      assign_content_address
+
       pre_install_checks
 
       run_pre_install_hooks

lib/bundler/rubygems_integration.rb

@@ -144,7 +144,12 @@ def ext_lock
 
     def spec_from_gem(path)
       require "rubygems/package"
-      Gem::Package.new(path).spec
+      package = Gem::Package.new(path)
+      spec = package.spec
+      if package.respond_to?(:content_address)
+        spec.content_address = package.content_address
+      end
+      spec
     end
 
     def build_gem(gem_dir, spec)

lib/bundler/source/rubygems.rb

@@ -196,6 +196,8 @@ def download(spec, options = {})
              "the security policy didn't allow it, with the message: #{e.message}"
           end
 
+          s.content_address = spec.content_address if spec.content_address
+
           spec.__swap__(s)
         end