diff --git a/lib/bundler/cli/pristine.rb b/lib/bundler/cli/pristine.rb index 152ac985ac49..41503f7ad472 100644 --- a/lib/bundler/cli/pristine.rb +++ b/lib/bundler/cli/pristine.rb @@ -46,6 +46,14 @@ def run next if git_sources.include?(source) git_sources << source + when Source::Path + extension_dir = source.extension_dir_for(spec) + unless extension_dir + Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is sourced from local path.") + next + end + + FileUtils.rm_rf extension_dir else Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is sourced from local path.") next diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 368a85c1812f..21da3a7237bc 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -76,6 +76,10 @@ The following is a list of all configuration keys and their purpose\. You can le .IP "\(bu" 4 \fBbin\fR (\fBBUNDLE_BIN\fR): If configured, \fBbundle binstubs\fR will install executables from gems in the bundle to the specified directory\. Otherwise it will create them in a \fBbin\fR directory relative to the Gemfile directory\. These executables run in Bundler's context\. If used, you might add this directory to your environment's \fBPATH\fR variable\. For instance, if the \fBrails\fR gem comes with a \fBrails\fR executable, \fBbundle binstubs\fR will create a \fBbin/rails\fR executable that ensures that all referred dependencies will be resolved using the bundled gems\. .IP "\(bu" 4 +\fBbuild_path_extensions\fR (\fBBUNDLE_BUILD_PATH_EXTENSIONS\fR): Whether Bundler should build the native extensions of gems sourced from a local path\. Off by default, since building them means compiling code out of a directory Bundler does not own\. It can also be set for a single gem, as \fBbuild_path_extensions\.\fR (\fBBUNDLE_BUILD_PATH_EXTENSIONS__\fR), which takes precedence over the global setting\. +.IP +Extensions are built out of tree: the gem is copied into a directory under \fBBUNDLE_PATH\fR and compiled there, so that nothing is ever written into your checkout, and the build directory is put ahead of the gem's own \fBlib\fR on the load path\. A rebuild happens whenever the size or modification time of any file under an extension's directory changes, or when the flags configured through \fBbuild\.\fR change\. Run \fBbundle pristine \fR to force one\. +.IP "\(bu" 4 \fBcache_all\fR (\fBBUNDLE_CACHE_ALL\fR): Cache all gems, including path and git gems\. This needs to be explicitly before bundler 4, but will be the default on bundler 4\. .IP "\(bu" 4 \fBcache_all_platforms\fR (\fBBUNDLE_CACHE_ALL_PLATFORMS\fR): Cache gems for all platforms\. diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index 834356a2d064..75ef2c5fcfc9 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -122,6 +122,19 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). `rails` executable, `bundle binstubs` will create a `bin/rails` executable that ensures that all referred dependencies will be resolved using the bundled gems. +* `build_path_extensions` (`BUNDLE_BUILD_PATH_EXTENSIONS`): + Whether Bundler should build the native extensions of gems sourced from a + local path. Off by default, since building them means compiling code out of + a directory Bundler does not own. It can also be set for a single gem, as + `build_path_extensions.` (`BUNDLE_BUILD_PATH_EXTENSIONS__`), which + takes precedence over the global setting. + + Extensions are built out of tree: the gem is copied into a directory under + `BUNDLE_PATH` and compiled there, so that nothing is ever written into your + checkout, and the build directory is put ahead of the gem's own `lib` on the + load path. A rebuild happens whenever the size or modification time of any + file under an extension's directory changes, or when the flags configured + through `build.` change. Run `bundle pristine ` to force one. * `cache_all` (`BUNDLE_CACHE_ALL`): Cache all gems, including path and git gems. This needs to be explicitly before bundler 4, but will be the default on bundler 4. diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index 4ad2bdf46f04..f46feeb0bc71 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -223,6 +223,19 @@ def load_paths full_require_paths end + alias_method :rg_full_require_paths, :full_require_paths + def full_require_paths + @bundler_full_require_paths ||= begin # rubocop:disable Naming/MemoizedInstanceVariableName + paths = rg_full_require_paths + + if source.respond_to?(:path?) && source.path? && paths.include?(extension_dir) + [extension_dir] + (paths - [extension_dir]) + else + paths + end + end + end + alias_method :rg_extension_dir, :extension_dir def extension_dir # following instance variable is already used in original method diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index c439b5c01db4..1c29bc39ffb9 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -8,6 +8,7 @@ class Settings BOOL_KEYS = %w[ auto_install + build_path_extensions cache_all cache_all_platforms clean diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb index db6c0ff8df72..e55c921c4916 100644 --- a/lib/bundler/source/path.rb +++ b/lib/bundler/source/path.rb @@ -75,9 +75,10 @@ def name def install(spec, options = {}) using_message = "Using #{version_message(spec, options[:previous_spec])} from #{self}" + using_message += " with native extensions" if missing_extensions?(spec) using_message += " and installing its executables" unless spec.executables.empty? print_using_message using_message - generate_bin(spec, disable_extensions: true) + generate_bin(spec, disable_extensions: !missing_extensions?(spec), build_args: options[:build_args]) nil # no post-install message end @@ -119,6 +120,16 @@ def expanded_original_path @expanded_original_path ||= expand(original_path) end + def extension_dir_for(spec) + return unless build_extensions?(spec) + + @extension_dirs ||= {} + @extension_dirs[spec.full_name] ||= Bundler.install_path.join( + "extensions", Gem::Platform.local.to_s, Gem.extension_api_version, + "#{spec.full_name}-#{extension_digest(spec)}" + ).to_s + end + private def expanded_path @@ -172,6 +183,10 @@ def load_spec_files # consider that for activation and never makes sense to ignore it. spec.ignored = false + if extension_dir = extension_dir_for(spec) + spec.extension_dir = extension_dir + end + # Validation causes extension_dir to be calculated, which depends # on #source, so we validate here instead of load_gemspec validate_spec(spec) @@ -207,6 +222,55 @@ def load_spec_files index end + def build_extensions?(spec) + return false unless path? + return false if spec.extensions.empty? + + per_gem = Bundler.settings["build_path_extensions.#{spec.name}"] + return per_gem unless per_gem.nil? + + Bundler.settings[:build_path_extensions] || false + end + + def missing_extensions?(spec) + return false unless extension_dir_for(spec) + + spec.missing_extensions? + end + + def extension_digest(spec) + gem_dir = spec.full_gem_path + digest = SharedHelpers.digest(:SHA256).new + digest << Bundler.settings["build.#{spec.name}"].to_s + + extension_source_files(spec).each do |file| + digest << file.delete_prefix("#{gem_dir}/") << "\0" + stat = File.stat(file) + digest << "#{stat.size}-#{stat.mtime.to_i}-#{stat.mtime.nsec}" << "\0" + end + + digest.hexdigest[0, 12] + end + + def extension_source_files(spec) + files = spec.extensions.flat_map do |extension| + dir = File.dirname(extension) + base = dir == "." ? spec.full_gem_path : File.join(spec.full_gem_path, dir) + Gem::Util.glob_files_in_dir("**/*", base) + end + + files << spec.loaded_from.to_s + files.uniq.select {|file| File.file?(file) }.sort + end + + def extension_build_dir(spec) + return unless extension_dir_for(spec) + + Bundler.bundle_path.join( + "cache", "bundler", "path_extensions", File.basename(extension_dir_for(spec)) + ).to_s + end + def relative_path(path = self.path) if path.to_s.start_with?(root_path.to_s) return path.relative_path_from(root_path) @@ -235,7 +299,8 @@ def generate_bin(spec, options = {}) env_shebang: false, disable_extensions: options[:disable_extensions], build_args: options[:build_args], - bundler_extension_cache_path: extension_cache_path(spec) + bundler_extension_cache_path: extension_cache_path(spec), + extension_build_dir: extension_build_dir(spec) ) installer.post_install rescue Gem::InvalidSpecificationException => e diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb index 39765e5da229..fe63c7a1f0cb 100644 --- a/lib/bundler/source/path/installer.rb +++ b/lib/bundler/source/path/installer.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../../rubygems_gem_installer" +require_relative "../../vendored_fileutils" module Bundler class Source @@ -18,6 +19,7 @@ def initialize(spec, options = {}) @build_args = options[:build_args] || Bundler.rubygems.build_args @gem_bin_dir = "#{Bundler.rubygems.gem_dir}/bin" @disable_extensions = options[:disable_extensions] + @extension_build_dir = options[:extension_build_dir] @bin_dir = @gem_bin_dir end @@ -34,8 +36,39 @@ def post_install run_hooks(:post_install) end + def build_extensions + return super unless @extension_build_dir + + stage_extension_sources + + real_spec = @spec + @spec = staged_spec(real_spec) + begin + super + ensure + @spec = real_spec + FileUtils.rm_rf(@extension_build_dir) + end + end + private + def stage_extension_sources + SharedHelpers.filesystem_access(@extension_build_dir, :create) do |path| + FileUtils.rm_rf(path) + FileUtils.mkdir_p(path) + FileUtils.cp_r("#{@spec.full_gem_path}/.", path) + end + end + + def staged_spec(spec) + staged = spec.dup + staged.source = nil + staged.full_gem_path = @extension_build_dir + staged.extension_dir = spec.extension_dir + staged + end + def run_hooks(type) hooks_meth = "#{type}_hooks" return unless Gem.respond_to?(hooks_meth) diff --git a/spec/install/gemfile/path_extensions_spec.rb b/spec/install/gemfile/path_extensions_spec.rb new file mode 100644 index 000000000000..18c3472d84aa --- /dev/null +++ b/spec/install/gemfile/path_extensions_spec.rb @@ -0,0 +1,264 @@ +# frozen_string_literal: true + +RSpec.describe "bundle install with path sources that have extensions" do + before do + build_lib "foo", "1.0", path: lib_path("foo") do |s| + s.extensions = ["ext/extconf.rb"] + s.write "ext/extconf.rb", <<-RUBY + require "mkmf" + create_makefile("foo_c") + RUBY + + s.write "ext/foo.h", <<-C + #define FOO_VALUE 1 + C + + s.write "ext/foo.c", <<-C + #include "ruby.h" + #include "foo.h" + + void Init_foo_c(void) { + rb_define_global_const("FOO_C", INT2NUM(FOO_VALUE)); + } + C + + s.write "lib/foo.rb", <<-RUBY + require "foo_c" + RUBY + end + end + + let(:gemfile_source) do + <<-G + source "https://gem.repo1" + gem "foo", :path => "#{lib_path("foo")}" + G + end + + def extension_dirs + Pathname.glob(default_bundle_path("bundler/gems/extensions/*/*/foo-1.0-*")) + end + + # Everything the gem shipped with, so that specs can assert Bundler did not + # leave a Makefile, an object file or a `.so` behind in the user's checkout. + def checkout_files + Pathname.glob(lib_path("foo/**/*")).select(&:file?).map do |file| + file.relative_path_from(lib_path("foo")).to_s + end.sort + end + + it "does not build extensions by default" do + pristine_checkout = checkout_files + + install_gemfile gemfile_source + + expect(extension_dirs).to be_empty + expect(checkout_files).to eq(pristine_checkout) + + run "begin; require 'foo'; rescue LoadError => e; puts e.message; end" + expect(out).to include("cannot load such file -- foo_c") + end + + context "when path extension building is enabled" do + before { bundle_config "build_path_extensions true" } + + it "prints output when building native extensions" do + install_gemfile gemfile_source, verbose: true + + expect(out).to include("Using foo 1.0 from source at `#{lib_path("foo")}` with native extensions") + end + + it "builds the extension and makes it requirable" do + install_gemfile gemfile_source + + expect(extension_dirs.size).to eq(1) + + run "require 'foo'; puts FOO_C" + expect(out).to eq("1") + end + + it "builds out of tree, leaving the path gem's checkout untouched" do + pristine_checkout = checkout_files + + install_gemfile gemfile_source + + expect(checkout_files).to eq(pristine_checkout) + end + + it "puts the build directory ahead of the gem's own load paths" do + install_gemfile gemfile_source + + bundle %(exec ruby -e 'puts Gem.loaded_specs["foo"].full_require_paths') + expect(out.split("\n").take(2)).to eq([extension_dirs.first.to_s, lib_path("foo/lib").to_s]) + end + + it "does not rebuild the extension when nothing changed" do + install_gemfile gemfile_source + + built_at = extension_dirs.first.join("gem.build_complete").mtime + + bundle :install + + expect(extension_dirs.size).to eq(1) + expect(extension_dirs.first.join("gem.build_complete").mtime).to eq(built_at) + end + + it "rebuilds the extension when its sources change" do + install_gemfile gemfile_source + + run "require 'foo'; puts FOO_C" + expect(out).to eq("1") + + File.write(lib_path("foo/ext/foo.c"), <<-C) + #include "ruby.h" + + void Init_foo_c(void) { + rb_define_global_const("FOO_C", INT2NUM(2)); + } + C + + bundle :install + + run "require 'foo'; puts FOO_C" + expect(out).to eq("2") + end + + it "rebuilds the extension when a native extension file changes" do + install_gemfile gemfile_source + + run "require 'foo'; puts FOO_C" + expect(out).to eq("1") + + File.write(lib_path("foo/ext/foo.h"), <<-C) + #define FOO_VALUE 2 + C + + bundle :install + + run "require 'foo'; puts FOO_C" + expect(out).to eq("2") + end + + it "fails under bundle exec when the native extension is stale" do + install_gemfile gemfile_source + + bundle "exec ruby -e 'require \"foo\"; puts FOO_C'" + expect(out).to eq("1") + + File.write(lib_path("foo/ext/foo.h"), <<-C) + #define FOO_VALUE 2 + C + + bundle "exec ruby -e 'require \"foo\"; puts FOO_C'", raise_on_error: false + + expect(exitstatus).not_to eq(0) + expect(err_without_deprecations).to include("cannot load such file -- foo_c") + end + + it "behaves the same under bundle exec before install as when the built extension is stale" do + gemfile gemfile_source + + bundle "exec ruby -e 'require \"foo\"; puts FOO_C'", raise_on_error: false + expect(exitstatus).not_to eq(0) + before_install_error = err_without_deprecations + expect(before_install_error).to include("cannot load such file -- foo_c") + + install_gemfile gemfile_source + + File.write(lib_path("foo/ext/foo.h"), <<-C) + #define FOO_VALUE 2 + C + + bundle "exec ruby -e 'require \"foo\"; puts FOO_C'", raise_on_error: false + expect(exitstatus).not_to eq(0) + expect(err_without_deprecations).to eq(before_install_error) + end + + it "shows compilation errors when building native extensions" do + build_lib "bar", "1.0", path: lib_path("bar") do |s| + s.extensions = ["ext/extconf.rb"] + s.write "ext/extconf.rb", <<-RUBY + require "mkmf" + create_makefile("bar_c") + RUBY + s.write "ext/bar.c", <<-C + #include "ruby.h" + + void Init_bar_c(void) { + this will not compile + } + C + s.write "lib/bar.rb", "require 'bar_c'" + end + + install_gemfile <<-G, raise_on_error: false, verbose: true + source "https://gem.repo1" + gem "bar", :path => "#{lib_path("bar")}" + G + + expect(exitstatus).not_to eq(0) + expect(out).to include("Using bar 1.0 from source at `#{lib_path("bar")}` with native extensions") + expect(err_without_deprecations).to include("Gem::Ext::BuildError") + expect(err_without_deprecations).to match(/error:/i) + end + + it "passes the flags configured for the gem to the build" do + build_lib "bar", "1.0", path: lib_path("bar") do |s| + s.extensions = ["ext/extconf.rb"] + s.write "ext/extconf.rb", <<-RUBY + require "mkmf" + raise ArgumentError unless with_config("bar") == "hello" + create_makefile("bar_c") + RUBY + s.write "ext/bar.c", "#include \"ruby.h\"\nvoid Init_bar_c(void) {}\n" + s.write "lib/bar.rb", "require 'bar_c'" + end + + bundle_config "build.bar --with-bar=hello" + + install_gemfile <<-G + source "https://gem.repo1" + gem "bar", :path => "#{lib_path("bar")}" + G + + run "require 'bar'; puts 'built'" + expect(out).to eq("built") + end + + it "can be turned off again for a single gem" do + bundle_config "build_path_extensions.foo false" + + install_gemfile gemfile_source + + expect(extension_dirs).to be_empty + end + end + + context "when path extension building is enabled for a single gem" do + before { bundle_config "build_path_extensions.foo true" } + + it "builds that gem's extension" do + install_gemfile gemfile_source + + run "require 'foo'; puts FOO_C" + expect(out).to eq("1") + end + + it "leaves other path gems alone" do + build_lib "bar", "1.0", path: lib_path("bar") do |s| + s.extensions = ["ext/extconf.rb"] + s.write "ext/extconf.rb", "require 'mkmf'\ncreate_makefile('bar_c')\n" + s.write "ext/bar.c", "#include \"ruby.h\"\nvoid Init_bar_c(void) {}\n" + s.write "lib/bar.rb", "require 'bar_c'" + end + + install_gemfile <<-G + source "https://gem.repo1" + gem "foo", :path => "#{lib_path("foo")}" + gem "bar", :path => "#{lib_path("bar")}" + G + + expect(Pathname.glob(default_bundle_path("bundler/gems/extensions/*/*/bar-1.0-*"))).to be_empty + end + end +end diff --git a/spec/support/shards.rb b/spec/support/shards.rb index ccadff75faa9..ac7ef76eb85f 100644 --- a/spec/support/shards.rb +++ b/spec/support/shards.rb @@ -144,6 +144,7 @@ module Shards "spec/bundler/ci_detector_spec.rb", ], shard_d: [ + "spec/install/gemfile/path_extensions_spec.rb", "spec/bundler/rubygems_ext_spec.rb", "spec/bundler/resolver/cooldown_spec.rb", "spec/install/cooldown_spec.rb",