Make rake gem download source tarballs automatically

Previously `scripts/test-gem-build` would call `rake compile` just to
make mini_portile2 download the source tarballs. However, this
needlessly compiles abseil and libre2 before building the gem.

To improve this, split the loading of the recipes into a separate file
(`ext/re2/recipes.rb`). The `rake prepare` step invokes the `download`
method to download the required tarballs in preparation for building
the gem.

Now we add a `rake prepare` task that will run if `rake gem` is run.

Rakefile

@@ -5,6 +5,8 @@ require 'rspec/core/rake_task'
 require 'rake_compiler_dock'
 require 'yaml'
 
+require_relative 'ext/re2/recipes"
+
 CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'],
               FileList['**/extconf.h'],
               FileList['**/Makefile'],
@@ -18,6 +20,14 @@ CLOBBER.add("ports/*").exclude(%r{ports/archives$})
 
 RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec')
 
+task :prepare do
+  puts "Preparing project for gem building..."
+  recipes = load_recipes
+  recipes.each { |recipe| recipe.download }
+end
+
+task gem: :prepare
+
 Gem::PackageTask.new(RE2_GEM_SPEC) do |p|
   p.need_zip = false
   p.need_tar = false

ext/re2/extconf.rb

@@ -5,10 +5,7 @@
 # Released under the BSD Licence, please see LICENSE.txt
 
 require 'mkmf'
-
-PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
-
-REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.4" # keep this version in sync with the one in the gemspec
+require_relative 'recipes'
 
 RE2_HELP_MESSAGE = <<~HELP
   USAGE: ruby #{$0} [options]
@@ -212,62 +209,46 @@ def build_extension
   end
 end
 
-def process_recipe(name, version)
-  require "rubygems"
-  gem("mini_portile2", REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time
-  require "mini_portile2"
-  message("Using mini_portile version #{MiniPortile::VERSION}\n")
-
+def process_recipe(recipe)
   cross_build_p = config_cross_build?
   message "Cross build is #{cross_build_p ? "enabled" : "disabled"}.\n"
 
-  MiniPortileCMake.new(name, version).tap do |recipe|
-    recipe.host = target_host
-    target_dir = File.join(PACKAGE_ROOT_DIR, "ports")
-    # Ensure x64-mingw-ucrt and x64-mingw32 use different library paths since the host
-    # is the same (x86_64-w64-mingw32).
-    target_dir = File.join(target_dir, target_arch) if cross_build_p
-    recipe.target = target_dir
-
-    recipe.configure_options += [
-      # abseil needs a C++14 compiler
-      '-DCMAKE_CXX_STANDARD=17',
-      # needed for building the C extension shared library with -fPIC
-      '-DCMAKE_POSITION_INDEPENDENT_CODE=ON',
-      # ensures pkg-config and installed libraries will be in lib, not lib64
-      '-DCMAKE_INSTALL_LIBDIR=lib'
-    ]
-
-    yield recipe
-
-    checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed"
-
-    if File.exist?(checkpoint)
-      message("Building re2 with a packaged version of #{name}-#{version}.\n")
-    else
-      message(<<~EOM)
+  recipe.host = target_host
+  # Ensure x64-mingw-ucrt and x64-mingw32 use different library paths since the host
+  # is the same (x86_64-w64-mingw32).
+  recipe.target = File.join(recipe.target, target_arch) if cross_build_p
+
+  yield recipe
+
+  checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed"
+  name = recipe.name
+  version = recipe.version
+
+  if File.exist?(checkpoint)
+    message("Building re2 with a packaged version of #{name}-#{version}.\n")
+  else
+    message(<<~EOM)
         ---------- IMPORTANT NOTICE ----------
         Building re2 with a packaged version of #{name}-#{version}.
         Configuration options: #{recipe.configure_options.shelljoin}
       EOM
 
-      unless recipe.patch_files.empty?
-        message("The following patches are being applied:\n")
+    unless recipe.patch_files.empty?
+      message("The following patches are being applied:\n")
 
-        recipe.patch_files.each do |patch|
-          message("  - %s\n" % File.basename(patch))
-        end
+      recipe.patch_files.each do |patch|
+        message("  - %s\n" % File.basename(patch))
       end
-
-      # Use a temporary base directory to reduce filename lengths since
-      # Windows can hit a limit of 250 characters (CMAKE_OBJECT_PATH_MAX).
-      with_temp_dir { recipe.cook }
-
-      FileUtils.touch(checkpoint)
     end
 
-    recipe.activate
+    # Use a temporary base directory to reduce filename lengths since
+    # Windows can hit a limit of 250 characters (CMAKE_OBJECT_PATH_MAX).
+    with_temp_dir { recipe.cook }
+
+    FileUtils.touch(checkpoint)
   end
+
+  recipe.activate
 end
 
 def build_with_system_libraries
@@ -359,23 +340,15 @@ def add_static_ldflags(flags)
 def build_with_vendored_libraries
   message "Building re2 using packaged libraries.\n"
 
-  require 'yaml'
-  dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml'))
+  abseil_recipe, re2_recipe = load_recipes
 
-  abseil_recipe = process_recipe('abseil', dependencies['abseil']['version']) do |recipe|
-    recipe.files = [{
-      url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz",
-      sha256: dependencies['abseil']['sha256']
-    }]
+  process_recipe(abseil_recipe) do |recipe|
     recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON', '-DCMAKE_CXX_VISIBILITY_PRESET=hidden']
   end
 
-  re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe|
-    recipe.files = [{
-      url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz",
-      sha256: dependencies['libre2']['sha256']
-    }]
-    recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG', '-DCMAKE_CXX_VISIBILITY_PRESET=hidden']
+  process_recipe(re2_recipe) do |recipe|
+    recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG',
+                                 '-DCMAKE_CXX_VISIBILITY_PRESET=hidden']
   end
 
   dir_config("re2", File.join(re2_recipe.path, 'include'), File.join(re2_recipe.path, 'lib'))

ext/re2/recipes.rb

@@ -0,0 +1,43 @@
+PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+REQUIRED_MINI_PORTILE_VERSION = '~> 2.8.4' # keep this version in sync with the one in the gemspec
+
+def build_recipe(name, version)
+  require 'rubygems'
+  gem('mini_portile2', REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time
+  require 'mini_portile2'
+
+  MiniPortileCMake.new(name, version).tap do |recipe|
+    recipe.target = File.join(PACKAGE_ROOT_DIR, 'ports')
+    recipe.configure_options += [
+      # abseil needs a C++14 compiler
+      '-DCMAKE_CXX_STANDARD=17',
+      # needed for building the C extension shared library with -fPIC
+      '-DCMAKE_POSITION_INDEPENDENT_CODE=ON',
+      # ensures pkg-config and installed libraries will be in lib, not lib64
+      '-DCMAKE_INSTALL_LIBDIR=lib'
+    ]
+
+    yield recipe
+  end
+end
+
+def load_recipes
+  require 'yaml'
+  dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml'))
+
+  abseil_recipe = build_recipe('abseil', dependencies['abseil']['version']) do |recipe|
+    recipe.files = [{
+      url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz",
+      sha256: dependencies['abseil']['sha256']
+    }]
+  end
+
+  re2_recipe = build_recipe('libre2', dependencies['libre2']['version']) do |recipe|
+    recipe.files = [{
+      url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz",
+      sha256: dependencies['libre2']['sha256']
+    }]
+  end
+
+  [abseil_recipe, re2_recipe]
+end

re2.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |s|
     "dependencies.yml",
     "ext/re2/extconf.rb",
     "ext/re2/re2.cc",
+    "ext/re2/recipes.rb",
     "Gemfile",
     "lib/re2.rb",
     "lib/re2/scanner.rb",

scripts/test-gem-build

@@ -20,8 +20,7 @@ bundle install --local || bundle install
 bundle exec rake set-version-to-timestamp
 
 if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then
-  # TODO we're only compiling so that we retrieve tarballs, we can do better.
-  bundle exec rake clean compile
+  bundle exec rake clean
   bundle exec rake gem
 else
   bundle exec rake gem:${BUILD_NATIVE_GEM}:builder