4 files changed, 715 insertions, 0 deletions
diff --git a/ext/psych/extconf.rb b/ext/psych/extconf.rb
index 05e3f4a..02e0b7a 100644
--- a/ext/psych/extconf.rb
+++ b/ext/psych/extconf.rb
@@ -9,6 +9,14 @@ end
 yaml_source = with_config("libyaml-source-dir") || enable_config("bundled-libyaml", false)
 if yaml_source == true
   yaml_source = Dir.glob("#{$srcdir}/yaml{,-*}/").max_by {|n| File.basename(n).scan(/\d+/).map(&:to_i)}
+  unless yaml_source
+    require_relative '../../tool/extlibs.rb'
+    extlibs = ExtLibs.new(cache_dir: File.expand_path("../../tmp/download_cache", $srcdir))
+    unless extlibs.process_under($srcdir)
+      raise "failed to download libyaml source"
+    end
+    yaml_source, = Dir.glob("#{$srcdir}/yaml-*/")
+  end
 elsif yaml_source
   yaml_source = yaml_source.gsub(/\$\((\w+)\)|\$\{(\w+)\}/) {ENV[$1||$2]}
 end
diff --git a/ext/psych/extlibs b/ext/psych/extlibs
new file mode 100644
index 0000000..b4ac02e
--- /dev/null
+++ b/ext/psych/extlibs
@@ -0,0 +1,7 @@
+ver = 0.2.5
+
+https://github.com/yaml/libyaml/releases/download/$(ver)/yaml-$(ver).tar.gz \
+	rmd160:cc175ed640046722fb7790de828002633407b6b9 \
+	sha256:c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 \
+	sha512:dadd7d8e0d88b5ebab005e5d521d56d541580198aa497370966b98c904586e642a1cd4f3881094eb57624f218d50db77417bbfd0ffdce50340f011e35e8c4c02 \
+	#
diff --git a/tool/downloader.rb b/tool/downloader.rb
new file mode 100644
index 0000000..d3a9f75
--- /dev/null
+++ b/tool/downloader.rb
@@ -0,0 +1,415 @@
+# Used by configure and make to download or update mirrored Ruby and GCC
+# files. This will use HTTPS if possible, falling back to HTTP.
+
+require 'fileutils'
+require 'open-uri'
+require 'pathname'
+begin
+  require 'net/https'
+rescue LoadError
+  https = 'http'
+else
+  https = 'https'
+
+  # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old
+  # versions do not.  so, patching OpenSSL::X509::Store#add_file instead.
+  class OpenSSL::X509::Store
+    alias orig_add_file add_file
+    def add_file(pems)
+      Array(pems).each do |pem|
+        if File.directory?(pem)
+          add_path pem
+        else
+          orig_add_file pem
+        end
+      end
+    end
+  end
+  # since open-uri internally checks ssl_ca_cert using File.directory?,
+  # allow to accept an array.
+  class <<File
+    alias orig_directory? directory?
+    def File.directory? files
+      files.is_a?(Array) ? false : orig_directory?(files)
+    end
+  end
+end
+
+class Downloader
+  def self.https=(https)
+    @@https = https
+  end
+
+  def self.https?
+    @@https == 'https'
+  end
+
+  def self.https
+    @@https
+  end
+
+  class GNU < self
+    def self.download(name, *rest)
+      if https?
+        begin
+          super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest)
+        rescue => e
+          STDERR.puts "Download failed (#{e.message}), try another URL"
+          super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest)
+        end
+      else
+        super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest)
+      end
+    end
+  end
+
+  class RubyGems < self
+    def self.download(name, dir = nil, since = true, options = {})
+      require 'rubygems'
+      options = options.dup
+      options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__)))
+      super("https://rubygems.org/downloads/#{name}", name, dir, since, options)
+    end
+  end
+
+  Gems = RubyGems
+
+  class Unicode < self
+    INDEX = {}  # cache index file information across files in the same directory
+    UNICODE_PUBLIC = "https://www.unicode.org/Public/"
+
+    def self.download(name, dir = nil, since = true, options = {})
+      options = options.dup
+      unicode_beta = options.delete(:unicode_beta)
+      name_dir_part = name.sub(/[^\/]+$/, '')
+      if unicode_beta == 'YES'
+        if INDEX.size == 0
+          index_options = options.dup
+          index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file
+          index_data = File.read(under(dir, "index.html")) rescue nil
+          index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options)
+          INDEX[:index] = File.read(index_file)
+          since = true unless INDEX[:index] == index_data
+        end
+        file_base = File.basename(name, '.txt')
+        return if file_base == '.' # Use pre-generated headers and tables
+        beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/]
+        # make sure we always check for new versions of files,
+        # because they can easily change in the beta period
+        super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options)
+      else
+        index_file = Pathname.new(under(dir, name_dir_part+'index.html'))
+        if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/
+          raise "Although Unicode is not in beta, file #{index_file} exists. " +
+                "Remove all files in this directory and in .downloaded-cache/ " +
+                "because they may be leftovers from the beta period."
+        end
+        super(UNICODE_PUBLIC+name, name, dir, since, options)
+      end
+    end
+  end
+
+  def self.mode_for(data)
+    /\A#!/ =~ data ? 0755 : 0644
+  end
+
+  def self.http_options(file, since)
+    options = {}
+    if since
+      case since
+      when true
+        since = (File.mtime(file).httpdate rescue nil)
+      when Time
+        since = since.httpdate
+      end
+      if since
+        options['If-Modified-Since'] = since
+      end
+    end
+    options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content
+    options
+  end
+
+  def self.httpdate(date)
+    Time.httpdate(date)
+  rescue ArgumentError => e
+    # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4
+    # digit year, sometimes.
+    /\A\s*
+     (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20
+     (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20
+     (\d\d):(\d\d):(\d\d)\x20
+     GMT
+     \s*\z/ix =~ date or raise
+    warn e.message
+    Time.utc($3, $2, $1, $4, $5, $6)
+  end
+
+  # Downloader.download(url, name, [dir, [since]])
+  #
+  # Update a file from url if newer version is available.
+  # Creates the file if the file doesn't yet exist; however, the
+  # directory where the file is being created has to exist already.
+  # The +since+ parameter can take the following values, with associated meanings:
+  #  true ::
+  #    Take the last-modified time of the current file on disk, and only download
+  #    if the server has a file that was modified later. Download unconditionally
+  #    if we don't have the file yet. Default.
+  #  +some time value+ ::
+  #    Use this time value instead of the time of modification of the file on disk.
+  #  nil ::
+  #    Only download the file if it doesn't exist yet.
+  #  false ::
+  #    always download url regardless of whether we already have a file,
+  #    and regardless of modification times. (This is essentially just a waste of
+  #    network resources, except in the case that the file we have is somehow damaged.
+  #    Please note that using this recurringly might create or be seen as a
+  #    denial of service attack.)
+  #
+  # Example usage:
+  #   download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt',
+  #            'UnicodeData.txt', 'enc/unicode/data'
+  def self.download(url, name, dir = nil, since = true, options = {})
+    options = options.dup
+    url = URI(url)
+    dryrun = options.delete(:dryrun)
+    options.delete(:unicode_beta) # just to be on the safe side for gems and gcc
+
+    if name
+      file = Pathname.new(under(dir, name))
+    else
+      name = File.basename(url.path)
+    end
+    cache_save = options.delete(:cache_save) {
+      ENV["CACHE_SAVE"] != "no"
+    }
+    cache = cache_file(url, name, options.delete(:cache_dir))
+    file ||= cache
+    if since.nil? and file.exist?
+      if $VERBOSE
+        $stdout.puts "#{file} already exists"
+        $stdout.flush
+      end
+      if cache_save
+        save_cache(cache, file, name)
+      end
+      return file.to_path
+    end
+    if dryrun
+      puts "Download #{url} into #{file}"
+      return
+    end
+    if link_cache(cache, file, name, $VERBOSE)
+      return file.to_path
+    end
+    if !https? and URI::HTTPS === url
+      warn "*** using http instead of https ***"
+      url.scheme = 'http'
+      url = URI(url.to_s)
+    end
+    if $VERBOSE
+      $stdout.print "downloading #{name} ... "
+      $stdout.flush
+    end
+    mtime = nil
+    options = options.merge(http_options(file, since.nil? ? true : since))
+    begin
+      data = with_retry(10) do
+        data = url.read(options)
+        if mtime = data.meta["last-modified"]
+          mtime = Time.httpdate(mtime)
+        end
+        data
+      end
+    rescue OpenURI::HTTPError => http_error
+      if http_error.message =~ /^304 / # 304 Not Modified
+        if $VERBOSE
+          $stdout.puts "#{name} not modified"
+          $stdout.flush
+        end
+        return file.to_path
+      end
+      raise
+    rescue Timeout::Error
+      if since.nil? and file.exist?
+        puts "Request for #{url} timed out, using old version."
+        return file.to_path
+      end
+      raise
+    rescue SocketError
+      if since.nil? and file.exist?
+        puts "No network connection, unable to download #{url}, using old version."
+        return file.to_path
+      end
+      raise
+    end
+    dest = (cache_save && cache && !cache.exist? ? cache : file)
+    dest.parent.mkpath
+    dest.open("wb", 0600) do |f|
+      f.write(data)
+      f.chmod(mode_for(data))
+    end
+    if mtime
+      dest.utime(mtime, mtime)
+    end
+    if $VERBOSE
+      $stdout.puts "done"
+      $stdout.flush
+    end
+    if dest.eql?(cache)
+      link_cache(cache, file, name)
+    elsif cache_save
+      save_cache(cache, file, name)
+    end
+    return file.to_path
+  rescue => e
+    raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}"
+  end
+
+  def self.under(dir, name)
+    dir ? File.join(dir, File.basename(name)) : name
+  end
+
+  def self.cache_file(url, name, cache_dir = nil)
+    case cache_dir
+    when false
+      return nil
+    when nil
+      cache_dir = ENV['CACHE_DIR']
+      if !cache_dir or cache_dir.empty?
+        cache_dir = ".downloaded-cache"
+      end
+    end
+    Pathname.new(cache_dir) + (name || File.basename(URI(url).path))
+  end
+
+  def self.link_cache(cache, file, name, verbose = false)
+    return false unless cache and cache.exist?
+    return true if cache.eql?(file)
+    if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN']
+      begin
+        file.make_symlink(cache.relative_path_from(file.parent))
+      rescue SystemCallError
+      else
+        if verbose
+          $stdout.puts "made symlink #{name} to #{cache}"
+          $stdout.flush
+        end
+        return true
+      end
+    end
+    begin
+      file.make_link(cache)
+    rescue SystemCallError
+    else
+      if verbose
+        $stdout.puts "made link #{name} to #{cache}"
+        $stdout.flush
+      end
+      return true
+    end
+  end
+
+  def self.save_cache(cache, file, name)
+    return unless cache or cache.eql?(file)
+    begin
+      st = cache.stat
+    rescue
+      begin
+        file.rename(cache)
+      rescue
+        return
+      end
+    else
+      return unless st.mtime > file.lstat.mtime
+      file.unlink
+    end
+    link_cache(cache, file, name)
+  end
+
+  def self.with_retry(max_times, &block)
+    times = 0
+    begin
+      block.call
+    rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e
+      raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error
+      times += 1
+      if times <= max_times
+        $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..."
+        sleep(times ** 2)
+        retry
+      else
+        raise
+      end
+    end
+  end
+  private_class_method :with_retry
+end
+
+Downloader.https = https.freeze
+
+if $0 == __FILE__
+  since = true
+  options = {}
+  until ARGV.empty?
+    case ARGV[0]
+    when '-d'
+      destdir = ARGV[1]
+      ARGV.shift
+    when '-p'
+      # strip directory names from the name to download, and add the
+      # prefix instead.
+      prefix = ARGV[1]
+      ARGV.shift
+    when '-e'
+      since = nil
+    when '-a'
+      since = false
+    when '-n', '--dryrun'
+      options[:dryrun] = true
+    when '--cache-dir'
+      options[:cache_dir] = ARGV[1]
+      ARGV.shift
+    when '--unicode-beta'
+      options[:unicode_beta] = ARGV[1]
+      ARGV.shift
+    when /\A--cache-dir=(.*)/m
+      options[:cache_dir] = $1
+    when /\A-/
+      abort "#{$0}: unknown option #{ARGV[0]}"
+    else
+      break
+    end
+    ARGV.shift
+  end
+  dl = Downloader.constants.find do |name|
+    ARGV[0].casecmp(name.to_s) == 0
+  end unless ARGV.empty?
+  $VERBOSE = true
+  if dl
+    dl = Downloader.const_get(dl)
+    ARGV.shift
+    ARGV.each do |name|
+      dir = destdir
+      if prefix
+        name = name.sub(/\A\.\//, '')
+        destdir2 = destdir.sub(/\A\.\//, '')
+        if name.start_with?(destdir2+"/")
+          name = name[(destdir2.size+1)..-1]
+          if (dir = File.dirname(name)) == '.'
+            dir = destdir
+          else
+            dir = File.join(destdir, dir)
+          end
+        else
+          name = File.basename(name)
+        end
+        name = "#{prefix}/#{name}"
+      end
+      dl.download(name, dir, since, options)
+    end
+  else
+    abort "usage: #{$0} url name" unless ARGV.size == 2
+    Downloader.download(ARGV[0], ARGV[1], destdir, since, options)
+  end
+end
diff --git a/tool/extlibs.rb b/tool/extlibs.rb
new file mode 100755
index 0000000..5d1a2db
--- /dev/null
+++ b/tool/extlibs.rb
@@ -0,0 +1,285 @@
+#!/usr/bin/ruby
+
+# Used to download, extract and patch extension libraries (extlibs)
+# for Ruby. See common.mk for Ruby's usage.
+
+require 'digest'
+require_relative 'downloader'
+begin
+  require_relative 'lib/colorize'
+rescue LoadError
+end
+
+class ExtLibs
+  unless defined?(Colorize)
+    class Colorize
+      def pass(str) str; end
+      def fail(str) str; end
+    end
+  end
+
+  class Vars < Hash
+    def pattern
+      /\$\((#{Regexp.union(keys)})\)/
+    end
+
+    def expand(str)
+      if empty?
+        str
+      else
+        str.gsub(pattern) {self[$1]}
+      end
+    end
+  end
+
+  def initialize(mode = :all, cache_dir: nil)
+    @mode = mode
+    @cache_dir = cache_dir
+    @colorize = Colorize.new
+  end
+
+  def cache_file(url, cache_dir)
+    Downloader.cache_file(url, nil, cache_dir).to_path
+  end
+
+  def do_download(url, cache_dir)
+    Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir)
+  end
+
+  def do_checksum(cache, chksums)
+    chksums.each do |sum|
+      name, sum = sum.split(/:/)
+      if $VERBOSE
+        $stdout.print "checking #{name} of #{cache} ..."
+        $stdout.flush
+      end
+      hd = Digest(name.upcase).file(cache).hexdigest
+      if $VERBOSE
+        $stdout.print " "
+        $stdout.puts hd == sum ? @colorize.pass("OK") : @colorize.fail("NG")
+        $stdout.flush
+      end
+      unless hd == sum
+        raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}"
+      end
+    end
+  end
+
+  def do_extract(cache, dir)
+    if $VERBOSE
+      $stdout.puts "extracting #{cache} into #{dir}"
+      $stdout.flush
+    end
+    ext = File.extname(cache)
+    case ext
+    when '.gz', '.tgz'
+      f = IO.popen(["gzip", "-dc", cache])
+      cache = cache.chomp('.gz')
+    when '.bz2', '.tbz'
+      f = IO.popen(["bzip2", "-dc", cache])
+      cache = cache.chomp('.bz2')
+    when '.xz', '.txz'
+      f = IO.popen(["xz", "-dc", cache])
+      cache = cache.chomp('.xz')
+    else
+      inp = cache
+    end
+    inp ||= f.binmode
+    ext = File.extname(cache)
+    case ext
+    when '.tar', /\A\.t[gbx]z\z/
+      pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir)
+    when '.zip'
+      pid = Process.spawn("unzip", inp, "-d", dir)
+    end
+    f.close if f
+    Process.wait(pid)
+    $?.success? or raise "failed to extract #{cache}"
+  end
+
+  def do_patch(dest, patch, args)
+    if $VERBOSE
+      $stdout.puts "applying #{patch} under #{dest}"
+      $stdout.flush
+    end
+    Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args))
+    $?.success? or raise "failed to patch #{patch}"
+  end
+
+  def do_link(file, src, dest)
+    file = File.join(dest, file)
+    if (target = src).start_with?("/")
+      target = File.join([".."] * file.count("/"), src)
+    end
+    return unless File.exist?(File.expand_path(target, File.dirname(file)))
+    File.unlink(file) rescue nil
+    begin
+      File.symlink(target, file)
+    rescue
+    else
+      if $VERBOSE
+        $stdout.puts "linked #{target} to #{file}"
+        $stdout.flush
+      end
+      return
+    end
+    begin
+      src = src.sub(/\A\//, '')
+      File.copy_stream(src, file)
+    rescue
+      if $VERBOSE
+        $stdout.puts "failed to link #{src} to #{file}: #{$!.message}"
+      end
+    else
+      if $VERBOSE
+        $stdout.puts "copied #{src} to #{file}"
+      end
+    end
+  end
+
+  def do_exec(command, dir, dest)
+    dir = dir ? File.join(dest, dir) : dest
+    if $VERBOSE
+      $stdout.puts "running #{command.dump} under #{dir}"
+      $stdout.flush
+    end
+    system(command, chdir: dir) or raise "failed #{command.dump}"
+  end
+
+  def do_command(mode, dest, url, cache_dir, chksums)
+    extracted = false
+    base = /.*(?=\.tar(?:\.\w+)?\z)/
+
+    case mode
+    when :download
+      cache = do_download(url, cache_dir)
+      do_checksum(cache, chksums)
+    when :extract
+      cache = cache_file(url, cache_dir)
+      target = File.join(dest, File.basename(cache)[base])
+      unless File.directory?(target)
+        do_checksum(cache, chksums)
+        extracted = do_extract(cache, dest)
+      end
+    when :all
+      cache = do_download(url, cache_dir)
+      target = File.join(dest, File.basename(cache)[base])
+      unless File.directory?(target)
+        do_checksum(cache, chksums)
+        extracted = do_extract(cache, dest)
+      end
+    end
+    extracted
+  end
+
+  def process(list)
+    mode = @mode
+    cache_dir = @cache_dir
+    after_extract = (mode == :all or mode == :patch)
+    success = true
+    if $VERBOSE
+      $stdout.puts "downloading for #{list}"
+      $stdout.flush
+    end
+    vars = Vars.new
+    extracted = false
+    dest = File.dirname(list)
+    url = chksums = nil
+    IO.foreach(list) do |line|
+      line.sub!(/\s*#.*/, '')
+      if /^(\w+)\s*=\s*(.*)/ =~ line
+        vars[$1] = vars.expand($2)
+        next
+      end
+      if chksums
+        chksums.concat(line.split)
+      elsif /^\t/ =~ line
+        if extracted and after_extract
+          patch, *args = line.split.map {|s| vars.expand(s)}
+          do_patch(dest, patch, args)
+        end
+        next
+      elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line
+        if extracted and after_extract
+          command = vars.expand($2.strip)
+          chdir = $1 and chdir = vars.expand(chdir)
+          do_exec(command, chdir, dest)
+        end
+        next
+      elsif /->/ =~ line
+        if extracted and after_extract
+          link, file = $`.strip, $'.strip
+          do_link(vars.expand(link), vars.expand(file), dest)
+        end
+        next
+      else
+        url, *chksums = line.split(' ')
+      end
+      if chksums.last == '\\'
+        chksums.pop
+        next
+      end
+      unless url
+        chksums = nil
+        next
+      end
+      url = vars.expand(url)
+      begin
+        extracted = do_command(mode, dest, url, cache_dir, chksums)
+      rescue => e
+        warn e.full_message
+        success = false
+      end
+      url = chksums = nil
+    end
+    success
+  end
+
+  def process_under(dir)
+    success = true
+    Dir.glob("#{dir}/**/extlibs") do |list|
+      success &= process(list)
+    end
+    success
+  end
+
+  def self.run(argv)
+    cache_dir = nil
+    mode = :all
+    until argv.empty?
+      case argv[0]
+      when '--download'
+        mode = :download
+      when '--extract'
+        mode = :extract
+      when '--patch'
+        mode = :patch
+      when '--all'
+        mode = :all
+      when '--cache'
+        argv.shift
+        cache_dir = argv[0]
+      when /\A--cache=/
+        cache_dir = $'
+      when '--'
+        argv.shift
+        break
+      when /\A-/
+        warn "unknown option: #{argv[0]}"
+        return false
+      else
+        break
+      end
+      argv.shift
+    end
+
+    extlibs = new(mode, cache_dir: cache_dir)
+    argv.inject(true) do |success, dir|
+      success & extlibs.process_under(dir)
+    end
+  end
+end
+
+if $0 == __FILE__
+  exit ExtLibs.run(ARGV)
+end