diff options
author | Nobuyoshi Nakada <nobu@ruby-lang.org> | 2022-03-19 22:53:55 +0900 |
---|---|---|
committer | Nobuyoshi Nakada <nobu@ruby-lang.org> | 2022-03-27 19:39:01 +0900 |
commit | 03f809b8fae5b5fc3a2acc6cc278dc72aff75e2d (patch) | |
tree | 770e4e10e572e448f3a82ce5a38a126c0476bd38 | |
parent | c69ddddf0cbdcd72b1f1b1fbfed79686a0ae1dff (diff) | |
download | psych-03f809b8fae5b5fc3a2acc6cc278dc72aff75e2d.tar.gz |
Download libyaml source
-rw-r--r-- | ext/psych/extconf.rb | 8 | ||||
-rw-r--r-- | ext/psych/extlibs | 7 | ||||
-rw-r--r-- | tool/downloader.rb | 415 | ||||
-rwxr-xr-x | tool/extlibs.rb | 285 |
4 files changed, 715 insertions, 0 deletions
diff --git a/ext/psych/extconf.rb b/ext/psych/extconf.rb index 05e3f4a..02e0b7a 100644 --- a/ext/psych/extconf.rb +++ b/ext/psych/extconf.rb @@ -9,6 +9,14 @@ end yaml_source = with_config("libyaml-source-dir") || enable_config("bundled-libyaml", false) if yaml_source == true yaml_source = Dir.glob("#{$srcdir}/yaml{,-*}/").max_by {|n| File.basename(n).scan(/\d+/).map(&:to_i)} + unless yaml_source + require_relative '../../tool/extlibs.rb' + extlibs = ExtLibs.new(cache_dir: File.expand_path("../../tmp/download_cache", $srcdir)) + unless extlibs.process_under($srcdir) + raise "failed to download libyaml source" + end + yaml_source, = Dir.glob("#{$srcdir}/yaml-*/") + end elsif yaml_source yaml_source = yaml_source.gsub(/\$\((\w+)\)|\$\{(\w+)\}/) {ENV[$1||$2]} end diff --git a/ext/psych/extlibs b/ext/psych/extlibs new file mode 100644 index 0000000..b4ac02e --- /dev/null +++ b/ext/psych/extlibs @@ -0,0 +1,7 @@ +ver = 0.2.5 + +https://github.com/yaml/libyaml/releases/download/$(ver)/yaml-$(ver).tar.gz \ + rmd160:cc175ed640046722fb7790de828002633407b6b9 \ + sha256:c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 \ + sha512:dadd7d8e0d88b5ebab005e5d521d56d541580198aa497370966b98c904586e642a1cd4f3881094eb57624f218d50db77417bbfd0ffdce50340f011e35e8c4c02 \ + # diff --git a/tool/downloader.rb b/tool/downloader.rb new file mode 100644 index 0000000..d3a9f75 --- /dev/null +++ b/tool/downloader.rb @@ -0,0 +1,415 @@ +# Used by configure and make to download or update mirrored Ruby and GCC +# files. This will use HTTPS if possible, falling back to HTTP. + +require 'fileutils' +require 'open-uri' +require 'pathname' +begin + require 'net/https' +rescue LoadError + https = 'http' +else + https = 'https' + + # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old + # versions do not. so, patching OpenSSL::X509::Store#add_file instead. + class OpenSSL::X509::Store + alias orig_add_file add_file + def add_file(pems) + Array(pems).each do |pem| + if File.directory?(pem) + add_path pem + else + orig_add_file pem + end + end + end + end + # since open-uri internally checks ssl_ca_cert using File.directory?, + # allow to accept an array. + class <<File + alias orig_directory? directory? + def File.directory? files + files.is_a?(Array) ? false : orig_directory?(files) + end + end +end + +class Downloader + def self.https=(https) + @@https = https + end + + def self.https? + @@https == 'https' + end + + def self.https + @@https + end + + class GNU < self + def self.download(name, *rest) + if https? + begin + super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) + rescue => e + STDERR.puts "Download failed (#{e.message}), try another URL" + super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) + end + else + super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + end + end + end + + class RubyGems < self + def self.download(name, dir = nil, since = true, options = {}) + require 'rubygems' + options = options.dup + options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + end + end + + Gems = RubyGems + + class Unicode < self + INDEX = {} # cache index file information across files in the same directory + UNICODE_PUBLIC = "https://www.unicode.org/Public/" + + def self.download(name, dir = nil, since = true, options = {}) + options = options.dup + unicode_beta = options.delete(:unicode_beta) + name_dir_part = name.sub(/[^\/]+$/, '') + if unicode_beta == 'YES' + if INDEX.size == 0 + index_options = options.dup + index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + index_data = File.read(under(dir, "index.html")) rescue nil + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + INDEX[:index] = File.read(index_file) + since = true unless INDEX[:index] == index_data + end + file_base = File.basename(name, '.txt') + return if file_base == '.' # Use pre-generated headers and tables + beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] + # make sure we always check for new versions of files, + # because they can easily change in the beta period + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + else + index_file = Pathname.new(under(dir, name_dir_part+'index.html')) + if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ + raise "Although Unicode is not in beta, file #{index_file} exists. " + + "Remove all files in this directory and in .downloaded-cache/ " + + "because they may be leftovers from the beta period." + end + super(UNICODE_PUBLIC+name, name, dir, since, options) + end + end + end + + def self.mode_for(data) + /\A#!/ =~ data ? 0755 : 0644 + end + + def self.http_options(file, since) + options = {} + if since + case since + when true + since = (File.mtime(file).httpdate rescue nil) + when Time + since = since.httpdate + end + if since + options['If-Modified-Since'] = since + end + end + options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content + options + end + + def self.httpdate(date) + Time.httpdate(date) + rescue ArgumentError => e + # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4 + # digit year, sometimes. + /\A\s* + (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20 + (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20 + (\d\d):(\d\d):(\d\d)\x20 + GMT + \s*\z/ix =~ date or raise + warn e.message + Time.utc($3, $2, $1, $4, $5, $6) + end + + # Downloader.download(url, name, [dir, [since]]) + # + # Update a file from url if newer version is available. + # Creates the file if the file doesn't yet exist; however, the + # directory where the file is being created has to exist already. + # The +since+ parameter can take the following values, with associated meanings: + # true :: + # Take the last-modified time of the current file on disk, and only download + # if the server has a file that was modified later. Download unconditionally + # if we don't have the file yet. Default. + # +some time value+ :: + # Use this time value instead of the time of modification of the file on disk. + # nil :: + # Only download the file if it doesn't exist yet. + # false :: + # always download url regardless of whether we already have a file, + # and regardless of modification times. (This is essentially just a waste of + # network resources, except in the case that the file we have is somehow damaged. + # Please note that using this recurringly might create or be seen as a + # denial of service attack.) + # + # Example usage: + # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', + # 'UnicodeData.txt', 'enc/unicode/data' + def self.download(url, name, dir = nil, since = true, options = {}) + options = options.dup + url = URI(url) + dryrun = options.delete(:dryrun) + options.delete(:unicode_beta) # just to be on the safe side for gems and gcc + + if name + file = Pathname.new(under(dir, name)) + else + name = File.basename(url.path) + end + cache_save = options.delete(:cache_save) { + ENV["CACHE_SAVE"] != "no" + } + cache = cache_file(url, name, options.delete(:cache_dir)) + file ||= cache + if since.nil? and file.exist? + if $VERBOSE + $stdout.puts "#{file} already exists" + $stdout.flush + end + if cache_save + save_cache(cache, file, name) + end + return file.to_path + end + if dryrun + puts "Download #{url} into #{file}" + return + end + if link_cache(cache, file, name, $VERBOSE) + return file.to_path + end + if !https? and URI::HTTPS === url + warn "*** using http instead of https ***" + url.scheme = 'http' + url = URI(url.to_s) + end + if $VERBOSE + $stdout.print "downloading #{name} ... " + $stdout.flush + end + mtime = nil + options = options.merge(http_options(file, since.nil? ? true : since)) + begin + data = with_retry(10) do + data = url.read(options) + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end + data + end + rescue OpenURI::HTTPError => http_error + if http_error.message =~ /^304 / # 304 Not Modified + if $VERBOSE + $stdout.puts "#{name} not modified" + $stdout.flush + end + return file.to_path + end + raise + rescue Timeout::Error + if since.nil? and file.exist? + puts "Request for #{url} timed out, using old version." + return file.to_path + end + raise + rescue SocketError + if since.nil? and file.exist? + puts "No network connection, unable to download #{url}, using old version." + return file.to_path + end + raise + end + dest = (cache_save && cache && !cache.exist? ? cache : file) + dest.parent.mkpath + dest.open("wb", 0600) do |f| + f.write(data) + f.chmod(mode_for(data)) + end + if mtime + dest.utime(mtime, mtime) + end + if $VERBOSE + $stdout.puts "done" + $stdout.flush + end + if dest.eql?(cache) + link_cache(cache, file, name) + elsif cache_save + save_cache(cache, file, name) + end + return file.to_path + rescue => e + raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}" + end + + def self.under(dir, name) + dir ? File.join(dir, File.basename(name)) : name + end + + def self.cache_file(url, name, cache_dir = nil) + case cache_dir + when false + return nil + when nil + cache_dir = ENV['CACHE_DIR'] + if !cache_dir or cache_dir.empty? + cache_dir = ".downloaded-cache" + end + end + Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) + end + + def self.link_cache(cache, file, name, verbose = false) + return false unless cache and cache.exist? + return true if cache.eql?(file) + if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] + begin + file.make_symlink(cache.relative_path_from(file.parent)) + rescue SystemCallError + else + if verbose + $stdout.puts "made symlink #{name} to #{cache}" + $stdout.flush + end + return true + end + end + begin + file.make_link(cache) + rescue SystemCallError + else + if verbose + $stdout.puts "made link #{name} to #{cache}" + $stdout.flush + end + return true + end + end + + def self.save_cache(cache, file, name) + return unless cache or cache.eql?(file) + begin + st = cache.stat + rescue + begin + file.rename(cache) + rescue + return + end + else + return unless st.mtime > file.lstat.mtime + file.unlink + end + link_cache(cache, file, name) + end + + def self.with_retry(max_times, &block) + times = 0 + begin + block.call + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e + raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error + times += 1 + if times <= max_times + $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..." + sleep(times ** 2) + retry + else + raise + end + end + end + private_class_method :with_retry +end + +Downloader.https = https.freeze + +if $0 == __FILE__ + since = true + options = {} + until ARGV.empty? + case ARGV[0] + when '-d' + destdir = ARGV[1] + ARGV.shift + when '-p' + # strip directory names from the name to download, and add the + # prefix instead. + prefix = ARGV[1] + ARGV.shift + when '-e' + since = nil + when '-a' + since = false + when '-n', '--dryrun' + options[:dryrun] = true + when '--cache-dir' + options[:cache_dir] = ARGV[1] + ARGV.shift + when '--unicode-beta' + options[:unicode_beta] = ARGV[1] + ARGV.shift + when /\A--cache-dir=(.*)/m + options[:cache_dir] = $1 + when /\A-/ + abort "#{$0}: unknown option #{ARGV[0]}" + else + break + end + ARGV.shift + end + dl = Downloader.constants.find do |name| + ARGV[0].casecmp(name.to_s) == 0 + end unless ARGV.empty? + $VERBOSE = true + if dl + dl = Downloader.const_get(dl) + ARGV.shift + ARGV.each do |name| + dir = destdir + if prefix + name = name.sub(/\A\.\//, '') + destdir2 = destdir.sub(/\A\.\//, '') + if name.start_with?(destdir2+"/") + name = name[(destdir2.size+1)..-1] + if (dir = File.dirname(name)) == '.' + dir = destdir + else + dir = File.join(destdir, dir) + end + else + name = File.basename(name) + end + name = "#{prefix}/#{name}" + end + dl.download(name, dir, since, options) + end + else + abort "usage: #{$0} url name" unless ARGV.size == 2 + Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + end +end diff --git a/tool/extlibs.rb b/tool/extlibs.rb new file mode 100755 index 0000000..5d1a2db --- /dev/null +++ b/tool/extlibs.rb @@ -0,0 +1,285 @@ +#!/usr/bin/ruby + +# Used to download, extract and patch extension libraries (extlibs) +# for Ruby. See common.mk for Ruby's usage. + +require 'digest' +require_relative 'downloader' +begin + require_relative 'lib/colorize' +rescue LoadError +end + +class ExtLibs + unless defined?(Colorize) + class Colorize + def pass(str) str; end + def fail(str) str; end + end + end + + class Vars < Hash + def pattern + /\$\((#{Regexp.union(keys)})\)/ + end + + def expand(str) + if empty? + str + else + str.gsub(pattern) {self[$1]} + end + end + end + + def initialize(mode = :all, cache_dir: nil) + @mode = mode + @cache_dir = cache_dir + @colorize = Colorize.new + end + + def cache_file(url, cache_dir) + Downloader.cache_file(url, nil, cache_dir).to_path + end + + def do_download(url, cache_dir) + Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + end + + def do_checksum(cache, chksums) + chksums.each do |sum| + name, sum = sum.split(/:/) + if $VERBOSE + $stdout.print "checking #{name} of #{cache} ..." + $stdout.flush + end + hd = Digest(name.upcase).file(cache).hexdigest + if $VERBOSE + $stdout.print " " + $stdout.puts hd == sum ? @colorize.pass("OK") : @colorize.fail("NG") + $stdout.flush + end + unless hd == sum + raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" + end + end + end + + def do_extract(cache, dir) + if $VERBOSE + $stdout.puts "extracting #{cache} into #{dir}" + $stdout.flush + end + ext = File.extname(cache) + case ext + when '.gz', '.tgz' + f = IO.popen(["gzip", "-dc", cache]) + cache = cache.chomp('.gz') + when '.bz2', '.tbz' + f = IO.popen(["bzip2", "-dc", cache]) + cache = cache.chomp('.bz2') + when '.xz', '.txz' + f = IO.popen(["xz", "-dc", cache]) + cache = cache.chomp('.xz') + else + inp = cache + end + inp ||= f.binmode + ext = File.extname(cache) + case ext + when '.tar', /\A\.t[gbx]z\z/ + pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) + when '.zip' + pid = Process.spawn("unzip", inp, "-d", dir) + end + f.close if f + Process.wait(pid) + $?.success? or raise "failed to extract #{cache}" + end + + def do_patch(dest, patch, args) + if $VERBOSE + $stdout.puts "applying #{patch} under #{dest}" + $stdout.flush + end + Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args)) + $?.success? or raise "failed to patch #{patch}" + end + + def do_link(file, src, dest) + file = File.join(dest, file) + if (target = src).start_with?("/") + target = File.join([".."] * file.count("/"), src) + end + return unless File.exist?(File.expand_path(target, File.dirname(file))) + File.unlink(file) rescue nil + begin + File.symlink(target, file) + rescue + else + if $VERBOSE + $stdout.puts "linked #{target} to #{file}" + $stdout.flush + end + return + end + begin + src = src.sub(/\A\//, '') + File.copy_stream(src, file) + rescue + if $VERBOSE + $stdout.puts "failed to link #{src} to #{file}: #{$!.message}" + end + else + if $VERBOSE + $stdout.puts "copied #{src} to #{file}" + end + end + end + + def do_exec(command, dir, dest) + dir = dir ? File.join(dest, dir) : dest + if $VERBOSE + $stdout.puts "running #{command.dump} under #{dir}" + $stdout.flush + end + system(command, chdir: dir) or raise "failed #{command.dump}" + end + + def do_command(mode, dest, url, cache_dir, chksums) + extracted = false + base = /.*(?=\.tar(?:\.\w+)?\z)/ + + case mode + when :download + cache = do_download(url, cache_dir) + do_checksum(cache, chksums) + when :extract + cache = cache_file(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + when :all + cache = do_download(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + end + extracted + end + + def process(list) + mode = @mode + cache_dir = @cache_dir + after_extract = (mode == :all or mode == :patch) + success = true + if $VERBOSE + $stdout.puts "downloading for #{list}" + $stdout.flush + end + vars = Vars.new + extracted = false + dest = File.dirname(list) + url = chksums = nil + IO.foreach(list) do |line| + line.sub!(/\s*#.*/, '') + if /^(\w+)\s*=\s*(.*)/ =~ line + vars[$1] = vars.expand($2) + next + end + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and after_extract + patch, *args = line.split.map {|s| vars.expand(s)} + do_patch(dest, patch, args) + end + next + elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line + if extracted and after_extract + command = vars.expand($2.strip) + chdir = $1 and chdir = vars.expand(chdir) + do_exec(command, chdir, dest) + end + next + elsif /->/ =~ line + if extracted and after_extract + link, file = $`.strip, $'.strip + do_link(vars.expand(link), vars.expand(file), dest) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + unless url + chksums = nil + next + end + url = vars.expand(url) + begin + extracted = do_command(mode, dest, url, cache_dir, chksums) + rescue => e + warn e.full_message + success = false + end + url = chksums = nil + end + success + end + + def process_under(dir) + success = true + Dir.glob("#{dir}/**/extlibs") do |list| + success &= process(list) + end + success + end + + def self.run(argv) + cache_dir = nil + mode = :all + until argv.empty? + case argv[0] + when '--download' + mode = :download + when '--extract' + mode = :extract + when '--patch' + mode = :patch + when '--all' + mode = :all + when '--cache' + argv.shift + cache_dir = argv[0] + when /\A--cache=/ + cache_dir = $' + when '--' + argv.shift + break + when /\A-/ + warn "unknown option: #{argv[0]}" + return false + else + break + end + argv.shift + end + + extlibs = new(mode, cache_dir: cache_dir) + argv.inject(true) do |success, dir| + success & extlibs.process_under(dir) + end + end +end + +if $0 == __FILE__ + exit ExtLibs.run(ARGV) +end |