Class: Gem::Indexer (Ruby 2.4.2)
Description: Class : Gem::Indexer - Ruby 2.4.2

Support for the Ruby 2.4 series has ended. See here for reference.

### Home Classes Methods

### In Files

- rubygems/indexer.rb

### Parent

Object

### Methods

- ::new
- #build_indices
- #build_marshal_gemspecs
- #build_modern_index
- #build_modern_indices
- #compact_specs
- #compress
- #compress_indices
- #gem_file_list
- #generate_index
- #gzip
- #install_indices
- #make_temp_directories
- #map_gems_to_specs
- #paranoid
- #update_index
- #update_specs_index

### Included Modules

- Gem::UserInteraction

### Class/Module Index

Quicksearch

- Gem::AvailableSet
- Gem::BasicSpecification
- Gem::Command
- Gem::CommandLineError
- Gem::CommandManager
- Gem::Commands
- Gem::Commands::BuildCommand
- Gem::Commands::CertCommand
- Gem::Commands::CheckCommand
- Gem::Commands::CleanupCommand
- Gem::Commands::ContentsCommand
- Gem::Commands::DependencyCommand
- Gem::Commands::EnvironmentCommand
- Gem::Commands::FetchCommand
- Gem::Commands::GenerateIndexCommand
- Gem::Commands::HelpCommand
- Gem::Commands::InstallCommand
- Gem::Commands::ListCommand
- Gem::Commands::LockCommand
- Gem::Commands::MirrorCommand
- Gem::Commands::OpenCommand
- Gem::Commands::OutdatedCommand
- Gem::Commands::OwnerCommand
- Gem::Commands::PristineCommand
- Gem::Commands::PushCommand
- Gem::Commands::QueryCommand
- Gem::Commands::RdocCommand
- Gem::Commands::SearchCommand
- Gem::Commands::ServerCommand
- Gem::Commands::SetupCommand
- Gem::Commands::SourcesCommand
- Gem::Commands::SpecificationCommand
- Gem::Commands::StaleCommand
- Gem::Commands::UninstallCommand
- Gem::Commands::UnpackCommand
- Gem::Commands::UpdateCommand
- Gem::Commands::WhichCommand
- Gem::Commands::YankCommand
- Gem::ConfigFile
- Gem::ConflictError
- Gem::ConsoleUI
- Gem::DefaultUserInteraction
- Gem::Dependency
- Gem::DependencyError
- Gem::DependencyInstaller
- Gem::DependencyList
- Gem::DependencyRemovalException
- Gem::DependencyResolutionError
- Gem::Deprecate
- Gem::Doctor
- Gem::DocumentError
- Gem::EndOfYAMLException
- Gem::ErrorReason
- Gem::Exception
- Gem::Ext
- Gem::Ext::BuildError
- Gem::Ext::Builder
- Gem::Ext::CmakeBuilder
- Gem::Ext::ConfigureBuilder
- Gem::Ext::ExtConfBuilder
- Gem::Ext::RakeBuilder
- Gem::FakeFetcher
- Gem::FilePermissionError
- Gem::FormatException
- Gem::GemNotFoundException
- Gem::GemNotInHomeException
- Gem::GemRunner
- Gem::GemcutterUtilities
- Gem::ImpossibleDependenciesError
- Gem::Indexer
- Gem::InstallError
- Gem::InstallUpdateOptions
- Gem::Installer
- Gem::Installer::FakePackage
- Gem::InstallerTestCase
- Gem::InvalidSpecificationException
- Gem::Licenses
- Gem::List
- Gem::LoadError
- Gem::LocalRemoteOptions
- Gem::MissingSpecError
- Gem::MissingSpecVersionError
- Gem::MockGemUi
- Gem::MockGemUi::InputEOFError
- Gem::MockGemUi::SystemExitException
- Gem::MockGemUi::TTY
- Gem::MockGemUi::TermError
- Gem::NameTuple
- Gem::NoAliasYAMLTree
- Gem::OperationNotSupportedError
- Gem::Package::DigestIO
- Gem::Package::Error
- Gem::Package::FormatError
- Gem::Package::NonSeekableIO
- Gem::Package::Old
- Gem::Package::PathError
- Gem::Package::TarHeader
- Gem::Package::TarInvalidError
- Gem::Package::TarReader
- Gem::Package::TarReader::Entry
- Gem::Package::TarReader::UnexpectedEOF
- Gem::Package::TarTestCase
- Gem::Package::TarWriter
- Gem::Package::TarWriter::BoundedStream
- Gem::Package::TarWriter::FileOverflow
- Gem::Package::TarWriter::RestrictedStream
- Gem::Package::TooLongFileName
- Gem::PackageTask
- Gem::PathSupport
- Gem::Platform
- Gem::PlatformMismatch
- Gem::RemoteError
- Gem::RemoteFetcher
- Gem::RemoteFetcher::FetchError
- Gem::RemoteFetcher::UnknownHostError
- Gem::RemoteInstallationCancelled
- Gem::RemoteInstallationSkipped
- Gem::RemoteSourceException
- Gem::RequestSet
- Gem::RequestSet::GemDependencyAPI
- Gem::RequestSet::Lockfile
- Gem::RequestSet::Lockfile::ParseError
- Gem::RequestSet::Lockfile::Parser
- Gem::RequestSet::Lockfile::Tokenizer
- Gem::Requirement
- Gem::Requirement::BadRequirementError
- Gem::Resolver
- Gem::Resolver::APISet
- Gem::Resolver::APISpecification
- Gem::Resolver::ActivationRequest
- Gem::Resolver::BestSet
- Gem::Resolver::ComposedSet
- Gem::Resolver::Conflict
- Gem::Resolver::CurrentSet
- Gem::Resolver::DependencyRequest
- Gem::Resolver::GitSet
- Gem::Resolver::GitSpecification
- Gem::Resolver::IndexSet
- Gem::Resolver::IndexSpecification
- Gem::Resolver::InstalledSpecification
- Gem::Resolver::InstallerSet
- Gem::Resolver::LocalSpecification
- Gem::Resolver::LockSet
- Gem::Resolver::LockSpecification
- Gem::Resolver::Molinillo
- Gem::Resolver::Molinillo::CircularDependencyError
- Gem::Resolver::Molinillo::Delegates
- Gem::Resolver::Molinillo::Delegates::ResolutionState
- Gem::Resolver::Molinillo::Delegates::SpecificationProvider
- Gem::Resolver::Molinillo::DependencyGraph
- Gem::Resolver::Molinillo::DependencyGraph::Action
- Gem::Resolver::Molinillo::DependencyGraph::AddEdgeNoCircular
- Gem::Resolver::Molinillo::DependencyGraph::DeleteEdge
- Gem::Resolver::Molinillo::DependencyGraph::DetachVertexNamed
- Gem::Resolver::Molinillo::DependencyGraph::Log
- Gem::Resolver::Molinillo::DependencyGraph::Tag
- Gem::Resolver::Molinillo::DependencyGraph::Vertex
- Gem::Resolver::Molinillo::DependencyState
- Gem::Resolver::Molinillo::NoSuchDependencyError
- Gem::Resolver::Molinillo::PossibilityState
- Gem::Resolver::Molinillo::ResolutionState
- Gem::Resolver::Molinillo::Resolver
- Gem::Resolver::Molinillo::Resolver::Resolution
- Gem::Resolver::Molinillo::ResolverError
- Gem::Resolver::Molinillo::SpecificationProvider
- Gem::Resolver::Molinillo::UI
- Gem::Resolver::Molinillo::VersionConflict
- Gem::Resolver::RequirementList
- Gem::Resolver::Set
- Gem::Resolver::SourceSet
- Gem::Resolver::SpecSpecification
- Gem::Resolver::Specification
- Gem::Resolver::Stats
- Gem::Resolver::VendorSet
- Gem::Resolver::VendorSpecification
- Gem::RubyVersionMismatch
- Gem::Security
- Gem::Security::Exception
- Gem::Security::Policy
- Gem::Security::Signer
- Gem::Security::TrustDir
- Gem::Server
- Gem::SilentUI
- Gem::Source
- Gem::Source::Git
- Gem::Source::Installed
- Gem::Source::Local
- Gem::Source::Lock
- Gem::Source::SpecificFile
- Gem::Source::Vendor
- Gem::SourceFetchProblem
- Gem::SourceList
- Gem::SpecFetcher
- Gem::SpecificGemNotFoundException
- Gem::Specification
- Gem::StreamUI
- Gem::StreamUI::SilentDownloadReporter
- Gem::StreamUI::SilentProgressReporter
- Gem::StreamUI::SimpleProgressReporter
- Gem::StreamUI::VerboseDownloadReporter
- Gem::StreamUI::VerboseProgressReporter
- Gem::StubSpecification
- Gem::SystemExitException
- Gem::TestCase
- Gem::TestCase::SpecFetcherSetup
- Gem::TestCase::StaticSet
- Gem::Text
- Gem::Uninstaller
- Gem::UnsatisfiableDependencyError
- Gem::UriFormatter
- Gem::UserInteraction
- Gem::Util
- Gem::Validator
- Gem::VerificationError
- Gem::Version
- Gem::VersionOption
- Kernel
- Object
- TempIO
- YAML::Syck
- YAML::Syck

No matching classes.

# Gem::Indexer

Top level class for building the gem repository index.

### Attributes

build_modern[RW]

Build indexes for RubyGems 1.2.0 and newer when true

dest_directory[R]

Index install location

dest_latest_specs_index[R]

Latest specs index install location

dest_prerelease_specs_index[R]

Prerelease specs index install location

dest_specs_index[R]

Specs index install location

directory[R]

Index build directory

### Public Class Methods

new(directory, options = {}) click to toggle source

Create an indexer that will index the gems in `directory`.

                   # File rubygems/indexer.rb, line 52
    def initialize(directory, options = {})
      require 'fileutils'
      require 'tmpdir'
      require 'zlib'

      unless defined?(Builder::XChar) then
        raise "Gem::Indexer requires that the XML Builder library be installed:" +
              "\n\tgem install builder"
      end

      options = { :build_modern => true }.merge options

      @build_modern = options[:build_modern]

      @dest_directory = directory
      @directory = File.join(Dir.tmpdir, "gem_generate_index_#{$$}")

      marshal_name = "Marshal.#{Gem.marshal_version}"

      @master_index = File.join @directory, 'yaml'
      @marshal_index = File.join @directory, marshal_name

      @quick_dir = File.join @directory, 'quick'
      @quick_marshal_dir = File.join @quick_dir, marshal_name
      @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH

      @quick_index = File.join @quick_dir, 'index'
      @latest_index = File.join @quick_dir, 'latest_index'

      @specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
      @latest_specs_index =
        File.join(@directory, "latest_specs.#{Gem.marshal_version}")
      @prerelease_specs_index =
        File.join(@directory, "prerelease_specs.#{Gem.marshal_version}")
      @dest_specs_index =
        File.join(@dest_directory, "specs.#{Gem.marshal_version}")
      @dest_latest_specs_index =
        File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}")
      @dest_prerelease_specs_index =
        File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}")

      @files = []
    end

### Public Instance Methods

build_indices() click to toggle source

Build various indices

                   # File rubygems/indexer.rb, line 99
    def build_indices
      specs = map_gems_to_specs gem_file_list
      Gem::Specification._resort! specs
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern

      compress_indices
    end

build_marshal_gemspecs(specs) click to toggle source

Builds Marshal quick index gemspecs.

                   # File rubygems/indexer.rb, line 111
    def build_marshal_gemspecs specs
      count = specs.count
      progress = ui.progress_reporter count,
                                      "Generating Marshal quick index gemspecs for #{count} gems",
                                      "Complete"

      files = []

      Gem.time 'Generated Marshal quick index gemspecs' do
        specs.each do |spec|
          next if spec.default_gem?
          spec_file_name = "#{spec.original_name}.gemspec.rz"
          marshal_name = File.join @quick_marshal_dir, spec_file_name

          marshal_zipped = Gem.deflate Marshal.dump(spec)
          open marshal_name, 'wb' do |io| io.write marshal_zipped end

          files << marshal_name

          progress.updated spec.original_name
        end

        progress.done
      end

      @files << @quick_marshal_dir

      files
    end

build_modern_index(index, file, name) click to toggle source

Build a single index for RubyGems 1.2 and newer

                   # File rubygems/indexer.rb, line 144
    def build_modern_index(index, file, name)
      say "Generating #{name} index"

      Gem.time "Generated #{name} index" do
        open(file, 'wb') do |io|
          specs = index.map do |*spec|
            # We have to splat here because latest_specs is an array, while the
            # others are hashes.
            spec = spec.flatten.last
            platform = spec.original_platform

            # win32-api-1.0.4-x86-mswin32-60
            unless String === platform then
              alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
              next
            end

            platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
            [spec.name, spec.version, platform]
          end

          specs = compact_specs(specs)
          Marshal.dump(specs, io)
        end
      end
    end

build_modern_indices(specs) click to toggle source

Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease

                   # File rubygems/indexer.rb, line 174
    def build_modern_indices specs
      prerelease, released = specs.partition { |s|
        s.version.prerelease?
      }
      latest_specs =
        Gem::Specification._latest_specs specs

      build_modern_index(released.sort, @specs_index, 'specs')
      build_modern_index(latest_specs.sort, @latest_specs_index, 'latest specs')
      build_modern_index(prerelease.sort, @prerelease_specs_index,
                         'prerelease specs')

      @files += [@specs_index,
                 "#{@specs_index}.gz",
                 @latest_specs_index,
                 "#{@latest_specs_index}.gz",
                 @prerelease_specs_index,
                 "#{@prerelease_specs_index}.gz"]
    end

compact_specs(specs) click to toggle source

Compacts Marshal output for the specs index data source by using identical objects as much as possible.

                   # File rubygems/indexer.rb, line 242
    def compact_specs(specs)
      names = {}
      versions = {}
      platforms = {}

      specs.map do |(name, version, platform)|
        names[name] = name unless names.include? name
        versions[version] = version unless versions.include? version
        platforms[platform] = platform unless platforms.include? platform

        [names[name], versions[version], platforms[platform]]
      end
    end

compress(filename, extension) click to toggle source

Compress `filename` with `extension`.

                   # File rubygems/indexer.rb, line 259
    def compress(filename, extension)
      data = Gem.read_binary filename

      zipped = Gem.deflate data

      open "#{filename}.#{extension}", 'wb' do |io|
        io.write zipped
      end
    end

compress_indices() click to toggle source

Compresses indices on disk

                   # File rubygems/indexer.rb, line 226
    def compress_indices
      say "Compressing indices"

      Gem.time 'Compressed indices' do
        if @build_modern then
          gzip @specs_index
          gzip @latest_specs_index
          gzip @prerelease_specs_index
        end
      end
    end

gem_file_list() click to toggle source

List of gem file names to index.

                   # File rubygems/indexer.rb, line 272
    def gem_file_list
      Dir[File.join(@dest_directory, "gems", '*.gem')]
    end

generate_index() click to toggle source

Builds and installs indices.

                   # File rubygems/indexer.rb, line 279
    def generate_index
      make_temp_directories
      build_indices
      install_indices
    rescue SignalException
    ensure
      FileUtils.rm_rf @directory
    end

gzip(filename) click to toggle source

Zlib::GzipWriter wrapper that gzips `filename` on disk.

                   # File rubygems/indexer.rb, line 291
    def gzip(filename)
      Zlib::GzipWriter.open "#{filename}.gz" do |io|
        io.write Gem.read_binary(filename)
      end
    end

install_indices() click to toggle source

Install generated indices into the destination directory.

                   # File rubygems/indexer.rb, line 300
    def install_indices
      verbose = Gem.configuration.really_verbose

      say "Moving index into production dir #{@dest_directory}" if verbose

      files = @files
      files.delete @quick_marshal_dir if files.include? @quick_dir

      if files.include? @quick_marshal_dir and not files.include? @quick_dir then
        files.delete @quick_marshal_dir

        dst_name = File.join(@dest_directory, @quick_marshal_dir_base)

        FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose
        FileUtils.rm_rf dst_name, :verbose => verbose
        FileUtils.mv(@quick_marshal_dir, dst_name,
                     :verbose => verbose, :force => true)
      end

      files = files.map do |path|
        path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
      end

      files.each do |file|
        src_name = File.join @directory, file
        dst_name = File.join @dest_directory, file

        FileUtils.rm_rf dst_name, :verbose => verbose
        FileUtils.mv(src_name, @dest_directory,
                     :verbose => verbose, :force => true)
      end
    end

make_temp_directories() click to toggle source

Make directories for index generation

                   # File rubygems/indexer.rb, line 336
    def make_temp_directories
      FileUtils.rm_rf @directory
      FileUtils.mkdir_p @directory, :mode => 0700
      FileUtils.mkdir_p @quick_marshal_dir
    end

map_gems_to_specs(gems) click to toggle source

                   # File rubygems/indexer.rb, line 194
    def map_gems_to_specs gems
      gems.map { |gemfile|
        if File.size(gemfile) == 0 then
          alert_warning "Skipping zero-length gem: #{gemfile}"
          next
        end

        begin
          spec = Gem::Package.new(gemfile).spec
          spec.loaded_from = gemfile

          spec.abbreviate
          spec.sanitize

          spec
        rescue SignalException
          alert_error "Received signal, exiting"
          raise
        rescue Exception => e
          msg = ["Unable to process #{gemfile}",
                 "#{e.message} (#{e.class})",
                 "\t#{e.backtrace.join "\n\t"}"].join("\n")
          alert_error msg
        end
      }.compact
    end

paranoid(path, extension) click to toggle source

Ensure `path` and path with `extension` are identical.

                   # File rubygems/indexer.rb, line 345
    def paranoid(path, extension)
      data = Gem.read_binary path
      compressed_data = Gem.read_binary "#{path}.#{extension}"

      unless data == Gem.inflate(compressed_data) then
        raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
      end
    end

update_index() click to toggle source

Perform an in-place update of the repository from newly added gems.

                   # File rubygems/indexer.rb, line 357
    def update_index
      make_temp_directories

      specs_mtime = File.stat(@dest_specs_index).mtime
      newest_mtime = Time.at 0

      updated_gems = gem_file_list.select do |gem|
        gem_mtime = File.stat(gem).mtime
        newest_mtime = gem_mtime if gem_mtime > newest_mtime
        gem_mtime >= specs_mtime
      end

      if updated_gems.empty? then
        say 'No new gems'
        terminate_interaction 0
      end

      specs = map_gems_to_specs updated_gems
      prerelease, released = specs.partition { |s| s.version.prerelease? }

      files = build_marshal_gemspecs specs

      Gem.time 'Updated indexes' do
        update_specs_index released, @dest_specs_index, @specs_index
        update_specs_index released, @dest_latest_specs_index, @latest_specs_index
        update_specs_index(prerelease,
                           @dest_prerelease_specs_index,
                           @prerelease_specs_index)
      end

      compress_indices

      verbose = Gem.configuration.really_verbose

      say "Updating production dir #{@dest_directory}" if verbose

      files << @specs_index
      files << "#{@specs_index}.gz"
      files << @latest_specs_index
      files << "#{@latest_specs_index}.gz"
      files << @prerelease_specs_index
      files << "#{@prerelease_specs_index}.gz"

      files = files.map do |path|
        path.sub(/^#{Regexp.escape @directory}\/?/, '') # HACK?
      end

      files.each do |file|
        src_name = File.join @directory, file
        dst_name = File.join @dest_directory, file # REFACTOR: duped above

        FileUtils.mv src_name, dst_name, :verbose => verbose,
                     :force => true

        File.utime newest_mtime, newest_mtime, dst_name
      end
    end

update_specs_index(index, source, dest) click to toggle source

Combines specs in `index` and `source` then writes out a new copy to `dest`. For a latest index, does not ensure the new file is minimal.

                   # File rubygems/indexer.rb, line 419
    def update_specs_index(index, source, dest)
      specs_index = Marshal.load Gem.read_binary(source)

      index.each do |spec|
        platform = spec.original_platform
        platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
        specs_index << [spec.name, spec.version, platform]
      end

      specs_index = compact_specs specs_index.uniq.sort

      open dest, 'wb' do |io|
        Marshal.dump specs_index, io
      end
    end