Module: UnicodeNormalize (Ruby 2.2.4)
Description: Module : UnicodeNormalize - Ruby 2.2.4

- Home
- Core 2.2.4
- Std-lib 2.2.4
- Downloads

Support for the Ruby 2.2 series has ended. See here for reference.

### Home Classes Methods

### In Files

- unicode_normalize/normalize.rb
- unicode_normalize/tables.rb

### Methods

- ::canonical_ordering_one
- ::hangul_comp_one
- ::hangul_decomp_one
- ::nfc_one
- ::nfd_one
- ::nfkd_one
- ::normalize
- ::normalized?

### Class/Module Index

Quicksearch

- String
- UnicodeNormalize

No matching classes.

# UnicodeNormalize

automatically generated by template/unicode_norm_gen.tmpl

### Constants

ACCENTS
CLASS_TABLE
COMPOSITION_TABLE
DECOMPOSITION_TABLE
KOMPATIBLE_TABLE
LBASE
LCOUNT
MAX_HASH_LENGTH

  Constant for max hash capacity to avoid DoS attack

NCOUNT
NF_HASH_C
NF_HASH_D
NF_HASH_K
REGEXP_C
REGEXP_C_STRING
REGEXP_D

  Regular Expressions and Hash Constants

REGEXP_D_STRING
REGEXP_K
REGEXP_K_STRING
SBASE

      Constants For Hangul

  for details such as the meaning of the identifiers below, please see www.unicode.org/versions/Unicode7.0.0/ch03.pdf, pp. 144/145

SCOUNT
TBASE
TCOUNT
UNICODE_ENCODINGS

  Unicode-based encodings (except UTF-8)

UNICODE_VERSION
VBASE
VCOUNT

### Public Class Methods

canonical_ordering_one(string) click to toggle source

Canonical Ordering

                   # File unicode_normalize/normalize.rb, line 71
    def self.canonical_ordering_one(string)
      sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
      (sorting.length-2).downto(0) do |i| # almost, but not exactly bubble sort
        (0..i).each do |j|
          later_class = sorting[j+1].last
          if 0<later_class and later_class<sorting[j].last
            sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
          end
        end
      end
      return sorting.collect(&:first).join('')
    end

hangul_comp_one(string) click to toggle source

                   # File unicode_normalize/normalize.rb, line 55
    def self.hangul_comp_one(string)
      length = string.length
      if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead  < LCOUNT and
                      0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
        lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
        if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
          (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
        else
          lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
        end
      else
        string
      end
    end

hangul_decomp_one(target) click to toggle source

Hangul Algorithm

                   # File unicode_normalize/normalize.rb, line 46
    def self.hangul_decomp_one(target)
      syllable_index = target.ord - SBASE
      return target if syllable_index < 0 || syllable_index >= SCOUNT
      l = LBASE + syllable_index / NCOUNT
      v = VBASE + (syllable_index % NCOUNT) / TCOUNT
      t = TBASE + syllable_index % TCOUNT
      (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
    end

nfc_one(string) click to toggle source

                   # File unicode_normalize/normalize.rb, line 94
    def self.nfc_one(string)
      nfd_string = nfd_one string
      start = nfd_string[0]
      last_class = CLASS_TABLE[start]-1
      accents = ''
      nfd_string[1..-1].each_char do |accent|
        accent_class = CLASS_TABLE[accent]
        if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
          start = composite
        else
          accents << accent
          last_class = accent_class
        end
      end
      hangul_comp_one(start+accents)
    end

nfd_one(string) click to toggle source

Normalization Forms for Patterns (not whole Strings)

                   # File unicode_normalize/normalize.rb, line 85
    def self.nfd_one(string)
      string = string.chars.map! {|c| DECOMPOSITION_TABLE[c] || c}.join('')
      canonical_ordering_one(hangul_decomp_one(string))
    end

nfkd_one(string) click to toggle source

                   # File unicode_normalize/normalize.rb, line 90
    def self.nfkd_one(string)
      string.chars.map! {|c| KOMPATIBLE_TABLE[c] || c}.join('')
    end

normalize(string, form = :nfc) click to toggle source

                   # File unicode_normalize/normalize.rb, line 111
    def self.normalize(string, form = :nfc)
      encoding = string.encoding
      case encoding
      when Encoding::UTF_8
        case form
        when :nfc then
          string.gsub REGEXP_C, NF_HASH_C
        when :nfd then
          string.gsub REGEXP_D, NF_HASH_D
        when :nfkc then
          string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
        when :nfkd then
          string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
        else
          raise ArgumentError, "Invalid normalization form #{form}."
        end
      when Encoding::US_ASCII
        string
      when *UNICODE_ENCODINGS
        normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
      else
        raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
      end
    end

normalized?(string, form = :nfc) click to toggle source

                   # File unicode_normalize/normalize.rb, line 136
    def self.normalized?(string, form = :nfc)
      encoding = string.encoding
      case encoding
      when Encoding::UTF_8
        case form
        when :nfc then
          string.scan REGEXP_C do |match|
            return false  if NF_HASH_C[match] != match
          end
          true
        when :nfd then
          string.scan REGEXP_D do |match|
            return false  if NF_HASH_D[match] != match
          end
          true
        when :nfkc then
          normalized?(string, :nfc) and string !~ REGEXP_K
        when :nfkd then
          normalized?(string, :nfd) and string !~ REGEXP_K
        else
          raise ArgumentError, "Invalid normalization form #{form}."
        end
      when Encoding::US_ASCII
        true
      when *UNICODE_ENCODINGS
        normalized? string.encode(Encoding::UTF_8), form
      else
        raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
      end
    end