gen_case_lookups.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

# Create lookup tables and tests for Unicode Case Folding transformations using
# the `CaseFolding.txt` Unicode Data File, which is subject to these constraints:
#
# 1. Certain documents and files on this website contain a legend indicating
#    that "Modification is permitted." Any person is hereby authorized,
#    without fee, to modify such documents and files to create derivative
#    works conforming to the Unicode® Standard, subject to Terms and
#    Conditions herein.
# 2. Any person is hereby authorized, without fee, to view, use, reproduce,
#    and distribute all documents and files, subject to the Terms and
#    Conditions herein.
# 3. Further specifications of rights and restrictions pertaining to the use
#    of the Unicode DATA FILES and SOFTWARE can be found in the Unicode Data
#    Files and Software License.
#
# The Unicode Data Files and Software License, which can be found at
# <https://www.unicode.org/license.txt> is included in this repository. The
# license requires one of:
#
# (a) this copyright and permission notice appear with all copies of the
#     Data Files or Software, or
# (b) this copyright and permission notice appear in associated Documentation.
#
# `focaccia` distributes this license as `LICENSE-UNICODE` in crate bundles
# and includes `AND Unicode-DFS-2016` in the `Cargo.toml` SPDX license
# expression. See: https://spdx.org/licenses/Unicode-DFS-2016.html.
#
# The generated sources created by this script are subject to both the MIT
# License contained in this repository, `LICENSE`, and the Unicode Data Files
# and Software License, `LICENSE-UNICODE`.

require 'fileutils'

mappings = File.readlines('CaseFolding.txt')

casefolding_version = mappings[0..2].map { |line| "// #{line.delete_prefix('# ')}" }.join.chomp

char_mappings = Hash.new { |hash, key| hash[key] = {} }

mappings.each do |line|
  next if line.empty?
  next if line[0] == '#'

  encoded, = line.split('#', 2)
  code, status, map_to = encoded.split(';').map(&:strip)

  next unless %w[C F T].include?(status)

  code = code.to_i(16)
  map_to = map_to.split.map { |char| char.to_i(16) }

  mode = :full
  mode = :turkic if status == 'T'

  char_mappings[code][mode] = map_to
end

range_start = char_mappings.keys.min
last = char_mappings.keys.min

ranges = []

char_mappings.keys.sort.drop(1).each do |key|
  if key == last + 1 &&
     char_mappings[key][:full].length == char_mappings[last][:full].length &&
     char_mappings[key][:full].first == char_mappings[last][:full].first + 1
    last = key
    next
  end
  ranges << { start: range_start, end: last, span: last - range_start + 1 }
  range_start = key
  last = key
end
ranges << { start: range_start, end: last, span: last - range_start + 1 }

ranges.each do |range|
  start = range[:start]
  last = range[:end]

  start_offset = char_mappings[start][:full][0] - start
  (start..last).each do |char|
    char_offset = char_mappings[char][:full][0] - char
    raise "unequal offset in char range: #{range.inspect}" unless char_offset == start_offset
  end
  range[:offset] = start_offset

  puts range.inspect
end

rs = File.open('src/folding/mapping/lookup.rs', 'w')

rs.puts(<<~AUTOGEN)
  // @generated
  //
  // This source is autogenerated. Do not modify it directly.
  // To make modifications to this code, see `scripts/gen_case_lookups.rb`.
  //
  // Last generated on #{Time.now.utc}.
  //
  // Unicode version:
  #{casefolding_version}

  use super::{Mapping, Mode};

  #[must_use]
  #[allow(clippy::match_same_arms)]
  #[allow(clippy::too_many_lines)]
  pub const fn lookup(c: char, mode: Mode) -> Mapping {
      let codepoint = c as u32;
      let char_bytes = codepoint.to_be_bytes();
      let mid_byte = char_bytes[2];
      let high_bytes = u16::from_be_bytes([char_bytes[0], char_bytes[1]]);
      match (high_bytes, mid_byte) {
          (0x0000, 0x00) => match c {
              // Turkic mapping in ASCII range
              // 0049; T; 0131; # LATIN CAPITAL LETTER I
              '\\u{0049}' if matches!(mode, Mode::Turkic) => Mapping::Single(0x0131),
              c if c.is_ascii() => Mapping::Single(c.to_ascii_lowercase() as u32),
AUTOGEN

last_high_bytes = 0x00
last_mid_byte = 0x00

ranges.each do |range|
  next if range[:end] < 128

  start = range[:start]
  last = range[:end]
  offset = range[:offset]
  mapping = char_mappings[start]

  mid_byte = ((start >> 8) & 0xFF)
  high_bytes = ((start >> 16) & 0xFFFF)

  if high_bytes != last_high_bytes || mid_byte != last_mid_byte
    rs.puts '            _ => Mapping::Single(codepoint),'
    rs.puts '        },'
    rs.puts "        (0x#{high_bytes.to_s(16).upcase.rjust(4, '0')}, 0x#{mid_byte.to_s(16).upcase.rjust(2, '0')}) => match c {"
    last_high_bytes = high_bytes
    last_mid_byte = mid_byte
  end

  if mapping.key?(:turkic) && mapping.key?(:full)
    raise unless (last - start).zero?

    char = start.to_s(16).upcase.rjust(4, '0')
    full = mapping[:full].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }
    case full.length
    when 1
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Full) => Mapping::Single(0x#{full[0]}),"
    when 2
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Full) => Mapping::Double(0x#{full[0]}, 0x#{full[1]}),"
    when 3
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Full) => Mapping::Triple(0x#{full[0]}, 0x#{full[1]}}, 0x#{full[2]}),"
    else
      raise "Unsupported mapping length: #{map.inspect} for code #{code}"
    end
    turkic = mapping[:turkic].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }
    case turkic.length
    when 1
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Turkic) => Mapping::Single(0x#{turkic[0]}),"
    when 2
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Turkic) => Mapping::Double(0x#{turkic[0]}, 0x#{turkic[1]}),"
    when 3
      rs.puts "            '\\u{#{char}}' if matches!(mode, Mode::Turkic) => Mapping::Triple(0x#{turkic[0]}, 0x#{turkic[1]}, 0x#{turkic[2]}),"
    else
      raise "Unsupported mapping length: #{map.inspect} for code #{code}"
    end
  elsif mapping.key?(:full) && !offset.nil?
    full = mapping[:full].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }

    base = start.to_s(16).upcase.rjust(4, '0')
    op = '+'
    op_offset = offset
    if offset.negative?
      op = '-'
      op_offset = -offset
    end
    op_offset = op_offset.to_s(16).rjust(4, '0')
    if (last - start).zero? && full.length == 1
      rs.puts "            '\\u{#{base}}' => Mapping::Single(0x#{full[0]}),"
    elsif full.length == 1
      finish = last.to_s(16).upcase.rjust(4, '0')
      rs.puts "            '\\u{#{base}}'..='\\u{#{finish}}' => Mapping::Single(codepoint #{op} 0x#{op_offset}),"
    elsif (last - start).zero? && full.length == 2
      rs.puts "            '\\u{#{base}}' => Mapping::Double(0x#{full[0]}, 0x#{full[1]}),"
    elsif full.length == 2
      finish = last.to_s(16).upcase.rjust(4, '0')
      rs.puts "            '\\u{#{base}}'..='\\u{#{finish}}' => Mapping::Double(codepoint #{op} 0x#{op_offset}, 0x#{full[1]}),"
    elsif (last - start).zero? && full.length == 3
      rs.puts "            '\\u{#{base}}' => Mapping::Triple(0x#{full[0]}, 0x#{full[1]}, 0x#{full[2]}),"
    elsif full.length == 3
      finish = last.to_s(16).upcase.rjust(4, '0')
      rs.puts "            '\\u{#{base}}'..='\\u{#{finish}}' => Mapping::Triple(codepoint #{op} 0x#{op_offset}, 0x#{full[1]}, 0x#{full[2]}),"
    end
  elsif mapping.key?(:full)
    char = start.to_s(16).upcase.rjust(4, '0')
    map = mapping[:full].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }
    case map.length
    when 1
      rs.puts "            '\\u{#{char}}' => Mapping::Single(0x#{map[0]}),"
    when 2
      rs.puts "            '\\u{#{char}}' => Mapping::Double(0x#{map[0]}, 0x#{map[1]}),"
    when 3
      rs.puts "            '\\u{#{char}}' => Mapping::Triple(0x#{map[0]}, 0x#{map[1]}, 0x#{map[2]}),"
    else
      raise "Unsupported mapping length: #{map.inspect} for code #{code}"
    end
  else
    raise 'unsupported full/turkic mapping combination'
  end
end

rs.puts '            _ => Mapping::Single(codepoint),'
rs.puts '        },'
rs.puts '        _ => Mapping::Single(codepoint),'

rs.puts '    }'
rs.puts '}'
rs.close

FileUtils.mkdir_p('src/exhaustive')

rs = File.open('src/exhaustive/full.rs', 'w')

rs.puts(<<~AUTOGEN)
  // @generated
  //
  // This source is autogenerated. Do not modify it directly.
  // To make modifications to this code, see `scripts/gen_case_lookups.rb`.
  //
  // Last generated on #{Time.now.utc}.
  //
  // Unicode version:
  #{casefolding_version}

  #[must_use]
  #[allow(clippy::match_same_arms)]
  #[allow(clippy::too_many_lines)]
  fn lookup_naive(c: char, buf: &mut [u8; 4]) -> &str {
      match c {
AUTOGEN

char_mappings.keys.sort.each do |from|
  mapping = char_mappings[from]

  char = from.to_s(16).upcase.rjust(4, '0')
  full = mapping[:full].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }

  case full.length
  when 1
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\","
  when 2
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\\u{#{full[1]}}\","
  when 3
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\\u{#{full[1]}}\\u{#{full[2]}}\","
  else
    raise "Unsupported mapping length: #{map.inspect} for code #{code}"
  end
end

rs.puts '        _ => c.encode_utf8(buf),'
rs.puts '    }'
rs.puts '}'
rs.puts

rs.puts(<<~TEST)
  #[cfg(test)]
  mod tests {
      use core::cmp::Ordering;

      use super::lookup_naive;
      use crate::{unicode_full_case_eq, unicode_full_casecmp};

      #[test]
      fn full_case_folding() {
          let mut enc = [0; 4];
          let mut buf = [0; 4];
          for ch in '\\0'..=char::MAX {
              let left = ch.encode_utf8(&mut enc);
              let right = lookup_naive(ch, &mut buf);
              #[rustfmt::skip]
              assert!(unicode_full_case_eq(left, right), "For: {}, expected: {}, got: {}.", ch, left, right);
              #[rustfmt::skip]
              assert!( unicode_full_case_eq(right, left), "For: {}, expected: {}, got: {}.", ch, right, left);
              #[rustfmt::skip]
              assert_eq!( unicode_full_casecmp(left, right), Ordering::Equal, "For: {}, expected: {}, got: {}.", ch, left, right);
              #[rustfmt::skip]
              assert_eq!( unicode_full_casecmp(right, left), Ordering::Equal, "For: {}, expected: {}, got: {}.", ch, right, left);
          }
      }
  }
TEST
rs.close

rs = File.open('src/exhaustive/turkic.rs', 'w')

rs.puts(<<~AUTOGEN)
  // @generated
  //
  // This source is autogenerated. Do not modify it directly.
  // To make modifications to this code, see `scripts/gen_case_lookups.rb`.
  //
  // Last generated on #{Time.now.utc}.
  //
  // Unicode version:
  #{casefolding_version}

  #[must_use]
  #[allow(clippy::match_same_arms)]
  #[allow(clippy::too_many_lines)]
  fn lookup_naive(c: char, buf: &mut [u8; 4]) -> &str {
      match c {
AUTOGEN

char_mappings.keys.sort.each do |from|
  mapping = char_mappings[from]

  char = from.to_s(16).upcase.rjust(4, '0')
  full =
    if mapping[:turkic]
      mapping[:turkic].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }
    else
      mapping[:full].map { |ch| ch.to_s(16).upcase.rjust(4, '0') }
    end

  case full.length
  when 1
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\","
  when 2
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\\u{#{full[1]}}\","
  when 3
    rs.puts "        '\\u{#{char}}' => \"\\u{#{full[0]}}\\u{#{full[1]}}\\u{#{full[2]}}\","
  else
    raise "Unsupported mapping length: #{map.inspect} for code #{code}"
  end
end

rs.puts '        _ => c.encode_utf8(buf),'
rs.puts '    }'
rs.puts '}'
rs.puts

rs.puts(<<~TEST)
  #[cfg(test)]
  mod tests {
      use core::cmp::Ordering;

      use super::lookup_naive;
      use crate::{unicode_full_turkic_case_eq, unicode_full_turkic_casecmp};

      #[test]
      fn turkic_case_folding() {
          let mut enc = [0; 4];
          let mut buf = [0; 4];
          for ch in '\\0'..=char::MAX {
              let left = ch.encode_utf8(&mut enc);
              let right = lookup_naive(ch, &mut buf);
              #[rustfmt::skip]
              assert!(unicode_full_turkic_case_eq(left, right), "For: {}, expected: {}, got: {}.", ch, left, right);
              #[rustfmt::skip]
              assert!(unicode_full_turkic_case_eq(right, left), "For: {}, expected: {}, got: {}.", ch, right, left);
              #[rustfmt::skip]
              assert_eq!(unicode_full_turkic_casecmp(left, right), Ordering::Equal, "For: {}, expected: {}, got: {}.", ch, left, right);
              #[rustfmt::skip]
              assert_eq!(unicode_full_turkic_casecmp(right, left), Ordering::Equal, "For: {}, expected: {}, got: {}.", ch, right, left);
          }
      }
  }
TEST
rs.close