text.nx

// std.text: UTF-8 text by code point and by grapheme cluster, written in
// Nexium, with the Unicode tables it needs.
//
// `import std.text` then:
//
//     let n = text.char_count("héllo")            // 5 code points, not 6 bytes
//     for c in text.scalars("héllo") { ... }       // chars (Unicode scalars)
//     let g = text.grapheme_count("🇯🇵 e\u{301}")   // 3: a flag, a space, an é
//     for g in text.graphemes(s) { ... }           // what a reader counts as characters
//     let w = text.width("日本語")                   // 6 columns on a terminal
//     let cell = text.pad_right(text.truncate_width(name, 20), 20)
//     let s = text.to_upper("straße")               // "STRASSE"
//     let same = text.eq_ignore_case("Straße", "STRASSE")
//     let t = text.truncate("héllo wörld", 5)       // "héllo", never mid-character
//
// Strings are bytes; this module reads them as UTF-8, tolerating bad input
// (an invalid byte decodes as U+FFFD and advances one byte). Grapheme
// clusters are Unicode's extended grapheme clusters (UAX #29): a letter and
// its marks, a Hangul syllable, an emoji sequence joined by ZWJ, a flag, an
// Indic conjunct. Case mapping is Unicode's full mapping for every script,
// without the rules that depend on a language (Turkish and Lithuanian i);
// `to_lower` writes a final sigma where a word ends. `width` counts
// terminal columns as terminals draw them: East Asian wide and fullwidth
// characters and emoji sequences take two, marks and zero-width characters
// none. The tables at the end are generated by scripts/unicode_tables.py
// (`UNICODE_VERSION` says from which version).

pub const REPLACEMENT: u32 = 0xFFFD

/// A decoded code point and the number of bytes it took.
pub struct Decoded {
    cp: u32
    len: usize
}

/// Decode the code point starting at byte `i`. Invalid input yields
/// U+FFFD with length 1 so callers always make progress.
pub fn decode_at(s: []u8, i: usize) -> Decoded {
    if i >= s.len { return Decoded{ .cp = REPLACEMENT, .len = 0 } }
    let b0 = s[i]
    if b0 < 0x80 { return Decoded{ .cp = b0 as u32, .len = 1 } }
    var need: usize = 0
    var cp: u32 = 0
    var min: u32 = 0
    if b0 >= 0xC2 and b0 <= 0xDF {
        need = 1
        cp = (b0 & 0x1F) as u32
        min = 0x80
    } else if b0 >= 0xE0 and b0 <= 0xEF {
        need = 2
        cp = (b0 & 0x0F) as u32
        min = 0x800
    } else if b0 >= 0xF0 and b0 <= 0xF4 {
        need = 3
        cp = (b0 & 0x07) as u32
        min = 0x10000
    } else {
        return Decoded{ .cp = REPLACEMENT, .len = 1 }
    }
    var k: usize = 1
    while k <= need {
        if i + k >= s.len { return Decoded{ .cp = REPLACEMENT, .len = 1 } }
        let b = s[i + k]
        if b < 0x80 or b > 0xBF { return Decoded{ .cp = REPLACEMENT, .len = 1 } }
        cp = (cp << 6) | ((b & 0x3F) as u32)
        k += 1
    }
    if cp < min or cp > 0x10FFFF or (cp >= 0xD800 and cp <= 0xDFFF) { return Decoded{ .cp = REPLACEMENT, .len = 1 } }
    return Decoded{ .cp = cp, .len = need + 1 }
}

/// Append a code point as UTF-8.
pub fn push(out: *mut String, cp: u32) {
    if cp < 0x80 {
        out.push_byte(cp as u8)
    } else if cp < 0x800 {
        out.push_byte((0xC0 | (cp >> 6)) as u8)
        out.push_byte((0x80 | (cp & 0x3F)) as u8)
    } else if cp < 0x10000 {
        out.push_byte((0xE0 | (cp >> 12)) as u8)
        out.push_byte((0x80 | ((cp >> 6) & 0x3F)) as u8)
        out.push_byte((0x80 | (cp & 0x3F)) as u8)
    } else {
        out.push_byte((0xF0 | (cp >> 18)) as u8)
        out.push_byte((0x80 | ((cp >> 12) & 0x3F)) as u8)
        out.push_byte((0x80 | ((cp >> 6) & 0x3F)) as u8)
        out.push_byte((0x80 | (cp & 0x3F)) as u8)
    }
}

/// A code point as a String.
pub fn encode(cp: u32) -> String {
    var out = String.new()
    push(&mut out, cp)
    return out
}

/// Is the text well-formed UTF-8?
pub fn is_valid(s: []u8) -> bool {
    var i: usize = 0
    while i < s.len {
        let d = decode_at(s, i)
        if d.cp == REPLACEMENT and (d.len == 1 and s[i] != 0xEF) { return false }
        i += d.len
    }
    return true
}

/// All code points.
pub fn chars(s: []u8) -> List(u32) {
    var out = List(u32).new()
    var i: usize = 0
    while i < s.len {
        let d = decode_at(s, i)
        out.append(d.cp)
        i += d.len
    }
    return out
}

/// The number of code points.
pub fn char_count(s: []u8) -> usize {
    var n: usize = 0
    var i: usize = 0
    while i < s.len {
        i += decode_at(s, i).len
        n += 1
    }
    return n
}

/// The byte offset of the `n`th code point (or `s.len` when past the end).
pub fn byte_offset(s: []u8, n: usize) -> usize {
    var i: usize = 0
    var k: usize = 0
    while i < s.len and k < n {
        i += decode_at(s, i).len
        k += 1
    }
    return i
}

/// The `n`th code point, or null.
pub fn char_at(s: []u8, n: usize) -> ?u32 {
    let i = byte_offset(s, n)
    if i >= s.len { return null }
    return decode_at(s, i).cp
}

/// Code points `from` (inclusive) to `to` (exclusive), as a slice of `s`.
pub fn slice(s: []u8, from: usize, to: usize) -> []u8 {
    let a = byte_offset(s, from)
    let b = byte_offset(s, to)
    if b < a { return s[a..a] }
    return s[a..b]
}

/// The first `n` code points; never cuts a character in half.
pub fn truncate(s: []u8, n: usize) -> []u8 {
    return s[0..byte_offset(s, n)]
}

/// The code points in reverse order.
pub fn reverse(s: []u8) -> String {
    let cps = chars(s)
    var out = String.with_capacity(s.len)
    var i = cps.len
    while i > 0 {
        i -= 1
        push(&mut out, cps[i])
    }
    return out
}

/// The Unicode scalars, as `char`s (an invalid byte reads as U+FFFD).
pub fn scalars(s: []u8) -> List(char) {
    var out = List(char).new()
    var i: usize = 0
    while i < s.len {
        let d = decode_at(s, i)
        out.append(d.cp as char)
        i += d.len
    }
    return out
}

// ------------------------------------------------------------------ lookups

// the index of the range in sorted, disjoint `los`..`his` that holds `cp`
fn find_range(los: []u32, his: []u32, cp: u32) -> ?usize {
    var lo: usize = 0
    var hi = los.len
    while lo < hi {
        let mid = lo + (hi - lo) / 2
        if cp < los[mid] {
            hi = mid
        } else if cp > his[mid] {
            lo = mid + 1
        } else {
            return mid
        }
    }
    return null
}

// where `cp` is in the sorted `cps`
fn find_cp(cps: []u32, cp: u32) -> ?usize {
    var lo: usize = 0
    var hi = cps.len
    while lo < hi {
        let mid = lo + (hi - lo) / 2
        if cp < cps[mid] {
            hi = mid
        } else if cp > cps[mid] {
            lo = mid + 1
        } else {
            return mid
        }
    }
    return null
}

// ------------------------------------------------------------------ graphemes

// grapheme cluster break classes (UAX #29), with Extended_Pictographic and
// Indic_Conjunct_Break folded in as the generated table has them
const GB_OTHER: u8 = 0
const GB_CR: u8 = 1
const GB_LF: u8 = 2
const GB_CONTROL: u8 = 3
const GB_EXTEND: u8 = 4
const GB_ZWJ: u8 = 5
const GB_RI: u8 = 6
const GB_PREPEND: u8 = 7
const GB_SPACING: u8 = 8
const GB_L: u8 = 9
const GB_V: u8 = 10
const GB_T: u8 = 11
const GB_LV: u8 = 12
const GB_LVT: u8 = 13
// Extended_Pictographic (its break class is Other)
const GB_PICT: u8 = 14
// Indic_Conjunct_Break=Consonant (Other)
const GB_CONSONANT: u8 = 15
// Indic_Conjunct_Break=Extend and =Linker (both Extend)
const GB_INCB_EXTEND: u8 = 16
const GB_LINKER: u8 = 17

fn gb_kind(cp: u32) -> u8 {
    if cp < 0x80 {
        if cp == 0x0D { return GB_CR }
        if cp == 0x0A { return GB_LF }
        if cp < 0x20 or cp == 0x7F { return GB_CONTROL }
        return GB_OTHER
    }
    // a Hangul syllable is LV every 28th from U+AC00, LVT between
    if cp >= 0xAC00 and cp <= 0xD7A3 { return if (cp - 0xAC00) % 28 == 0 { GB_LV } else { GB_LVT } }
    if let i = find_range(GB_LO[..], GB_HI[..], cp) { return GB_KIND[i] }
    return GB_OTHER
}

fn gb_extend(k: u8) -> bool {
    return k == GB_EXTEND or k == GB_INCB_EXTEND or k == GB_LINKER
}

// No break between `prev` and `cur`: rules GB3 to GB13, with `ri` the
// regional indicators in a row up to `prev`, `pict` 2 after a pictograph,
// its extenders and a ZWJ, and `conj` 2 after a consonant and extenders or
// linkers with a linker among them. Anything else breaks (GB999).
fn joins(prev: u8, cur: u8, ri: usize, pict: u8, conj: u8) -> bool {
    if prev == GB_CR and cur == GB_LF { return true }
    if prev == GB_CR or prev == GB_LF or prev == GB_CONTROL { return false }
    if cur == GB_CR or cur == GB_LF or cur == GB_CONTROL { return false }
    if prev == GB_L and (cur == GB_L or cur == GB_V or cur == GB_LV or cur == GB_LVT) { return true }
    if (prev == GB_LV or prev == GB_V) and (cur == GB_V or cur == GB_T) { return true }
    if (prev == GB_LVT or prev == GB_T) and cur == GB_T { return true }
    if gb_extend(cur) or cur == GB_ZWJ or cur == GB_SPACING { return true }
    if prev == GB_PREPEND { return true }
    if cur == GB_CONSONANT and conj == 2 { return true }
    if cur == GB_PICT and prev == GB_ZWJ and pict == 2 { return true }
    return cur == GB_RI and prev == GB_RI and ri % 2 == 1
}

/// The byte offset where the grapheme cluster that starts at byte `i`
/// ends. A grapheme cluster is what a reader takes for one character: a
/// letter and its marks, a Hangul syllable, an emoji sequence joined by
/// ZWJ, a flag, an Indic conjunct, CR LF (Unicode's extended grapheme
/// clusters, UAX #29).
pub fn grapheme_end(s: []u8, i: usize) -> usize {
    if i >= s.len { return s.len }
    let first = decode_at(s, i)
    var prev = gb_kind(first.cp)
    var j = i + first.len
    var ri: usize = if prev == GB_RI { 1 } else { 0 }
    var pict: u8 = if prev == GB_PICT { 1 } else { 0 }
    var conj: u8 = if prev == GB_CONSONANT { 1 } else { 0 }
    while j < s.len {
        let d = decode_at(s, j)
        let cur = gb_kind(d.cp)
        if !joins(prev, cur, ri, pict, conj) { break }
        ri = if cur == GB_RI { ri + 1 } else { 0 }
        if cur == GB_PICT {
            pict = 1
        } else if pict == 1 and cur == GB_ZWJ {
            pict = 2
        } else if !(pict == 1 and gb_extend(cur)) {
            pict = 0
        }
        if cur == GB_CONSONANT {
            conj = 1
        } else if conj >= 1 and cur == GB_LINKER {
            conj = 2
        } else if !(conj >= 1 and (cur == GB_INCB_EXTEND or cur == GB_ZWJ)) {
            conj = 0
        }
        prev = cur
        j += d.len
    }
    return j
}

/// The grapheme clusters, as slices of the text.
pub fn graphemes(s: []u8) -> List([]u8) {
    var out = List([]u8).new()
    var i: usize = 0
    while i < s.len {
        let j = grapheme_end(s, i)
        out.append(s[i..j])
        i = j
    }
    return out
}

/// The number of grapheme clusters: the characters a reader counts.
pub fn grapheme_count(s: []u8) -> usize {
    var n: usize = 0
    var i: usize = 0
    while i < s.len {
        i = grapheme_end(s, i)
        n += 1
    }
    return n
}

/// The first `n` grapheme clusters; never splits one.
pub fn truncate_graphemes(s: []u8, n: usize) -> []u8 {
    var i: usize = 0
    var k: usize = 0
    while i < s.len and k < n {
        i = grapheme_end(s, i)
        k += 1
    }
    return s[0..i]
}

// ------------------------------------------------------------------ width

/// Does the code point take no columns (combining and enclosing marks,
/// format and zero-width characters, controls, Hangul vowels and finals)?
pub fn is_zero_width(cp: u32) -> bool {
    if cp < 0x20 or (cp >= 0x7F and cp <= 0x9F) { return true }
    if cp < 0x300 { return false }
    return find_range(ZERO_LO[..], ZERO_HI[..], cp) != null
}

/// Does the code point take two columns (East Asian wide and fullwidth,
/// emoji among them)?
pub fn is_wide(cp: u32) -> bool {
    if cp < 0x1100 { return false }
    return find_range(WIDE_LO[..], WIDE_HI[..], cp) != null
}

/// Columns a code point takes on a terminal: 0, 1 or 2.
pub fn char_width(cp: u32) -> usize {
    if is_zero_width(cp) { return 0 }
    if is_wide(cp) { return 2 }
    return 1
}

// The columns of one grapheme cluster: its first character's, and two for
// a flag and for an emoji sequence (a wide character in it, or U+FE0F
// asking for the emoji form).
fn cluster_width(g: []u8) -> usize {
    let first = decode_at(g, 0)
    if first.len >= g.len { return char_width(first.cp) }
    if is_wide(first.cp) or (first.cp >= 0x1F1E6 and first.cp <= 0x1F1FF) { return 2 }
    var i = first.len
    while i < g.len {
        let d = decode_at(g, i)
        if d.cp == 0xFE0F or is_wide(d.cp) { return 2 }
        i += d.len
    }
    return char_width(first.cp)
}

/// Columns the text takes on a terminal, grapheme cluster by cluster, as
/// terminals draw it: `e` with an accent is one, a family emoji two.
pub fn width(s: []u8) -> usize {
    var w: usize = 0
    var i: usize = 0
    while i < s.len {
        let j = grapheme_end(s, i)
        w += cluster_width(s[i..j])
        i = j
    }
    return w
}

/// Pad on the right to `columns` terminal columns (by width, not bytes).
pub fn pad_right(s: []u8, columns: usize) -> String {
    var out = String.from(s)
    var w = width(s)
    while w < columns {
        out.push_byte(' ')
        w += 1
    }
    return out
}

/// Pad on the left to `columns` terminal columns: the text right-aligned.
pub fn pad_left(s: []u8, columns: usize) -> String {
    var out = String.new()
    var w = width(s)
    while w < columns {
        out.push_byte(' ')
        w += 1
    }
    out.append(s)
    return out
}

/// The longest start of the text that fits in `columns` terminal columns,
/// cut between grapheme clusters.
pub fn truncate_width(s: []u8, columns: usize) -> []u8 {
    var w: usize = 0
    var i: usize = 0
    while i < s.len {
        let j = grapheme_end(s, i)
        let cw = cluster_width(s[i..j])
        if w + cw > columns { break }
        w += cw
        i = j
    }
    return s[0..i]
}

// ------------------------------------------------------------------ case

// `cp` by a table of runs (`lo`, `lo + step`, ... `hi` each add their
// delta), or `cp` itself
fn map_runs(los: []u32, his: []u32, deltas: []i32, steps: []u8, cp: u32) -> u32 {
    if let i = find_range(los, his, cp) {
        if (cp - los[i]) % (steps[i] as u32) == 0 { return ((cp as i64) + (deltas[i] as i64)) as u32 }
    }
    return cp
}

/// Upper-case a code point by Unicode's simple mapping, one code point to
/// one: `ß` stays `ß` here, where `to_upper` writes `SS`.
pub fn upper_char(cp: u32) -> u32 {
    if cp < 0x80 { return if cp >= 'a' as u32 and cp <= 'z' as u32 { cp - 32 } else { cp } }
    return map_runs(UPPER_LO[..], UPPER_HI[..], UPPER_DELTA[..], UPPER_STEP[..], cp)
}

/// Lower-case a code point by Unicode's simple mapping.
pub fn lower_char(cp: u32) -> u32 {
    if cp < 0x80 { return if cp >= 'A' as u32 and cp <= 'Z' as u32 { cp + 32 } else { cp } }
    return map_runs(LOWER_LO[..], LOWER_HI[..], LOWER_DELTA[..], LOWER_STEP[..], cp)
}

/// Case-fold a code point by Unicode's simple folding (`ς` and `σ` fold
/// alike); `fold` does the full folding of a text.
pub fn fold_char(cp: u32) -> u32 {
    if cp < 0x80 { return lower_char(cp) }
    if let i = find_cp(FOLD_CP[..], cp) { return FOLD_TO[i] }
    return lower_char(cp)
}

// the code points a mapping to more than one gives `cp`, or null
fn special_of(cps: []u32, at: []u16, to: []u32, cp: u32) -> ?[]u32 {
    if let i = find_cp(cps, cp) { return to[(at[i] as usize)..(at[i + 1] as usize)] }
    return null
}

// a letter with a case, for the final sigma
fn is_cased(cp: u32) -> bool {
    return upper_char(cp) != cp or lower_char(cp) != cp
}

// may the code point sit inside a word without ending it: a mark, a format
// character, an apostrophe, a period (Unicode's Case_Ignorable, roughly)
fn is_case_ignorable(cp: u32) -> bool {
    if cp == 0x27 or cp == 0x2E or cp == 0x3A or cp == 0xAD or cp == 0xB7 or cp == 0x2018 or cp == 0x2019 or cp == 0x2024 { return true }
    return cp >= 0x300 and is_zero_width(cp)
}

// Does the capital sigma at byte `i` end a word: a cased letter before it
// (past ignorable ones) and none after?
fn final_sigma(s: []u8, i: usize, len: usize) -> bool {
    var j = i
    var before = false
    while j > 0 {
        var k = j - 1
        while k > 0 and (s[k] & 0xC0) == 0x80 { k -= 1 }
        let d = decode_at(s, k)
        j = k
        if is_case_ignorable(d.cp) { continue }
        before = is_cased(d.cp)
        break
    }
    if !before { return false }
    var n = i + len
    while n < s.len {
        let d = decode_at(s, n)
        n += d.len
        if is_case_ignorable(d.cp) { continue }
        return !is_cased(d.cp)
    }
    return true
}

fn map_full(s: []u8, upper: bool) -> String {
    var out = String.with_capacity(s.len)
    var i: usize = 0
    while i < s.len {
        let d = decode_at(s, i)
        if upper {
            if let m = special_of(UPPER_SPECIAL_CP[..], UPPER_SPECIAL_AT[..], UPPER_SPECIAL_TO[..], d.cp) {
                for c in m { push(&mut out, c) }
            } else {
                push(&mut out, upper_char(d.cp))
            }
        } else if d.cp == 0x3A3 {
            push(&mut out, if final_sigma(s, i, d.len) { 0x3C2 } else { 0x3C3 })
        } else {
            if let m = special_of(LOWER_SPECIAL_CP[..], LOWER_SPECIAL_AT[..], LOWER_SPECIAL_TO[..], d.cp) {
                for c in m { push(&mut out, c) }
            } else {
                push(&mut out, lower_char(d.cp))
            }
        }
        i += d.len
    }
    return out
}

/// Upper-case by Unicode's full mapping, every script: `straße` becomes
/// `STRASSE`, `fi` becomes `FI`.
pub fn to_upper(s: []u8) -> String {
    return map_full(s, true)
}

/// Lower-case by Unicode's full mapping, with `ς` for a sigma that ends a
/// word: `ΟΔΟΣ` becomes `οδος`.
pub fn to_lower(s: []u8) -> String {
    return map_full(s, false)
}

/// Case-fold by Unicode's full folding, to compare or look up text without
/// regard to case: `Straße`, `STRASSE` and `strasse` fold alike.
pub fn fold(s: []u8) -> String {
    var out = String.with_capacity(s.len)
    var i: usize = 0
    while i < s.len {
        let d = decode_at(s, i)
        if let m = special_of(FOLD_SPECIAL_CP[..], FOLD_SPECIAL_AT[..], FOLD_SPECIAL_TO[..], d.cp) {
            for c in m { push(&mut out, c) }
        } else {
            push(&mut out, fold_char(d.cp))
        }
        i += d.len
    }
    return out
}

/// Compare ignoring case, by full case folding.
pub fn eq_ignore_case(a: []u8, b: []u8) -> bool {
    let fa = fold(a)
    let fb = fold(b)
    return fa[..] == fb[..]
}

// ------------------------------------------------------------------ tests

test "decode and encode" {
    expect_eq(char_count("héllo"), 5)
    expect_eq(char_count("日本語"), 3)
    expect_eq(chars("aé日😀").len, 4)
    expect_eq(chars("aé日😀")[3], 0x1F600)
    expect_eq(encode(0xE9), "é")
    expect_eq(encode(0x65E5), "日")
    expect_eq(encode(0x1F600), "😀")
    expect(is_valid("héllo 日本"))
    expect(!is_valid(b"\xff\xfe"))
    expect(!is_valid(b"\xe6\x97"))
    let bad = decode_at(b"\xff", 0)
    expect_eq(bad.cp, REPLACEMENT)
    expect_eq(bad.len, 1)
    expect_eq(char_at("héllo", 1).?, 0xE9)
    expect(char_at("hé", 2) == null)
    expect_eq(slice("héllo wörld", 6, 11), "wörld")
    expect_eq(truncate("héllo wörld", 5), "héllo")
    expect_eq(reverse("héllo"), "olléh")
    expect_eq(byte_offset("héllo", 2), 3)
}

test "scalars" {
    let cs = scalars("aé日😀")
    expect_eq(cs.len, 4)
    expect_eq(cs[1], 'é')
    expect_eq(cs[3], '😀')
    let bad = scalars(b"a\xffb")
    expect_eq(bad[1] as u32, REPLACEMENT)
}

test "grapheme clusters" {
    // a letter and its accent, CR LF, flags, a family, a skin tone, a
    // syllable in jamo and precomposed, a Devanagari conjunct (GB9c), a
    // prepended mark, a ZWJ between letters
    expect_eq(grapheme_count("e\u{301}"), 1)
    expect_eq(grapheme_count("\r\n"), 1)
    expect_eq(grapheme_count("\n\r"), 2)
    expect_eq(grapheme_count("🇯🇵🇫🇷"), 2)
    expect_eq(grapheme_count("🇯🇵🇫"), 2)
    expect_eq(grapheme_count("👨\u{200D}👩\u{200D}👧"), 1)
    expect_eq(grapheme_count("👍🏽"), 1)
    expect_eq(grapheme_count("\u{1100}\u{1161}\u{11A8}"), 1)
    expect_eq(grapheme_count("각"), 1)
    expect_eq(grapheme_count("\u{915}\u{94D}\u{937}\u{93F}"), 1)
    expect_eq(grapheme_count("\u{915}\u{94D}"), 1)
    expect_eq(grapheme_count("\u{600}1"), 1)
    expect_eq(grapheme_count("a\u{200D}b"), 2)
    expect_eq(grapheme_count("héllo"), 5)
    let gs = graphemes("e\u{301}🇯🇵x")
    expect_eq(gs.len, 3)
    expect_eq(gs[0], "e\u{301}")
    expect_eq(gs[1], "🇯🇵")
    expect_eq(truncate_graphemes("e\u{301}e\u{301}e", 2), "e\u{301}e\u{301}")
    expect_eq(grapheme_end("ab", 0), 1)
    expect_eq(grapheme_end("ab", 2), 2)
    expect_eq(grapheme_count(""), 0)
}

test "width" {
    expect_eq(width("hello"), 5)
    expect_eq(width("日本語"), 6)
    expect_eq(width("e\u{301}"), 1)
    expect_eq(width("a\u{200B}b"), 2)
    expect_eq(char_width(0x1F600), 2)
    expect_eq(char_width(0x591), 0)
    expect_eq(char_width(0xFF21), 2)
    expect_eq(pad_right("日本", 6), "日本  ")
    // emoji sequences and flags take two columns, however many code points
    expect_eq(width("👨\u{200D}👩\u{200D}👧"), 2)
    expect_eq(width("🇯🇵"), 2)
    expect_eq(width("\u{2764}\u{FE0F}"), 2)
    expect_eq(width("\u{2764}"), 1)
    expect_eq(width("\u{1100}\u{1161}\u{11A8}"), 2)
    expect_eq(pad_left("日本", 6), "  日本")
    expect_eq(truncate_width("日本語", 5), "日本")
    expect_eq(truncate_width("ab日", 3), "ab")
    expect_eq(truncate_width("e\u{301}x", 1), "e\u{301}")
}

test "case" {
    expect_eq(to_upper("héllo wörld"), "HÉLLO WÖRLD")
    expect_eq(to_lower("ÀÉÎÕÜ"), "àéîõü")
    expect_eq(to_upper("straße"), "STRASSE")
    expect_eq(to_upper("αβγ"), "ΑΒΓ")
    expect_eq(to_lower("ΑΒΓΣ"), "αβγς")
    expect_eq(to_lower("ΣΑΣ ΣΑΣ."), "σας σας.")
    expect_eq(to_lower("Σ"), "σ")
    expect_eq(to_upper("привет"), "ПРИВЕТ")
    expect_eq(to_lower("ПРИВЕТ ЁЖ"), "привет ёж")
    expect_eq(to_upper("łódź"), "ŁÓDŹ")
    expect_eq(to_lower("ŁÓDŹ"), "łódź")
    // scripts the old offsets did not reach: Armenian, Georgian, Deseret
    expect_eq(to_upper("բարի"), "ԲԱՐԻ")
    expect_eq(to_lower("ᲐᲑᲒ"), "აბგ")
    expect_eq(to_upper("აბგ"), "ᲐᲑᲒ")
    expect_eq(to_upper("𐐨"), "𐐀")
    expect_eq(to_upper("fi"), "FI")
    expect_eq(to_lower("İ"), "i\u{307}")
    expect_eq(to_upper("Dž"), "DŽ")
    expect(eq_ignore_case("Héllo", "hÉLLO"))
    expect(!eq_ignore_case("Héllo", "hello"))
    expect(eq_ignore_case("Straße", "STRASSE"))
    expect(eq_ignore_case("ΣΑΣ", "σας"))
    expect_eq(fold("Straße"), "strasse")
    expect_eq(fold("fi"), "fi")
    expect_eq(upper_char('a' as u32), 'A' as u32)
    expect_eq(lower_char(0x3A3), 0x3C3)
    expect_eq(upper_char(0xDF), 0xDF)
    expect_eq(fold_char(0x3C2), 0x3C3)
}

// ---------------------------------------------- generated: Unicode tables
// by scripts/unicode_tables.py from the Unicode Character Database 16.0.0;
// run it again rather than editing these.

pub const UNICODE_VERSION: []u8 = "16.0.0"

// grapheme cluster break classes of the code points from U+0080 on, the
// Hangul syllables aside (GB_* above)
const GB_LO: [678]u32 = [
    0x80, 0xA9, 0xAD, 0xAE, 0x300, 0x483, 0x591, 0x5BF,
    0x5C1, 0x5C4, 0x5C7, 0x600, 0x610, 0x61C, 0x64B, 0x670,
    0x6D6, 0x6DD, 0x6DF, 0x6E7, 0x6EA, 0x70F, 0x711, 0x730,
    0x7A6, 0x7EB, 0x7FD, 0x816, 0x81B, 0x825, 0x829, 0x859,
    0x890, 0x897, 0x8CA, 0x8E2, 0x8E3, 0x903, 0x915, 0x93A,
    0x93B, 0x93C, 0x93E, 0x941, 0x949, 0x94D, 0x94E, 0x951,
    0x958, 0x962, 0x978, 0x981, 0x982, 0x995, 0x9AA, 0x9B2,
    0x9B6, 0x9BC, 0x9BE, 0x9BF, 0x9C1, 0x9C7, 0x9CB, 0x9CD,
    0x9D7, 0x9DC, 0x9DF, 0x9E2, 0x9F0, 0x9FE, 0xA01, 0xA03,
    0xA3C, 0xA3E, 0xA41, 0xA47, 0xA4B, 0xA51, 0xA70, 0xA75,
    0xA81, 0xA83, 0xA95, 0xAAA, 0xAB2, 0xAB5, 0xABC, 0xABE,
    0xAC1, 0xAC7, 0xAC9, 0xACB, 0xACD, 0xAE2, 0xAF9, 0xAFA,
    0xB01, 0xB02, 0xB15, 0xB2A, 0xB32, 0xB35, 0xB3C, 0xB3E,
    0xB40, 0xB41, 0xB47, 0xB4B, 0xB4D, 0xB55, 0xB5C, 0xB5F,
    0xB62, 0xB71, 0xB82, 0xBBE, 0xBBF, 0xBC0, 0xBC1, 0xBC6,
    0xBCA, 0xBCD, 0xBD7, 0xC00, 0xC01, 0xC04, 0xC15, 0xC2A,
    0xC3C, 0xC3E, 0xC41, 0xC46, 0xC4A, 0xC4D, 0xC55, 0xC58,
    0xC62, 0xC81, 0xC82, 0xCBC, 0xCBE, 0xCBF, 0xCC1, 0xCC2,
    0xCC3, 0xCC6, 0xCCA, 0xCD5, 0xCE2, 0xCF3, 0xD00, 0xD02,
    0xD15, 0xD3B, 0xD3E, 0xD3F, 0xD41, 0xD46, 0xD4A, 0xD4D,
    0xD4E, 0xD57, 0xD62, 0xD81, 0xD82, 0xDCA, 0xDCF, 0xDD0,
    0xDD2, 0xDD6, 0xDD8, 0xDDF, 0xDF2, 0xE31, 0xE33, 0xE34,
    0xE47, 0xEB1, 0xEB3, 0xEB4, 0xEC8, 0xF18, 0xF35, 0xF37,
    0xF39, 0xF3E, 0xF71, 0xF7F, 0xF80, 0xF86, 0xF8D, 0xF99,
    0xFC6, 0x102D, 0x1031, 0x1032, 0x1039, 0x103B, 0x103D, 0x1056,
    0x1058, 0x105E, 0x1071, 0x1082, 0x1084, 0x1085, 0x108D, 0x109D,
    0x1100, 0x1160, 0x11A8, 0x135D, 0x1712, 0x1732, 0x1752, 0x1772,
    0x17B4, 0x17B6, 0x17B7, 0x17BE, 0x17C6, 0x17C7, 0x17C9, 0x17DD,
    0x180B, 0x180E, 0x180F, 0x1885, 0x18A9, 0x1920, 0x1923, 0x1927,
    0x1929, 0x1930, 0x1932, 0x1933, 0x1939, 0x1A17, 0x1A19, 0x1A1B,
    0x1A55, 0x1A56, 0x1A57, 0x1A58, 0x1A60, 0x1A62, 0x1A65, 0x1A6D,
    0x1A73, 0x1A7F, 0x1AB0, 0x1B00, 0x1B04, 0x1B34, 0x1B3E, 0x1B42,
    0x1B6B, 0x1B80, 0x1B82, 0x1BA1, 0x1BA2, 0x1BA6, 0x1BA8, 0x1BE6,
    0x1BE7, 0x1BE8, 0x1BEA, 0x1BED, 0x1BEE, 0x1BEF, 0x1C24, 0x1C2C,
    0x1C34, 0x1C36, 0x1CD0, 0x1CD4, 0x1CE1, 0x1CE2, 0x1CED, 0x1CF4,
    0x1CF7, 0x1CF8, 0x1DC0, 0x200B, 0x200C, 0x200D, 0x200E, 0x2028,
    0x203C, 0x2049, 0x2060, 0x20D0, 0x2122, 0x2139, 0x2194, 0x21A9,
    0x231A, 0x2328, 0x2388, 0x23CF, 0x23E9, 0x23F8, 0x24C2, 0x25AA,
    0x25B6, 0x25C0, 0x25FB, 0x2600, 0x2607, 0x2614, 0x2690, 0x2708,
    0x2714, 0x2716, 0x271D, 0x2721, 0x2728, 0x2733, 0x2744, 0x2747,
    0x274C, 0x274E, 0x2753, 0x2757, 0x2763, 0x2795, 0x27A1, 0x27B0,
    0x27BF, 0x2934, 0x2B05, 0x2B1B, 0x2B50, 0x2B55, 0x2CEF, 0x2D7F,
    0x2DE0, 0x302A, 0x3030, 0x303D, 0x3099, 0x3297, 0x3299, 0xA66F,
    0xA674, 0xA69E, 0xA6F0, 0xA802, 0xA806, 0xA80B, 0xA823, 0xA825,
    0xA827, 0xA82C, 0xA880, 0xA8B4, 0xA8C4, 0xA8E0, 0xA8FF, 0xA926,
    0xA947, 0xA952, 0xA953, 0xA960, 0xA980, 0xA983, 0xA9B3, 0xA9B4,
    0xA9B6, 0xA9BA, 0xA9BC, 0xA9BE, 0xA9C0, 0xA9E5, 0xAA29, 0xAA2F,
    0xAA31, 0xAA33, 0xAA35, 0xAA43, 0xAA4C, 0xAA4D, 0xAA7C, 0xAAB0,
    0xAAB2, 0xAAB7, 0xAABE, 0xAAC1, 0xAAEB, 0xAAEC, 0xAAEE, 0xAAF5,
    0xAAF6, 0xABE3, 0xABE5, 0xABE6, 0xABE8, 0xABE9, 0xABEC, 0xABED,
    0xD7B0, 0xD7CB, 0xFB1E, 0xFE00, 0xFE20, 0xFEFF, 0xFF9E, 0xFFF0,
    0x101FD, 0x102E0, 0x10376, 0x10A01, 0x10A05, 0x10A0C, 0x10A38, 0x10A3F,
    0x10AE5, 0x10D24, 0x10D69, 0x10EAB, 0x10EFC, 0x10F46, 0x10F82, 0x11000,
    0x11001, 0x11002, 0x11038, 0x11070, 0x11073, 0x1107F, 0x11082, 0x110B0,
    0x110B3, 0x110B7, 0x110B9, 0x110BD, 0x110C2, 0x110CD, 0x11100, 0x11127,
    0x1112C, 0x1112D, 0x11145, 0x11173, 0x11180, 0x11182, 0x111B3, 0x111B6,
    0x111BF, 0x111C0, 0x111C2, 0x111C9, 0x111CE, 0x111CF, 0x1122C, 0x1122F,
    0x11232, 0x11234, 0x1123E, 0x11241, 0x112DF, 0x112E0, 0x112E3, 0x11300,
    0x11302, 0x1133B, 0x1133E, 0x1133F, 0x11340, 0x11341, 0x11347, 0x1134B,
    0x1134D, 0x11357, 0x11362, 0x11366, 0x11370, 0x113B8, 0x113B9, 0x113BB,
    0x113C2, 0x113C5, 0x113C7, 0x113CA, 0x113CC, 0x113CE, 0x113D1, 0x113D2,
    0x113E1, 0x11435, 0x11438, 0x11440, 0x11442, 0x11445, 0x11446, 0x1145E,
    0x114B0, 0x114B1, 0x114B3, 0x114B9, 0x114BA, 0x114BB, 0x114BD, 0x114BE,
    0x114BF, 0x114C1, 0x114C2, 0x115AF, 0x115B0, 0x115B2, 0x115B8, 0x115BC,
    0x115BE, 0x115BF, 0x115DC, 0x11630, 0x11633, 0x1163B, 0x1163D, 0x1163E,
    0x1163F, 0x116AB, 0x116AC, 0x116AD, 0x116AE, 0x116B0, 0x1171D, 0x1171E,
    0x1171F, 0x11722, 0x11726, 0x11727, 0x1182C, 0x1182F, 0x11838, 0x11839,
    0x11930, 0x11931, 0x11937, 0x1193B, 0x1193F, 0x11940, 0x11941, 0x11942,
    0x11943, 0x119D1, 0x119D4, 0x119DA, 0x119DC, 0x119E0, 0x119E4, 0x11A01,
    0x11A33, 0x11A39, 0x11A3A, 0x11A3B, 0x11A47, 0x11A51, 0x11A57, 0x11A59,
    0x11A84, 0x11A8A, 0x11A97, 0x11A98, 0x11C2F, 0x11C30, 0x11C38, 0x11C3E,
    0x11C3F, 0x11C92, 0x11CA9, 0x11CAA, 0x11CB1, 0x11CB2, 0x11CB4, 0x11CB5,
    0x11D31, 0x11D3A, 0x11D3C, 0x11D3F, 0x11D46, 0x11D47, 0x11D8A, 0x11D90,
    0x11D93, 0x11D95, 0x11D96, 0x11D97, 0x11EF3, 0x11EF5, 0x11F00, 0x11F02,
    0x11F03, 0x11F34, 0x11F36, 0x11F3E, 0x11F40, 0x11F5A, 0x13430, 0x13440,
    0x13447, 0x1611E, 0x1612A, 0x1612D, 0x16AF0, 0x16B30, 0x16D63, 0x16D67,
    0x16F4F, 0x16F51, 0x16F8F, 0x16FE4, 0x16FF0, 0x1BC9D, 0x1BCA0, 0x1CF00,
    0x1CF30, 0x1D165, 0x1D16D, 0x1D173, 0x1D17B, 0x1D185, 0x1D1AA, 0x1D242,
    0x1DA00, 0x1DA3B, 0x1DA75, 0x1DA84, 0x1DA9B, 0x1DAA1, 0x1E000, 0x1E008,
    0x1E01B, 0x1E023, 0x1E026, 0x1E08F, 0x1E130, 0x1E2AE, 0x1E2EC, 0x1E4EC,
    0x1E5EE, 0x1E8D0, 0x1E944, 0x1F000, 0x1F10D, 0x1F12F, 0x1F16C, 0x1F17E,
    0x1F18E, 0x1F191, 0x1F1AD, 0x1F1E6, 0x1F201, 0x1F21A, 0x1F22F, 0x1F232,
    0x1F23C, 0x1F249, 0x1F3FB, 0x1F400, 0x1F546, 0x1F680, 0x1F774, 0x1F7D5,
    0x1F80C, 0x1F848, 0x1F85A, 0x1F888, 0x1F8AE, 0x1F90C, 0x1F93C, 0x1F947,
    0x1FC00, 0xE0000, 0xE0020, 0xE0080, 0xE0100, 0xE01F0,
]

const GB_HI: [678]u32 = [
    0x9F, 0xA9, 0xAD, 0xAE, 0x36F, 0x489, 0x5BD, 0x5BF,
    0x5C2, 0x5C5, 0x5C7, 0x605, 0x61A, 0x61C, 0x65F, 0x670,
    0x6DC, 0x6DD, 0x6E4, 0x6E8, 0x6ED, 0x70F, 0x711, 0x74A,
    0x7B0, 0x7F3, 0x7FD, 0x819, 0x823, 0x827, 0x82D, 0x85B,
    0x891, 0x89F, 0x8E1, 0x8E2, 0x902, 0x903, 0x939, 0x93A,
    0x93B, 0x93C, 0x940, 0x948, 0x94C, 0x94D, 0x94F, 0x957,
    0x95F, 0x963, 0x97F, 0x981, 0x983, 0x9A8, 0x9B0, 0x9B2,
    0x9B9, 0x9BC, 0x9BE, 0x9C0, 0x9C4, 0x9C8, 0x9CC, 0x9CD,
    0x9D7, 0x9DD, 0x9DF, 0x9E3, 0x9F1, 0x9FE, 0xA02, 0xA03,
    0xA3C, 0xA40, 0xA42, 0xA48, 0xA4D, 0xA51, 0xA71, 0xA75,
    0xA82, 0xA83, 0xAA8, 0xAB0, 0xAB3, 0xAB9, 0xABC, 0xAC0,
    0xAC5, 0xAC8, 0xAC9, 0xACC, 0xACD, 0xAE3, 0xAF9, 0xAFF,
    0xB01, 0xB03, 0xB28, 0xB30, 0xB33, 0xB39, 0xB3C, 0xB3F,
    0xB40, 0xB44, 0xB48, 0xB4C, 0xB4D, 0xB57, 0xB5D, 0xB5F,
    0xB63, 0xB71, 0xB82, 0xBBE, 0xBBF, 0xBC0, 0xBC2, 0xBC8,
    0xBCC, 0xBCD, 0xBD7, 0xC00, 0xC03, 0xC04, 0xC28, 0xC39,
    0xC3C, 0xC40, 0xC44, 0xC48, 0xC4C, 0xC4D, 0xC56, 0xC5A,
    0xC63, 0xC81, 0xC83, 0xCBC, 0xCBE, 0xCC0, 0xCC1, 0xCC2,
    0xCC4, 0xCC8, 0xCCD, 0xCD6, 0xCE3, 0xCF3, 0xD01, 0xD03,
    0xD3A, 0xD3C, 0xD3E, 0xD40, 0xD44, 0xD48, 0xD4C, 0xD4D,
    0xD4E, 0xD57, 0xD63, 0xD81, 0xD83, 0xDCA, 0xDCF, 0xDD1,
    0xDD4, 0xDD6, 0xDDE, 0xDDF, 0xDF3, 0xE31, 0xE33, 0xE3A,
    0xE4E, 0xEB1, 0xEB3, 0xEBC, 0xECE, 0xF19, 0xF35, 0xF37,
    0xF39, 0xF3F, 0xF7E, 0xF7F, 0xF84, 0xF87, 0xF97, 0xFBC,
    0xFC6, 0x1030, 0x1031, 0x1037, 0x103A, 0x103C, 0x103E, 0x1057,
    0x1059, 0x1060, 0x1074, 0x1082, 0x1084, 0x1086, 0x108D, 0x109D,
    0x115F, 0x11A7, 0x11FF, 0x135F, 0x1715, 0x1734, 0x1753, 0x1773,
    0x17B5, 0x17B6, 0x17BD, 0x17C5, 0x17C6, 0x17C8, 0x17D3, 0x17DD,
    0x180D, 0x180E, 0x180F, 0x1886, 0x18A9, 0x1922, 0x1926, 0x1928,
    0x192B, 0x1931, 0x1932, 0x1938, 0x193B, 0x1A18, 0x1A1A, 0x1A1B,
    0x1A55, 0x1A56, 0x1A57, 0x1A5E, 0x1A60, 0x1A62, 0x1A6C, 0x1A72,
    0x1A7C, 0x1A7F, 0x1ACE, 0x1B03, 0x1B04, 0x1B3D, 0x1B41, 0x1B44,
    0x1B73, 0x1B81, 0x1B82, 0x1BA1, 0x1BA5, 0x1BA7, 0x1BAD, 0x1BE6,
    0x1BE7, 0x1BE9, 0x1BEC, 0x1BED, 0x1BEE, 0x1BF3, 0x1C2B, 0x1C33,
    0x1C35, 0x1C37, 0x1CD2, 0x1CE0, 0x1CE1, 0x1CE8, 0x1CED, 0x1CF4,
    0x1CF7, 0x1CF9, 0x1DFF, 0x200B, 0x200C, 0x200D, 0x200F, 0x202E,
    0x203C, 0x2049, 0x206F, 0x20F0, 0x2122, 0x2139, 0x2199, 0x21AA,
    0x231B, 0x2328, 0x2388, 0x23CF, 0x23F3, 0x23FA, 0x24C2, 0x25AB,
    0x25B6, 0x25C0, 0x25FE, 0x2605, 0x2612, 0x2685, 0x2705, 0x2712,
    0x2714, 0x2716, 0x271D, 0x2721, 0x2728, 0x2734, 0x2744, 0x2747,
    0x274C, 0x274E, 0x2755, 0x2757, 0x2767, 0x2797, 0x27A1, 0x27B0,
    0x27BF, 0x2935, 0x2B07, 0x2B1C, 0x2B50, 0x2B55, 0x2CF1, 0x2D7F,
    0x2DFF, 0x302F, 0x3030, 0x303D, 0x309A, 0x3297, 0x3299, 0xA672,
    0xA67D, 0xA69F, 0xA6F1, 0xA802, 0xA806, 0xA80B, 0xA824, 0xA826,
    0xA827, 0xA82C, 0xA881, 0xA8C3, 0xA8C5, 0xA8F1, 0xA8FF, 0xA92D,
    0xA951, 0xA952, 0xA953, 0xA97C, 0xA982, 0xA983, 0xA9B3, 0xA9B5,
    0xA9B9, 0xA9BB, 0xA9BD, 0xA9BF, 0xA9C0, 0xA9E5, 0xAA2E, 0xAA30,
    0xAA32, 0xAA34, 0xAA36, 0xAA43, 0xAA4C, 0xAA4D, 0xAA7C, 0xAAB0,
    0xAAB4, 0xAAB8, 0xAABF, 0xAAC1, 0xAAEB, 0xAAED, 0xAAEF, 0xAAF5,
    0xAAF6, 0xABE4, 0xABE5, 0xABE7, 0xABE8, 0xABEA, 0xABEC, 0xABED,
    0xD7C6, 0xD7FB, 0xFB1E, 0xFE0F, 0xFE2F, 0xFEFF, 0xFF9F, 0xFFFB,
    0x101FD, 0x102E0, 0x1037A, 0x10A03, 0x10A06, 0x10A0F, 0x10A3A, 0x10A3F,
    0x10AE6, 0x10D27, 0x10D6D, 0x10EAC, 0x10EFF, 0x10F50, 0x10F85, 0x11000,
    0x11001, 0x11002, 0x11046, 0x11070, 0x11074, 0x11081, 0x11082, 0x110B2,
    0x110B6, 0x110B8, 0x110BA, 0x110BD, 0x110C2, 0x110CD, 0x11102, 0x1112B,
    0x1112C, 0x11134, 0x11146, 0x11173, 0x11181, 0x11182, 0x111B5, 0x111BE,
    0x111BF, 0x111C0, 0x111C3, 0x111CC, 0x111CE, 0x111CF, 0x1122E, 0x11231,
    0x11233, 0x11237, 0x1123E, 0x11241, 0x112DF, 0x112E2, 0x112EA, 0x11301,
    0x11303, 0x1133C, 0x1133E, 0x1133F, 0x11340, 0x11344, 0x11348, 0x1134C,
    0x1134D, 0x11357, 0x11363, 0x1136C, 0x11374, 0x113B8, 0x113BA, 0x113C0,
    0x113C2, 0x113C5, 0x113C9, 0x113CA, 0x113CD, 0x113D0, 0x113D1, 0x113D2,
    0x113E2, 0x11437, 0x1143F, 0x11441, 0x11444, 0x11445, 0x11446, 0x1145E,
    0x114B0, 0x114B2, 0x114B8, 0x114B9, 0x114BA, 0x114BC, 0x114BD, 0x114BE,
    0x114C0, 0x114C1, 0x114C3, 0x115AF, 0x115B1, 0x115B5, 0x115BB, 0x115BD,
    0x115BE, 0x115C0, 0x115DD, 0x11632, 0x1163A, 0x1163C, 0x1163D, 0x1163E,
    0x11640, 0x116AB, 0x116AC, 0x116AD, 0x116AF, 0x116B7, 0x1171D, 0x1171E,
    0x1171F, 0x11725, 0x11726, 0x1172B, 0x1182E, 0x11837, 0x11838, 0x1183A,
    0x11930, 0x11935, 0x11938, 0x1193E, 0x1193F, 0x11940, 0x11941, 0x11942,
    0x11943, 0x119D3, 0x119D7, 0x119DB, 0x119DF, 0x119E0, 0x119E4, 0x11A0A,
    0x11A38, 0x11A39, 0x11A3A, 0x11A3E, 0x11A47, 0x11A56, 0x11A58, 0x11A5B,
    0x11A89, 0x11A96, 0x11A97, 0x11A99, 0x11C2F, 0x11C36, 0x11C3D, 0x11C3E,
    0x11C3F, 0x11CA7, 0x11CA9, 0x11CB0, 0x11CB1, 0x11CB3, 0x11CB4, 0x11CB6,
    0x11D36, 0x11D3A, 0x11D3D, 0x11D45, 0x11D46, 0x11D47, 0x11D8E, 0x11D91,
    0x11D94, 0x11D95, 0x11D96, 0x11D97, 0x11EF4, 0x11EF6, 0x11F01, 0x11F02,
    0x11F03, 0x11F35, 0x11F3A, 0x11F3F, 0x11F42, 0x11F5A, 0x1343F, 0x13440,
    0x13455, 0x16129, 0x1612C, 0x1612F, 0x16AF4, 0x16B36, 0x16D63, 0x16D6A,
    0x16F4F, 0x16F87, 0x16F92, 0x16FE4, 0x16FF1, 0x1BC9E, 0x1BCA3, 0x1CF2D,
    0x1CF46, 0x1D169, 0x1D172, 0x1D17A, 0x1D182, 0x1D18B, 0x1D1AD, 0x1D244,
    0x1DA36, 0x1DA6C, 0x1DA75, 0x1DA84, 0x1DA9F, 0x1DAAF, 0x1E006, 0x1E018,
    0x1E021, 0x1E024, 0x1E02A, 0x1E08F, 0x1E136, 0x1E2AE, 0x1E2EF, 0x1E4EF,
    0x1E5EF, 0x1E8D6, 0x1E94A, 0x1F0FF, 0x1F10F, 0x1F12F, 0x1F171, 0x1F17F,
    0x1F18E, 0x1F19A, 0x1F1E5, 0x1F1FF, 0x1F20F, 0x1F21A, 0x1F22F, 0x1F23A,
    0x1F23F, 0x1F3FA, 0x1F3FF, 0x1F53D, 0x1F64F, 0x1F6FF, 0x1F77F, 0x1F7FF,
    0x1F80F, 0x1F84F, 0x1F85F, 0x1F88F, 0x1F8FF, 0x1F93A, 0x1F945, 0x1FAFF,
    0x1FFFD, 0xE001F, 0xE007F, 0xE00FF, 0xE01EF, 0xE0FFF,
]

const GB_KIND: [678]u8 = [
    3, 14, 3, 14, 16, 16, 16, 16, 16, 16, 16, 7, 16, 3, 16, 16, 16, 7, 16, 16, 16, 7, 16, 16,
    16, 16, 16, 16, 16, 16, 16, 16, 7, 16, 16, 7, 16, 8, 15, 16, 8, 16, 8, 16, 8, 17, 8, 16,
    15, 16, 15, 16, 8, 15, 15, 15, 15, 16, 16, 8, 16, 8, 8, 17, 16, 15, 15, 16, 15, 16, 16, 8,
    16, 8, 16, 16, 16, 16, 16, 16, 16, 8, 15, 15, 15, 15, 16, 8, 16, 16, 8, 8, 17, 16, 15, 16,
    16, 8, 15, 15, 15, 15, 16, 16, 8, 16, 8, 8, 17, 16, 15, 15, 16, 15, 16, 16, 8, 16, 8, 8,
    8, 16, 16, 16, 8, 16, 15, 15, 16, 16, 8, 16, 16, 17, 16, 15, 16, 16, 8, 16, 8, 16, 8, 16,
    8, 16, 16, 16, 16, 8, 16, 8, 15, 16, 16, 8, 16, 8, 8, 17, 7, 16, 16, 16, 8, 16, 16, 8,
    16, 16, 8, 16, 8, 16, 8, 16, 16, 16, 8, 16, 16, 16, 16, 16, 16, 8, 16, 8, 16, 16, 16, 16,
    16, 16, 8, 16, 16, 8, 16, 8, 16, 16, 16, 16, 8, 16, 16, 16, 9, 10, 11, 16, 16, 16, 16, 16,
    16, 8, 16, 8, 16, 8, 16, 16, 16, 3, 16, 16, 16, 16, 8, 16, 8, 8, 16, 8, 16, 16, 8, 16,
    8, 16, 8, 16, 16, 16, 16, 8, 16, 16, 16, 16, 8, 16, 8, 16, 16, 16, 8, 8, 16, 8, 16, 16,
    8, 16, 8, 16, 8, 16, 8, 16, 8, 16, 16, 16, 8, 16, 16, 16, 8, 16, 16, 3, 4, 5, 3, 3,
    14, 14, 3, 16, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
    14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 16, 16,
    16, 16, 14, 14, 16, 14, 14, 16, 16, 16, 16, 16, 16, 16, 8, 16, 8, 16, 8, 8, 16, 16, 16, 16,
    16, 8, 16, 9, 16, 8, 16, 8, 16, 8, 16, 8, 16, 16, 16, 8, 16, 8, 16, 16, 16, 8, 16, 16,
    16, 16, 16, 16, 8, 16, 8, 8, 16, 8, 16, 8, 16, 8, 8, 16, 10, 11, 16, 16, 16, 3, 16, 3,
    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 8, 16, 8, 16, 16, 16, 16, 8, 8,
    16, 8, 16, 7, 16, 7, 16, 16, 8, 16, 8, 16, 16, 8, 8, 16, 8, 16, 7, 16, 8, 16, 8, 16,
    8, 16, 16, 16, 16, 8, 16, 16, 8, 16, 16, 8, 16, 8, 8, 8, 16, 16, 8, 16, 16, 16, 8, 16,
    16, 16, 16, 8, 8, 16, 7, 16, 16, 8, 16, 8, 16, 8, 16, 16, 16, 8, 16, 8, 16, 8, 16, 8,
    16, 8, 16, 16, 8, 16, 8, 16, 8, 16, 16, 8, 16, 8, 16, 8, 16, 16, 8, 16, 8, 16, 16, 8,
    16, 16, 8, 16, 8, 16, 8, 16, 16, 8, 8, 16, 7, 8, 7, 8, 16, 8, 16, 16, 8, 16, 8, 16,
    16, 8, 7, 16, 16, 16, 8, 16, 7, 16, 8, 16, 8, 16, 16, 8, 16, 16, 8, 16, 8, 16, 8, 16,
    16, 16, 16, 16, 7, 16, 8, 16, 8, 16, 8, 16, 16, 8, 16, 7, 8, 8, 16, 8, 16, 16, 3, 16,
    16, 16, 8, 16, 16, 16, 10, 10, 16, 8, 16, 16, 16, 16, 3, 16, 16, 16, 16, 3, 16, 16, 16, 16,
    16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 14, 14, 14, 14, 14,
    14, 14, 14, 6, 14, 14, 14, 14, 14, 14, 16, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
    14, 3, 16, 3, 16, 3,
]

// no columns: nonspacing and enclosing marks, format characters but the
// soft hyphen, Hangul vowels and finals
const ZERO_LO: [372]u32 = [
    0x300, 0x483, 0x591, 0x5BF, 0x5C1, 0x5C4, 0x5C7, 0x600,
    0x610, 0x61C, 0x64B, 0x670, 0x6D6, 0x6DF, 0x6E7, 0x6EA,
    0x70F, 0x711, 0x730, 0x7A6, 0x7EB, 0x7FD, 0x816, 0x81B,
    0x825, 0x829, 0x859, 0x890, 0x897, 0x8CA, 0x93A, 0x93C,
    0x941, 0x94D, 0x951, 0x962, 0x981, 0x9BC, 0x9C1, 0x9CD,
    0x9E2, 0x9FE, 0xA01, 0xA3C, 0xA41, 0xA47, 0xA4B, 0xA51,
    0xA70, 0xA75, 0xA81, 0xABC, 0xAC1, 0xAC7, 0xACD, 0xAE2,
    0xAFA, 0xB01, 0xB3C, 0xB3F, 0xB41, 0xB4D, 0xB55, 0xB62,
    0xB82, 0xBC0, 0xBCD, 0xC00, 0xC04, 0xC3C, 0xC3E, 0xC46,
    0xC4A, 0xC55, 0xC62, 0xC81, 0xCBC, 0xCBF, 0xCC6, 0xCCC,
    0xCE2, 0xD00, 0xD3B, 0xD41, 0xD4D, 0xD62, 0xD81, 0xDCA,
    0xDD2, 0xDD6, 0xE31, 0xE34, 0xE47, 0xEB1, 0xEB4, 0xEC8,
    0xF18, 0xF35, 0xF37, 0xF39, 0xF71, 0xF80, 0xF86, 0xF8D,
    0xF99, 0xFC6, 0x102D, 0x1032, 0x1039, 0x103D, 0x1058, 0x105E,
    0x1071, 0x1082, 0x1085, 0x108D, 0x109D, 0x1160, 0x135D, 0x1712,
    0x1732, 0x1752, 0x1772, 0x17B4, 0x17B7, 0x17C6, 0x17C9, 0x17DD,
    0x180B, 0x1885, 0x18A9, 0x1920, 0x1927, 0x1932, 0x1939, 0x1A17,
    0x1A1B, 0x1A56, 0x1A58, 0x1A60, 0x1A62, 0x1A65, 0x1A73, 0x1A7F,
    0x1AB0, 0x1B00, 0x1B34, 0x1B36, 0x1B3C, 0x1B42, 0x1B6B, 0x1B80,
    0x1BA2, 0x1BA8, 0x1BAB, 0x1BE6, 0x1BE8, 0x1BED, 0x1BEF, 0x1C2C,
    0x1C36, 0x1CD0, 0x1CD4, 0x1CE2, 0x1CED, 0x1CF4, 0x1CF8, 0x1DC0,
    0x200B, 0x202A, 0x2060, 0x2066, 0x20D0, 0x2CEF, 0x2D7F, 0x2DE0,
    0x302A, 0x3099, 0xA66F, 0xA674, 0xA69E, 0xA6F0, 0xA802, 0xA806,
    0xA80B, 0xA825, 0xA82C, 0xA8C4, 0xA8E0, 0xA8FF, 0xA926, 0xA947,
    0xA980, 0xA9B3, 0xA9B6, 0xA9BC, 0xA9E5, 0xAA29, 0xAA31, 0xAA35,
    0xAA43, 0xAA4C, 0xAA7C, 0xAAB0, 0xAAB2, 0xAAB7, 0xAABE, 0xAAC1,
    0xAAEC, 0xAAF6, 0xABE5, 0xABE8, 0xABED, 0xD7B0, 0xD7CB, 0xFB1E,
    0xFE00, 0xFE20, 0xFEFF, 0xFFF9, 0x101FD, 0x102E0, 0x10376, 0x10A01,
    0x10A05, 0x10A0C, 0x10A38, 0x10A3F, 0x10AE5, 0x10D24, 0x10D69, 0x10EAB,
    0x10EFC, 0x10F46, 0x10F82, 0x11001, 0x11038, 0x11070, 0x11073, 0x1107F,
    0x110B3, 0x110B9, 0x110BD, 0x110C2, 0x110CD, 0x11100, 0x11127, 0x1112D,
    0x11173, 0x11180, 0x111B6, 0x111C9, 0x111CF, 0x1122F, 0x11234, 0x11236,
    0x1123E, 0x11241, 0x112DF, 0x112E3, 0x11300, 0x1133B, 0x11340, 0x11366,
    0x11370, 0x113BB, 0x113CE, 0x113D0, 0x113D2, 0x113E1, 0x11438, 0x11442,
    0x11446, 0x1145E, 0x114B3, 0x114BA, 0x114BF, 0x114C2, 0x115B2, 0x115BC,
    0x115BF, 0x115DC, 0x11633, 0x1163D, 0x1163F, 0x116AB, 0x116AD, 0x116B0,
    0x116B7, 0x1171D, 0x1171F, 0x11722, 0x11727, 0x1182F, 0x11839, 0x1193B,
    0x1193E, 0x11943, 0x119D4, 0x119DA, 0x119E0, 0x11A01, 0x11A33, 0x11A3B,
    0x11A47, 0x11A51, 0x11A59, 0x11A8A, 0x11A98, 0x11C30, 0x11C38, 0x11C3F,
    0x11C92, 0x11CAA, 0x11CB2, 0x11CB5, 0x11D31, 0x11D3A, 0x11D3C, 0x11D3F,
    0x11D47, 0x11D90, 0x11D95, 0x11D97, 0x11EF3, 0x11F00, 0x11F36, 0x11F40,
    0x11F42, 0x11F5A, 0x13430, 0x13447, 0x1611E, 0x1612D, 0x16AF0, 0x16B30,
    0x16D63, 0x16D67, 0x16F4F, 0x16F8F, 0x16FE4, 0x1BC9D, 0x1BCA0, 0x1CF00,
    0x1CF30, 0x1D167, 0x1D173, 0x1D185, 0x1D1AA, 0x1D242, 0x1DA00, 0x1DA3B,
    0x1DA75, 0x1DA84, 0x1DA9B, 0x1DAA1, 0x1E000, 0x1E008, 0x1E01B, 0x1E023,
    0x1E026, 0x1E08F, 0x1E130, 0x1E2AE, 0x1E2EC, 0x1E4EC, 0x1E5EE, 0x1E8D0,
    0x1E944, 0xE0001, 0xE0020, 0xE0100,
]

const ZERO_HI: [372]u32 = [
    0x36F, 0x489, 0x5BD, 0x5BF, 0x5C2, 0x5C5, 0x5C7, 0x605,
    0x61A, 0x61C, 0x65F, 0x670, 0x6DD, 0x6E4, 0x6E8, 0x6ED,
    0x70F, 0x711, 0x74A, 0x7B0, 0x7F3, 0x7FD, 0x819, 0x823,
    0x827, 0x82D, 0x85B, 0x891, 0x89F, 0x902, 0x93A, 0x93C,
    0x948, 0x94D, 0x957, 0x963, 0x981, 0x9BC, 0x9C4, 0x9CD,
    0x9E3, 0x9FE, 0xA02, 0xA3C, 0xA42, 0xA48, 0xA4D, 0xA51,
    0xA71, 0xA75, 0xA82, 0xABC, 0xAC5, 0xAC8, 0xACD, 0xAE3,
    0xAFF, 0xB01, 0xB3C, 0xB3F, 0xB44, 0xB4D, 0xB56, 0xB63,
    0xB82, 0xBC0, 0xBCD, 0xC00, 0xC04, 0xC3C, 0xC40, 0xC48,
    0xC4D, 0xC56, 0xC63, 0xC81, 0xCBC, 0xCBF, 0xCC6, 0xCCD,
    0xCE3, 0xD01, 0xD3C, 0xD44, 0xD4D, 0xD63, 0xD81, 0xDCA,
    0xDD4, 0xDD6, 0xE31, 0xE3A, 0xE4E, 0xEB1, 0xEBC, 0xECE,
    0xF19, 0xF35, 0xF37, 0xF39, 0xF7E, 0xF84, 0xF87, 0xF97,
    0xFBC, 0xFC6, 0x1030, 0x1037, 0x103A, 0x103E, 0x1059, 0x1060,
    0x1074, 0x1082, 0x1086, 0x108D, 0x109D, 0x11FF, 0x135F, 0x1714,
    0x1733, 0x1753, 0x1773, 0x17B5, 0x17BD, 0x17C6, 0x17D3, 0x17DD,
    0x180F, 0x1886, 0x18A9, 0x1922, 0x1928, 0x1932, 0x193B, 0x1A18,
    0x1A1B, 0x1A56, 0x1A5E, 0x1A60, 0x1A62, 0x1A6C, 0x1A7C, 0x1A7F,
    0x1ACE, 0x1B03, 0x1B34, 0x1B3A, 0x1B3C, 0x1B42, 0x1B73, 0x1B81,
    0x1BA5, 0x1BA9, 0x1BAD, 0x1BE6, 0x1BE9, 0x1BED, 0x1BF1, 0x1C33,
    0x1C37, 0x1CD2, 0x1CE0, 0x1CE8, 0x1CED, 0x1CF4, 0x1CF9, 0x1DFF,
    0x200F, 0x202E, 0x2064, 0x206F, 0x20F0, 0x2CF1, 0x2D7F, 0x2DFF,
    0x302D, 0x309A, 0xA672, 0xA67D, 0xA69F, 0xA6F1, 0xA802, 0xA806,
    0xA80B, 0xA826, 0xA82C, 0xA8C5, 0xA8F1, 0xA8FF, 0xA92D, 0xA951,
    0xA982, 0xA9B3, 0xA9B9, 0xA9BD, 0xA9E5, 0xAA2E, 0xAA32, 0xAA36,
    0xAA43, 0xAA4C, 0xAA7C, 0xAAB0, 0xAAB4, 0xAAB8, 0xAABF, 0xAAC1,
    0xAAED, 0xAAF6, 0xABE5, 0xABE8, 0xABED, 0xD7C6, 0xD7FB, 0xFB1E,
    0xFE0F, 0xFE2F, 0xFEFF, 0xFFFB, 0x101FD, 0x102E0, 0x1037A, 0x10A03,
    0x10A06, 0x10A0F, 0x10A3A, 0x10A3F, 0x10AE6, 0x10D27, 0x10D6D, 0x10EAC,
    0x10EFF, 0x10F50, 0x10F85, 0x11001, 0x11046, 0x11070, 0x11074, 0x11081,
    0x110B6, 0x110BA, 0x110BD, 0x110C2, 0x110CD, 0x11102, 0x1112B, 0x11134,
    0x11173, 0x11181, 0x111BE, 0x111CC, 0x111CF, 0x11231, 0x11234, 0x11237,
    0x1123E, 0x11241, 0x112DF, 0x112EA, 0x11301, 0x1133C, 0x11340, 0x1136C,
    0x11374, 0x113C0, 0x113CE, 0x113D0, 0x113D2, 0x113E2, 0x1143F, 0x11444,
    0x11446, 0x1145E, 0x114B8, 0x114BA, 0x114C0, 0x114C3, 0x115B5, 0x115BD,
    0x115C0, 0x115DD, 0x1163A, 0x1163D, 0x11640, 0x116AB, 0x116AD, 0x116B5,
    0x116B7, 0x1171D, 0x1171F, 0x11725, 0x1172B, 0x11837, 0x1183A, 0x1193C,
    0x1193E, 0x11943, 0x119D7, 0x119DB, 0x119E0, 0x11A0A, 0x11A38, 0x11A3E,
    0x11A47, 0x11A56, 0x11A5B, 0x11A96, 0x11A99, 0x11C36, 0x11C3D, 0x11C3F,
    0x11CA7, 0x11CB0, 0x11CB3, 0x11CB6, 0x11D36, 0x11D3A, 0x11D3D, 0x11D45,
    0x11D47, 0x11D91, 0x11D95, 0x11D97, 0x11EF4, 0x11F01, 0x11F3A, 0x11F40,
    0x11F42, 0x11F5A, 0x13440, 0x13455, 0x16129, 0x1612F, 0x16AF4, 0x16B36,
    0x16D63, 0x16D6A, 0x16F4F, 0x16F92, 0x16FE4, 0x1BC9E, 0x1BCA3, 0x1CF2D,
    0x1CF46, 0x1D169, 0x1D182, 0x1D18B, 0x1D1AD, 0x1D244, 0x1DA36, 0x1DA6C,
    0x1DA75, 0x1DA84, 0x1DA9F, 0x1DAAF, 0x1E006, 0x1E018, 0x1E021, 0x1E024,
    0x1E02A, 0x1E08F, 0x1E136, 0x1E2AE, 0x1E2EF, 0x1E4EF, 0x1E5EF, 0x1E8D6,
    0x1E94A, 0xE0001, 0xE007F, 0xE01EF,
]

// two columns: East Asian wide and fullwidth
const WIDE_LO: [122]u32 = [
    0x1100, 0x231A, 0x2329, 0x23E9, 0x23F0, 0x23F3, 0x25FD, 0x2614,
    0x2630, 0x2648, 0x267F, 0x268A, 0x2693, 0x26A1, 0x26AA, 0x26BD,
    0x26C4, 0x26CE, 0x26D4, 0x26EA, 0x26F2, 0x26F5, 0x26FA, 0x26FD,
    0x2705, 0x270A, 0x2728, 0x274C, 0x274E, 0x2753, 0x2757, 0x2795,
    0x27B0, 0x27BF, 0x2B1B, 0x2B50, 0x2B55, 0x2E80, 0x2E9B, 0x2F00,
    0x2FF0, 0x3041, 0x3099, 0x3105, 0x3131, 0x3190, 0x31EF, 0x3220,
    0x3250, 0xA490, 0xA960, 0xAC00, 0xF900, 0xFE10, 0xFE30, 0xFE54,
    0xFE68, 0xFF01, 0xFFE0, 0x16FE0, 0x16FF0, 0x17000, 0x18800, 0x18CFF,
    0x1AFF0, 0x1AFF5, 0x1AFFD, 0x1B000, 0x1B132, 0x1B150, 0x1B155, 0x1B164,
    0x1B170, 0x1D300, 0x1D360, 0x1F004, 0x1F0CF, 0x1F18E, 0x1F191, 0x1F200,
    0x1F210, 0x1F240, 0x1F250, 0x1F260, 0x1F300, 0x1F32D, 0x1F337, 0x1F37E,
    0x1F3A0, 0x1F3CF, 0x1F3E0, 0x1F3F4, 0x1F3F8, 0x1F440, 0x1F442, 0x1F4FF,
    0x1F54B, 0x1F550, 0x1F57A, 0x1F595, 0x1F5A4, 0x1F5FB, 0x1F680, 0x1F6CC,
    0x1F6D0, 0x1F6D5, 0x1F6DC, 0x1F6EB, 0x1F6F4, 0x1F7E0, 0x1F7F0, 0x1F90C,
    0x1F93C, 0x1F947, 0x1FA70, 0x1FA80, 0x1FA8F, 0x1FACE, 0x1FADF, 0x1FAF0,
    0x20000, 0x30000,
]

const WIDE_HI: [122]u32 = [
    0x115F, 0x231B, 0x232A, 0x23EC, 0x23F0, 0x23F3, 0x25FE, 0x2615,
    0x2637, 0x2653, 0x267F, 0x268F, 0x2693, 0x26A1, 0x26AB, 0x26BE,
    0x26C5, 0x26CE, 0x26D4, 0x26EA, 0x26F3, 0x26F5, 0x26FA, 0x26FD,
    0x2705, 0x270B, 0x2728, 0x274C, 0x274E, 0x2755, 0x2757, 0x2797,
    0x27B0, 0x27BF, 0x2B1C, 0x2B50, 0x2B55, 0x2E99, 0x2EF3, 0x2FD5,
    0x303E, 0x3096, 0x30FF, 0x312F, 0x318E, 0x31E5, 0x321E, 0x3247,
    0xA48C, 0xA4C6, 0xA97C, 0xD7A3, 0xFAFF, 0xFE19, 0xFE52, 0xFE66,
    0xFE6B, 0xFF60, 0xFFE6, 0x16FE4, 0x16FF1, 0x187F7, 0x18CD5, 0x18D08,
    0x1AFF3, 0x1AFFB, 0x1AFFE, 0x1B122, 0x1B132, 0x1B152, 0x1B155, 0x1B167,
    0x1B2FB, 0x1D356, 0x1D376, 0x1F004, 0x1F0CF, 0x1F18E, 0x1F19A, 0x1F202,
    0x1F23B, 0x1F248, 0x1F251, 0x1F265, 0x1F320, 0x1F335, 0x1F37C, 0x1F393,
    0x1F3CA, 0x1F3D3, 0x1F3F0, 0x1F3F4, 0x1F43E, 0x1F440, 0x1F4FC, 0x1F53D,
    0x1F54E, 0x1F567, 0x1F57A, 0x1F596, 0x1F5A4, 0x1F64F, 0x1F6C5, 0x1F6CC,
    0x1F6D2, 0x1F6D7, 0x1F6DF, 0x1F6EC, 0x1F6FC, 0x1F7EB, 0x1F7F0, 0x1F93A,
    0x1F945, 0x1F9FF, 0x1FA7C, 0x1FA89, 0x1FAC6, 0x1FADC, 0x1FAE9, 0x1FAF8,
    0x2FFFD, 0x3FFFD,
]

// the simple lowercase mapping: LO, LO + STEP, ... HI each add DELTA
const LOWER_LO: [187]u32 = [
    0x41, 0xC0, 0xD8, 0x100, 0x130, 0x132, 0x139, 0x14A,
    0x178, 0x179, 0x181, 0x182, 0x186, 0x187, 0x189, 0x18B,
    0x18E, 0x18F, 0x190, 0x191, 0x193, 0x194, 0x196, 0x197,
    0x198, 0x19C, 0x19D, 0x19F, 0x1A0, 0x1A6, 0x1A7, 0x1A9,
    0x1AC, 0x1AE, 0x1AF, 0x1B1, 0x1B3, 0x1B7, 0x1B8, 0x1BC,
    0x1C4, 0x1C5, 0x1C7, 0x1C8, 0x1CA, 0x1CB, 0x1DE, 0x1F1,
    0x1F2, 0x1F6, 0x1F7, 0x1F8, 0x220, 0x222, 0x23A, 0x23B,
    0x23D, 0x23E, 0x241, 0x243, 0x244, 0x245, 0x246, 0x370,
    0x376, 0x37F, 0x386, 0x388, 0x38C, 0x38E, 0x391, 0x3A3,
    0x3CF, 0x3D8, 0x3F4, 0x3F7, 0x3F9, 0x3FA, 0x3FD, 0x400,
    0x410, 0x460, 0x48A, 0x4C0, 0x4C1, 0x4D0, 0x531, 0x10A0,
    0x10C7, 0x10CD, 0x13A0, 0x13F0, 0x1C89, 0x1C90, 0x1CBD, 0x1E00,
    0x1E9E, 0x1EA0, 0x1F08, 0x1F18, 0x1F28, 0x1F38, 0x1F48, 0x1F59,
    0x1F68, 0x1F88, 0x1F98, 0x1FA8, 0x1FB8, 0x1FBA, 0x1FBC, 0x1FC8,
    0x1FCC, 0x1FD8, 0x1FDA, 0x1FE8, 0x1FEA, 0x1FEC, 0x1FF8, 0x1FFA,
    0x1FFC, 0x2126, 0x212A, 0x212B, 0x2132, 0x2160, 0x2183, 0x24B6,
    0x2C00, 0x2C60, 0x2C62, 0x2C63, 0x2C64, 0x2C67, 0x2C6D, 0x2C6E,
    0x2C6F, 0x2C70, 0x2C72, 0x2C75, 0x2C7E, 0x2C80, 0x2CEB, 0x2CF2,
    0xA640, 0xA680, 0xA722, 0xA732, 0xA779, 0xA77D, 0xA77E, 0xA78B,
    0xA78D, 0xA790, 0xA796, 0xA7AA, 0xA7AB, 0xA7AC, 0xA7AD, 0xA7AE,
    0xA7B0, 0xA7B1, 0xA7B2, 0xA7B3, 0xA7B4, 0xA7C4, 0xA7C5, 0xA7C6,
    0xA7C7, 0xA7CB, 0xA7CC, 0xA7D0, 0xA7D6, 0xA7DC, 0xA7F5, 0xFF21,
    0x10400, 0x104B0, 0x10570, 0x1057C, 0x1058C, 0x10594, 0x10C80, 0x10D50,
    0x118A0, 0x16E40, 0x1E900,
]

const LOWER_HI: [187]u32 = [
    0x5A, 0xD6, 0xDE, 0x12E, 0x130, 0x136, 0x147, 0x176,
    0x178, 0x17D, 0x181, 0x184, 0x186, 0x187, 0x18A, 0x18B,
    0x18E, 0x18F, 0x190, 0x191, 0x193, 0x194, 0x196, 0x197,
    0x198, 0x19C, 0x19D, 0x19F, 0x1A4, 0x1A6, 0x1A7, 0x1A9,
    0x1AC, 0x1AE, 0x1AF, 0x1B2, 0x1B5, 0x1B7, 0x1B8, 0x1BC,
    0x1C4, 0x1C5, 0x1C7, 0x1C8, 0x1CA, 0x1DB, 0x1EE, 0x1F1,
    0x1F4, 0x1F6, 0x1F7, 0x21E, 0x220, 0x232, 0x23A, 0x23B,
    0x23D, 0x23E, 0x241, 0x243, 0x244, 0x245, 0x24E, 0x372,
    0x376, 0x37F, 0x386, 0x38A, 0x38C, 0x38F, 0x3A1, 0x3AB,
    0x3CF, 0x3EE, 0x3F4, 0x3F7, 0x3F9, 0x3FA, 0x3FF, 0x40F,
    0x42F, 0x480, 0x4BE, 0x4C0, 0x4CD, 0x52E, 0x556, 0x10C5,
    0x10C7, 0x10CD, 0x13EF, 0x13F5, 0x1C89, 0x1CBA, 0x1CBF, 0x1E94,
    0x1E9E, 0x1EFE, 0x1F0F, 0x1F1D, 0x1F2F, 0x1F3F, 0x1F4D, 0x1F5F,
    0x1F6F, 0x1F8F, 0x1F9F, 0x1FAF, 0x1FB9, 0x1FBB, 0x1FBC, 0x1FCB,
    0x1FCC, 0x1FD9, 0x1FDB, 0x1FE9, 0x1FEB, 0x1FEC, 0x1FF9, 0x1FFB,
    0x1FFC, 0x2126, 0x212A, 0x212B, 0x2132, 0x216F, 0x2183, 0x24CF,
    0x2C2F, 0x2C60, 0x2C62, 0x2C63, 0x2C64, 0x2C6B, 0x2C6D, 0x2C6E,
    0x2C6F, 0x2C70, 0x2C72, 0x2C75, 0x2C7F, 0x2CE2, 0x2CED, 0x2CF2,
    0xA66C, 0xA69A, 0xA72E, 0xA76E, 0xA77B, 0xA77D, 0xA786, 0xA78B,
    0xA78D, 0xA792, 0xA7A8, 0xA7AA, 0xA7AB, 0xA7AC, 0xA7AD, 0xA7AE,
    0xA7B0, 0xA7B1, 0xA7B2, 0xA7B3, 0xA7C2, 0xA7C4, 0xA7C5, 0xA7C6,
    0xA7C9, 0xA7CB, 0xA7CC, 0xA7D0, 0xA7DA, 0xA7DC, 0xA7F5, 0xFF3A,
    0x10427, 0x104D3, 0x1057A, 0x1058A, 0x10592, 0x10595, 0x10CB2, 0x10D65,
    0x118BF, 0x16E5F, 0x1E921,
]

const LOWER_DELTA: [187]i32 = [
    32, 32, 32, 1, -199, 1, 1, 1, -121, 1, 210, 1,
    206, 1, 205, 1, 79, 202, 203, 1, 205, 207, 211, 209,
    1, 211, 213, 214, 1, 218, 1, 218, 1, 218, 1, 217,
    1, 219, 1, 1, 2, 1, 2, 1, 2, 1, 1, 2,
    1, -97, -56, 1, -130, 1, 10795, 1, -163, 10792, 1, -195,
    69, 71, 1, 1, 1, 116, 38, 37, 64, 63, 32, 32,
    8, 1, -60, 1, -7, 1, -130, 80, 32, 1, 1, 15,
    1, 1, 48, 7264, 7264, 7264, 38864, 8, 1, -3008, -3008, 1,
    -7615, 1, -8, -8, -8, -8, -8, -8, -8, -8, -8, -8,
    -8, -74, -9, -86, -9, -8, -100, -8, -112, -7, -128, -126,
    -9, -7517, -8383, -8262, 28, 16, 1, 26, 48, 1, -10743, -3814,
    -10727, 1, -10780, -10749, -10783, -10782, 1, 1, -10815, 1, 1, 1,
    1, 1, 1, 1, 1, -35332, 1, 1, -42280, 1, 1, -42308,
    -42319, -42315, -42305, -42308, -42258, -42282, -42261, 928, 1, -48, -42307, -35384,
    1, -42343, 1, 1, 1, -42561, 1, 32, 40, 40, 39, 39,
    39, 39, 64, 32, 32, 32, 34,
]

const LOWER_STEP: [187]u8 = [
    1, 1, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1,
    2, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
    1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1,
    2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1,
    2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
]

// the simple uppercase mapping: LO, LO + STEP, ... HI each add DELTA
const UPPER_LO: [205]u32 = [
    0x61, 0xB5, 0xE0, 0xF8, 0xFF, 0x101, 0x131, 0x133,
    0x13A, 0x14B, 0x17A, 0x17F, 0x180, 0x183, 0x188, 0x18C,
    0x192, 0x195, 0x199, 0x19A, 0x19B, 0x19E, 0x1A1, 0x1A8,
    0x1AD, 0x1B0, 0x1B4, 0x1B9, 0x1BD, 0x1BF, 0x1C5, 0x1C6,
    0x1C8, 0x1C9, 0x1CB, 0x1CC, 0x1CE, 0x1DD, 0x1DF, 0x1F2,
    0x1F3, 0x1F5, 0x1F9, 0x223, 0x23C, 0x23F, 0x242, 0x247,
    0x250, 0x251, 0x252, 0x253, 0x254, 0x256, 0x259, 0x25B,
    0x25C, 0x260, 0x261, 0x263, 0x264, 0x265, 0x266, 0x268,
    0x269, 0x26A, 0x26B, 0x26C, 0x26F, 0x271, 0x272, 0x275,
    0x27D, 0x280, 0x282, 0x283, 0x287, 0x288, 0x289, 0x28A,
    0x28C, 0x292, 0x29D, 0x29E, 0x345, 0x371, 0x377, 0x37B,
    0x3AC, 0x3AD, 0x3B1, 0x3C2, 0x3C3, 0x3CC, 0x3CD, 0x3D0,
    0x3D1, 0x3D5, 0x3D6, 0x3D7, 0x3D9, 0x3F0, 0x3F1, 0x3F2,
    0x3F3, 0x3F5, 0x3F8, 0x3FB, 0x430, 0x450, 0x461, 0x48B,
    0x4C2, 0x4CF, 0x4D1, 0x561, 0x10D0, 0x10FD, 0x13F8, 0x1C80,
    0x1C81, 0x1C82, 0x1C83, 0x1C85, 0x1C86, 0x1C87, 0x1C88, 0x1C8A,
    0x1D79, 0x1D7D, 0x1D8E, 0x1E01, 0x1E9B, 0x1EA1, 0x1F00, 0x1F10,
    0x1F20, 0x1F30, 0x1F40, 0x1F51, 0x1F60, 0x1F70, 0x1F72, 0x1F76,
    0x1F78, 0x1F7A, 0x1F7C, 0x1F80, 0x1F90, 0x1FA0, 0x1FB0, 0x1FB3,
    0x1FBE, 0x1FC3, 0x1FD0, 0x1FE0, 0x1FE5, 0x1FF3, 0x214E, 0x2170,
    0x2184, 0x24D0, 0x2C30, 0x2C61, 0x2C65, 0x2C66, 0x2C68, 0x2C73,
    0x2C76, 0x2C81, 0x2CEC, 0x2CF3, 0x2D00, 0x2D27, 0x2D2D, 0xA641,
    0xA681, 0xA723, 0xA733, 0xA77A, 0xA77F, 0xA78C, 0xA791, 0xA794,
    0xA797, 0xA7B5, 0xA7C8, 0xA7CD, 0xA7D1, 0xA7D7, 0xA7F6, 0xAB53,
    0xAB70, 0xFF41, 0x10428, 0x104D8, 0x10597, 0x105A3, 0x105B3, 0x105BB,
    0x10CC0, 0x10D70, 0x118C0, 0x16E60, 0x1E922,
]

const UPPER_HI: [205]u32 = [
    0x7A, 0xB5, 0xF6, 0xFE, 0xFF, 0x12F, 0x131, 0x137,
    0x148, 0x177, 0x17E, 0x17F, 0x180, 0x185, 0x188, 0x18C,
    0x192, 0x195, 0x199, 0x19A, 0x19B, 0x19E, 0x1A5, 0x1A8,
    0x1AD, 0x1B0, 0x1B6, 0x1B9, 0x1BD, 0x1BF, 0x1C5, 0x1C6,
    0x1C8, 0x1C9, 0x1CB, 0x1CC, 0x1DC, 0x1DD, 0x1EF, 0x1F2,
    0x1F3, 0x1F5, 0x21F, 0x233, 0x23C, 0x240, 0x242, 0x24F,
    0x250, 0x251, 0x252, 0x253, 0x254, 0x257, 0x259, 0x25B,
    0x25C, 0x260, 0x261, 0x263, 0x264, 0x265, 0x266, 0x268,
    0x269, 0x26A, 0x26B, 0x26C, 0x26F, 0x271, 0x272, 0x275,
    0x27D, 0x280, 0x282, 0x283, 0x287, 0x288, 0x289, 0x28B,
    0x28C, 0x292, 0x29D, 0x29E, 0x345, 0x373, 0x377, 0x37D,
    0x3AC, 0x3AF, 0x3C1, 0x3C2, 0x3CB, 0x3CC, 0x3CE, 0x3D0,
    0x3D1, 0x3D5, 0x3D6, 0x3D7, 0x3EF, 0x3F0, 0x3F1, 0x3F2,
    0x3F3, 0x3F5, 0x3F8, 0x3FB, 0x44F, 0x45F, 0x481, 0x4BF,
    0x4CE, 0x4CF, 0x52F, 0x586, 0x10FA, 0x10FF, 0x13FD, 0x1C80,
    0x1C81, 0x1C82, 0x1C84, 0x1C85, 0x1C86, 0x1C87, 0x1C88, 0x1C8A,
    0x1D79, 0x1D7D, 0x1D8E, 0x1E95, 0x1E9B, 0x1EFF, 0x1F07, 0x1F15,
    0x1F27, 0x1F37, 0x1F45, 0x1F57, 0x1F67, 0x1F71, 0x1F75, 0x1F77,
    0x1F79, 0x1F7B, 0x1F7D, 0x1F87, 0x1F97, 0x1FA7, 0x1FB1, 0x1FB3,
    0x1FBE, 0x1FC3, 0x1FD1, 0x1FE1, 0x1FE5, 0x1FF3, 0x214E, 0x217F,
    0x2184, 0x24E9, 0x2C5F, 0x2C61, 0x2C65, 0x2C66, 0x2C6C, 0x2C73,
    0x2C76, 0x2CE3, 0x2CEE, 0x2CF3, 0x2D25, 0x2D27, 0x2D2D, 0xA66D,
    0xA69B, 0xA72F, 0xA76F, 0xA77C, 0xA787, 0xA78C, 0xA793, 0xA794,
    0xA7A9, 0xA7C3, 0xA7CA, 0xA7CD, 0xA7D1, 0xA7DB, 0xA7F6, 0xAB53,
    0xABBF, 0xFF5A, 0x1044F, 0x104FB, 0x105A1, 0x105B1, 0x105B9, 0x105BC,
    0x10CF2, 0x10D85, 0x118DF, 0x16E7F, 0x1E943,
]

const UPPER_DELTA: [205]i32 = [
    -32, 743, -32, -32, 121, -1, -232, -1, -1, -1, -1, -300,
    195, -1, -1, -1, -1, 97, -1, 163, 42561, 130, -1, -1,
    -1, -1, -1, -1, -1, 56, -1, -2, -1, -2, -1, -2,
    -1, -79, -1, -1, -2, -1, -1, -1, -1, 10815, -1, -1,
    10783, 10780, 10782, -210, -206, -205, -202, -203, 42319, -205, 42315, -207,
    42343, 42280, 42308, -209, -211, 42308, 10743, 42305, -211, 10749, -213, -214,
    10727, -218, 42307, -218, 42282, -218, -69, -217, -71, -219, 42261, 42258,
    84, -1, -1, 130, -38, -37, -32, -31, -32, -64, -63, -62,
    -57, -47, -54, -8, -1, -86, -80, 7, -116, -96, -1, -1,
    -32, -80, -1, -1, -1, -15, -1, -48, 3008, 3008, -8, -6254,
    -6253, -6244, -6242, -6243, -6236, -6181, 35266, -1, 35332, 3814, 35384, -1,
    -59, -1, 8, 8, 8, 8, 8, 8, 8, 74, 86, 100,
    128, 112, 126, 8, 8, 8, 8, 9, -7205, 9, 8, 8,
    7, 9, -28, -16, -1, -26, -48, -1, -10795, -10792, -1, -1,
    -1, -1, -1, -1, -7264, -7264, -7264, -1, -1, -1, -1, -1,
    -1, -1, -1, 48, -1, -1, -1, -1, -1, -1, -1, -928,
    -38864, -32, -40, -40, -39, -39, -39, -39, -64, -32, -32, -32,
    -34,
]

const UPPER_STEP: [205]u8 = [
    1, 1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1,
    1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 2, 2, 1, 1, 1, 2,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 2, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1,
    1, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
]

// where case folding differs from the simple lowercase mapping
const FOLD_CP: [198]u32 = [
    0xB5, 0x130, 0x17F, 0x345, 0x3C2, 0x3D0, 0x3D1, 0x3D5,
    0x3D6, 0x3F0, 0x3F1, 0x3F5, 0x13A0, 0x13A1, 0x13A2, 0x13A3,
    0x13A4, 0x13A5, 0x13A6, 0x13A7, 0x13A8, 0x13A9, 0x13AA, 0x13AB,
    0x13AC, 0x13AD, 0x13AE, 0x13AF, 0x13B0, 0x13B1, 0x13B2, 0x13B3,
    0x13B4, 0x13B5, 0x13B6, 0x13B7, 0x13B8, 0x13B9, 0x13BA, 0x13BB,
    0x13BC, 0x13BD, 0x13BE, 0x13BF, 0x13C0, 0x13C1, 0x13C2, 0x13C3,
    0x13C4, 0x13C5, 0x13C6, 0x13C7, 0x13C8, 0x13C9, 0x13CA, 0x13CB,
    0x13CC, 0x13CD, 0x13CE, 0x13CF, 0x13D0, 0x13D1, 0x13D2, 0x13D3,
    0x13D4, 0x13D5, 0x13D6, 0x13D7, 0x13D8, 0x13D9, 0x13DA, 0x13DB,
    0x13DC, 0x13DD, 0x13DE, 0x13DF, 0x13E0, 0x13E1, 0x13E2, 0x13E3,
    0x13E4, 0x13E5, 0x13E6, 0x13E7, 0x13E8, 0x13E9, 0x13EA, 0x13EB,
    0x13EC, 0x13ED, 0x13EE, 0x13EF, 0x13F0, 0x13F1, 0x13F2, 0x13F3,
    0x13F4, 0x13F5, 0x13F8, 0x13F9, 0x13FA, 0x13FB, 0x13FC, 0x13FD,
    0x1C80, 0x1C81, 0x1C82, 0x1C83, 0x1C84, 0x1C85, 0x1C86, 0x1C87,
    0x1C88, 0x1E9B, 0x1FBE, 0x1FD3, 0x1FE3, 0xAB70, 0xAB71, 0xAB72,
    0xAB73, 0xAB74, 0xAB75, 0xAB76, 0xAB77, 0xAB78, 0xAB79, 0xAB7A,
    0xAB7B, 0xAB7C, 0xAB7D, 0xAB7E, 0xAB7F, 0xAB80, 0xAB81, 0xAB82,
    0xAB83, 0xAB84, 0xAB85, 0xAB86, 0xAB87, 0xAB88, 0xAB89, 0xAB8A,
    0xAB8B, 0xAB8C, 0xAB8D, 0xAB8E, 0xAB8F, 0xAB90, 0xAB91, 0xAB92,
    0xAB93, 0xAB94, 0xAB95, 0xAB96, 0xAB97, 0xAB98, 0xAB99, 0xAB9A,
    0xAB9B, 0xAB9C, 0xAB9D, 0xAB9E, 0xAB9F, 0xABA0, 0xABA1, 0xABA2,
    0xABA3, 0xABA4, 0xABA5, 0xABA6, 0xABA7, 0xABA8, 0xABA9, 0xABAA,
    0xABAB, 0xABAC, 0xABAD, 0xABAE, 0xABAF, 0xABB0, 0xABB1, 0xABB2,
    0xABB3, 0xABB4, 0xABB5, 0xABB6, 0xABB7, 0xABB8, 0xABB9, 0xABBA,
    0xABBB, 0xABBC, 0xABBD, 0xABBE, 0xABBF, 0xFB05,
]

const FOLD_TO: [198]u32 = [
    0x3BC, 0x130, 0x73, 0x3B9, 0x3C3, 0x3B2, 0x3B8, 0x3C6,
    0x3C0, 0x3BA, 0x3C1, 0x3B5, 0x13A0, 0x13A1, 0x13A2, 0x13A3,
    0x13A4, 0x13A5, 0x13A6, 0x13A7, 0x13A8, 0x13A9, 0x13AA, 0x13AB,
    0x13AC, 0x13AD, 0x13AE, 0x13AF, 0x13B0, 0x13B1, 0x13B2, 0x13B3,
    0x13B4, 0x13B5, 0x13B6, 0x13B7, 0x13B8, 0x13B9, 0x13BA, 0x13BB,
    0x13BC, 0x13BD, 0x13BE, 0x13BF, 0x13C0, 0x13C1, 0x13C2, 0x13C3,
    0x13C4, 0x13C5, 0x13C6, 0x13C7, 0x13C8, 0x13C9, 0x13CA, 0x13CB,
    0x13CC, 0x13CD, 0x13CE, 0x13CF, 0x13D0, 0x13D1, 0x13D2, 0x13D3,
    0x13D4, 0x13D5, 0x13D6, 0x13D7, 0x13D8, 0x13D9, 0x13DA, 0x13DB,
    0x13DC, 0x13DD, 0x13DE, 0x13DF, 0x13E0, 0x13E1, 0x13E2, 0x13E3,
    0x13E4, 0x13E5, 0x13E6, 0x13E7, 0x13E8, 0x13E9, 0x13EA, 0x13EB,
    0x13EC, 0x13ED, 0x13EE, 0x13EF, 0x13F0, 0x13F1, 0x13F2, 0x13F3,
    0x13F4, 0x13F5, 0x13F0, 0x13F1, 0x13F2, 0x13F3, 0x13F4, 0x13F5,
    0x432, 0x434, 0x43E, 0x441, 0x442, 0x442, 0x44A, 0x463,
    0xA64B, 0x1E61, 0x3B9, 0x390, 0x3B0, 0x13A0, 0x13A1, 0x13A2,
    0x13A3, 0x13A4, 0x13A5, 0x13A6, 0x13A7, 0x13A8, 0x13A9, 0x13AA,
    0x13AB, 0x13AC, 0x13AD, 0x13AE, 0x13AF, 0x13B0, 0x13B1, 0x13B2,
    0x13B3, 0x13B4, 0x13B5, 0x13B6, 0x13B7, 0x13B8, 0x13B9, 0x13BA,
    0x13BB, 0x13BC, 0x13BD, 0x13BE, 0x13BF, 0x13C0, 0x13C1, 0x13C2,
    0x13C3, 0x13C4, 0x13C5, 0x13C6, 0x13C7, 0x13C8, 0x13C9, 0x13CA,
    0x13CB, 0x13CC, 0x13CD, 0x13CE, 0x13CF, 0x13D0, 0x13D1, 0x13D2,
    0x13D3, 0x13D4, 0x13D5, 0x13D6, 0x13D7, 0x13D8, 0x13D9, 0x13DA,
    0x13DB, 0x13DC, 0x13DD, 0x13DE, 0x13DF, 0x13E0, 0x13E1, 0x13E2,
    0x13E3, 0x13E4, 0x13E5, 0x13E6, 0x13E7, 0x13E8, 0x13E9, 0x13EA,
    0x13EB, 0x13EC, 0x13ED, 0x13EE, 0x13EF, 0xFB06,
]

// upper mappings to more than one code point: CP[i] becomes TO[AT[i]..AT[i + 1]]
const UPPER_SPECIAL_CP: [102]u32 = [
    0xDF, 0x149, 0x1F0, 0x390, 0x3B0, 0x587, 0x1E96, 0x1E97,
    0x1E98, 0x1E99, 0x1E9A, 0x1F50, 0x1F52, 0x1F54, 0x1F56, 0x1F80,
    0x1F81, 0x1F82, 0x1F83, 0x1F84, 0x1F85, 0x1F86, 0x1F87, 0x1F88,
    0x1F89, 0x1F8A, 0x1F8B, 0x1F8C, 0x1F8D, 0x1F8E, 0x1F8F, 0x1F90,
    0x1F91, 0x1F92, 0x1F93, 0x1F94, 0x1F95, 0x1F96, 0x1F97, 0x1F98,
    0x1F99, 0x1F9A, 0x1F9B, 0x1F9C, 0x1F9D, 0x1F9E, 0x1F9F, 0x1FA0,
    0x1FA1, 0x1FA2, 0x1FA3, 0x1FA4, 0x1FA5, 0x1FA6, 0x1FA7, 0x1FA8,
    0x1FA9, 0x1FAA, 0x1FAB, 0x1FAC, 0x1FAD, 0x1FAE, 0x1FAF, 0x1FB2,
    0x1FB3, 0x1FB4, 0x1FB6, 0x1FB7, 0x1FBC, 0x1FC2, 0x1FC3, 0x1FC4,
    0x1FC6, 0x1FC7, 0x1FCC, 0x1FD2, 0x1FD3, 0x1FD6, 0x1FD7, 0x1FE2,
    0x1FE3, 0x1FE4, 0x1FE6, 0x1FE7, 0x1FF2, 0x1FF3, 0x1FF4, 0x1FF6,
    0x1FF7, 0x1FFC, 0xFB00, 0xFB01, 0xFB02, 0xFB03, 0xFB04, 0xFB05,
    0xFB06, 0xFB13, 0xFB14, 0xFB15, 0xFB16, 0xFB17,
]

const UPPER_SPECIAL_AT: [103]u16 = [
    0, 2, 4, 6, 9, 12, 14, 16, 18, 20, 22, 24, 26, 29, 32, 35,
    37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67,
    69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99,
    101, 103, 105, 107, 109, 111, 113, 115, 117, 119, 121, 123, 125, 127, 129, 131,
    133, 135, 137, 139, 142, 144, 146, 148, 150, 152, 155, 157, 160, 163, 165, 168,
    171, 174, 176, 178, 181, 183, 185, 187, 189, 192, 194, 196, 198, 200, 203, 206,
    208, 210, 212, 214, 216, 218, 220,
]

const UPPER_SPECIAL_TO: [220]u32 = [
    0x53, 0x53, 0x2BC, 0x4E, 0x4A, 0x30C, 0x399, 0x308,
    0x301, 0x3A5, 0x308, 0x301, 0x535, 0x552, 0x48, 0x331,
    0x54, 0x308, 0x57, 0x30A, 0x59, 0x30A, 0x41, 0x2BE,
    0x3A5, 0x313, 0x3A5, 0x313, 0x300, 0x3A5, 0x313, 0x301,
    0x3A5, 0x313, 0x342, 0x1F08, 0x399, 0x1F09, 0x399, 0x1F0A,
    0x399, 0x1F0B, 0x399, 0x1F0C, 0x399, 0x1F0D, 0x399, 0x1F0E,
    0x399, 0x1F0F, 0x399, 0x1F08, 0x399, 0x1F09, 0x399, 0x1F0A,
    0x399, 0x1F0B, 0x399, 0x1F0C, 0x399, 0x1F0D, 0x399, 0x1F0E,
    0x399, 0x1F0F, 0x399, 0x1F28, 0x399, 0x1F29, 0x399, 0x1F2A,
    0x399, 0x1F2B, 0x399, 0x1F2C, 0x399, 0x1F2D, 0x399, 0x1F2E,
    0x399, 0x1F2F, 0x399, 0x1F28, 0x399, 0x1F29, 0x399, 0x1F2A,
    0x399, 0x1F2B, 0x399, 0x1F2C, 0x399, 0x1F2D, 0x399, 0x1F2E,
    0x399, 0x1F2F, 0x399, 0x1F68, 0x399, 0x1F69, 0x399, 0x1F6A,
    0x399, 0x1F6B, 0x399, 0x1F6C, 0x399, 0x1F6D, 0x399, 0x1F6E,
    0x399, 0x1F6F, 0x399, 0x1F68, 0x399, 0x1F69, 0x399, 0x1F6A,
    0x399, 0x1F6B, 0x399, 0x1F6C, 0x399, 0x1F6D, 0x399, 0x1F6E,
    0x399, 0x1F6F, 0x399, 0x1FBA, 0x399, 0x391, 0x399, 0x386,
    0x399, 0x391, 0x342, 0x391, 0x342, 0x399, 0x391, 0x399,
    0x1FCA, 0x399, 0x397, 0x399, 0x389, 0x399, 0x397, 0x342,
    0x397, 0x342, 0x399, 0x397, 0x399, 0x399, 0x308, 0x300,
    0x399, 0x308, 0x301, 0x399, 0x342, 0x399, 0x308, 0x342,
    0x3A5, 0x308, 0x300, 0x3A5, 0x308, 0x301, 0x3A1, 0x313,
    0x3A5, 0x342, 0x3A5, 0x308, 0x342, 0x1FFA, 0x399, 0x3A9,
    0x399, 0x38F, 0x399, 0x3A9, 0x342, 0x3A9, 0x342, 0x399,
    0x3A9, 0x399, 0x46, 0x46, 0x46, 0x49, 0x46, 0x4C,
    0x46, 0x46, 0x49, 0x46, 0x46, 0x4C, 0x53, 0x54,
    0x53, 0x54, 0x544, 0x546, 0x544, 0x535, 0x544, 0x53B,
    0x54E, 0x546, 0x544, 0x53D,
]

// lower mappings to more than one code point: CP[i] becomes TO[AT[i]..AT[i + 1]]
const LOWER_SPECIAL_CP: [1]u32 = [
    0x130,
]

const LOWER_SPECIAL_AT: [2]u16 = [
    0, 2,
]

const LOWER_SPECIAL_TO: [2]u32 = [
    0x69, 0x307,
]

// fold mappings to more than one code point: CP[i] becomes TO[AT[i]..AT[i + 1]]
const FOLD_SPECIAL_CP: [104]u32 = [
    0xDF, 0x130, 0x149, 0x1F0, 0x390, 0x3B0, 0x587, 0x1E96,
    0x1E97, 0x1E98, 0x1E99, 0x1E9A, 0x1E9E, 0x1F50, 0x1F52, 0x1F54,
    0x1F56, 0x1F80, 0x1F81, 0x1F82, 0x1F83, 0x1F84, 0x1F85, 0x1F86,
    0x1F87, 0x1F88, 0x1F89, 0x1F8A, 0x1F8B, 0x1F8C, 0x1F8D, 0x1F8E,
    0x1F8F, 0x1F90, 0x1F91, 0x1F92, 0x1F93, 0x1F94, 0x1F95, 0x1F96,
    0x1F97, 0x1F98, 0x1F99, 0x1F9A, 0x1F9B, 0x1F9C, 0x1F9D, 0x1F9E,
    0x1F9F, 0x1FA0, 0x1FA1, 0x1FA2, 0x1FA3, 0x1FA4, 0x1FA5, 0x1FA6,
    0x1FA7, 0x1FA8, 0x1FA9, 0x1FAA, 0x1FAB, 0x1FAC, 0x1FAD, 0x1FAE,
    0x1FAF, 0x1FB2, 0x1FB3, 0x1FB4, 0x1FB6, 0x1FB7, 0x1FBC, 0x1FC2,
    0x1FC3, 0x1FC4, 0x1FC6, 0x1FC7, 0x1FCC, 0x1FD2, 0x1FD3, 0x1FD6,
    0x1FD7, 0x1FE2, 0x1FE3, 0x1FE4, 0x1FE6, 0x1FE7, 0x1FF2, 0x1FF3,
    0x1FF4, 0x1FF6, 0x1FF7, 0x1FFC, 0xFB00, 0xFB01, 0xFB02, 0xFB03,
    0xFB04, 0xFB05, 0xFB06, 0xFB13, 0xFB14, 0xFB15, 0xFB16, 0xFB17,
]

const FOLD_SPECIAL_AT: [105]u16 = [
    0, 2, 4, 6, 8, 11, 14, 16, 18, 20, 22, 24, 26, 28, 30, 33,
    36, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67,
    69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99,
    101, 103, 105, 107, 109, 111, 113, 115, 117, 119, 121, 123, 125, 127, 129, 131,
    133, 135, 137, 139, 141, 143, 146, 148, 150, 152, 154, 156, 159, 161, 164, 167,
    169, 172, 175, 178, 180, 182, 185, 187, 189, 191, 193, 196, 198, 200, 202, 204,
    207, 210, 212, 214, 216, 218, 220, 222, 224,
]

const FOLD_SPECIAL_TO: [224]u32 = [
    0x73, 0x73, 0x69, 0x307, 0x2BC, 0x6E, 0x6A, 0x30C,
    0x3B9, 0x308, 0x301, 0x3C5, 0x308, 0x301, 0x565, 0x582,
    0x68, 0x331, 0x74, 0x308, 0x77, 0x30A, 0x79, 0x30A,
    0x61, 0x2BE, 0x73, 0x73, 0x3C5, 0x313, 0x3C5, 0x313,
    0x300, 0x3C5, 0x313, 0x301, 0x3C5, 0x313, 0x342, 0x1F00,
    0x3B9, 0x1F01, 0x3B9, 0x1F02, 0x3B9, 0x1F03, 0x3B9, 0x1F04,
    0x3B9, 0x1F05, 0x3B9, 0x1F06, 0x3B9, 0x1F07, 0x3B9, 0x1F00,
    0x3B9, 0x1F01, 0x3B9, 0x1F02, 0x3B9, 0x1F03, 0x3B9, 0x1F04,
    0x3B9, 0x1F05, 0x3B9, 0x1F06, 0x3B9, 0x1F07, 0x3B9, 0x1F20,
    0x3B9, 0x1F21, 0x3B9, 0x1F22, 0x3B9, 0x1F23, 0x3B9, 0x1F24,
    0x3B9, 0x1F25, 0x3B9, 0x1F26, 0x3B9, 0x1F27, 0x3B9, 0x1F20,
    0x3B9, 0x1F21, 0x3B9, 0x1F22, 0x3B9, 0x1F23, 0x3B9, 0x1F24,
    0x3B9, 0x1F25, 0x3B9, 0x1F26, 0x3B9, 0x1F27, 0x3B9, 0x1F60,
    0x3B9, 0x1F61, 0x3B9, 0x1F62, 0x3B9, 0x1F63, 0x3B9, 0x1F64,
    0x3B9, 0x1F65, 0x3B9, 0x1F66, 0x3B9, 0x1F67, 0x3B9, 0x1F60,
    0x3B9, 0x1F61, 0x3B9, 0x1F62, 0x3B9, 0x1F63, 0x3B9, 0x1F64,
    0x3B9, 0x1F65, 0x3B9, 0x1F66, 0x3B9, 0x1F67, 0x3B9, 0x1F70,
    0x3B9, 0x3B1, 0x3B9, 0x3AC, 0x3B9, 0x3B1, 0x342, 0x3B1,
    0x342, 0x3B9, 0x3B1, 0x3B9, 0x1F74, 0x3B9, 0x3B7, 0x3B9,
    0x3AE, 0x3B9, 0x3B7, 0x342, 0x3B7, 0x342, 0x3B9, 0x3B7,
    0x3B9, 0x3B9, 0x308, 0x300, 0x3B9, 0x308, 0x301, 0x3B9,
    0x342, 0x3B9, 0x308, 0x342, 0x3C5, 0x308, 0x300, 0x3C5,
    0x308, 0x301, 0x3C1, 0x313, 0x3C5, 0x342, 0x3C5, 0x308,
    0x342, 0x1F7C, 0x3B9, 0x3C9, 0x3B9, 0x3CE, 0x3B9, 0x3C9,
    0x342, 0x3C9, 0x342, 0x3B9, 0x3C9, 0x3B9, 0x66, 0x66,
    0x66, 0x69, 0x66, 0x6C, 0x66, 0x66, 0x69, 0x66,
    0x66, 0x6C, 0x73, 0x74, 0x73, 0x74, 0x574, 0x576,
    0x574, 0x565, 0x574, 0x56B, 0x57E, 0x576, 0x574, 0x56D,
]

// ---------------------------------------------- end of the generated tables