diff --git a/config/unicode17_universal_glyph_set.yml b/config/unicode17_universal_glyph_set.yml index 654a7f3..ed8b599 100644 --- a/config/unicode17_universal_glyph_set.yml +++ b/config/unicode17_universal_glyph_set.yml @@ -1,6 +1,6 @@ --- unicode_version: 17.0.0 -ucode_version: 0.5.0 +ucode_version: 0.5.1 generated_at: '2026-06-28T00:00:00Z' default_sources: - kind: fontist diff --git a/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb b/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb index a848567..00dd5a3 100644 --- a/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb +++ b/lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb @@ -56,11 +56,17 @@ class CodepointMapper # about block scope). # @param force_positional_for_font_ids [Set] Type0 # font object IDs that always trigger positional attribution + # @param trace_cache [PageTraceCache, nil] consulted by + # {#needs_positional?} for the partial-ToUnicode-coverage + # case (the font has glyphs the CMap doesn't list). nil = + # skip that check (legacy behavior). def initialize(strategies:, block_range: nil, - force_positional_for_font_ids: Set.new) + force_positional_for_font_ids: Set.new, + trace_cache: nil) @strategies = strategies @block_range = block_range @force_positional_for_font_ids = force_positional_for_font_ids + @trace_cache = trace_cache end # Convenience builder — wires up the default 3-strategy chain @@ -71,7 +77,9 @@ def initialize(strategies:, block_range: nil, # @param trace_cache [PageTraceCache, nil] when provided, the # TraceStrategy shares this cache (lets the caller reuse the # traced pages for downstream concerns like Catalog's - # location lookup). nil = construct internally. + # location lookup) AND the orchestrator consults it for + # gap detection (ToUnicode partial-coverage case). nil = + # construct internally. # @return [CodepointMapper] def self.build(source:, correlator_configs:, indexer:, block_range: nil, force_positional_for_font_ids: Set.new, @@ -93,7 +101,8 @@ def self.build(source:, correlator_configs:, indexer:, ] new(strategies: strategies, block_range: block_range, - force_positional_for_font_ids: force_positional_for_font_ids) + force_positional_for_font_ids: force_positional_for_font_ids, + trace_cache: trace_cache) end # @param descriptor [RawFontDescriptor] @@ -136,7 +145,7 @@ def run_intrinsic(descriptor, strategies) {} end - # Positional strategies are gated behind three conditions, any + # Positional strategies are gated behind four conditions, any # of which triggers them: # # 1. Caller explicitly listed this font in @@ -146,10 +155,17 @@ def run_intrinsic(descriptor, strategies) # 3. The intrinsic result fell entirely outside the # caller's block scope — the font's CMap encoded the # wrong codepoints (Option 1 auto-detect, e.g. U1F200). + # 4. The intrinsic result covered fewer glyphs than the + # trace cache observes on the page — the font's CMap is + # partial (Unicode Consortium ships orphan glyphs whose + # CIDs the ToUnicode stream forgot to map, e.g. the + # last glyph of Mongolian Supplement U+1166C and Myanmar + # Extended-A U+116DA in the v17.0 charts). def needs_positional?(descriptor, intrinsic_result) return true if @force_positional_for_font_ids.include?(descriptor.font_obj_id) return true if intrinsic_result.empty? return true if intrinsic_out_of_scope?(intrinsic_result) + return true if intrinsic_has_uncovered_gids?(descriptor, intrinsic_result) false end @@ -162,6 +178,21 @@ def intrinsic_out_of_scope?(intrinsic_result) intrinsic_result.keys.all? { |cp| !@block_range.include?(cp) } end + # Detects the partial-ToUnicode-coverage case: the font's + # CMap names N codepoints, but the page actually renders >N + # glyphs from this font (orphan glyphs whose CIDs the CMap + # forgot). Positional attribution via the chart's hex labels + # is the only way to recover them. + def intrinsic_has_uncovered_gids?(descriptor, intrinsic_result) + return false unless @trace_cache + return false if intrinsic_result.empty? + + covered_gids = intrinsic_result.values.to_set + @trace_cache.distinct_gids_for(descriptor.base_font).any? do |gid| + !covered_gids.include?(gid) + end + end + # Positional chain: union of all positional strategies' results. # Within positional, earlier strategies win on conflict (chain # order expresses caller preference — CorrelatorStrategy diff --git a/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb b/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb index e1b6e50..af41d9d 100644 --- a/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb +++ b/lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb @@ -4,6 +4,7 @@ require "ucode/glyphs/embedded_fonts/mutool" require "ucode/glyphs/embedded_fonts/trace_parser" +require "ucode/glyphs/embedded_fonts/trace_glyph" module Ucode module Glyphs @@ -24,6 +25,15 @@ module EmbeddedFonts # Lazy: glyphs are only fetched when a strategy first asks for # them. PDFs where every font has /ToUnicode never trigger the # trace at all. + # + # ## mutool font-name truncation + # + # mutool trace emits font names truncated to 31 chars (PDF + # base-font-name limit). The BaseFont dict may carry the full + # original name (e.g. `HBBJCP+Uni11660Mongoliansupplement`). + # All name comparisons in this class go through + # {TraceGlyph.name_match?} so the truncation doesn't silently + # break lookups for long-named fonts. class PageTraceCache # @param pdf [Pathname, String] # @param page_count [Integer] total pages in the PDF @@ -55,7 +65,7 @@ def each_page_for(base_font) glyphs_by_page.each_with_index do |glyphs, idx| next if idx.zero? - present = glyphs.any? { |g| g.font_name == base_font } + present = glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) } next unless present present_in_any = true @@ -68,8 +78,27 @@ def each_page_for(base_font) # @return [Boolean] true if any page references this font def references_font?(base_font) glyphs_by_page.any? do |page_glyphs| - page_glyphs.any? { |g| g.font_name == base_font } + page_glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) } + end + end + + # Returns the set of distinct GIDs rendered on any page for + # the given font. Used by {CodepointMapper#needs_positional?} + # to detect the partial-ToUnicode-coverage case (font ships + # more glyphs than its CMap admits). + # + # @param base_font [String] + # @return [Set] + def distinct_gids_for(base_font) + gids = Set.new + glyphs_by_page.each do |page_glyphs| + page_glyphs.each do |g| + next unless TraceGlyph.name_match?(g.font_name, base_font) + + gids << g.gid if g.gid + end end + gids end # Locate the first occurrence of a specific (font, gid) pair @@ -87,7 +116,7 @@ def find_glyph(base_font:, gid:) next if idx.zero? match = page_glyphs.find do |g| - g.font_name == base_font && g.gid == gid + TraceGlyph.name_match?(g.font_name, base_font) && g.gid == gid end return { page: idx, x: match.x, y: match.y } if match end diff --git a/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb b/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb index 2b85f7d..d47bdb9 100644 --- a/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb +++ b/lib/ucode/glyphs/embedded_fonts/trace_correlator.rb @@ -48,7 +48,7 @@ def correlate(trace_glyphs) private def select_specimens(trace_glyphs) - trace_glyphs.select { |g| g.font_name == @specimen_font_name } + trace_glyphs.select { |g| TraceGlyph.name_match?(g.font_name, @specimen_font_name) } end def select_labels(trace_glyphs) @@ -93,7 +93,7 @@ def detect_label_font(trace_glyphs) def select_hex_candidates(trace_glyphs) trace_glyphs.select do |g| - g.font_name != @specimen_font_name && + !TraceGlyph.name_match?(g.font_name, @specimen_font_name) && g.unicode&.match?(/\A[0-9A-Fa-f]\z/) end end diff --git a/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb b/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb index bac99fd..cb42a75 100644 --- a/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb +++ b/lib/ucode/glyphs/embedded_fonts/trace_glyph.rb @@ -3,6 +3,16 @@ module Ucode module Glyphs module EmbeddedFonts + # mutool trace emits font names truncated to 31 characters + # (PDF base-font-name limit, see PDF 32000-1 §7.9.6). The + # BaseFont dict, however, may carry the full original name. + # `HBBJCP+Uni11660Mongoliansupplement` (34 chars) is emitted + # by mutool trace as `HBBJCP+Uni11660Mongoliansupplem`. The + # helpers below let callers compare trace-side names against + # catalog-side names without tripping on the truncation. + TRACE_NAME_LIMIT = 31 + private_constant :TRACE_NAME_LIMIT + # Value object for one glyph emitted by `mutool trace`. # # Each `` element in the trace XML maps to one TraceGlyph: @@ -21,7 +31,31 @@ module EmbeddedFonts :y, :unicode, keyword_init: true, - ) + ) do + class << self + # @param name [String, nil] a BaseFont name from `mutool info` + # or a trace-emitted font name + # @return [String, nil] the name truncated to the trace limit + def normalize_name(name) + return nil if name.nil? + + name.length <= TRACE_NAME_LIMIT ? name : name[0, TRACE_NAME_LIMIT] + end + + # True when `a` and `b` resolve to the same normalized + # name. Treats `nil` as never matching (avoids accidental + # collision on missing names). + # + # @param a [String, nil] + # @param b [String, nil] + # @return [Boolean] + def name_match?(a, b) + return false if a.nil? || b.nil? + + normalize_name(a) == normalize_name(b) + end + end + end end end end diff --git a/spec/ucode/glyphs/embedded_fonts/page_trace_cache_spec.rb b/spec/ucode/glyphs/embedded_fonts/page_trace_cache_spec.rb index 130fe85..d8da6eb 100644 --- a/spec/ucode/glyphs/embedded_fonts/page_trace_cache_spec.rb +++ b/spec/ucode/glyphs/embedded_fonts/page_trace_cache_spec.rb @@ -151,4 +151,43 @@ def call(pdf, page) expect(result_b[:x]).to eq(700.0) end end + + # Long font names trip mutool's 31-char trace-output truncation. + # The catalog sees the full BaseFont name from `mutool info` + # (e.g. `HBBJCP+Uni11660Mongoliansupplement`); the trace emits + # `HBBJCP+Uni11660Mongoliansupplem`. All PageTraceCache lookups + # must compare via TraceGlyph.name_match? or long-named fonts + # silently disappear from positional correlation. + describe "long font name tolerance" do + let(:full_name) { "HBBJCP+Uni11660Mongoliansupplement" } # 34 chars + let(:truncated_name) { "HBBJCP+Uni11660Mongoliansupplem" } # 31 chars + let(:mutool) do + StubTraceMutool.new( + by_page: { + 1 => %(), + 2 => %(), + }, + ) + end + let(:cache) { described_class.new(pdf: pdf, page_count: 2, mutool: mutool) } + + it "#each_page_for matches the font via its full BaseFont name" do + yielded = [] + cache.each_page_for(full_name) { |p, _g| yielded << p } + expect(yielded).to contain_exactly(1, 2) + end + + it "#references_font? returns true for the full BaseFont name" do + expect(cache.references_font?(full_name)).to be(true) + end + + it "#distinct_gids_for enumerates GIDs across pages" do + expect(cache.distinct_gids_for(full_name)).to eq(Set.new([96, 108])) + end + + it "#find_glyph locates the (font, gid) pair via the full name" do + result = cache.find_glyph(base_font: full_name, gid: 108) + expect(result).to eq({ page: 2, x: 335.34, y: 520.32 }) + end + end end