fix(glyphs): close two positional-correlation gaps (#57, #58)

config/unicode17_universal_glyph_set.yml

@@ -1,6 +1,6 @@
 ---
 unicode_version: 17.0.0
-ucode_version: 0.5.0
+ucode_version: 0.5.1
 generated_at: '2026-06-28T00:00:00Z'
 default_sources:
 - kind: fontist

lib/ucode/glyphs/embedded_fonts/codepoint_mapper.rb

@@ -56,11 +56,17 @@ class CodepointMapper
         #   about block scope).
         # @param force_positional_for_font_ids [Set<Integer>] Type0
         #   font object IDs that always trigger positional attribution
+        # @param trace_cache [PageTraceCache, nil] consulted by
+        #   {#needs_positional?} for the partial-ToUnicode-coverage
+        #   case (the font has glyphs the CMap doesn't list). nil =
+        #   skip that check (legacy behavior).
         def initialize(strategies:, block_range: nil,
-                       force_positional_for_font_ids: Set.new)
+                       force_positional_for_font_ids: Set.new,
+                       trace_cache: nil)
           @strategies = strategies
           @block_range = block_range
           @force_positional_for_font_ids = force_positional_for_font_ids
+          @trace_cache = trace_cache
         end
 
         # Convenience builder — wires up the default 3-strategy chain
@@ -71,7 +77,9 @@ def initialize(strategies:, block_range: nil,
         # @param trace_cache [PageTraceCache, nil] when provided, the
         #   TraceStrategy shares this cache (lets the caller reuse the
         #   traced pages for downstream concerns like Catalog's
-        #   location lookup). nil = construct internally.
+        #   location lookup) AND the orchestrator consults it for
+        #   gap detection (ToUnicode partial-coverage case). nil =
+        #   construct internally.
         # @return [CodepointMapper]
         def self.build(source:, correlator_configs:, indexer:,
                        block_range: nil, force_positional_for_font_ids: Set.new,
@@ -93,7 +101,8 @@ def self.build(source:, correlator_configs:, indexer:,
           ]
           new(strategies: strategies,
               block_range: block_range,
-              force_positional_for_font_ids: force_positional_for_font_ids)
+              force_positional_for_font_ids: force_positional_for_font_ids,
+              trace_cache: trace_cache)
         end
 
         # @param descriptor [RawFontDescriptor]
@@ -136,7 +145,7 @@ def run_intrinsic(descriptor, strategies)
           {}
         end
 
-        # Positional strategies are gated behind three conditions, any
+        # Positional strategies are gated behind four conditions, any
         # of which triggers them:
         #
         #   1. Caller explicitly listed this font in
@@ -146,10 +155,17 @@ def run_intrinsic(descriptor, strategies)
         #   3. The intrinsic result fell entirely outside the
         #      caller's block scope — the font's CMap encoded the
         #      wrong codepoints (Option 1 auto-detect, e.g. U1F200).
+        #   4. The intrinsic result covered fewer glyphs than the
+        #      trace cache observes on the page — the font's CMap is
+        #      partial (Unicode Consortium ships orphan glyphs whose
+        #      CIDs the ToUnicode stream forgot to map, e.g. the
+        #      last glyph of Mongolian Supplement U+1166C and Myanmar
+        #      Extended-A U+116DA in the v17.0 charts).
         def needs_positional?(descriptor, intrinsic_result)
           return true if @force_positional_for_font_ids.include?(descriptor.font_obj_id)
           return true if intrinsic_result.empty?
           return true if intrinsic_out_of_scope?(intrinsic_result)
+          return true if intrinsic_has_uncovered_gids?(descriptor, intrinsic_result)
 
           false
         end
@@ -162,6 +178,21 @@ def intrinsic_out_of_scope?(intrinsic_result)
           intrinsic_result.keys.all? { |cp| !@block_range.include?(cp) }
         end
 
+        # Detects the partial-ToUnicode-coverage case: the font's
+        # CMap names N codepoints, but the page actually renders >N
+        # glyphs from this font (orphan glyphs whose CIDs the CMap
+        # forgot). Positional attribution via the chart's hex labels
+        # is the only way to recover them.
+        def intrinsic_has_uncovered_gids?(descriptor, intrinsic_result)
+          return false unless @trace_cache
+          return false if intrinsic_result.empty?
+
+          covered_gids = intrinsic_result.values.to_set
+          @trace_cache.distinct_gids_for(descriptor.base_font).any? do |gid|
+            !covered_gids.include?(gid)
+          end
+        end
+
         # Positional chain: union of all positional strategies' results.
         # Within positional, earlier strategies win on conflict (chain
         # order expresses caller preference — CorrelatorStrategy

lib/ucode/glyphs/embedded_fonts/page_trace_cache.rb

@@ -4,6 +4,7 @@
 
 require "ucode/glyphs/embedded_fonts/mutool"
 require "ucode/glyphs/embedded_fonts/trace_parser"
+require "ucode/glyphs/embedded_fonts/trace_glyph"
 
 module Ucode
   module Glyphs
@@ -24,6 +25,15 @@ module EmbeddedFonts
       # Lazy: glyphs are only fetched when a strategy first asks for
       # them. PDFs where every font has /ToUnicode never trigger the
       # trace at all.
+      #
+      # ## mutool font-name truncation
+      #
+      # mutool trace emits font names truncated to 31 chars (PDF
+      # base-font-name limit). The BaseFont dict may carry the full
+      # original name (e.g. `HBBJCP+Uni11660Mongoliansupplement`).
+      # All name comparisons in this class go through
+      # {TraceGlyph.name_match?} so the truncation doesn't silently
+      # break lookups for long-named fonts.
       class PageTraceCache
         # @param pdf [Pathname, String]
         # @param page_count [Integer] total pages in the PDF
@@ -55,7 +65,7 @@ def each_page_for(base_font)
           glyphs_by_page.each_with_index do |glyphs, idx|
             next if idx.zero?
 
-            present = glyphs.any? { |g| g.font_name == base_font }
+            present = glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) }
             next unless present
 
             present_in_any = true
@@ -68,8 +78,27 @@ def each_page_for(base_font)
         # @return [Boolean] true if any page references this font
         def references_font?(base_font)
           glyphs_by_page.any? do |page_glyphs|
-            page_glyphs.any? { |g| g.font_name == base_font }
+            page_glyphs.any? { |g| TraceGlyph.name_match?(g.font_name, base_font) }
+          end
+        end
+
+        # Returns the set of distinct GIDs rendered on any page for
+        # the given font. Used by {CodepointMapper#needs_positional?}
+        # to detect the partial-ToUnicode-coverage case (font ships
+        # more glyphs than its CMap admits).
+        #
+        # @param base_font [String]
+        # @return [Set<Integer>]
+        def distinct_gids_for(base_font)
+          gids = Set.new
+          glyphs_by_page.each do |page_glyphs|
+            page_glyphs.each do |g|
+              next unless TraceGlyph.name_match?(g.font_name, base_font)
+
+              gids << g.gid if g.gid
+            end
           end
+          gids
         end
 
         # Locate the first occurrence of a specific (font, gid) pair
@@ -87,7 +116,7 @@ def find_glyph(base_font:, gid:)
             next if idx.zero?
 
             match = page_glyphs.find do |g|
-              g.font_name == base_font && g.gid == gid
+              TraceGlyph.name_match?(g.font_name, base_font) && g.gid == gid
             end
             return { page: idx, x: match.x, y: match.y } if match
           end

lib/ucode/glyphs/embedded_fonts/trace_correlator.rb

@@ -48,7 +48,7 @@ def correlate(trace_glyphs)
         private
 
         def select_specimens(trace_glyphs)
-          trace_glyphs.select { |g| g.font_name == @specimen_font_name }
+          trace_glyphs.select { |g| TraceGlyph.name_match?(g.font_name, @specimen_font_name) }
         end
 
         def select_labels(trace_glyphs)
@@ -93,7 +93,7 @@ def detect_label_font(trace_glyphs)
 
         def select_hex_candidates(trace_glyphs)
           trace_glyphs.select do |g|
-            g.font_name != @specimen_font_name &&
+            !TraceGlyph.name_match?(g.font_name, @specimen_font_name) &&
               g.unicode&.match?(/\A[0-9A-Fa-f]\z/)
           end
         end

lib/ucode/glyphs/embedded_fonts/trace_glyph.rb

@@ -3,6 +3,16 @@
 module Ucode
   module Glyphs
     module EmbeddedFonts
+      # mutool trace emits font names truncated to 31 characters
+      # (PDF base-font-name limit, see PDF 32000-1 §7.9.6). The
+      # BaseFont dict, however, may carry the full original name.
+      # `HBBJCP+Uni11660Mongoliansupplement` (34 chars) is emitted
+      # by mutool trace as `HBBJCP+Uni11660Mongoliansupplem`. The
+      # helpers below let callers compare trace-side names against
+      # catalog-side names without tripping on the truncation.
+      TRACE_NAME_LIMIT = 31
+      private_constant :TRACE_NAME_LIMIT
+
       # Value object for one glyph emitted by `mutool trace`.
       #
       # Each `<g>` element in the trace XML maps to one TraceGlyph:
@@ -21,7 +31,31 @@ module EmbeddedFonts
         :y,
         :unicode,
         keyword_init: true,
-      )
+      ) do
+        class << self
+          # @param name [String, nil] a BaseFont name from `mutool info`
+          #   or a trace-emitted font name
+          # @return [String, nil] the name truncated to the trace limit
+          def normalize_name(name)
+            return nil if name.nil?
+
+            name.length <= TRACE_NAME_LIMIT ? name : name[0, TRACE_NAME_LIMIT]
+          end
+
+          # True when `a` and `b` resolve to the same normalized
+          # name. Treats `nil` as never matching (avoids accidental
+          # collision on missing names).
+          #
+          # @param a [String, nil]
+          # @param b [String, nil]
+          # @return [Boolean]
+          def name_match?(a, b)
+            return false if a.nil? || b.nil?
+
+            normalize_name(a) == normalize_name(b)
+          end
+        end
+      end
     end
   end
 end

spec/ucode/glyphs/embedded_fonts/page_trace_cache_spec.rb

@@ -151,4 +151,43 @@ def call(pdf, page)
       expect(result_b[:x]).to eq(700.0)
     end
   end
+
+  # Long font names trip mutool's 31-char trace-output truncation.
+  # The catalog sees the full BaseFont name from `mutool info`
+  # (e.g. `HBBJCP+Uni11660Mongoliansupplement`); the trace emits
+  # `HBBJCP+Uni11660Mongoliansupplem`. All PageTraceCache lookups
+  # must compare via TraceGlyph.name_match? or long-named fonts
+  # silently disappear from positional correlation.
+  describe "long font name tolerance" do
+    let(:full_name) { "HBBJCP+Uni11660Mongoliansupplement" } # 34 chars
+    let(:truncated_name) { "HBBJCP+Uni11660Mongoliansupplem" } # 31 chars
+    let(:mutool) do
+      StubTraceMutool.new(
+        by_page: {
+          1 => %(<document><span font="#{truncated_name}"><g glyph="96" x="336.3" y="694.2"/></span></document>),
+          2 => %(<document><span font="#{truncated_name}"><g glyph="108" x="335.34" y="520.32"/></span></document>),
+        },
+      )
+    end
+    let(:cache) { described_class.new(pdf: pdf, page_count: 2, mutool: mutool) }
+
+    it "#each_page_for matches the font via its full BaseFont name" do
+      yielded = []
+      cache.each_page_for(full_name) { |p, _g| yielded << p }
+      expect(yielded).to contain_exactly(1, 2)
+    end
+
+    it "#references_font? returns true for the full BaseFont name" do
+      expect(cache.references_font?(full_name)).to be(true)
+    end
+
+    it "#distinct_gids_for enumerates GIDs across pages" do
+      expect(cache.distinct_gids_for(full_name)).to eq(Set.new([96, 108]))
+    end
+
+    it "#find_glyph locates the (font, gid) pair via the full name" do
+      result = cache.find_glyph(base_font: full_name, gid: 108)
+      expect(result).to eq({ page: 2, x: 335.34, y: 520.32 })
+    end
+  end
 end