From 66ee5a9db3ac49fc45fe4afe11794b4dcfff82c3 Mon Sep 17 00:00:00 2001
From: palmoni5 <d8401948@gmail.com>
Date: Thu, 10 Sep 2026 02:49:22 +0300
Subject: [PATCH 1/2] fix(sefaria): separate the dibbur with a dash in Tosafot
 volumes that end it with a period
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sefaria's Talmud commentaries separate the dibbur hamatchil from the comment with a spaced dash, and both the reader and the line_dh index rely on it. Twelve Tosafot volumes (Bava Batra, Menachot, Niddah, Shevuot, Sukkah, Rosh Hashanah, Moed Katan, Beitzah, Chagigah, Taanit, Makkot, Horayot) end the dibbur with a period instead, so they had no dibburim at all and read as one run-on sentence.

SefariaDashlessDibburim rewrites the first '. ' of a content line of those books to ' – ' when the first sentence is 1-12 words, carries no tag, and is followed by text; lines that already contain a spaced dash are untouched. 88-99% of each volume's content lines fit. Applied right after cleanSefariaLine in the payload reader (so the line is flagged as modified for anchor purposes) and summarised per book at the end of the import.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 .../sefariasqlite/SefariaBookPayloadReader.kt |  2 +-
 .../sefariasqlite/SefariaDashlessDibburim.kt  | 63 +++++++++++++++++++
 .../sefariasqlite/SefariaDirectImporter.kt    |  1 +
 .../SefariaDashlessDibburimTest.kt            | 55 ++++++++++++++++
 4 files changed, 120 insertions(+), 1 deletion(-)
 create mode 100644 generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
 create mode 100644 generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt

diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
index 60a7aa94..ef30d8ae 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
@@ -668,7 +668,7 @@ internal class SefariaBookPayloadReader(
         if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) {
             val content = leafPrimitive?.takeIf { it.isString }?.content
             if (!content.isNullOrEmpty()) {
-                val cleaned = cleanSefariaLine(content)
+                val cleaned = SefariaDashlessDibburim.separate(bookHeTitle, cleanSefariaLine(content))
                 if (cleaned.isNotEmpty()) {
                     output += linePrefix + cleaned
                     if (cleanShifts != null) {
diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
new file mode 100644
index 00000000..e43c4a42
--- /dev/null
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
@@ -0,0 +1,63 @@
+package io.github.kdroidfilter.seforimlibrary.sefariasqlite
+
+import co.touchlab.kermit.Logger
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * Sefaria's Talmud commentaries separate the dibbur hamatchil from the comment
+ * with a spaced dash (`דיבור – פירוש`). The Tosafot volumes listed here end it
+ * with a period instead, so neither the reader nor the `line_dh` index can
+ * tell the dibbur from the first sentence. Rewrites the first `. ` of such a
+ * line to ` – ` so these volumes read and index like the rest.
+ */
+internal object SefariaDashlessDibburim {
+
+    /** Sefaria `heTitle`s whose lines end the dibbur with a period (≥ 88% of content lines fit). */
+    val bookHeTitles: Set<String> = setOf(
+        "תוספות על בבא בתרא",
+        "תוספות על מנחות",
+        "תוספות על נדה",
+        "תוספות על שבועות",
+        "תוספות על סוכה",
+        "תוספות על ראש השנה",
+        "תוספות על מועד קטן",
+        "תוספות על ביצה",
+        "תוספות על חגיגה",
+        "תוספות על תענית",
+        "תוספות על מכות",
+        "תוספות על הוריות",
+    )
+
+    /** Longer first sentences are commentary, not a quoted dibbur. */
+    private const val MAX_DIBBUR_WORDS = 12
+
+    private const val SEPARATOR = " – "
+
+    private val SPACED_DASH = Regex("""\s[-–—]\s""")
+    private val WHITESPACE = Regex("""\s+""")
+
+    private val separatedByBook = ConcurrentHashMap<String, Int>()
+
+    /** Returns [line] with its dibbur separated by a dash, or unchanged when the book or line does not fit. */
+    fun separate(bookHeTitle: String, line: String): String {
+        if (bookHeTitle !in bookHeTitles) return line
+        if (line.startsWith("<h", ignoreCase = true) || SPACED_DASH.containsMatchIn(line)) return line
+        val cut = line.indexOf(". ")
+        if (cut <= 0) return line
+        val dibbur = line.substring(0, cut)
+        if ('<' in dibbur) return line
+        val words = WHITESPACE.split(dibbur.trim()).count { it.isNotEmpty() }
+        if (words !in 1..MAX_DIBBUR_WORDS) return line
+        val comment = line.substring(cut + 2)
+        if (comment.isBlank()) return line
+        separatedByBook.merge(bookHeTitle, 1, Int::plus)
+        return dibbur + SEPARATOR + comment
+    }
+
+    fun logSummary(logger: Logger) {
+        for (title in bookHeTitles) {
+            val n = separatedByBook[title] ?: 0
+            logger.i { "Dashless dibburim: '$title' — $n lines separated" }
+        }
+    }
+}
diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
index 32b66166..633783e6 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
@@ -449,6 +449,7 @@ class SefariaDirectImporter(
         }
 
         logger.i { "Inserted all books and lines" }
+        SefariaDashlessDibburim.logSummary(logger)
 
         // Build the title→bookId index in two global phases (all primaries, then
         // all aliases) so a primary title always beats any alias regardless of
diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt
new file mode 100644
index 00000000..6de78c74
--- /dev/null
+++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt
@@ -0,0 +1,55 @@
+package io.github.kdroidfilter.seforimlibrary.sefariasqlite
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class SefariaDashlessDibburimTest {
+
+    private val book = "תוספות על סוכה"
+
+    @Test
+    fun `first sentence becomes the dibbur, separated by a spaced en dash`() {
+        // Verbatim from Tosafot on Sukkah 2a in seforim.db.
+        assertEquals(
+            "מאי שנא גבי סוכה דתני פסולה ומאי שנא גבי מבוי דתני תקנתא – והא דלא פריך מאי שנא גבי הדס",
+            SefariaDashlessDibburim.separate(
+                book,
+                "מאי שנא גבי סוכה דתני פסולה ומאי שנא גבי מבוי דתני תקנתא. והא דלא פריך מאי שנא גבי הדס",
+            ),
+        )
+    }
+
+    @Test
+    fun `only the first period is touched`() {
+        assertEquals(
+            "אמר רבה – קצת קשה. ותירץ דאין הכי נמי.",
+            SefariaDashlessDibburim.separate(book, "אמר רבה. קצת קשה. ותירץ דאין הכי נמי."),
+        )
+    }
+
+    @Test
+    fun `a line that already has a dash is left alone`() {
+        val line = "סוכה דאורייתא תני פסולה – פרש\"י דשייך למיתני בה לשון פסול. ועוד"
+        assertEquals(line, SefariaDashlessDibburim.separate(book, line))
+    }
+
+    @Test
+    fun `a long first sentence is commentary, not a dibbur`() {
+        val line = "ואם תאמר מאי שנא גבי סוכה דתני פסולה ומאי שנא גבי מבוי דתני תקנתא והא דלא פריך. ויש לומר"
+        assertEquals(line, SefariaDashlessDibburim.separate(book, line))
+    }
+
+    @Test
+    fun `headings, tagged prefixes and lines without a sentence break are left alone`() {
+        assertEquals("<h2>דף ב.</h2>", SefariaDashlessDibburim.separate(book, "<h2>דף ב.</h2>"))
+        assertEquals("<b>תוספות</b>. ביאור", SefariaDashlessDibburim.separate(book, "<b>תוספות</b>. ביאור"))
+        assertEquals("תוספות", SefariaDashlessDibburim.separate(book, "תוספות"))
+        assertEquals("אמר רבה. ", SefariaDashlessDibburim.separate(book, "אמר רבה. "))
+    }
+
+    @Test
+    fun `books outside the list are never changed`() {
+        val line = "מאימתי קורין. משעה שהכהנים נכנסין לאכול"
+        assertEquals(line, SefariaDashlessDibburim.separate("רש\"י על ברכות", line))
+    }
+}

From c5117c0a53a71d1bf6352fea142523f59dd013f2 Mon Sep 17 00:00:00 2001
From: ypl <7353755@gmail.com>
Date: Thu, 10 Sep 2026 16:38:17 +0300
Subject: [PATCH 2/2] fix(sefaria): reject structural dashless markers

---
 .../sefariasqlite/SefariaDashlessDibburim.kt          | 11 ++++++++++-
 .../sefariasqlite/SefariaDirectImporter.kt            |  1 +
 .../sefariasqlite/SefariaDashlessDibburimTest.kt      |  9 +++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
index e43c4a42..bfbba469 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburim.kt
@@ -1,6 +1,7 @@
 package io.github.kdroidfilter.seforimlibrary.sefariasqlite
 
 import co.touchlab.kermit.Logger
+import io.github.kdroidfilter.seforimlibrary.common.dh.DhExtractor
 import java.util.concurrent.ConcurrentHashMap
 
 /**
@@ -50,10 +51,18 @@ internal object SefariaDashlessDibburim {
         if (words !in 1..MAX_DIBBUR_WORDS) return line
         val comment = line.substring(cut + 2)
         if (comment.isBlank()) return line
+        val separated = dibbur + SEPARATOR + comment
+        // Keep this source repair aligned with the downstream index. In
+        // particular, structural markers such as `מתני'` and `(הג"ה` must not
+        // be rewritten merely because they happen to end with a period.
+        if (DhExtractor.extract(separated, DhExtractor.Format.DASH) == null) return line
         separatedByBook.merge(bookHeTitle, 1, Int::plus)
-        return dibbur + SEPARATOR + comment
+        return separated
     }
 
+    /** Starts a fresh per-import summary; this object also serves reusable readers in the same JVM. */
+    fun resetSummary() = separatedByBook.clear()
+
     fun logSummary(logger: Logger) {
         for (title in bookHeTitles) {
             val n = separatedByBook[title] ?: 0
diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
index 508f2b99..011be530 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt
@@ -51,6 +51,7 @@ class SefariaDirectImporter(
         private set
 
     suspend fun import() = coroutineScope {
+        SefariaDashlessDibburim.resetSummary()
         val dbRoot = findDatabaseExportRoot(exportRoot)
         val jsonDir = dbRoot.resolve("json")
         val schemaDir = dbRoot.resolve("schemas")
diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt
index 6de78c74..5a58d4c9 100644
--- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt
+++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDashlessDibburimTest.kt
@@ -47,6 +47,15 @@ class SefariaDashlessDibburimTest {
         assertEquals("אמר רבה. ", SefariaDashlessDibburim.separate(book, "אמר רבה. "))
     }
 
+    @Test
+    fun `structural markers from the real corpus are not rewritten as dibburim`() {
+        // Verbatim from Tosafot on Bava Batra 139b and 163b in seforim.db.
+        val mishnah = "מתני'. והבנות יזונו. מה שהזכיר רשב\"ם"
+        val gloss = "(הג\"ה. שיטה ומחצה. נראה ליישב כגון שחתומים עדים"
+        assertEquals(mishnah, SefariaDashlessDibburim.separate("תוספות על בבא בתרא", mishnah))
+        assertEquals(gloss, SefariaDashlessDibburim.separate("תוספות על בבא בתרא", gloss))
+    }
+
     @Test
     fun `books outside the list are never changed`() {
         val line = "מאימתי קורין. משעה שהכהנים נכנסין לאכול"