Comment parse

src/main/kotlin/org/bsplines/ltexls/parsing/CodeAnnotatedTextBuilder.kt

@@ -26,6 +26,17 @@ abstract class CodeAnnotatedTextBuilder(
   val codeLanguageId: String,
 ) : AnnotatedTextBuilder() {
   abstract fun addCode(code: String): CodeAnnotatedTextBuilder
+  open fun addComment(
+    code: Array<String>,
+    markups: Array<Triple<String, Int, Int>>,
+  ): CodeAnnotatedTextBuilder {
+    for ((index, markup) in markups.withIndex()) {
+      this.addMarkup(markup.first, "\n")
+      this.addCode(code[index])
+    }
+
+    return this
+  }
 
   @Suppress("UNUSED_PARAMETER")
   open fun setSettings(settings: Settings) {
@@ -38,23 +49,28 @@ abstract class CodeAnnotatedTextBuilder(
         "bib",
         "bibtex",
         -> LatexAnnotatedTextBuilder(codeLanguageId)
+
         "git-commit",
         "gitcommit",
         -> GitCommitAnnotatedTextBuilder(codeLanguageId)
+
         "html",
         "xhtml",
         -> HtmlAnnotatedTextBuilder(codeLanguageId)
+
         "context",
         "context.tex",
         "latex",
         "plaintex",
         "rsweave",
         "tex",
         -> LatexAnnotatedTextBuilder(codeLanguageId)
+
         "markdown",
         "quarto",
         "rmd",
         -> MarkdownAnnotatedTextBuilder(codeLanguageId)
+
         "nop" -> NopAnnotatedTextBuilder(codeLanguageId)
         "org" -> OrgAnnotatedTextBuilder(codeLanguageId)
         "plaintext" -> PlaintextAnnotatedTextBuilder(codeLanguageId)

src/main/kotlin/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilder.kt

@@ -7,6 +7,7 @@
 
 package org.bsplines.ltexls.parsing.markdown
 
+import com.vladsch.flexmark.ast.FencedCodeBlock
 import com.vladsch.flexmark.ext.definition.DefinitionExtension
 import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension
 import com.vladsch.flexmark.ext.gitlab.GitLabExtension
@@ -36,6 +37,8 @@ class MarkdownAnnotatedTextBuilder(
   private var firstCellInTableRow = false
   private val nodeTypeStack = ArrayDeque<String>()
   private var language: String = "en-US"
+  private var shadowMarkups = listOf<Triple<String, Int, Int>>()
+  private var shadowOffset = 0
   private val nodeSignatures: MutableList<MarkdownNodeSignature> = ArrayList(
     MarkdownAnnotatedTextBuilderDefaults.DEFAULT_MARKDOWN_NODE_SIGNATURES,
   )
@@ -70,7 +73,9 @@ class MarkdownAnnotatedTextBuilder(
     return result
   }
 
-  private fun addMarkup(newPos: Int) {
+  private fun addMarkup(finalPos: Int) {
+    var newPos = finalPos
+
     val inParagraph: Boolean = isInNodeType("Paragraph")
 
     while ((this.pos < this.code.length) && (this.pos < newPos)) {
@@ -83,19 +88,26 @@ class MarkdownAnnotatedTextBuilder(
       }
 
       if (curPos > this.pos) super.addMarkup(this.code.substring(this.pos, curPos))
-      super.addMarkup(this.code.substring(curPos, curPos + 1), (if (inParagraph) " " else "\n"))
-      this.pos = curPos + 1
+      this.pos = curPos
+      val tmpShadowOffset = shadowOffset
+      if (removeComment()) {
+        newPos += (shadowOffset - tmpShadowOffset)
+      } else {
+        super.addMarkup(this.code.substring(curPos, curPos + 1), (if (inParagraph) " " else "\n"))
+        this.pos += 1
+      }
     }
 
     if (newPos > pos) {
       super.addMarkup(this.code.substring(this.pos, newPos))
       this.pos = newPos
+      removeComment()
     }
   }
 
   private fun addMarkup(node: Node, interpretAs: String) {
-    addMarkup(node.startOffset)
-    val newPos: Int = node.endOffset
+    addMarkup(node.startOffset + shadowOffset)
+    val newPos: Int = node.endOffset + shadowOffset
     super.addMarkup(this.code.substring(this.pos, newPos), interpretAs)
     this.pos = newPos
   }
@@ -116,8 +128,7 @@ class MarkdownAnnotatedTextBuilder(
 
     if (Logging.LOGGER.isLoggable(Level.FINEST)) {
       Logging.LOGGER.finest(
-        "flexmarkAst = "
-          + AstCollectingVisitor().collectAndGetAstText(document),
+        "flexmarkAst = " + AstCollectingVisitor().collectAndGetAstText(document),
       )
     }
 
@@ -129,6 +140,30 @@ class MarkdownAnnotatedTextBuilder(
     return this
   }
 
+  override fun addComment(
+    code: Array<String>,
+    markups: Array<Triple<String, Int, Int>>,
+  ): CodeAnnotatedTextBuilder {
+    var fullCode = ""
+    var clearCode = ""
+
+    for ((index, markup) in markups.withIndex()) {
+      fullCode += markup.first + code[index]
+      clearCode += code[index] + "\n"
+    }
+
+    this.code = fullCode
+    this.pos = 0
+    this.shadowMarkups = markups.toList()
+    this.shadowOffset = 0
+
+    visitChildren(this.parser.parse(clearCode))
+
+    if (this.pos < this.code.length) addMarkup(this.code.length)
+
+    return this
+  }
+
   private fun visit(node: Node) {
     val nodeType: String = node.javaClass.simpleName
 
@@ -143,29 +178,67 @@ class MarkdownAnnotatedTextBuilder(
     }
 
     if (isInIgnoredNodeType()) {
-      addMarkup(node.endOffset)
+      addMarkup(node.endOffset + shadowOffset)
     } else if (isDummyNodeType(nodeType)) {
       addMarkup(node, generateDummy())
     } else if (nodeType == "Text") {
-      addMarkup(node.startOffset)
-      addText(node.endOffset)
+      addMarkup(node.startOffset + shadowOffset)
+      addText(node.endOffset + shadowOffset)
     } else if (nodeType == "HtmlEntity") {
       addMarkup(node, Escaping.unescapeHtml(node.chars))
     } else {
-      if (nodeType == "Paragraph") addMarkup(node.startOffset)
+      if (nodeType == "Paragraph") {
+        addMarkup(node.startOffset + shadowOffset)
+      } else if (nodeType == "FencedCodeBlock") {
+        val block = node as FencedCodeBlock
+        addMarkup(pos + block.openingMarker.count() + block.info.count())
+      }
       this.nodeTypeStack.addLast(nodeType)
       visitChildren(node)
       this.nodeTypeStack.removeLastOrNull()
+      if (nodeType == "FencedCodeBlock") {
+        addMarkup(pos + (node as FencedCodeBlock).closingMarker.count() - 1)
+      }
       if (nodeType == "DefinitionTerm") super.addMarkup("", ".")
     }
   }
 
   private fun visitChildren(node: Node) {
     for (child: Node in node.children) {
+      removeComment()
       visit(child)
     }
   }
 
+  private fun removeComment(): Boolean {
+    var removed = false
+
+    while (shadowMarkups.isNotEmpty()) {
+      val shadowMarkup = shadowMarkups.first()
+
+      if (shadowMarkup.second == pos) {
+        removed = true
+
+        super.addMarkup(shadowMarkup.first, "\n")
+
+        val offset = shadowMarkup.third - shadowMarkup.second
+        // we add new line in markdown code
+        shadowOffset += if (shadowMarkup.first.firstOrNull() == '\n') {
+          offset - 1
+        } else {
+          offset
+        }
+
+        pos += offset
+        shadowMarkups = shadowMarkups.drop(1)
+      } else {
+        break
+      }
+    }
+
+    return removed
+  }
+
   override fun setSettings(settings: Settings) {
     this.language = settings.languageShortCode
 
@@ -181,6 +254,7 @@ class MarkdownAnnotatedTextBuilder(
           dummyGenerator = DummyGenerator.getInstance(plural = plural, vowel = vowel)
           MarkdownNodeSignature.Action.Dummy
         }
+
         else -> continue
       }
 

src/main/kotlin/org/bsplines/ltexls/parsing/program/ProgramAnnotatedTextBuilder.kt

@@ -63,19 +63,27 @@ class ProgramAnnotatedTextBuilder(
     )
     var curPos = 0
 
+    var code = arrayOf<String>()
+    var markups = arrayOf<Triple<String, Int, Int>>()
+
     for (matchResult: MatchResult in lineContentsRegex.findAll(comment)) {
       val matchGroup: MatchGroup = matchResult.groups[1] ?: continue
 
-      var lastPos = curPos
-      curPos = matchGroup.range.first
-      annotatedTextBuilder.addMarkup(comment.substring(lastPos, curPos), "\n")
+      markups += Triple(
+        comment.substring(curPos, matchGroup.range.first),
+        curPos,
+        matchGroup.range.first,
+      )
 
-      lastPos = curPos
       curPos = matchGroup.range.last + 1
-      annotatedTextBuilder.addCode(comment.substring(lastPos, curPos))
+
+      code += comment.substring(matchGroup.range.first, curPos)
     }
 
+    annotatedTextBuilder.addComment(code, markups)
+
     if (curPos < comment.length) annotatedTextBuilder.addMarkup(comment.substring(curPos))
+
     return this
   }
 

src/main/kotlin/org/bsplines/ltexls/parsing/program/ProgramCommentRegexs.kt

@@ -27,7 +27,7 @@ data class ProgramCommentRegexs(
     if (this.lineCommentRegexString != null) {
       if (builder.isNotEmpty()) builder.append("|")
       builder.append(
-        "(?<lineComment>(?:^[ \t]*" + this.lineCommentRegexString + "[ \t](?:.*?)$(?:\r?\n)?)+)",
+        "(?<lineComment>(?:^[ \t]*" + this.lineCommentRegexString + "(?:[ \t].*?)?$(?:\r?\n)?)+)",
       )
     }
 
@@ -68,8 +68,13 @@ data class ProgramCommentRegexs(
       val lineCommentRegexString: String?
 
       when (codeLanguageId) {
+        "rust" -> {
+          blockCommentStartRegexString = "/\\*\\*?"
+          blockCommentEndRegexString = "\\*\\*?/"
+          lineCommentRegexString = "//[/!]?"
+        }
         "c", "cpp", "csharp", "dart", "fsharp", "go", "groovy", "java", "javascript",
-        "javascriptreact", "kotlin", "php", "rust", "scala", "swift", "typescript",
+        "javascriptreact", "kotlin", "php", "scala", "swift", "typescript",
         "typescriptreact", "verilog",
         -> {
           blockCommentStartRegexString = "/\\*\\*?"