Fix heading auto-link hijacking nested link labels (#668)

src/Markdig.Tests/TestAutoLinks.cs

@@ -21,4 +21,28 @@ public class TestAutoLinks
 
         Assert.That(html, Is.EqualTo(expected).IgnoreWhiteSpace);
     }
+
+    // https://github.com/xoofx/markdig/issues/668
+    // A heading's implicit reference must not resolve inside another still-open
+    // link bracket, which would break the outer link.
+    [Test]
+    public void TestAutoIdentifierHeadingLinkDoesNotHijackNestedLinkLabel()
+    {
+        var markdown = "# Testing Markdown\n\n" +
+                        "### Header\n\n" +
+                        "[Testing [Header]](https://www.bing.com)\n\n" +
+                        "[Testing [test]](https://www.google.com)\n";
+        var expected = "<h1 id=\"testing-markdown\">Testing Markdown</h1>\n" +
+                        "<h3 id=\"header\">Header</h3>\n" +
+                        "<p><a href=\"https://www.bing.com\">Testing [Header]</a></p>\n" +
+                        "<p><a href=\"https://www.google.com\">Testing [test]</a></p>";
+
+        var pipeline = new MarkdownPipelineBuilder()
+            .UseAutoLinks()
+            .UseAutoIdentifiers()
+            .Build();
+        var html = Markdown.ToHtml(markdown, pipeline);
+
+        Assert.That(html, Is.EqualTo(expected).IgnoreWhiteSpace);
+    }
 }

src/Markdig/Extensions/AutoIdentifiers/HeadingLinkReferenceDefinition.cs

@@ -24,4 +24,11 @@ public class HeadingLinkReferenceDefinition : LinkReferenceDefinition
     /// Gets or sets the heading related to this link reference definition.
     /// </summary>
     public HeadingBlock Heading { get; set; }
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// Created implicitly for every heading, so it must not resolve inside another
+    /// still-open link bracket, e.g. <c>[Some text [Heading]](url)</c>.
+    /// </remarks>
+    internal override bool AllowResolutionInsideOpenLink => false;
 }

src/Markdig/Parsers/Inlines/LinkInlineParser.cs

@@ -150,6 +150,13 @@ public class LinkInlineParser : InlineParser
             return false;
         }
 
+        // An implicit definition must not hijack a bracket nested inside a
+        // still-open link, which would break the outer one.
+        if (!linkRef.AllowResolutionInsideOpenLink && HasActiveAncestorLink(parent))
+        {
+            return false;
+        }
+
         Inline? link = null;
         // Try to use a callback directly defined on the LinkReferenceDefinition
         if (linkRef.CreateLinkInline != null)
@@ -430,6 +437,24 @@ public class LinkInlineParser : InlineParser
         }
     }
 
+    /// <summary>
+    /// Determines whether the delimiter is nested inside another link/image
+    /// delimiter that may still resolve.
+    /// </summary>
+    private static bool HasActiveAncestorLink(LinkDelimiterInline delimiter)
+    {
+        var inline = delimiter.Parent;
+        while (inline != null)
+        {
+            if (inline is LinkDelimiterInline { IsActive: true })
+            {
+                return true;
+            }
+            inline = inline.Parent;
+        }
+        return false;
+    }
+
     private static void MarkParentAsInactive(Inline? inline)
     {
         while (inline != null)

src/Markdig/Syntax/LinkReferenceDefinition.cs

@@ -136,6 +136,16 @@ public class LinkReferenceDefinition : LeafBlock
     /// </remarks>
     public CreateLinkInlineDelegate? CreateLinkInline { get; set; }
 
+    /// <summary>
+    /// Gets whether this definition may resolve into a link while its label is
+    /// nested inside another still-open link or image bracket.
+    /// </summary>
+    /// <remarks>
+    /// <c>true</c> for user-authored definitions, preserving CommonMark's "links
+    /// cannot contain links". Implicitly generated ones override it to <c>false</c>.
+    /// </remarks>
+    internal virtual bool AllowResolutionInsideOpenLink => true;
+
     /// <summary>
     /// Tries to the parse the specified text into a definition.
     /// </summary>