Fix heading auto-link hijacking nested link labels (#668)

src/Markdig.Tests/TestAutoLinks.cs

@@ -21,4 +21,29 @@ public class TestAutoLinks
 
         Assert.That(html, Is.EqualTo(expected).IgnoreWhiteSpace);
     }
+
+    // https://github.com/xoofx/markdig/issues/668
+    // A heading's implicit "AutoLink" reference (registered by the AutoIdentifiers extension) must not
+    // resolve when its label is nested inside another still-open link bracket, otherwise it silently
+    // breaks the outer, explicitly authored link.
+    [Test]
+    public void TestAutoIdentifierHeadingLinkDoesNotHijackNestedLinkLabel()
+    {
+        var markdown = "# Testing Markdown\n\n" +
+                        "### Header\n\n" +
+                        "[Testing [Header]](https://www.bing.com)\n\n" +
+                        "[Testing [test]](https://www.google.com)\n";
+        var expected = "<h1 id=\"testing-markdown\">Testing Markdown</h1>\n" +
+                        "<h3 id=\"header\">Header</h3>\n" +
+                        "<p><a href=\"https://www.bing.com\">Testing [Header]</a></p>\n" +
+                        "<p><a href=\"https://www.google.com\">Testing [test]</a></p>";
+
+        var pipeline = new MarkdownPipelineBuilder()
+            .UseAutoLinks()
+            .UseAutoIdentifiers()
+            .Build();
+        var html = Markdown.ToHtml(markdown, pipeline);
+
+        Assert.That(html, Is.EqualTo(expected).IgnoreWhiteSpace);
+    }
 }

src/Markdig/Extensions/AutoIdentifiers/HeadingLinkReferenceDefinition.cs

@@ -24,4 +24,12 @@ public class HeadingLinkReferenceDefinition : LinkReferenceDefinition
     /// Gets or sets the heading related to this link reference definition.
     /// </summary>
     public HeadingBlock Heading { get; set; }
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// This reference definition is created implicitly for every heading, so it must not resolve when its
+    /// label is nested inside another still-open link bracket, e.g. <c>[Some text [HeadingText]](url)</c>.
+    /// Otherwise it would silently break the outer, explicitly authored link.
+    /// </remarks>
+    internal override bool AllowResolutionInsideOpenLink => false;
 }

src/Markdig/Parsers/Inlines/LinkInlineParser.cs

@@ -150,6 +150,14 @@ public class LinkInlineParser : InlineParser
             return false;
         }
 
+        // Implicitly generated reference definitions (e.g. automatic heading anchors from the auto-identifier
+        // extension) must not hijack a bracket pair nested inside another still-open link, otherwise they would
+        // silently break the outer, explicitly authored link (see https://github.com/xoofx/markdig/issues/668).
+        if (!linkRef.AllowResolutionInsideOpenLink && HasActiveAncestorLink(parent))
+        {
+            return false;
+        }
+
         Inline? link = null;
         // Try to use a callback directly defined on the LinkReferenceDefinition
         if (linkRef.CreateLinkInline != null)
@@ -430,6 +438,24 @@ public class LinkInlineParser : InlineParser
         }
     }
 
+    /// <summary>
+    /// Determines whether the given link delimiter is nested inside another link/image delimiter that is
+    /// still active, i.e. it may still be resolved into a real link/image once fully parsed.
+    /// </summary>
+    private static bool HasActiveAncestorLink(LinkDelimiterInline delimiter)
+    {
+        var inline = delimiter.Parent;
+        while (inline != null)
+        {
+            if (inline is LinkDelimiterInline { IsActive: true })
+            {
+                return true;
+            }
+            inline = inline.Parent;
+        }
+        return false;
+    }
+
     private static void MarkParentAsInactive(Inline? inline)
     {
         while (inline != null)

src/Markdig/Syntax/LinkReferenceDefinition.cs

@@ -136,6 +136,21 @@ public class LinkReferenceDefinition : LeafBlock
     /// </remarks>
     public CreateLinkInlineDelegate? CreateLinkInline { get; set; }
 
+    /// <summary>
+    /// Gets a value indicating whether this reference definition is allowed to resolve into a link when its
+    /// label is nested inside another link or image bracket that is still open (and may still become a valid
+    /// link/image once fully parsed).
+    /// </summary>
+    /// <remarks>
+    /// This is <c>true</c> for regular, user-authored reference definitions so that CommonMark's "links cannot
+    /// contain links" resolution order is preserved. Implicitly generated reference definitions (e.g. the
+    /// automatic heading anchors created by the auto-identifier extension) override this to <c>false</c> so
+    /// they don't silently hijack a bracket pair that is part of a larger, still-open link label, such as
+    /// <c>[Some text [HeadingText]](https://example.com)</c> (see
+    /// https://github.com/xoofx/markdig/issues/668).
+    /// </remarks>
+    internal virtual bool AllowResolutionInsideOpenLink => true;
+
     /// <summary>
     /// Tries to the parse the specified text into a definition.
     /// </summary>