Recognize a VB exception filter's type test

vbc builds an exception filter as a single non-short-circuiting
expression: `isinst`, then the user's `when` conditions, combined with
`and`. DetectCatchWhenConditionBlocks only knew the branch chain csc
emits, so the type test stayed inside the filter and the handler kept the
`object` variable it has in IL:

    catch (object obj) when ((obj is Exception) & (num2 != 0) & (num == 0))
    {
        ProjectData.SetProjectError((Exception)obj);

A catch type has to derive from Exception, so that does not compile.

The conjunction form is matched too now, lifting the test to the catch
type as the block form already does. The other conjuncts stop running for
a non-matching exception once the test moves, so the filter has to be
pure for this to be invisible; PropagateExceptionVariable then drops the
castclass in the handler:

    catch (Exception ex) when ((num2 != 0) & (num == 0))
    {
        ProjectData.SetProjectError(ex);

Closes #3659

ICSharpCode.Decompiler.Tests/ILPrettyTestRunner.cs

@@ -267,6 +267,12 @@ namespace ICSharpCode.Decompiler.Tests
 			await Run();
 		}
 
+		[Test]
+		public async Task Issue3659()
+		{
+			await Run();
+		}
+
 		[Test]
 		public async Task Issue3466()
 		{

ICSharpCode.Decompiler.Tests/TestCases/ILPretty/Issue3659.cs

@@ -0,0 +1,65 @@
+using System;
+using Microsoft.VisualBasic.CompilerServices;
+
+namespace ICSharpCode.Decompiler.Tests.TestCases.ILPretty
+{
+	public class Issue3659
+	{
+		public static void Func(ref Issue3659 obj, object value)
+		{
+		}
+
+		public static int ShowMessage(string a, int b, string c, object d, int e)
+		{
+			return 0;
+		}
+
+		internal void VBFunction(object value)
+		{
+			int try0000_dispatch = -1;
+			int num2 = default;
+			int num = default;
+			while (true)
+			{
+				try
+				{
+					/*Note: ILSpy has introduced the following switch to emulate a goto from catch-block to try-block*/;
+					switch (try0000_dispatch)
+					{
+						default:
+						{
+							ProjectData.ClearProjectError();
+							num2 = 2;
+							Issue3659 obj = this;
+							Func(ref obj, value);
+							goto end_IL_0000;
+						}
+						case 40:
+							num = -1;
+							switch (num2)
+							{
+								case 2:
+									ShowMessage("VBFunction", 0, "Exception", null, 0);
+									goto end_IL_0000;
+							}
+							break;
+					}
+				}
+				catch (Exception ex) when ((num2 != 0) & (num == 0))
+				{
+					ProjectData.SetProjectError(ex);
+					try0000_dispatch = 40;
+					continue;
+				}
+				throw ProjectData.CreateProjectError(-214628237);
+				continue;
+				end_IL_0000:
+				break;
+			}
+			if (num != 0)
+			{
+				ProjectData.ClearProjectError();
+			}
+		}
+	}
+}

ICSharpCode.Decompiler.Tests/TestCases/ILPretty/Issue3659.il

@@ -0,0 +1,109 @@
+.assembly extern System.Runtime
+{
+	.publickeytoken = (B7 7A 5C 56 19 34 E0 89 )
+	.ver 4:0:0:0
+}
+.assembly extern Microsoft.VisualBasic.Core
+{
+	.publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A )
+	.ver 10:0:0:0
+}
+.assembly Issue3659 { .ver 0:0:0:0 }
+.module Issue3659.dll
+
+.class private auto ansi '<Module>'
+{
+}
+
+.class public auto ansi beforefieldinit ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659
+	extends [System.Runtime]System.Object
+{
+	.method public hidebysig static void Func(class ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659& obj, object 'value') cil managed
+	{
+		ret
+	}
+
+	.method public hidebysig static int32 ShowMessage(string a, int32 b, string c, object d, int32 e) cil managed
+	{
+		ldc.i4.0
+		ret
+	}
+
+	.method assembly hidebysig instance void VBFunction(object 'value') cil managed
+	{
+		.maxstack  5
+		.locals init (
+			[0] int32,
+			[1] int32,
+			[2] class ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659
+		)
+
+		.try
+		{
+			IL_0000:  call void [Microsoft.VisualBasic.Core]Microsoft.VisualBasic.CompilerServices.ProjectData::ClearProjectError()
+			IL_0005:  ldc.i4.2
+			IL_0006:  stloc.0
+			IL_0007:  ldarg.0
+			IL_0008:  stloc.2
+			IL_0009:  ldloca.s 2
+			IL_000b:  ldarg.1
+			IL_000c:  call void ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659::Func(class ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659&, object)
+			IL_0011:  br.s       IL_0026
+
+			IL_0013: ldstr "VBFunction"
+			IL_0018: ldc.i4.0
+			IL_0019: ldstr "Exception"
+			IL_001e: ldnull
+			IL_001f: ldc.i4.0
+			IL_0020: call int32 ICSharpCode.Decompiler.Tests.TestCases.ILPretty.Issue3659::ShowMessage(string, int32, string, object, int32)
+			IL_0025: pop
+
+			IL_0026:  leave.s    IL_0069
+
+			IL_0028: ldc.i4.m1
+			IL_0029: stloc.1
+			IL_002a: ldloc.0
+			IL_002b: switch (IL_003c, IL_003c, IL_0013)
+
+			IL_003c:  leave.s    IL_005e
+		}
+		filter
+		{
+			IL_003e:  isinst [System.Runtime]System.Exception
+			IL_0043:  ldnull
+			IL_0044:  cgt.un
+			IL_0046:  ldloc.0
+			IL_0047:  ldc.i4.0
+			IL_0048:  cgt.un
+			IL_004a:  and
+			IL_004b:  ldloc.1
+			IL_004c:  ldc.i4.0
+			IL_004d:  ceq
+			IL_004f:  and
+			IL_0050:  endfilter
+		}
+		{
+			IL_0052:  castclass [System.Runtime]System.Exception
+			IL_0057:  call void [Microsoft.VisualBasic.Core]Microsoft.VisualBasic.CompilerServices.ProjectData::SetProjectError(class [System.Runtime]System.Exception)
+			IL_005c:  leave.s IL_0028
+		}
+
+		IL_005e:  ldc.i4 -214628237
+		IL_0063:  call class [System.Runtime]System.Exception [Microsoft.VisualBasic.Core]Microsoft.VisualBasic.CompilerServices.ProjectData::CreateProjectError(int32)
+		IL_0068:  throw
+
+		IL_0069:  ldloc.1
+		IL_006a:  brfalse.s  IL_0071
+
+		IL_006c:  call void [Microsoft.VisualBasic.Core]Microsoft.VisualBasic.CompilerServices.ProjectData::ClearProjectError()
+
+		IL_0071:  ret
+	}
+
+	.method public hidebysig specialname rtspecialname instance void .ctor() cil managed
+	{
+		ldarg.0
+		call instance void [System.Runtime]System.Object::.ctor()
+		ret
+	}
+}

ICSharpCode.Decompiler/IL/Transforms/DetectCatchWhenConditionBlocks.cs

@@ -69,11 +69,79 @@ namespace ICSharpCode.Decompiler.IL.Transforms
 						container.SortBlocks(deleteUnreachableBlocks: true);
 					}
 
+					PropagateExceptionVariable(context, catchBlock);
+				}
+				else if (catchBlock.Filter is BlockContainer filterContainer
+					&& MatchTypeTestConjunction(catchBlock.Variable, filterContainer,
+						out var conjunctionType, out var typeTest)
+					&& conjunctionType.GetStackType() == catchBlock.Variable.StackType)
+				{
+					context.Step($"Detected catch-when type test for {catchBlock.Variable.Name}", typeTest);
+					catchBlock.Variable.Type = conjunctionType;
+					var remainder = typeTest.Right;
+					typeTest.ReplaceWith(remainder);
+					context.EndStep(remainder);
+
 					PropagateExceptionVariable(context, catchBlock);
 				}
 			}
 		}
 
+		/// <summary>
+		/// BlockContainer {
+		/// 	Block entryPoint (incoming: 1) {
+		/// 		leave container(bit.and(bit.and(comp(isinst T(ldloc exceptionVar) != ldnull), cond1), cond2))
+		/// 	}
+		/// }
+		/// The Visual Basic compiler emits the whole filter as one non-short-circuiting
+		/// expression rather than the block chain csc emits, so the type test sits in a
+		/// conjunction instead of a branch. On a match, typeTest is the innermost `bit.and`,
+		/// whose right operand is the filter that remains once the test moves to the catch type.
+		/// </summary>
+		bool MatchTypeTestConjunction(ILVariable exceptionVar, BlockContainer container,
+			out IType exceptionType, out BinaryNumericInstruction typeTest)
+		{
+			exceptionType = null;
+			typeTest = null;
+			var entryPoint = container.EntryPoint;
+			if (entryPoint == null || entryPoint.IncomingEdgeCount != 1 || entryPoint.Instructions.Count != 1)
+				return false;
+			if (!entryPoint.Instructions[0].MatchLeave(container, out var condition))
+				return false;
+			// Once the test moves to the catch type, the other conjuncts stop running for a
+			// non-matching exception, which is only invisible if they have no side effects.
+			if (!SemanticHelper.IsPure(condition.Flags))
+				return false;
+			// The test is the leftmost operand of a left-nested chain of `&`.
+			while (condition is BinaryNumericInstruction { Operator: BinaryNumericOperator.BitAnd } and)
+			{
+				if (MatchTypeTest(and.Left, exceptionVar, out exceptionType))
+				{
+					typeTest = and;
+					return true;
+				}
+				condition = and.Left;
+			}
+			return false;
+		}
+
+		/// <summary>
+		/// comp(isinst T(ldloc exceptionVar) != ldnull), however the compiler spelled the null test.
+		/// </summary>
+		static bool MatchTypeTest(ILInstruction condition, ILVariable exceptionVar, out IType exceptionType)
+		{
+			exceptionType = null;
+			if (condition is not Comp comp || !comp.Right.MatchLdNull())
+				return false;
+			// `cgt.un x, null` is how both compilers spell `x != null` for a reference.
+			if (comp.Kind != ComparisonKind.Inequality
+				&& !(comp.Kind == ComparisonKind.GreaterThan && comp.InputType == StackType.O))
+				return false;
+			if (!comp.Left.MatchIsInst(out var argument, out exceptionType))
+				return false;
+			return argument.MatchLdLoc(exceptionVar);
+		}
+
 		/// <summary>
 		/// catch E_189 : 0200007C System.Exception when (BlockContainer {
 		/// 	Block IL_0079 (incoming: 1) {