Automated stack corruption detection

source2/Compiler/Cosmos.Compiler.DebugStub/CmdSend.asm

@@ -142,3 +142,12 @@ Call DebugStub_ComWrite32
 DebugStub_SendPtr_Exit:
 Ret
 
+DebugStub_SendStackCorruptionOccurred:
+Mov AL, DebugStub_Const_Ds2Vs_StackCorruptionOccurred
+Call DebugStub_ComWriteAL
+
+Mov ESI, DebugStub_CallerEIP
+Call DebugStub_ComWrite32
+DebugStub_SendStackCorruptionOccurred_Exit:
+Ret
+

source2/Compiler/Cosmos.Compiler.DebugStub/CmdSend.xs

@@ -163,3 +163,16 @@ function SendPtr {
     ESI = EBP[8]
     ComWrite32()
 }
+
+// Input: Stack
+// Output: None
+// Modifies: EAX, ECX, EDX, ESI
+function SendStackCorruptionOccurred {
+    // Write the type
+    AL = #Ds2Vs_StackCorruptionOccurred
+    ComWriteAL()
+
+    // pointer value
+    ESI = @.CallerEIP
+    ComWrite32()
+}

source2/Compiler/Cosmos.Compiler.DebugStub/Consts.asm

@@ -51,4 +51,5 @@ DebugStub_Const_Ds2Vs_Frame equ 11
 DebugStub_Const_Ds2Vs_Stack equ 12
 DebugStub_Const_Ds2Vs_Pong equ 13
 DebugStub_Const_Ds2Vs_BreakPointAsm equ 14
+DebugStub_Const_Ds2Vs_StackCorruptionOccurred equ 15
 

source2/Compiler/Cosmos.Compiler.DebugStub/Consts.xs

@@ -55,3 +55,4 @@ const Ds2Vs_Frame = 11
 const Ds2Vs_Stack = 12
 const Ds2Vs_Pong = 13
 const Ds2Vs_BreakPointAsm = 14
+const Ds2Vs_StackCorruptionOccurred = 15

source2/Cosmos.Assembler/x86/Registers.cs

@@ -56,7 +56,8 @@ namespace Cosmos.Assembler.x86 {
         ST4,
         ST5,
         ST6,
-        ST7
+        ST7,
+        EIP,
     }
     public static class Registers {
         public const RegistersEnum EAX = RegistersEnum.EAX;
@@ -195,6 +196,8 @@ namespace Cosmos.Assembler.x86 {
             mRegToName.Add(ST5, "ST5");
             mRegToName.Add(ST6, "ST6");
             mRegToName.Add(ST7, "ST7");
+            mRegToName.Add(RegistersEnum.EIP, "EIP");
+            mNameToReg.Add("EIP", RegistersEnum.EIP);
             mNameToReg.Add("EAX", EAX);
             mNameToReg.Add("AX", AX);
             mNameToReg.Add("AH", AH);
@@ -382,7 +385,8 @@ namespace Cosmos.Assembler.x86 {
 
         public static bool Is32Bit(RegistersEnum aRegister)
         {
-            return aRegister == EAX || aRegister == EBX || aRegister == ECX || aRegister == EDX || aRegister == ESP || aRegister == EBP || aRegister == ESI || aRegister == EDI || aRegister == CR0 || aRegister == CR1 || aRegister == CR2 || aRegister == CR3 || aRegister == CR4;
+            return aRegister == EAX || aRegister == EBX || aRegister == ECX || aRegister == EDX || aRegister == ESP || aRegister == EBP || aRegister == ESI || aRegister == EDI || aRegister == CR0 || aRegister == CR1 || aRegister == CR2 || aRegister == CR3 || aRegister == CR4
+                || aRegister == RegistersEnum.EIP;
         }
 
         public static bool Is16Bit(RegistersEnum aRegister)

source2/Debug/Cosmos.Debug.Common/Consts.cs

@@ -29,6 +29,7 @@ namespace Cosmos.Debug.Common {
     public const byte Stack = 12;
     public const byte Pong = 13;
     public const byte BreakPointAsm = 14;
+      public const byte StackCorruptionOccurred = 15;
   }
 
   // Messages from Host (VS) to Guest (Cosmos)

source2/Debug/Cosmos.Debug.Common/DebugConnector.cs

@@ -26,6 +26,7 @@ namespace Cosmos.Debug.Common
         public Action<byte[]> CmdFrame;
         public Action<byte[]> CmdStack;
         public Action<byte[]> CmdPong;
+        public Action<UInt32> CmdStackCorruptionOccurred;
 
         protected byte mCurrentMsgType;
         protected AutoResetEvent mCmdWait = new AutoResetEvent(false);
@@ -436,6 +437,11 @@ namespace Cosmos.Debug.Common
                     Next(0, PacketPong);
                     break;
 
+                case Ds2Vs.StackCorruptionOccurred:
+                    DoDebugMsg("DC Recv: StackCorruptionOccurred");
+                    Next(4, PacketStackCorruptionOccurred);
+                    break;
+
                 default:
                     // Exceptions crash VS so use MsgBox instead
                     DoDebugMsg("Unknown debug command: " + mCurrentMsgType);
@@ -530,6 +536,15 @@ namespace Cosmos.Debug.Common
             WaitForMessage();
         }
 
+        protected void PacketStackCorruptionOccurred(byte[] aPacket)
+        {
+            if (CmdStackCorruptionOccurred != null)
+            {
+                CmdStackCorruptionOccurred(GetUInt32(aPacket, 0));
+            }
+            WaitForMessage();
+        }
+
         protected void PacketStack(byte[] aPacket)
         {
             if (CmdStack != null)

source2/Debug/Cosmos.Debug.VSDebugEngine/AD7.Impl/AD7Process.cs

@@ -303,7 +303,12 @@ namespace Cosmos.Debug.VSDebugEngine
             mDbgConnector.CmdFrame += new Action<byte[]>(DbgCmdFrame);
             mDbgConnector.CmdStack += new Action<byte[]>(DbgCmdStack);
             mDbgConnector.CmdPong += new Action<byte[]>(DbgCmdPong);
+            mDbgConnector.CmdStackCorruptionOccurred += DbgCmdStackCorruptionOccurred;
+        }
 
+        private void DbgCmdStackCorruptionOccurred(uint lastEIPAddress)
+        {
+            MessageBox.Show(String.Format("Stack corruption occurred at address 0x{0:X8}! Halting now.", lastEIPAddress));
         }
 
         internal AD7Process(NameValueCollection aDebugInfo, EngineCallback aCallback, AD7Engine aEngine, IDebugPort2 aPort)

source2/IL2CPU/Cosmos.IL2CPU/AppAssembler.cs

@@ -330,6 +330,21 @@ namespace Cosmos.IL2CPU
                     }
                 }
             }
+            //if (DebugMode == )
+            {
+                // if debugstub is active, emit a stack corruption detection. at this point EBP and ESP should have the same value. 
+                // if not, we should somehow break here.
+                new Compare {SourceReg = RegistersEnum.EBP, DestinationReg = RegistersEnum.ESP};
+                new ConditionalJump {Condition = ConditionalTestEnum.Equal, DestinationLabel = ILOp.GetMethodLabel(aMethod) + EndOfMethodLabelNameException + "__2"};
+                new ClrInterruptFlag();
+                // don't remove the call. It seems pointless, but we need it to retrieve the EIP value
+                new Call {DestinationLabel = ILOp.GetMethodLabel(aMethod) + EndOfMethodLabelNameException + "__Break_on_location"};
+                new Assembler.Label(ILOp.GetMethodLabel(aMethod) + EndOfMethodLabelNameException + "__Break_on_location");
+                new Pop {DestinationReg = RegistersEnum.EAX};
+                new Mov {DestinationRef = ElementReference.New("DebugStub_CallerEIP"), DestinationIsIndirect = true, SourceReg = RegistersEnum.EAX};
+                new Call { DestinationLabel = "DebugStub_SendStackCorruptionOccurred" };
+                new Halt();
+            }
             new Cosmos.Assembler.Label(ILOp.GetMethodLabel(aMethod) + EndOfMethodLabelNameException + "__2");
             new Pop { DestinationReg = Registers.EBP };
             var xRetSize = ((int)xTotalArgsSize) - ((int)xReturnSize);