Make CDB load custom dumpers.

Load in a 'well-defined' (temporary) breakpoint
at main().

share/qtcreator/gdbmacros/gdbmacros.cpp

@@ -2529,7 +2529,11 @@ void *qDumpObjectData440(
     int protocolVersion,
     int token,
     void *data,
+#ifdef Q_CC_MSVC // CDB cannot handle boolean parameters
+    int dumpChildren,
+#else
     bool dumpChildren,
+#endif
     int extraInt0,
     int extraInt1,
     int extraInt2,

src/plugins/debugger/cdb/cdbbreakpoint.cpp

@@ -48,7 +48,8 @@ static const char sourceFileQuoteC = '`';
 
 CDBBreakPoint::CDBBreakPoint() :
     ignoreCount(0),
-    lineNumber(-1)
+    lineNumber(-1),
+    oneShot(false)
 {
 }
 
@@ -57,7 +58,8 @@ CDBBreakPoint::CDBBreakPoint(const BreakpointData &bpd) :
      condition(bpd.condition),
      ignoreCount(0),
      funcName(bpd.funcName),
-     lineNumber(-1)
+     lineNumber(-1),
+     oneShot(false)
 {
     if (!bpd.ignoreCount.isEmpty())
         ignoreCount = bpd.ignoreCount.toInt();
@@ -75,6 +77,10 @@ int CDBBreakPoint::compare(const CDBBreakPoint& rhs) const
         return 1;
     if (lineNumber < rhs.lineNumber)
         return -1;
+    if (oneShot && !rhs.oneShot)
+        return 1;
+    if (!oneShot && rhs.oneShot)
+        return -1;
     if (const int fileCmp = fileName.compare(rhs.fileName))
         return fileCmp;
     if (const int  funcCmp = funcName.compare(rhs.funcName))
@@ -87,7 +93,8 @@ int CDBBreakPoint::compare(const CDBBreakPoint& rhs) const
 void CDBBreakPoint::clear()
 {
      ignoreCount = 0;
-     clearExpressionData();
+     oneShot = false;
+     clearExpressionData();     
 }
 
 void CDBBreakPoint::clearExpressionData()
@@ -110,6 +117,8 @@ QDebug operator<<(QDebug dbg, const CDBBreakPoint &bp)
         nsp << " condition='" << bp.condition << '\'';
     if (bp.ignoreCount)
         nsp << " ignoreCount=" << bp.ignoreCount;
+    if (bp.oneShot)
+        nsp << " oneShot";
     return dbg;
 }
 
@@ -144,7 +153,10 @@ bool CDBBreakPoint::apply(CIDebugBreakpoint *ibp, QString *errorMessage) const
     }
     // Pass Count is ignoreCount + 1
     ibp->SetPassCount(ignoreCount + 1u);
-    ibp->AddFlags(DEBUG_BREAKPOINT_ENABLED);
+    ULONG flags = DEBUG_BREAKPOINT_ENABLED;
+    if (oneShot)
+        flags |= DEBUG_BREAKPOINT_ONE_SHOT;
+    ibp->AddFlags(flags);
     return true;
 }
 
@@ -190,6 +202,10 @@ bool CDBBreakPoint::retrieve(CIDebugBreakpoint *ibp, QString *errorMessage)
     ibp->GetPassCount(&ignoreCount);
     if (ignoreCount)
         ignoreCount--;
+    ULONG flags = 0;
+    ibp->GetFlags(&flags);
+    if (flags & DEBUG_BREAKPOINT_ONE_SHOT)
+        oneShot = true;
     const QString expr = QString::fromUtf16(wszBuf);
     if (!parseExpression(expr)) {
         *errorMessage = QString::fromLatin1("Parsing of '%1' failed.").arg(expr);

src/plugins/debugger/cdb/cdbbreakpoint.h

@@ -83,6 +83,7 @@ struct CDBBreakPoint
     unsigned long ignoreCount;    // ignore count associated with breakpoint
     int lineNumber;     // line in source file
     QString funcName;       // name of containing function
+    bool oneShot;
 };
 
 QDebug operator<<(QDebug, const CDBBreakPoint &bp);

src/plugins/debugger/cdb/cdbdebugengine.cpp

@@ -53,6 +53,7 @@
 #include <utils/consoleprocess.h>
 
 #include <QtCore/QDebug>
+#include <QtCore/QTimer>
 #include <QtCore/QTimerEvent>
 #include <QtCore/QFileInfo>
 #include <QtCore/QDir>
@@ -96,13 +97,13 @@ QString msgDebugEngineComResult(HRESULT hr)
         case E_UNEXPECTED:
         return QLatin1String("E_UNEXPECTED");
         case E_NOTIMPL:
-        return QLatin1String("E_NOTIMPL");
+        return QLatin1String("E_NOTIMPL");        
     }
     if (hr == HRESULT_FROM_WIN32(ERROR_ACCESS_DENIED))
         return QLatin1String("ERROR_ACCESS_DENIED");;
     if (hr == HRESULT_FROM_NT(STATUS_CONTROL_C_EXIT))
         return QLatin1String("STATUS_CONTROL_C_EXIT");
-    return Core::Utils::winErrorMessage(HRESULT_CODE(hr));
+    return QLatin1String("E_FAIL ") + Core::Utils::winErrorMessage(HRESULT_CODE(hr));
 }
 
 static QString msgStackIndexOutOfRange(int idx, int size)
@@ -450,14 +451,16 @@ void CdbDebugEnginePrivate::clearDisplay()
 bool CdbDebugEngine::startDebugger()
 {
     m_d->clearDisplay();
+    const DebuggerStartMode mode = m_d->m_debuggerManager->startMode();
     // Figure out dumper. @TODO: same in gdb...
-    bool dumperEnabled = false && m_d->m_debuggerManager->qtDumperLibraryEnabled();
-    const QString dumperLibName = QDir::toNativeSeparators(m_d->m_debuggerManager->qtDumperLibraryName());
+    const QString dumperLibName = QDir::toNativeSeparators(m_d->m_debuggerManagerAccess->qtDumperLibraryName());
+    bool dumperEnabled = mode != AttachCore && !dumperLibName.isEmpty()
+                         && m_d->m_debuggerManagerAccess->qtDumperLibraryEnabled();
     if (dumperEnabled) {
         const QFileInfo fi(dumperLibName);
         if (!fi.isFile()) {
             const QString msg = tr("The dumper library '%1' does not exist.").arg(dumperLibName);
-            m_d->m_debuggerManager->showQtDumperLibraryWarning(msg);
+            m_d->m_debuggerManagerAccess->showQtDumperLibraryWarning(msg);
             dumperEnabled = false;
         }
     }
@@ -466,7 +469,6 @@ bool CdbDebugEngine::startDebugger()
     QString errorMessage;
     bool rc = false;
     m_d->clearForRun();
-    const DebuggerStartMode mode = m_d->m_debuggerManager->startMode();
     switch (mode) {
     case AttachExternal:
         rc = startAttachDebugger(m_d->m_debuggerManager->m_attachedPID, &errorMessage);
@@ -561,6 +563,52 @@ bool CdbDebugEngine::startDebuggerWithExecutable(DebuggerStartMode sm, QString *
     return true;
 }
 
+// check for a breakpoint at 'main()'
+static inline bool hasBreakPointAtMain(const BreakHandler *bp)
+{
+    if (const int count = bp->size()) {
+        // check all variations, resolved or not
+        const QString main = QLatin1String("main");
+        const QString qMain = QLatin1String("qMain");
+        const QString moduleMainPattern = QLatin1String("!main");
+        for (int i = 0; i < count ; i++) {
+            const QString &function = bp->at(i)->funcName;
+            if (function == main || function == qMain || function.endsWith(moduleMainPattern))
+                return true;
+        }
+    }
+    return false;
+}
+
+void CdbDebugEnginePrivate::processCreatedAttached(ULONG64 processHandle, ULONG64 initialThreadHandle)
+{
+    setDebuggeeHandles(reinterpret_cast<HANDLE>(processHandle), reinterpret_cast<HANDLE>(initialThreadHandle));
+    m_debuggerManagerAccess->notifyInferiorRunning();
+    ULONG currentThreadId;
+    if (SUCCEEDED(m_cif.debugSystemObjects->GetThreadIdByHandle(initialThreadHandle, &currentThreadId))) {
+        m_currentThreadId = currentThreadId;
+    } else {
+        m_currentThreadId = 0;
+    }
+    // Set initial breakpoints
+    if (m_debuggerManagerAccess->breakHandler()->hasPendingBreakpoints())
+        m_engine->attemptBreakpointSynchronization();    
+    // At any event, we want a temporary breakpoint at main() to load
+    // the dumpers.
+    if (m_dumper.state() == CdbDumperHelper::NotLoaded) {
+        if (!hasBreakPointAtMain(m_debuggerManagerAccess->breakHandler())) {
+            CDBBreakPoint mainBP;
+            // Do not resolve at this point in the rare event someone
+            // has main in a module
+            mainBP.funcName = QLatin1String("main");            
+            mainBP.oneShot = true;
+            QString errorMessage;
+            if (!mainBP.add(m_cif.debugControl, &errorMessage))
+                m_debuggerManagerAccess->showQtDumperLibraryWarning(errorMessage);
+        }
+    }
+}
+
 void CdbDebugEngine::processTerminated(unsigned long exitCode)
 {
     if (debugCDB)
@@ -859,13 +907,18 @@ void CdbDebugEngine::continueInferior()
 }
 
 // Continue process without notifications
-bool CdbDebugEnginePrivate::continueInferiorProcess(QString *errorMessage)
+bool CdbDebugEnginePrivate::continueInferiorProcess(QString *errorMessagePtr /* = 0 */)
 {
     if (debugCDB)
         qDebug() << Q_FUNC_INFO;
     const HRESULT hr = m_cif.debugControl->SetExecutionStatus(DEBUG_STATUS_GO);
     if (FAILED(hr)) {
-        *errorMessage = msgComFailed("SetExecutionStatus", hr);
+        const QString errorMessage = msgComFailed("SetExecutionStatus", hr);
+        if (errorMessagePtr) {
+            *errorMessagePtr = errorMessage;
+        } else {
+            qWarning("continueInferiorProcess: %s\n", qPrintable(errorMessage));
+        }
         return false;
     }
     return  true;
@@ -992,10 +1045,10 @@ void CdbDebugEngine::executeDebuggerCommand(const QString &command)
 
 bool CdbDebugEnginePrivate::executeDebuggerCommand(CIDebugControl *ctrl, const QString &command, QString *errorMessage)
 {
-    if (debugCDB)
-        qDebug() << Q_FUNC_INFO << command;
     // output to all clients, else we do not see anything
     const HRESULT hr = ctrl->ExecuteWide(DEBUG_OUTCTL_ALL_CLIENTS, command.utf16(), 0);
+    if (debugCDB)
+        qDebug() << "executeDebuggerCommand" << command << SUCCEEDED(hr);
     if (FAILED(hr)) {
         *errorMessage = QString::fromLatin1("Unable to execute '%1': %2").
                         arg(command, msgDebugEngineComResult(hr));
@@ -1300,10 +1353,19 @@ void CdbDebugEnginePrivate::handleDebugEvent()
 
     switch (mode) {
     case BreakEventHandle:
+    case BreakEventMain:
+        if (mode == BreakEventMain)
+            m_dumper.load(m_debuggerManager, m_debuggerManagerAccess);
         m_debuggerManagerAccess->notifyInferiorStopped();
         updateThreadList();
         updateStackTrace();
         break;
+    case BreakEventMainLoadDumpers:
+        // Temp stop to load dumpers    
+        m_dumper.load(m_debuggerManager, m_debuggerManagerAccess);
+        m_engine->startWatchTimer();
+        continueInferiorProcess();
+        break;
     case BreakEventIgnoreOnce:
         m_engine->startWatchTimer();
         break;
@@ -1417,17 +1479,6 @@ void CdbDebugEnginePrivate::handleModuleLoad(const QString &name)
     if (debugCDB>2)
         qDebug() << Q_FUNC_INFO << "\n    " << name;
     updateModules();
-    // Call the dumper helper hook and notify about progress.
-    bool ignoreNextBreakPoint;
-    if (m_dumper.moduleLoadHook(name, &ignoreNextBreakPoint)) {
-        if (m_dumper.state() == CdbDumperHelper::Loaded)
-            m_debuggerManagerAccess->showDebuggerOutput(QLatin1String(dumperPrefixC), QString::fromLatin1("Dumpers loaded: %1").arg(m_dumper.library()));
-    } else {
-        m_debuggerManager->showQtDumperLibraryWarning(m_dumper.errorMessage());
-        m_debuggerManagerAccess->showDebuggerOutput(QLatin1String(dumperPrefixC), QString::fromLatin1("Unable to load dumpers: %1").arg(m_dumper.errorMessage()));
-    }
-    if (ignoreNextBreakPoint)
-        m_breakEventMode = BreakEventIgnoreOnce;
 }
 
 void CdbDebugEnginePrivate::handleBreakpointEvent(PDEBUG_BREAKPOINT2 pBP)
@@ -1435,6 +1486,17 @@ void CdbDebugEnginePrivate::handleBreakpointEvent(PDEBUG_BREAKPOINT2 pBP)
     Q_UNUSED(pBP)
     if (debugCDB)
         qDebug() << Q_FUNC_INFO;
+    // Did we hit main() and did the user want that or is that just
+    // our internal BP to load the dumpers?
+    QString errorMessage;
+    CDBBreakPoint bp;
+    if (bp.retrieve(pBP, &errorMessage) && !bp.funcName.isEmpty()) {
+        if (bp.funcName == QLatin1String("main") || bp.funcName.endsWith(QLatin1String("!main"))) {
+            m_breakEventMode = bp.oneShot ? BreakEventMainLoadDumpers : BreakEventMain;
+        }
+        if (debugCDB)
+            qDebug() << bp << " b-mode=" << m_breakEventMode;
+    }
 }
 
 void CdbDebugEngine::reloadSourceFiles()

src/plugins/debugger/cdb/cdbdebugengine_p.h

@@ -98,6 +98,10 @@ struct CdbDebugEnginePrivate
     enum HandleBreakEventMode { // Special modes for break event handler.
         BreakEventHandle,
         BreakEventIgnoreOnce,
+        // We hit main (and the user intended it)
+        BreakEventMain,
+        // We hit main (and the user did not intend it, just load dumpers)
+        BreakEventMainLoadDumpers,
         BreakEventSyncBreakPoints,
     };
 
@@ -107,6 +111,7 @@ struct CdbDebugEnginePrivate
     bool init(QString *errorMessage);
     ~CdbDebugEnginePrivate();
 
+    void processCreatedAttached(ULONG64 processHandle, ULONG64 initialThreadHandle);
     void setDebuggeeHandles(HANDLE hDebuggeeProcess,  HANDLE hDebuggeeThread);
 
     bool isDebuggeeRunning() const { return m_watchTimer != -1; }
@@ -125,7 +130,7 @@ struct CdbDebugEnginePrivate
 
     bool interruptInterferiorProcess(QString *errorMessage);
 
-    bool continueInferiorProcess(QString *errorMessage);
+    bool continueInferiorProcess(QString *errorMessage = 0);
     bool continueInferior(QString *errorMessage);
 
     bool attemptBreakpointSynchronization(QString *errorMessage);

src/plugins/debugger/cdb/cdbdebugeventcallback.cpp

@@ -247,7 +247,8 @@ void formatException(const EXCEPTION_RECORD64 *e, QTextStream &str)
         break;
     case EXCEPTION_ACCESS_VIOLATION: {
             const bool writeOperation = e->ExceptionInformation[0];
-            str << (writeOperation ? "write access violation" : "read access violation");
+            str << (writeOperation ? "write" : "read")
+                << " access violation at: 0x" << e->ExceptionInformation[1];
     }
         break;
     case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
@@ -342,8 +343,10 @@ STDMETHODIMP CdbDebugEventCallback::Exception(
     QString msg;
     {
         QTextStream str(&msg);
-        formatException(Exception, m_pEngine->m_d->m_cif, str);
+        formatException(Exception, m_pEngine->m_d->m_cif, str);        
     }
+    if (debugCDB)
+        qDebug() << Q_FUNC_INFO << '\n' << msg;
     m_pEngine->m_d->m_debuggerManagerAccess->showApplicationOutput(msg);
     return S_OK;
 }
@@ -402,18 +405,7 @@ STDMETHODIMP CdbDebugEventCallback::CreateProcess(
     Q_UNUSED(StartOffset)
     if (debugCDB)
         qDebug() << Q_FUNC_INFO << ModuleName;
-
-    m_pEngine->m_d->setDebuggeeHandles(reinterpret_cast<HANDLE>(Handle), reinterpret_cast<HANDLE>(InitialThreadHandle));
-    m_pEngine->m_d->m_debuggerManagerAccess->notifyInferiorRunning();
-
-    ULONG currentThreadId;
-    if (SUCCEEDED(m_pEngine->m_d->m_cif.debugSystemObjects->GetThreadIdByHandle(InitialThreadHandle, &currentThreadId)))
-        m_pEngine->m_d->m_currentThreadId = currentThreadId;
-    else
-        m_pEngine->m_d->m_currentThreadId = 0;
-    // Set initial breakpoints
-    if (m_pEngine->m_d->m_debuggerManagerAccess->breakHandler()->hasPendingBreakpoints())
-        m_pEngine->attemptBreakpointSynchronization();
+    m_pEngine->m_d->processCreatedAttached(Handle, InitialThreadHandle);
     return S_OK;
 }
 

src/plugins/debugger/cdb/cdbdumperhelper.h

@@ -30,55 +30,77 @@
 #ifndef CDBDUMPERHELPER_H
 #define CDBDUMPERHELPER_H
 
-#include <QtCore/QString>
+#include <QtCore/QStringList>
 
 namespace Debugger {
 namespace Internal {
 
 struct CdbComInterfaces;
-
-// For code clarity, all the stuff related to custom dumpers
-// goes here.
-// "Custom dumper" is a library compiled against the current
-// Qt containing functions to evaluate values of Qt classes
-// (such as QString, taking pointers to their addresses).
-// The library must be loaded into the debuggee.
+class IDebuggerManagerAccessForEngines;
+class DebuggerManager;
+
+/* For code clarity, all the stuff related to custom dumpers
+ * goes here.
+ * "Custom dumper" is a library compiled against the current
+ * Qt containing functions to evaluate values of Qt classes
+ * (such as QString, taking pointers to their addresses).
+ * The library must be loaded into the debuggee.
+ * Loading the dumpers requires making the debuggee call functions
+ * (LoadLibrary() and the dumper functions). This only works if the
+ * debuggee is in a 'well-defined' breakpoint state (such as at 'main()').
+ * Calling the load functions from an IDebugEvent callback causes
+ * WaitForEvent() to fail with unknown errors. Calling the load functions from an
+ * non 'well-defined' (arbitrary) breakpoint state will cause LoadLibrary
+ * to trigger an access violations.
+ * Currently, we call the load function when stopping at 'main()' for which
+ * we set a temporary break point if the user does not want to stop there. */
 
 class CdbDumperHelper
 {
+    Q_DISABLE_COPY(CdbDumperHelper)
 public:
    enum State {
         Disabled,
         NotLoaded,
-        Loading,
         Loaded,
         Failed
     };
 
     explicit CdbDumperHelper(CdbComInterfaces *cif);
+    ~CdbDumperHelper();
+
+    State state() const { return m_state; }
+    operator bool() const { return m_state == Loaded; }
 
     // Call before starting the debugger
     void reset(const QString &library, bool enabled);
 
-    // Call from the module loaded event handler.
-    // It will load the dumper library and resolve the required symbols
-    // when appropriate.
-    bool moduleLoadHook(const QString &name, bool *ignoreNextBreakPoint);
-
-    State state() const { return m_state; }
-
-    QString errorMessage() const { return m_errorMessage; }
-    QString library() const { return m_library; }
+    // Call in a temporary breakpoint state to actually load.
+    void load(DebuggerManager *manager, IDebuggerManagerAccessForEngines *access);
 
 private:
+    struct DumperInputParameters;
+
+    void clearBuffer();
     bool resolveSymbols(QString *errorMessage);
+    bool getKnownTypes(QString *errorMessage);
+    bool callDumper(const DumperInputParameters &p, QByteArray *output, QString *errorMessage);
+    inline QString statusMessage() const;
 
     State m_state;
     CdbComInterfaces *m_cif;
 
     QString m_library;
     QString m_dumpObjectSymbol;
-    QString m_errorMessage;
+    QStringList m_knownTypes;
+    QString m_qtVersion;
+    QString m_qtNamespace;
+
+    quint64 m_inBufferAddress;
+    unsigned long m_inBufferSize;
+    quint64 m_outBufferAddress;
+    unsigned long m_outBufferSize;
+    char *m_buffer;
 };
 
 } // namespace Internal

src/plugins/debugger/cdb/cdbmodules.cpp

@@ -32,25 +32,47 @@
 #include "cdbdebugengine_p.h"
 
 #include <QtCore/QFileInfo>
+#include <QtCore/QRegExp>