C++: Clean up dev tools.

* Add -h and -help options describing the tools and their usage.

* Make the tools compile and run on Windows (MinGW, MSVC).

* Rename project dirs, executables and main source files to more
  meaningful names:
  - Use same base name for project dir, *.pro file, main source file
    and (if applicable) script file.
  - Use the prefix "cplusplus-".
  - The names are now:
      - gen-cpp-ast/generate-ast --> cplusplus-update-frontend
      - mkvisitor --> cplusplus-mkvisitor
      - cplusplus-dump/cplusplus0 --> cplusplus-ast2png

* Get rid of 'c++' shell scripts.

* Get rid of duplicates of 'conf.c++'. Rename to 'pp-configuration.inc'.

* Introduce src/tools/cplusplus-tools-utils containing common stuff
  that is used at least in two tools. 'pp-configuration.inc' can also be
  found here.

* cplusplus-update-frontend:
  - Print file paths of written files to stdout.
  - Convenience: Use default values referencing the appropriate dirs and
    files.

* cplusplus-mkvisitor:
  - Take only one argument, namely the path to AST.h.
  - Convenience: Use default path to AST.h.

* cplusplus-ast2png:
  - Make it run without LD_LIBRARY_PATH.
  - As the name suggests, generate image files in png format (needs
    'dot' from graphviz).
  - Convenience: Read from stdin, which useful for small snippets.

Change-Id: I79c4061fce4a1571c0588dfedd50d4a70715d9df
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

.gitignore

@@ -109,8 +109,10 @@ bin/qml2puppet.exe
 bin/qtpromaker
 bin/qtpromaker.exe
 share/doc/qtcreator/*.qch
-src/tools/gen-cpp-ast/generate-ast
-src/tools/mkvisitor/cplusplus0
+src/tools/cplusplus-mkvisitor/cplusplus-mkvisitor
+src/tools/cplusplus-mkvisitor/cplusplus-mkvisitor.exe
+src/tools/cplusplus-update-frontend/cplusplus-update-frontend
+src/tools/cplusplus-update-frontend/cplusplus-update-frontend.exe
 src/tools/qml/qmldump/qmldump
 src/tools/examplesscanner/examplesscanner
 src/tools/valgrindfake/valgrind-fake
@@ -118,13 +120,15 @@ bin/*.exe
 
 # Tests
 #------
-tests/manual/cplusplus-frontend/cplusplus0
-tests/manual/cplusplus-dump/cplusplus0
+tests/manual/cplusplus-frontend/cplusplus-frontend
+tests/manual/cplusplus-frontend/cplusplus-frontend.exe
 tests/manual/qml-ast2dot/qml-ast2dot
 tests/manual/debugger/simple/libsimple_test_plugin.*dylib
 tests/manual/debugger/simple/simple_test_app
 tests/manual/plain-cplusplus/plain-c++
 tests/manual/preprocessor/pp
+tests/tools/cplusplus-ast2png/cplusplus-ast2png
+tests/tools/cplusplus-ast2png/cplusplus-ast2png.exe
 tests/auto/cplusplus/codegen/tst_codegen
 tests/auto/cplusplus/ast/tst_ast
 tests/auto/cplusplus/codeformatter/tst_codeformatter

src/libs/3rdparty/cplusplus/AST.cpp

@@ -29,7 +29,7 @@
 
 /*
    All firstToken/lastToken methods below which have a doxygen comment with
-   \generated in it, will be re-generated when the tool "generate-ast" is run.
+   \generated in it, will be re-generated when the tool "cplusplus-update-frontend" is run.
 
    For methods which are hand-coded, or which should not be changed, make sure that
    the comment is gone.

src/tools/mkvisitor/main.cpp → src/tools/cplusplus-mkvisitor/cplusplus-mkvisitor.cpp

@@ -44,6 +44,8 @@
 #include <Overview.h>
 #include <LookupContext.h>
 
+#include "cplusplus-tools-utils.h"
+
 #include <QFile>
 #include <QList>
 #include <QCoreApplication>
@@ -422,35 +424,65 @@ protected:
     }
 };
 
+void printUsage()
+{
+    std::cout << "Usage: " << qPrintable(QFileInfo(qApp->arguments().at(0)).fileName())
+              << " [-v] [path to AST.h]\n\n"
+              << "Print a visitor class based on AST.h to stdout.\n\n";
+    const QString defaulPath = QFileInfo(PATH_AST_H).canonicalFilePath();
+    std::cout << "Default path: " << qPrintable(defaulPath) << '.' << "\n";
+}
+
 int main(int argc, char *argv[])
 {
     QCoreApplication app(argc, argv);
+    QStringList args = app.arguments();
+    args.removeFirst();
 
-    QStringList files = app.arguments();
-    files.removeFirst();
+    bool optionVerbose = false;
 
-    foreach (const QString &fileName, files) {
-        QFile file(fileName);
-        if (! file.open(QFile::ReadOnly))
-            continue;
+    // Process options & arguments
+    if (args.contains("-v")) {
+        optionVerbose = true;
+        args.removeOne("-v");
+    }
+    const bool helpRequested = args.contains("-h") || args.contains("-help");
+    if (helpRequested || args.count() >= 2) {
+        printUsage();
+        return helpRequested ? EXIT_SUCCESS : EXIT_FAILURE;
+    }
 
-        const QByteArray source = file.readAll();
-        file.close();
+    // Run the preprocessor
+    QString fileName = PATH_AST_H;
+    if (!args.isEmpty())
+        fileName = args.first();
 
-        Document::Ptr doc = Document::create(fileName);
-        //doc->control()->setDiagnosticClient(0);
-        doc->setUtf8Source(source);
-        doc->parse();
+    const QString fileNamePreprocessed = fileName + QLatin1String(".preprocessed");
+    CplusplusToolsUtils::SystemPreprocessor preprocessor(optionVerbose);
+    preprocessor.preprocessFile(fileName, fileNamePreprocessed);
 
-        doc->translationUnit()->blockErrors(true);
+    QFile file(fileNamePreprocessed);
+    if (! file.open(QFile::ReadOnly)) {
+        std::cerr << "Error: Could not open file \"" << qPrintable(file.fileName()) << "\"."
+                  << std::endl;
+        return EXIT_FAILURE;
+    }
 
-        doc->check();
-        Snapshot snapshot;
-	snapshot.insert(doc);
+    const QByteArray source = file.readAll();
+    file.close();
 
-	LookupContext context(doc, snapshot);
-	MkVisitor mkVisitor(context);
-    }
+    Document::Ptr doc = Document::create(fileName);
+    //doc->control()->setDiagnosticClient(0);
+    doc->setUtf8Source(source);
+    doc->parse();
+    doc->translationUnit()->blockErrors(true);
+    doc->check();
+
+    Snapshot snapshot;
+    snapshot.insert(doc);
+
+    LookupContext context(doc, snapshot);
+    MkVisitor mkVisitor(context);
 
     return EXIT_SUCCESS;
 }

src/tools/cplusplus-mkvisitor/cplusplus-mkvisitor.pro

+QT = core gui
+macx:CONFIG -= app_bundle
+win32:CONFIG += console
+TEMPLATE = app
+TARGET = cplusplus-mkvisitor
+DESTDIR = ./
+
+include(../../../qtcreator.pri)
+include(../../libs/cplusplus/cplusplus-lib.pri)
+include(../../../src/tools/cplusplus-tools-utils/cplusplus-tools-utils.pri)
+
+DEFINES += PATH_AST_H=\\\"$$PWD/../../libs/3rdparty/cplusplus/AST.h\\\"
+SOURCES += cplusplus-mkvisitor.cpp

src/tools/cplusplus-tools-utils/cplusplus-tools-utils.cpp

+/****************************************************************************
+**
+** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+
+#include "cplusplus-tools-utils.h"
+#include "environment.h"
+
+#include <QDebug>
+#include <QDir>
+#include <QFile>
+#include <QProcess>
+
+namespace CplusplusToolsUtils {
+
+QString portableExecutableName(const QString &executable)
+{
+#if defined(Q_OS_WIN)
+    return executable + QLatin1String(".exe");
+#else
+    return executable;
+#endif
+}
+
+void executeCommand(const QString &command, const QStringList &arguments, const QString &outputFile,
+                    bool verbose)
+{
+    QTextStream out(stderr);
+    if (command.isEmpty()) {
+        out << "Error: " << Q_FUNC_INFO << "Got empty command to execute." << endl;
+        exit(EXIT_FAILURE);
+    }
+
+    const QString fullCommand = command + QLatin1Char(' ') + arguments.join(QLatin1String(" "));
+    if (verbose)
+        out << "Executing: " << fullCommand << endl;
+
+    QProcess process;
+    if (!outputFile.isEmpty())
+        process.setStandardOutputFile(outputFile, QIODevice::Truncate);
+    process.start(command, arguments);
+    if (!process.waitForStarted()) {
+        out << QString("Error: Process \"%1\" did not start within timeout: %2.")
+                .arg(fullCommand, process.errorString())
+            << endl;
+        exit(EXIT_FAILURE);
+    }
+    if (!process.waitForFinished()) {
+        if (!verbose)
+            out << process.readAll() << endl;
+        out << QString("Error: Process \"%1\" did not finish within timeout.").arg(fullCommand)
+            << endl;
+        exit(EXIT_FAILURE);
+    }
+    const int exitCode = process.exitCode();
+    if (exitCode != 0) {
+        out << process.readAllStandardError() << endl;
+        out << QString("Error: Process \"%1\" finished with non zero exit value %2")
+            .arg(fullCommand, exitCode) << endl;
+        exit(EXIT_FAILURE);
+    }
+}
+
+SystemPreprocessor::SystemPreprocessor(bool verbose)
+    : m_verbose(verbose)
+{
+    m_knownCompilers[portableExecutableName("gcc")]
+        = QLatin1String("-DCPLUSPLUS_WITHOUT_QT -U__BLOCKS__ -xc++ -E -include");
+    m_knownCompilers[portableExecutableName("cl")]
+        = QLatin1String("/DCPLUSPLUS_WITHOUT_QT /U__BLOCKS__ /TP /E /I . /FI");
+
+    QMapIterator<QString, QString> i(m_knownCompilers);
+    while (i.hasNext()) {
+        i.next();
+        const QString executablePath
+            = Utils::Environment::systemEnvironment().searchInPath(i.key());
+        if (!executablePath.isEmpty()) {
+            m_compiler = i.key();
+            m_compilerArguments = i.value().split(QLatin1String(" "), QString::SkipEmptyParts);
+            m_compilerArguments
+                << QDir::toNativeSeparators(QLatin1String(PATH_PREPROCESSOR_CONFIG));
+            break;
+        }
+    }
+}
+
+void SystemPreprocessor::check() const
+{
+    QTextStream out(stderr);
+    if (!QFile::exists(PATH_PREPROCESSOR_CONFIG)) {
+        out << QString("Error: File \"%1\" does not exist.").arg(PATH_PREPROCESSOR_CONFIG) << endl;
+        exit(EXIT_FAILURE);
+    }
+    if (m_compiler.isEmpty()) {
+        const QString triedCompilers
+            = QStringList(m_knownCompilers.keys()).join(QLatin1String(", "));
+        out << QString("Error: No compiler found. Tried %1.").arg(triedCompilers) << endl;
+        exit(EXIT_FAILURE);
+    }
+}
+
+void SystemPreprocessor::preprocessFile(const QString &inputFile, const QString &outputFile) const
+{
+    check();
+    if (!QFile::exists(inputFile)) {
+        QTextStream out(stderr);
+        out << QString("Error: File \"%1\" does not exist.").arg(inputFile) << endl;
+        exit(EXIT_FAILURE);
+    }
+    const QStringList arguments = QStringList(m_compilerArguments)
+            << QDir::toNativeSeparators(inputFile);
+    executeCommand(m_compiler, arguments, outputFile, m_verbose);
+}
+
+} // namespace

src/tools/cplusplus-tools-utils/cplusplus-tools-utils.h

+/****************************************************************************
+**
+** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+
+#ifndef CPLUSPLUSTOOLSUTILS_H
+#define CPLUSPLUSTOOLSUTILS_H
+
+#include <QString>
+#include <QStringList>
+#include <QMap>
+
+namespace CplusplusToolsUtils {
+
+QString portableExecutableName(const QString &executable);
+void executeCommand(const QString &command, const QStringList &arguments, const QString &outputFile,
+                    bool verbose = false);
+
+// Preprocess a file by calling an external compiler in preprocessor mode (-E, /E).
+class SystemPreprocessor
+{
+public:
+    SystemPreprocessor(bool verbose = false);
+    void preprocessFile(const QString &inputFile, const QString &outputFile) const;
+
+private:
+    void check() const;
+
+    QMap<QString, QString> m_knownCompilers;
+    QString m_compiler; // Compiler that will be called in preprocessor mode
+    QStringList m_compilerArguments;
+    bool m_verbose;
+};
+
+} // namespace
+
+#endif // CPLUSPLUSTOOLSUTILS_H

src/tools/cplusplus-tools-utils/cplusplus-tools-utils.pri

+DEPENDPATH += $$PWD
+INCLUDEPATH += $$PWD $$PWD/../../libs/utils
+
+DEFINES += PATH_PREPROCESSOR_CONFIG=\\\"$$PWD/pp-configuration.inc\\\"
+DEFINES += QTCREATOR_UTILS_STATIC_LIB
+
+HEADERS += \
+    $$PWD/cplusplus-tools-utils.h \
+    $$PWD/../../libs/utils/environment.h
+
+SOURCES += \
+    $$PWD/cplusplus-tools-utils.cpp \
+    $$PWD/../../libs/utils/environment.cpp

tests/manual/cplusplus-frontend/conf.c++ → src/tools/cplusplus-tools-utils/pp-configuration.inc

 #define __extension__
 #define __context__
 #define __range__
-#define __asm(a...)
-#define __asm__(a...)
+#if !defined(_WIN32) && !defined(_WIN64)
+# define __asm(a...)
+# define __asm__(a...)
+# define __stdcall
+# define __fastcall
+#endif
 #define   restrict
 #define __restrict
 #define __restrict__
 // #define __weak
 #define __builtin_va_arg(a,b) ((b)0)
-#define __stdcall
-#define __fastcall
 #define __imag__
 #define __real__
 #define __complex__

src/tools/gen-cpp-ast/generate-ast.cpp → src/tools/cplusplus-update-frontend/cplusplus-update-frontend.cpp

@@ -91,12 +91,21 @@ static const char generatedHeader[] =
 "//  W A R N I N G\n"
 "//  -------------\n"
 "//\n"
-"// This file is automatically generated.\n"
+"// This file is automatically generated by \"cplusplus-update-frontend\".\n"
 "// Changes will be lost.\n"
 "//\n"
 "\n"
 ;
 
+static void closeAndPrintFilePath(QFile &file)
+{
+    if (file.isOpen()) {
+        const QString filePath = QFileInfo(file).canonicalFilePath();
+        std::cout << QDir::toNativeSeparators(filePath).toLatin1().constData() << std::endl;
+        file.close();
+    }
+}
+
 class ASTNodes
 {
 public:
@@ -226,6 +235,8 @@ public:
             "using namespace CPlusPlus;\n" << endl;
 
         accept(ast);
+
+        closeAndPrintFilePath(file);
     }
 
 protected:
@@ -354,7 +365,6 @@ public:
 
         QTextStream output(&file);
         out = &output;
-
         *out << copyrightHeader << generatedHeader <<
             "\n"
             "#include \"AST.h\"\n"
@@ -363,6 +373,8 @@ public:
             "using namespace CPlusPlus;\n" << endl;
 
         accept(ast);
+
+        closeAndPrintFilePath(file);
     }
 
 protected:
@@ -480,6 +492,8 @@ public:
                 << endl;
 
         accept(ast);
+
+        closeAndPrintFilePath(file);
     }
 
 protected:
@@ -627,7 +641,7 @@ public:
 
         accept(ast);
 
-        file.close();
+        closeAndPrintFilePath(file);
     }
 
 protected:
@@ -761,7 +775,7 @@ public:
 
         d.accept(unit->ast());
 
-        file.close();
+        closeAndPrintFilePath(file);
     }
 
 protected:
@@ -1220,6 +1234,7 @@ void generateAST_cpp(const Snapshot &snapshot, const QDir &cplusplusDir)
     if (file.open(QFile::WriteOnly)) {
         QTextStream out(&file);
         out << cpp_document.toPlainText();
+        closeAndPrintFilePath(file);
     }
 }
 
@@ -1312,6 +1327,8 @@ void generateASTVisitor_H(const Snapshot &, const QDir &cplusplusDir,
 "} // namespace CPlusPlus\n"
 "\n"
 "#endif // CPLUSPLUS_ASTVISITOR_H\n";
+
+  closeAndPrintFilePath(file);
 }
 
 void generateASTMatcher_H(const Snapshot &, const QDir &cplusplusDir,
@@ -1353,6 +1370,8 @@ void generateASTMatcher_H(const Snapshot &, const QDir &cplusplusDir,
 "} // namespace CPlusPlus\n"
 "\n"
 "#endif // CPLUSPLUS_ASTMATCHER_H\n";
+
+  closeAndPrintFilePath(file);
 }
 
 QStringList generateAST_H(const Snapshot &snapshot, const QDir &cplusplusDir, const QString &dumpersFile)
@@ -1437,6 +1456,7 @@ QStringList generateAST_H(const Snapshot &snapshot, const QDir &cplusplusDir, co
     if (file.open(QFile::WriteOnly)) {
         QTextStream out(&file);
         out << document.toPlainText();
+        closeAndPrintFilePath(file);
     }
 
     Accept0CG cg(cplusplusDir, AST_h_document->translationUnit());
@@ -1537,6 +1557,7 @@ void generateASTFwd_h(const Snapshot &snapshot, const QDir &cplusplusDir, const
     if (file.open(QFile::WriteOnly)) {
         QTextStream out(&file);
         out << document.toPlainText();
+        closeAndPrintFilePath(file);
     }
 }
 
@@ -1664,36 +1685,67 @@ void generateASTPatternBuilder_h(const QDir &cplusplusDir)
             << "} // end of namespace CPlusPlus" << endl
             << endl
             << "#endif // CPLUSPLUS_AST_PATTERN_BUILDER_H" << endl;
+
+    closeAndPrintFilePath(file);
+}
+
+void printUsage()
+{
+    const QByteArray executable = QFileInfo(qApp->arguments().first()).fileName().toLatin1();
+    std::cout << "Usage: " << executable.constData() << "\n"
+              << "       " << executable.constData() << " <frontend-dir> <dumpers-file>"
+              << "\n\n"
+              << "Generate appropriate header and source files of the C++ frontend accordingly\n"
+              << "to AST.h and print the paths of the written files. Run this tool after\n"
+              << "modifying AST.h."
+              << "\n\n";
+    const QString defaultPathCppFrontend = QFileInfo(PATH_CPP_FRONTEND).canonicalFilePath();
+    const QString defaultPathDumpersFile = QFileInfo(PATH_DUMPERS_FILE).canonicalFilePath();
+    std::cout << "Default values:" << "\n"
+              << "   frontend-dir: " << qPrintable(defaultPathCppFrontend) << "\n"
+              << "   dumpers-file: " << qPrintable(defaultPathDumpersFile) << "\n";
 }
 
 int main(int argc, char *argv[])
 {
     MyQApplication app(argc, argv);
+    QStringList args = app.arguments();
+    args.removeFirst();
+
+    QString pathCppFrontend = PATH_CPP_FRONTEND;
+    QString pathDumpersFile = PATH_DUMPERS_FILE;
+
+    const bool helpRequested = args.contains("-h") || args.contains("-help");
+    if (args.count() == 1 || args.count() >= 3 || helpRequested) {
+        printUsage();
+        return helpRequested ? EXIT_SUCCESS : EXIT_FAILURE;
+    } else if (args.count() == 2) {
+        pathCppFrontend = args.at(0);
+        pathDumpersFile = args.at(1);
+    }
 
-    QStringList files = app.arguments();
-    files.removeFirst();
-
-    if (files.size() != 1 && files.size() != 2) {
-        std::cerr << "Usage: cplusplus [path to C++ front-end]" << std::endl;
-        std::cerr << "   or: cplusplus [path to C++ front-end] [dumpers file name]" << std::endl;
+    QDir cplusplusDir(pathCppFrontend);
+    if (!QFile::exists(pathCppFrontend)) {
+        std::cerr << "Error: Directory \"" << qPrintable(cplusplusDir.absolutePath())
+                  << "\" does not exist." << std::endl;
         return EXIT_FAILURE;
     }
-
-    QDir cplusplusDir(files.first());
     if (!QFileInfo(cplusplusDir, QLatin1String("AST.h")).exists()) {
-        std::cerr << "Cannot find AST.h in " << qPrintable(cplusplusDir.absolutePath())
-                << std::endl;
+        std::cerr << "Error: Cannot find AST.h in \"" << qPrintable(cplusplusDir.absolutePath())
+                  << "\"." << std::endl;
+        return EXIT_FAILURE;
+    }
+    if (!QFile::exists(pathDumpersFile)) {
+        std::cerr << "Error: File \"" << qPrintable(pathDumpersFile)
+                  << "\" does not exist." << std::endl;
         return EXIT_FAILURE;
     }
-
-    QString dumpersFile;
-    if (files.size() == 2)
-        dumpersFile = files.last();
 
     Snapshot snapshot;
-    QStringList astDerivedClasses = generateAST_H(snapshot, cplusplusDir, dumpersFile);
+    QStringList astDerivedClasses = generateAST_H(snapshot, cplusplusDir, pathDumpersFile);
     astDerivedClasses.sort();
     generateASTFwd_h(snapshot, cplusplusDir, astDerivedClasses);
-
     generateASTPatternBuilder_h(cplusplusDir);
+
+    return EXIT_SUCCESS;
 }

src/tools/cplusplus-update-frontend/cplusplus-update-frontend.pro

+QT = core gui
+macx:CONFIG -= app_bundle
+win32:CONFIG += console
+TEMPLATE = app
+TARGET = cplusplus-update-frontend
+DESTDIR = ./