SSH: Add SFTP operations needed to implement remote FS traversal.

Change-Id: I3e7b52513211959a976545667e8e8372f2001c7e
Reviewed-by: hjk <qthjk@ovi.com>

src/libs/utils/ssh/sftpchannel.cpp

@@ -95,6 +95,9 @@ SftpChannel::SftpChannel(quint32 channelId,
         SIGNAL(initializationFailed(QString)), Qt::QueuedConnection);
     connect(d, SIGNAL(dataAvailable(Utils::SftpJobId, QString)), this,
         SIGNAL(dataAvailable(Utils::SftpJobId, QString)), Qt::QueuedConnection);
+    connect(d, SIGNAL(fileInfoAvailable(Utils::SftpJobId, QList<Utils::SftpFileInfo>)), this,
+        SIGNAL(fileInfoAvailable(Utils::SftpJobId, QList<Utils::SftpFileInfo>)),
+        Qt::QueuedConnection);
     connect(d, SIGNAL(finished(Utils::SftpJobId,QString)), this,
         SIGNAL(finished(Utils::SftpJobId,QString)), Qt::QueuedConnection);
     connect(d, SIGNAL(closed()), this, SIGNAL(closed()), Qt::QueuedConnection);
@@ -131,6 +134,12 @@ void SftpChannel::closeChannel()
     d->closeChannel();
 }
 
+SftpJobId SftpChannel::statFile(const QString &path)
+{
+    return d->createJob(Internal::SftpStatFile::Ptr(
+        new Internal::SftpStatFile(++d->m_nextJobId, path)));
+}
+
 SftpJobId SftpChannel::listDirectory(const QString &path)
 {
     return d->createJob(Internal::SftpListDir::Ptr(
@@ -453,6 +462,7 @@ void SftpChannelPrivate::handleStatus()
     case AbstractSftpOperation::MakeDir:
         handleMkdirStatus(it, response);
         break;
+    case AbstractSftpOperation::StatFile:
     case AbstractSftpOperation::RmDir:
     case AbstractSftpOperation::Rm:
     case AbstractSftpOperation::Rename:
@@ -702,10 +712,16 @@ void SftpChannelPrivate::handleName()
                 "Unexpected SSH_FXP_NAME packet.");
         }
 
+        QList<SftpFileInfo> fileInfoList;
         for (int i = 0; i < response.files.count(); ++i) {
             const SftpFile &file = response.files.at(i);
-            emit dataAvailable(op->jobId, file.fileName);
+
+            SftpFileInfo fileInfo;
+            fileInfo.name = file.fileName;
+            attributesToFileInfo(file.attributes, fileInfo);
+            fileInfoList << fileInfo;
         }
+        emit fileInfoAvailable(op->jobId, fileInfoList);
         sendData(m_outgoingPacket.generateReadDir(op->remoteHandle,
             op->jobId).rawData());
         break;
@@ -753,6 +769,18 @@ void SftpChannelPrivate::handleAttrs()
 {
     const SftpAttrsResponse &response = m_incomingPacket.asAttrsResponse();
     JobMap::Iterator it = lookupJob(response.requestId);
+
+    SftpStatFile::Ptr statOp = it.value().dynamicCast<SftpStatFile>();
+    if (statOp) {
+        SftpFileInfo fileInfo;
+        fileInfo.name = QFileInfo(statOp->path).fileName();
+        attributesToFileInfo(response.attrs, fileInfo);
+        emit fileInfoAvailable(it.key(), QList<SftpFileInfo>() << fileInfo);
+        emit finished(it.key());
+        m_jobs.erase(it);
+        return;
+    }
+
     AbstractSftpTransfer::Ptr transfer
         = it.value().dynamicCast<AbstractSftpTransfer>();
     if (!transfer || transfer->state != AbstractSftpTransfer::Open
@@ -866,6 +894,43 @@ void SftpChannelPrivate::sendTransferCloseHandle(const AbstractSftpTransfer::Ptr
     job->state = SftpDownload::CloseRequested;
 }
 
+void SftpChannelPrivate::attributesToFileInfo(const SftpFileAttributes &attributes,
+    SftpFileInfo &fileInfo) const
+{
+    if (attributes.sizePresent) {
+        fileInfo.sizeValid = true;
+        fileInfo.size = attributes.size;
+    }
+    if (attributes.permissionsPresent) {
+        if (attributes.permissions & 0x8000) // S_IFREG
+            fileInfo.type = FileTypeRegular;
+        else if (attributes.permissions & 0x4000) // S_IFDIR
+            fileInfo.type = FileTypeDirectory;
+        else
+            fileInfo.type = FileTypeOther;
+        fileInfo.permissionsValid = true;
+        fileInfo.permissions = 0;
+        if (attributes.permissions & 00001) // S_IXOTH
+            fileInfo.permissions |= QFile::ExeOther;
+        if (attributes.permissions & 00002) // S_IWOTH
+            fileInfo.permissions |= QFile::WriteOther;
+        if (attributes.permissions & 00004) // S_IROTH
+            fileInfo.permissions |= QFile::ReadOther;
+        if (attributes.permissions & 00010) // S_IXGRP
+            fileInfo.permissions |= QFile::ExeGroup;
+        if (attributes.permissions & 00020) // S_IWGRP
+            fileInfo.permissions |= QFile::WriteGroup;
+        if (attributes.permissions & 00040) // S_IRGRP
+            fileInfo.permissions |= QFile::ReadGroup;
+        if (attributes.permissions & 00100) // S_IXUSR
+            fileInfo.permissions |= QFile::ExeUser | QFile::ExeOwner;
+        if (attributes.permissions & 00200) // S_IWUSR
+            fileInfo.permissions |= QFile::WriteUser | QFile::WriteOwner;
+        if (attributes.permissions & 00400) // S_IRUSR
+            fileInfo.permissions |= QFile::ReadUser | QFile::ReadOwner;
+    }
+}
+
 void SftpChannelPrivate::removeTransferRequest(const JobMap::Iterator &it)
 {
     --it.value().staticCast<AbstractSftpTransfer>()->inFlightCount;

src/libs/utils/ssh/sftpchannel.h

@@ -66,6 +66,7 @@ public:
     void initialize();
     void closeChannel();
 
+    SftpJobId statFile(const QString &path);
     SftpJobId listDirectory(const QString &dirPath);
     SftpJobId createDirectory(const QString &dirPath);
     SftpJobId removeDirectory(const QString &dirPath);
@@ -91,13 +92,16 @@ signals:
     // error.isEmpty <=> finished successfully
     void finished(Utils::SftpJobId job, const QString &error = QString());
 
-    /*
-     * This signal is only emitted by the "List Directory" operation,
-     * one file at a time.
      // TODO: Also emit for each file copied by uploadDir().
-     */
     void dataAvailable(Utils::SftpJobId job, const QString &data);
 
+    /*
+     * This signal is emitted as a result of:
+     *     - statFile() (with the list having exactly one element)
+     *     - listDirectory() (potentially more than once)
+     */
+    void fileInfoAvailable(Utils::SftpJobId job, const QList<Utils::SftpFileInfo> &fileInfoList);
+
 private:
     SftpChannel(quint32 channelId, Internal::SshSendFacility &sendFacility);
 

src/libs/utils/ssh/sftpchannel_p.h

@@ -65,6 +65,7 @@ signals:
     void closed();
     void finished(Utils::SftpJobId job, const QString &error = QString());
     void dataAvailable(Utils::SftpJobId job, const QString &data);
+    void fileInfoAvailable(Utils::SftpJobId job, const QList<Utils::SftpFileInfo> &fileInfoList);
 
 private:
     typedef QMap<SftpJobId, AbstractSftpOperation::Ptr> JobMap;
@@ -116,6 +117,8 @@ private:
     void sendTransferCloseHandle(const AbstractSftpTransfer::Ptr &job,
         quint32 requestId);
 
+    void attributesToFileInfo(const SftpFileAttributes &attributes, SftpFileInfo &fileInfo) const;
+
     JobMap::Iterator lookupJob(SftpJobId id);
     JobMap m_jobs;
     SftpOutgoingPacket m_outgoingPacket;

src/libs/utils/ssh/sftpdefs.h

@@ -35,7 +35,8 @@
 
 #include <utils/utils_global.h>
 
-#include <QtCore/QtGlobal>
+#include <QFile>
+#include <QString>
 
 namespace Utils {
 
@@ -46,6 +47,23 @@ enum SftpOverwriteMode {
     SftpOverwriteExisting, SftpAppendToExisting, SftpSkipExisting
 };
 
+enum SftpFileType { FileTypeRegular, FileTypeDirectory, FileTypeOther, FileTypeUnknown };
+
+class QTCREATOR_UTILS_EXPORT SftpFileInfo
+{
+public:
+    SftpFileInfo() : type(FileTypeUnknown), sizeValid(false), permissionsValid(false) { }
+
+    QString name;
+    SftpFileType type;
+    quint64 size;
+    QFile::Permissions permissions;
+
+    // The RFC allows an SFTP server not to support any file attributes beyond the name.
+    bool sizeValid;
+    bool permissionsValid;
+};
+
 } // namespace Utils
 
 #endif // SFTPDEFS_H

src/libs/utils/ssh/sftpoperation.cpp

@@ -46,6 +46,16 @@ AbstractSftpOperation::AbstractSftpOperation(SftpJobId jobId) : jobId(jobId)
 AbstractSftpOperation::~AbstractSftpOperation() { }
 
 
+SftpStatFile::SftpStatFile(SftpJobId jobId, const QString &path)
+    : AbstractSftpOperation(jobId), path(path)
+{
+}
+
+SftpOutgoingPacket &SftpStatFile::initialPacket(SftpOutgoingPacket &packet)
+{
+    return packet.generateStat(path, jobId);
+}
+
 SftpMakeDir::SftpMakeDir(SftpJobId jobId, const QString &path,
     const SftpUploadDir::Ptr &parentJob)
     : AbstractSftpOperation(jobId), parentJob(parentJob), remoteDir(path)

src/libs/utils/ssh/sftpoperation_p.h

@@ -53,7 +53,7 @@ struct AbstractSftpOperation
 {
     typedef QSharedPointer<AbstractSftpOperation> Ptr;
     enum Type {
-        ListDir, MakeDir, RmDir, Rm, Rename, CreateLink, CreateFile, Download, UploadFile
+        StatFile, ListDir, MakeDir, RmDir, Rm, Rename, CreateLink, CreateFile, Download, UploadFile
     };
 
     AbstractSftpOperation(SftpJobId jobId);
@@ -70,6 +70,17 @@ private:
 
 struct SftpUploadDir;
 
+struct SftpStatFile : public AbstractSftpOperation
+{
+    typedef QSharedPointer<SftpStatFile> Ptr;
+
+    SftpStatFile(SftpJobId jobId, const QString &path);
+    virtual Type type() const { return StatFile; }
+    virtual SftpOutgoingPacket &initialPacket(SftpOutgoingPacket &packet);
+
+    const QString path;
+};
+
 struct SftpMakeDir : public AbstractSftpOperation
 {
     typedef QSharedPointer<SftpMakeDir> Ptr;

src/libs/utils/ssh/sftpoutgoingpacket.cpp

@@ -60,6 +60,11 @@ SftpOutgoingPacket &SftpOutgoingPacket::generateInit(quint32 version)
     return init(SSH_FXP_INIT, 0).appendInt(version).finalize();
 }
 
+SftpOutgoingPacket &SftpOutgoingPacket::generateStat(const QString &path, quint32 requestId)
+{
+    return init(SSH_FXP_LSTAT, requestId).appendString(path).finalize();
+}
+
 SftpOutgoingPacket &SftpOutgoingPacket::generateOpenDir(const QString &path,
     quint32 requestId)
 {

src/libs/utils/ssh/sftpoutgoingpacket_p.h

@@ -44,6 +44,7 @@ class SftpOutgoingPacket : public AbstractSftpPacket
 public:
     SftpOutgoingPacket();
     SftpOutgoingPacket &generateInit(quint32 version);
+    SftpOutgoingPacket &generateStat(const QString &path, quint32 requestId);
     SftpOutgoingPacket &generateOpenDir(const QString &path, quint32 requestId);
     SftpOutgoingPacket &generateReadDir(const QByteArray &handle,
         quint32 requestId);

src/libs/utils/ssh/sshconnection.cpp

@@ -76,6 +76,8 @@ namespace {
             Botan::LibraryInitializer::initialize("thread_safe=true");
             qRegisterMetaType<Utils::SshError>("Utils::SshError");
             qRegisterMetaType<Utils::SftpJobId>("Utils::SftpJobId");
+            qRegisterMetaType<Utils::SftpFileInfo>("Utils::SftpFileInfo");
+            qRegisterMetaType<QList <Utils::SftpFileInfo> >("QList<Utils::SftpFileInfo>");
             staticInitializationsDone = true;
         }
     }

tests/manual/ssh/sftp/sftptest.cpp

@@ -46,7 +46,11 @@ SftpTest::SftpTest(const Parameters &params)
     : m_parameters(params), m_state(Inactive), m_error(false),
       m_bigFileUploadJob(SftpInvalidJob),
       m_bigFileDownloadJob(SftpInvalidJob),
-      m_bigFileRemovalJob(SftpInvalidJob)
+      m_bigFileRemovalJob(SftpInvalidJob),
+      m_mkdirJob(SftpInvalidJob),
+      m_statDirJob(SftpInvalidJob),
+      m_lsDirJob(SftpInvalidJob),
+      m_rmDirJob(SftpInvalidJob)
 {
 }
 
@@ -85,6 +89,9 @@ void SftpTest::handleConnected()
             SLOT(handleChannelInitializationFailure(QString)));
         connect(m_channel.data(), SIGNAL(finished(Utils::SftpJobId, QString)),
             this, SLOT(handleJobFinished(Utils::SftpJobId, QString)));
+        connect(m_channel.data(),
+            SIGNAL(fileInfoAvailable(Utils::SftpJobId, QList<Utils::SftpFileInfo>)),
+            SLOT(handleFileInfo(Utils::SftpJobId, QList<Utils::SftpFileInfo>)));
         connect(m_channel.data(), SIGNAL(closed()), this,
             SLOT(handleChannelClosed()));
         m_state = InitializingChannel;
@@ -107,14 +114,16 @@ void SftpTest::handleDisconnected()
     else
         std::cout << "No errors encountered.";
     std::cout << std::endl;
-    qApp->quit();
+    qApp->exit(m_error ? EXIT_FAILURE : EXIT_SUCCESS);
 }
 
 void SftpTest::handleError()
 {
     std::cerr << "Encountered SSH error: "
         << qPrintable(m_connection->errorString()) << "." << std::endl;
-    qApp->quit();
+    m_error = true;
+    m_state = Disconnecting;
+    qApp->exit(EXIT_FAILURE);
 }
 
 void SftpTest::handleChannelInitialized()
@@ -384,17 +393,84 @@ void SftpTest::handleJobFinished(Utils::SftpJobId job, const QString &error)
         m_state = RemovingBig;
         break;
     }
-    case RemovingBig: {
+    case RemovingBig:
         if (!handleBigJobFinished(job, m_bigFileRemovalJob, error, "removing"))
             return;
-        const QString remoteFp
-            = remoteFilePath(QFileInfo(m_localBigFile->fileName()).fileName());
         std::cout << "Big files successfully removed. "
-            << "Now closing the SFTP channel..." << std::endl;
+            << "Now creating remote directory..." << std::endl;
+        m_remoteDirPath = QLatin1String("/tmp/sftptest-") + QDateTime::currentDateTime().toString();
+        m_mkdirJob = m_channel->createDirectory(m_remoteDirPath);
+        m_state = CreatingDir;
+        break;
+    case CreatingDir:
+        if (!handleJobFinished(job, m_mkdirJob, error, "creating remote directory"))
+            return;
+        std::cout << "Directory successfully created. Now checking directory attributes..."
+            << std::endl;
+        m_statDirJob = m_channel->statFile(m_remoteDirPath);
+        m_state = CheckingDirAttributes;
+        break;
+    case CheckingDirAttributes: {
+        if (!handleJobFinished(job, m_statDirJob, error, "checking directory attributes"))
+            return;
+        if (m_dirInfo.type != FileTypeDirectory) {
+            std::cerr << "Error: Newly created directory has file type " << m_dirInfo.type
+                << ", expected was " << FileTypeDirectory << "." << std::endl;
+            earlyDisconnectFromHost();
+            return;
+        }
+        const QString fileName = QFileInfo(m_remoteDirPath).fileName();
+        if (m_dirInfo.name != fileName) {
+            std::cerr << "Error: Remote directory reports file name '"
+                << qPrintable(m_dirInfo.name) << "', expected '" << qPrintable(fileName) << "'."
+                << std::endl;
+            earlyDisconnectFromHost();
+            return;
+        }
+        std::cout << "Directory attributes ok. Now checking directory contents..." << std::endl;
+        m_lsDirJob = m_channel->listDirectory(m_remoteDirPath);
+        m_state = CheckingDirContents;
+        break;
+    }
+    case CheckingDirContents:
+        if (!handleJobFinished(job, m_lsDirJob, error, "checking directory contents"))
+            return;
+        if (m_dirContents.count() != 2) {
+            std::cerr << "Error: Remote directory has " << m_dirContents.count()
+                << " entries, expected 2." << std::endl;
+            earlyDisconnectFromHost();
+            return;
+        }
+        foreach (const SftpFileInfo &fi, m_dirContents) {
+            if (fi.type != FileTypeDirectory) {
+                std::cerr << "Error: Remote directory has entry of type " << fi.type
+                    << ", expected " << FileTypeDirectory << "." << std::endl;
+                earlyDisconnectFromHost();
+                return;
+            }
+            if (fi.name != QLatin1String(".") && fi.name != QLatin1String("..")) {
+                std::cerr << "Error: Remote directory has entry '" << qPrintable(fi.name)
+                    << "', expected '.' or '..'." << std::endl;
+                earlyDisconnectFromHost();
+                return;
+            }
+        }
+        if (m_dirContents.first().name == m_dirContents.last().name) {
+            std::cerr << "Error: Remote directory has two entries of the same name." << std::endl;
+            earlyDisconnectFromHost();
+            return;
+        }
+        std::cout << "Directory contents ok. Now removing directory..." << std::endl;
+        m_rmDirJob = m_channel->removeDirectory(m_remoteDirPath);
+        m_state = RemovingDir;
+        break;
+    case RemovingDir:
+        if (!handleJobFinished(job, m_rmDirJob, error, "removing directory"))
+            return;
+        std::cout << "Directory successfully removed. Now closing the SFTP channel..." << std::endl;
         m_state = ChannelClosing;
         m_channel->closeChannel();
         break;
-    }
     case Disconnecting:
         break;
     default:
@@ -406,6 +482,32 @@ void SftpTest::handleJobFinished(Utils::SftpJobId job, const QString &error)
     }
 }
 
+void SftpTest::handleFileInfo(SftpJobId job, const QList<SftpFileInfo> &fileInfoList)
+{
+    switch (m_state) {
+    case CheckingDirAttributes: {
+        static int count = 0;
+        if (!checkJobId(job, m_statDirJob, "checking directory attributes"))
+            return;
+        if (++count > 1) {
+            std::cerr << "Error: More than one reply for directory attributes check." << std::endl;
+            earlyDisconnectFromHost();
+            return;
+        }
+        m_dirInfo = fileInfoList.first();
+        break;
+    }
+    case CheckingDirContents:
+        if (!checkJobId(job, m_lsDirJob, "checking directory contents"))
+            return;
+        m_dirContents << fileInfoList;
+        break;
+    default:
+        std::cerr << "Error: Unexpected file info in state " << m_state << "." << std::endl;
+        earlyDisconnectFromHost();
+    }
+}
+
 void SftpTest::removeFile(const FilePtr &file, bool remoteToo)
 {
     if (!file)
@@ -438,6 +540,17 @@ void SftpTest::earlyDisconnectFromHost()
     m_connection->disconnectFromHost();
 }
 
+bool SftpTest::checkJobId(SftpJobId job, SftpJobId expectedJob, const char *activity)
+{
+    if (job != expectedJob) {
+        std::cerr << "Error " << activity << ": Expected job id " << expectedJob
+           << ", got job id " << job << '.' << std::endl;
+        earlyDisconnectFromHost();
+        return false;
+    }
+    return true;
+}
+
 void SftpTest::removeFiles(bool remoteToo)
 {
     foreach (const FilePtr &file, m_localSmallFiles)
@@ -465,6 +578,19 @@ bool SftpTest::handleJobFinished(SftpJobId job, JobMap &jobMap,
     return true;
 }
 
+bool SftpTest::handleJobFinished(SftpJobId job, SftpJobId expectedJob, const QString &error,
+    const char *activity)
+{
+    if (!checkJobId(job, expectedJob, activity))
+        return false;
+    if (!error.isEmpty()) {
+        std::cerr << "Error " << activity << ": " << qPrintable(error) << "." << std::endl;
+        earlyDisconnectFromHost();
+        return false;
+    }
+    return true;
+}
+
 bool SftpTest::handleBigJobFinished(SftpJobId job, SftpJobId expectedJob,
     const QString &error, const char *activity)
 {

tests/manual/ssh/sftp/sftptest.h

@@ -61,14 +61,16 @@ private slots:
     void handleChannelInitialized();
     void handleChannelInitializationFailure(const QString &reason);
     void handleJobFinished(Utils::SftpJobId job, const QString &error);
+    void handleFileInfo(Utils::SftpJobId job, const QList<Utils::SftpFileInfo> &fileInfoList);
     void handleChannelClosed();
 
 private:
     typedef QHash<Utils::SftpJobId, QString> JobMap;
     typedef QSharedPointer<QFile> FilePtr;
-    enum State { Inactive, Connecting, InitializingChannel, UploadingSmall,
-        DownloadingSmall, RemovingSmall, UploadingBig, DownloadingBig,
-        RemovingBig, ChannelClosing, Disconnecting
+    enum State {
+        Inactive, Connecting, InitializingChannel, UploadingSmall, DownloadingSmall,
+        RemovingSmall, UploadingBig, DownloadingBig, RemovingBig, CreatingDir,
+        CheckingDirAttributes, CheckingDirContents, RemovingDir, ChannelClosing, Disconnecting
     };
 
     void removeFile(const FilePtr &filePtr, bool remoteToo);
@@ -76,8 +78,11 @@ private:
     QString cmpFileName(const QString &localFileName) const;
     QString remoteFilePath(const QString &localFileName) const;
     void earlyDisconnectFromHost();
+    bool checkJobId(Utils::SftpJobId job, Utils::SftpJobId expectedJob, const char *activity);
     bool handleJobFinished(Utils::SftpJobId job, JobMap &jobMap,
         const QString &error, const char *activity);
+    bool handleJobFinished(Utils::SftpJobId job, Utils::SftpJobId expectedJob, const QString &error,
+        const char *activity);
     bool handleBigJobFinished(Utils::SftpJobId job, Utils::SftpJobId expectedJob,
         const QString &error, const char *activity);
     bool compareFiles(QFile *orig, QFile *copy);
@@ -95,7 +100,14 @@ private:
     Utils::SftpJobId m_bigFileUploadJob;
     Utils::SftpJobId m_bigFileDownloadJob;
     Utils::SftpJobId m_bigFileRemovalJob;
+    Utils::SftpJobId m_mkdirJob;
+    Utils::SftpJobId m_statDirJob;
+    Utils::SftpJobId m_lsDirJob;
+    Utils::SftpJobId m_rmDirJob;
     QElapsedTimer m_bigJobTimer;
+    QString m_remoteDirPath;
+    Utils::SftpFileInfo m_dirInfo;
+    QList<Utils::SftpFileInfo> m_dirContents;
 };