Implement the network part of the support for divelogs.de

This implements support for:
 * uploading a zip file containing dives - untested
   (the zip file must have been prepared elsewhere)
 * downloading the dive list and the dive XML files

The networking part is finished, but it's missing the actual import of
the XML files sent by divelogs.de.

Signed-off-by: Thiago Macieira <thiago@macieira.org>

3 files changed, 327 insertions, 6 deletions

diff --git a/qt-ui/mainwindow.cpp b/qt-ui/mainwindow.cpp
index 101219725..b216240fa 100644
--- a/qt-ui/mainwindow.cpp
+++ b/qt-ui/mainwindow.cpp
@@ -269,7 +269,7 @@ void MainWindow::on_actionDownloadWeb_triggered()
 
 void MainWindow::on_actionDivelogs_de_triggered()
 {
-	DivelogsDeWebServices::instance()->exec();
+	DivelogsDeWebServices::instance()->downloadDives();
 }
 
 void MainWindow::on_actionEditDeviceNames_triggered()
diff --git a/qt-ui/subsurfacewebservices.cpp b/qt-ui/subsurfacewebservices.cpp
index 5756e806d..6a6da1853 100644
--- a/qt-ui/subsurfacewebservices.cpp
+++ b/qt-ui/subsurfacewebservices.cpp
@@ -3,16 +3,25 @@
 #include "mainwindow.h"
 
 #include <libxml/parser.h>
+#include <zip.h>
 
+#include <QDir>
+#include <QHttpMultiPart>
+#include <QMessageBox>
 #include <QNetworkAccessManager>
 #include <QNetworkReply>
 #include <QDebug>
 #include <QSettings>
+#include <QXmlStreamReader>
 #include <qdesktopservices.h>
 
 #include "../dive.h"
 #include "../divelist.h"
 
+#ifdef Q_OS_UNIX
+#  include <unistd.h> // for dup(2)
+#endif
+
 struct dive_table gps_location_table;
 static bool merge_locations_into_dives(void);
 
@@ -95,6 +104,7 @@ WebServices::WebServices(QWidget* parent, Qt::WindowFlags f): QDialog(parent, f)
 	ui.setupUi(this);
 	connect(ui.buttonBox, SIGNAL(clicked(QAbstractButton*)), this, SLOT(buttonClicked(QAbstractButton*)));
 	connect(ui.download, SIGNAL(clicked(bool)), this, SLOT(startDownload()));
+	connect(ui.upload, SIGNAL(clicked(bool)), this, SLOT(startUpload()));
 	connect(&timeout, SIGNAL(timeout()), this, SLOT(downloadTimedOut()));
 	ui.buttonBox->button(QDialogButtonBox::Apply)->setEnabled(false);
 	timeout.setSingleShot(true);
@@ -109,6 +119,13 @@ void WebServices::hidePassword()
 void WebServices::hideUpload()
 {
 	ui.upload->hide();
+	ui.download->show();
+}
+
+void WebServices::hideDownload()
+{
+	ui.download->hide();
+	ui.upload->show();
 }
 
 QNetworkAccessManager *WebServices::manager()
@@ -125,7 +142,7 @@ void WebServices::downloadTimedOut()
 	reply->deleteLater();
 	reply = NULL;
 	resetState();
-	ui.status->setText(tr("Download timed out"));
+	ui.status->setText(tr("Operation timed out"));
 }
 
 void WebServices::updateProgress(qint64 current, qint64 total)
@@ -144,7 +161,7 @@ void WebServices::updateProgress(qint64 current, qint64 total)
 	}
 	ui.progressBar->setRange(0, total);
 	ui.progressBar->setValue(current);
-	ui.status->setText(tr("Downloading..."));
+	ui.status->setText(tr("Transfering data..."));
 
 	// reset the timer: 30 seconds after we last got any data
 	timeout.start();
@@ -164,6 +181,9 @@ void WebServices::connectSignalsForDownload(QNetworkReply *reply)
 void WebServices::resetState()
 {
 	ui.download->setEnabled(true);
+	ui.upload->setEnabled(true);
+	ui.userID->setEnabled(true);
+	ui.password->setEnabled(true);
 	ui.progressBar->reset();
 	ui.progressBar->setRange(0,1);
 	ui.status->setText(QString());
@@ -252,7 +272,7 @@ void SubsurfaceWebServices::downloadFinished()
 	downloadedData = reply->readAll();
 
 	ui.download->setEnabled(true);
-	ui.status->setText(tr("Download Finished"));
+	ui.status->setText(tr("Download finished"));
 
 	uint resultCode = download_dialog_parse_response(downloadedData);
 	setStatusText(resultCode);
@@ -325,6 +345,92 @@ end:
 // #
 // #
 
+struct DiveListResult
+{
+	QString errorCondition;
+	QString errorDetails;
+	QByteArray idList; // comma-separated, suitable to be sent in the fetch request
+	int idCount;
+};
+
+static DiveListResult parseDiveLogsDeDiveList(const QByteArray &xmlData)
+{
+    /* XML format seems to be:
+     * <DiveDateReader version="1.0">
+     *   <DiveDates>
+     *     <date diveLogsId="nnn" lastModified="YYYY-MM-DD hh:mm:ss">DD.MM.YYYY hh:mm</date>
+     *     [repeat <date></date>]
+     *   </DiveDates>
+     * </DiveDateReader>
+     */
+    QXmlStreamReader reader(xmlData);
+    const QString invalidXmlError = DivelogsDeWebServices::tr("Invalid response from server");
+    bool seenDiveDates = false;
+    DiveListResult result;
+    result.idCount = 0;
+
+    if (reader.readNextStartElement() && reader.name() != "DiveDateReader") {
+	    result.errorCondition = invalidXmlError;
+	    result.errorDetails =
+			    DivelogsDeWebServices::tr("Expected XML tag 'DiveDateReader', got instead '%1")
+			    .arg(reader.name().toString());
+	    goto out;
+    }
+
+    while (reader.readNextStartElement()) {
+	    if (reader.name() != "DiveDates") {
+		    if (reader.name() == "Login") {
+			    QString status = reader.readElementText();
+			    // qDebug() << "Login status:" << status;
+
+			    // Note: there has to be a better way to determine a successful login...
+			    if (status == "failed") {
+				    result.errorCondition = "Login failed";
+				    goto out;
+			    }
+		    } else {
+			    // qDebug() << "Skipping" << reader.name();
+		    }
+		    continue;
+	    }
+
+	    // process <DiveDates>
+	    seenDiveDates = true;
+	    while (reader.readNextStartElement()) {
+		    if (reader.name() != "date") {
+			    // qDebug() << "Skipping" << reader.name();
+			    continue;
+		    }
+		    QStringRef id = reader.attributes().value("divelogsId");
+		    // qDebug() << "Found" << reader.name() << "with id =" << id;
+		    if (!id.isEmpty()) {
+			    result.idList += id.toLatin1();
+			    result.idList += ',';
+			    ++result.idCount;
+		    }
+
+		    reader.skipCurrentElement();
+	    }
+    }
+
+    // chop the ending comma, if any
+    result.idList.chop(1);
+
+    if (!seenDiveDates) {
+	    result.errorCondition = invalidXmlError;
+	    result.errorDetails = DivelogsDeWebServices::tr("Expected XML tag 'DiveDates' not found");
+    }
+
+out:
+    if (reader.hasError()) {
+	    // if there was an XML error, overwrite the result or other error conditions
+	    result.errorCondition = invalidXmlError;
+	    result.errorDetails = DivelogsDeWebServices::tr("Malformed XML response. Line %1: %2")
+				  .arg(reader.lineNumber()).arg(reader.errorString());
+    }
+    return result;
+}
+
 DivelogsDeWebServices* DivelogsDeWebServices::instance()
 {
 	static DivelogsDeWebServices *self = new DivelogsDeWebServices(mainWindow());
@@ -332,24 +438,217 @@ DivelogsDeWebServices* DivelogsDeWebServices::instance()
 	return self;
 }
 
-DivelogsDeWebServices::DivelogsDeWebServices(QWidget* parent, Qt::WindowFlags f): WebServices(parent, f)
+void DivelogsDeWebServices::downloadDives()
 {
+	hideUpload();
+	exec();
+}
 
+void DivelogsDeWebServices::uploadDives(QIODevice *dldContent)
+{
+	QHttpMultiPart mp(QHttpMultiPart::FormDataType);
+	QHttpPart part;
+	part.setRawHeader("Content-Disposition", "form-data; name=\"userfile\"");
+	part.setBodyDevice(dldContent);
+	mp.append(part);
+
+	multipart = &mp;
+	hideDownload();
+	exec();
+	multipart = NULL;
+
+	delete reply; // we need to ensure it has stopped using our QHttpMultiPart
+}
+
+DivelogsDeWebServices::DivelogsDeWebServices(QWidget* parent, Qt::WindowFlags f): WebServices(parent, f)
+{
+	QSettings s;
+	ui.userID->setText(s.value("divelogde_user").toString());
+	ui.password->setText(s.value("divelogde_pass").toString());
+	hideUpload();
 }
 
 void DivelogsDeWebServices::startUpload()
 {
+	ui.status->setText(tr("Uploading dive list..."));
+	ui.progressBar->setRange(0,0); // this makes the progressbar do an 'infinite spin'
+	ui.upload->setEnabled(false);
+	ui.userID->setEnabled(false);
+	ui.password->setEnabled(false);
 
+	QNetworkRequest request;
+	request.setUrl(QUrl("https://divelogs.de/DivelogsDirectImport.php"));
+	request.setRawHeader("Accept", "text/xml, application/xml");
+
+	QHttpPart part;
+	part.setRawHeader("Content-Disposition", "form-data; name=\"user\"");
+	part.setBody(ui.userID->text().toUtf8());
+	multipart->append(part);
+
+	part.setRawHeader("Content-Disposition", "form-data; name=\"pass\"");
+	part.setBody(ui.password->text().toUtf8());
+	multipart->append(part);
+
+	reply = manager()->post(request, multipart);
+	connect(reply, SIGNAL(finished()), this, SLOT(uploadFinished()));
+	connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
+		SLOT(uploadError(QNetworkReply::NetworkError)));
+	connect(reply, SIGNAL(uploadProgress(qint64,qint64)), this,
+		SLOT(updateProgress(qint64,qint64)));
+
+	timeout.start(30000); // 30s
 }
 
 void DivelogsDeWebServices::startDownload()
 {
+	ui.status->setText(tr("Downloading dive list..."));
+	ui.progressBar->setRange(0,0); // this makes the progressbar do an 'infinite spin'
+	ui.download->setEnabled(false);
+	ui.userID->setEnabled(false);
+	ui.password->setEnabled(false);
+
+	QNetworkRequest request;
+	request.setUrl(QUrl("https://divelogs.de/xml_available_dives.php"));
+	request.setRawHeader("Accept", "text/xml, application/xml");
+	request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
+
+#if QT_VERSION < QT_VERSION_CHECK(5,0,0)
+	QUrl body;
+	body.addQueryItem("user", ui.userID->text());
+	body.addQueryItem("pass", ui.password->text());
+
+	reply = manager()->post(request, body.encodedQuery());
+#else
+	QUrlQuery body;
+	body.addQueryItem("user", ui.userID->text());
+	body.addQueryItem("pass", ui.password->text());
+
+	reply = manager()->post(request, body.query(QUrl::FullyEncoded).toLatin1())
+#endif
+	connect(reply, SIGNAL(finished()), this, SLOT(listDownloadFinished()));
+	connect(reply, SIGNAL(error(QNetworkReply::NetworkError)),
+		this, SLOT(downloadError(QNetworkReply::NetworkError)));
+
+	timeout.start(30000); // 30s
+}
+
+void DivelogsDeWebServices::listDownloadFinished()
+{
+	if (!reply)
+		return;
+	QByteArray xmlData = reply->readAll();
+	reply->deleteLater();
+	reply = NULL;
+
+	// parse the XML data we downloaded
+	DiveListResult diveList = parseDiveLogsDeDiveList(xmlData);
+	if (!diveList.errorCondition.isEmpty()) {
+		// error condition
+		resetState();
+		ui.status->setText(diveList.errorCondition);
+		return;
+	}
+
+	ui.status->setText(tr("Downloading %1 dives...").arg(diveList.idCount));
+
+	QNetworkRequest request;
+//	request.setUrl(QUrl("https://divelogs.de/DivelogsDirectExport.php"));
+	request.setUrl(QUrl("http://divelogs.de/DivelogsDirectExport.php"));
+	request.setRawHeader("Accept", "application/zip, */*");
+	request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
+
+#if QT_VERSION < QT_VERSION_CHECK(5,0,0)
+	QUrl body;
+	body.addQueryItem("user", ui.userID->text());
+	body.addQueryItem("pass", ui.password->text());
+	body.addQueryItem("ids", diveList.idList);
+
+	reply = manager()->post(request, body.encodedQuery());
+#else
+	QUrlQuery body;
+	body.addQueryItem("user", ui.userID->text());
+	body.addQueryItem("pass", ui.password->text());
+	body.addQueryItem("ids", diveList.idList);
+
+	reply = manager()->post(request, body.query(QUrl::FullyEncoded).toLatin1())
+#endif
+
+	connect(reply, SIGNAL(readyRead()), this, SLOT(saveToZipFile()));
+	connectSignalsForDownload(reply);
+}
+
+void DivelogsDeWebServices::saveToZipFile()
+{
+	if (!zipFile.isOpen()) {
+		zipFile.setFileTemplate(QDir::tempPath() + "/import-XXXXXX.dld");
+		zipFile.open();
+	}
 
+	zipFile.write(reply->readAll());
 }
 
 void DivelogsDeWebServices::downloadFinished()
 {
+	if (!reply)
+		return;
+
+	ui.download->setEnabled(true);
+	ui.status->setText(tr("Download finished - %1").arg(reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString()));
+	reply->deleteLater();
+	reply = NULL;
+
+	int errorcode;
+	zipFile.seek(0);
+#ifdef Q_OS_UNIX
+	int duppedfd = dup(zipFile.handle());
+	struct zip *zip = zip_fdopen(duppedfd, 0, &errorcode);
+	if (!zip)
+		::close(duppedfd);
+#else
+	struct zip *zip = zip_open(zipFile.fileName(), 0, &errorcode);
+#endif
+	if (!zip) {
+		char buf[512];
+		zip_error_to_str(buf, sizeof(buf), errorcode, errno);
+		QMessageBox::critical(this, tr("Corrupted download"),
+				      tr("The archive could not be opened:\n%1").arg(QString::fromLocal8Bit(buf)));
+		zipFile.close();
+		return;
+	}
 
+	quint64 entries = zip_get_num_entries(zip, 0);
+	for (quint64 i = 0; i < entries; ++i) {
+		struct zip_file *zip_file = zip_fopen_index(zip, i, 0);
+		if (!zip_file) {
+			QMessageBox::critical(this, tr("Corrupted download"),
+					      tr("The archive contains corrupt data:\n%1").arg(QString::fromLocal8Bit(zip_strerror(zip))));
+			goto close_zip;
+		}
+
+		// ### FIXME: What do I do with this?
+
+		zip_fclose(zip_file);
+	}
+
+close_zip:
+	zip_close(zip);
+	zipFile.close();
+}
+
+void DivelogsDeWebServices::uploadFinished()
+{
+	if (!reply)
+		return;
+
+	ui.progressBar->setRange(0,1);
+	ui.upload->setEnabled(true);
+	ui.status->setText(tr("Upload finished"));
+
+	// check what the server sent us: it might contain
+	// an error condition, such as a failed login
+	QByteArray xmlData = reply->readAll();
+
+	// ### FIXME: what's the format?
 }
 
 void DivelogsDeWebServices::setStatusText(int status)
@@ -357,12 +656,21 @@ void DivelogsDeWebServices::setStatusText(int status)
 
 }
 
-void DivelogsDeWebServices::downloadError(QNetworkReply::NetworkError error)
+void DivelogsDeWebServices::downloadError(QNetworkReply::NetworkError)
 {
+	resetState();
+	ui.status->setText(tr("Download error: %1").arg(reply->errorString()));
+	reply->deleteLater();
+	reply = NULL;
+}
 
+void DivelogsDeWebServices::uploadError(QNetworkReply::NetworkError error)
+{
+	downloadError(error);
 }
 
 void DivelogsDeWebServices::buttonClicked(QAbstractButton* button)
 {
 
 }
+
diff --git a/qt-ui/subsurfacewebservices.h b/qt-ui/subsurfacewebservices.h
index 89c41e4dd..3c41b08d1 100644
--- a/qt-ui/subsurfacewebservices.h
+++ b/qt-ui/subsurfacewebservices.h
@@ -3,6 +3,7 @@
 
 #include <QDialog>
 #include <QNetworkReply>
+#include <QTemporaryFile>
 #include <QTimer>
 #include <libxml/tree.h>
 
@@ -10,6 +11,7 @@
 
 class QAbstractButton;
 class QNetworkReply;
+class QHttpMultiPart;
 
 class WebServices : public QDialog{
 	Q_OBJECT
@@ -17,6 +19,7 @@ public:
 	explicit WebServices(QWidget* parent = 0, Qt::WindowFlags f = 0);
 	void hidePassword();
 	void hideUpload();
+	void hideDownload();
 
 	static QNetworkAccessManager *manager();
 
@@ -32,6 +35,7 @@ protected slots:
 protected:
 	void resetState();
 	void connectSignalsForDownload(QNetworkReply *reply);
+	void connectSignalsForUpload();
 
 	Ui::WebServices ui;
 	QNetworkReply *reply;
@@ -61,18 +65,27 @@ class DivelogsDeWebServices : public WebServices {
 	Q_OBJECT
 public:
 	static DivelogsDeWebServices * instance();
+	void downloadDives();
+	void uploadDives(QIODevice *dldContent);
 
 private slots:
 	void startDownload();
 	void buttonClicked(QAbstractButton* button);
+	void saveToZipFile();
+	void listDownloadFinished();
 	void downloadFinished();
+	void uploadFinished();
 	void downloadError(QNetworkReply::NetworkError error);
+	void uploadError(QNetworkReply::NetworkError error);
 	void startUpload();
 private:
 	explicit DivelogsDeWebServices (QWidget* parent = 0, Qt::WindowFlags f = 0);
 	void setStatusText(int status);
 	void download_dialog_traverse_xml(xmlNodePtr node, unsigned int *download_status);
 	unsigned int download_dialog_parse_response(const QByteArray& length);
+
+	QHttpMultiPart *multipart;
+	QTemporaryFile zipFile;
 };
 
 #endif