From 223e1cfd1b02e84f2de9f274df1dbb506dae4883 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gr=C3=A9gory=20Oestreicher?= <greg@kamago.net>
Date: Mon, 19 Dec 2016 08:29:10 +0100
Subject: [PATCH] Embed images in articles
---
harbour-wallaread.pro | 6 +-
qml/harbour-wallaread.qml | 26 ++++++
qml/js/WallaBase.js | 129 ++++++++++++++++++++++++++++--
qml/types/Server.qml | 10 +--
src/harbour-wallaread.cpp | 2 +
src/imageembedder.cpp | 89 +++++++++++++++++++++
src/imageembedder.h | 43 ++++++++++
translations/harbour-wallaread.ts | 7 ++
8 files changed, 295 insertions(+), 17 deletions(-)
create mode 100644 src/imageembedder.cpp
create mode 100644 src/imageembedder.h
diff --git a/harbour-wallaread.pro b/harbour-wallaread.pro
index 6934099..cd09e33 100644
--- a/harbour-wallaread.pro
+++ b/harbour-wallaread.pro
@@ -15,7 +15,8 @@ TARGET = harbour-wallaread
CONFIG += sailfishapp c++11
SOURCES += src/harbour-wallaread.cpp \
- src/httprequester.cpp
+ src/httprequester.cpp \
+ src/imageembedder.cpp
OTHER_FILES += qml/harbour-wallaread.qml \
qml/cover/CoverPage.qml \
@@ -37,7 +38,8 @@ CONFIG += sailfishapp_i18n
#TRANSLATIONS += translations/harbour-wallaread-de.ts
HEADERS += \
- src/httprequester.h
+ src/httprequester.h \
+ src/imageembedder.h
DISTFILES += \
qml/js/WallaBase.js \
diff --git a/qml/harbour-wallaread.qml b/qml/harbour-wallaread.qml
index ff964d3..673a37e 100644
--- a/qml/harbour-wallaread.qml
+++ b/qml/harbour-wallaread.qml
@@ -21,10 +21,36 @@
import QtQuick 2.0
import Sailfish.Silica 1.0
+
+import harbour.wallaread 1.0
+
import "pages"
+import "./js/WallaBase.js" as WallaBase
+
ApplicationWindow
{
+ QtObject {
+ id: jsTimerSource
+
+ function setTimeout( cb, ms ) {
+ var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", jsTimerSource)
+ timer.repeat = false
+ timer.interval = ms
+ timer.triggered.connect( function() { cb(); timer.destroy(); } )
+ timer.start()
+ }
+ }
+
+ ImageEmbedder {
+ id: imageEmbedder
+ }
+
+ Component.onCompleted: {
+ WallaBase.setTimerSource( jsTimerSource )
+ WallaBase.setImageEmbedder( imageEmbedder )
+ }
+
initialPage: Component { ServersPage { } }
cover: Qt.resolvedUrl("cover/CoverPage.qml")
allowedOrientations: defaultAllowedOrientations
diff --git a/qml/js/WallaBase.js b/qml/js/WallaBase.js
index f20d490..95b9e2d 100644
--- a/qml/js/WallaBase.js
+++ b/qml/js/WallaBase.js
@@ -32,6 +32,28 @@ var ArticlesFilter = {
Starred: 4
}
+/*
+ Timer management, used for the various setTimeout() calls
+ */
+
+var _timerSource = null;
+
+function setTimerSource( source )
+{
+ _timerSource = source;
+}
+
+/*
+ Used to embed images in the articles
+ */
+
+var _imageEmbedder = null;
+
+function setImageEmbedder( embedder )
+{
+ _imageEmbedder = embedder;
+}
+
/*
Servers management
*/
@@ -277,7 +299,7 @@ function _sendAuthRequest( props, cb )
Articles management
*/
-function syncDeletedArticles( timerSource, props, cb )
+function syncDeletedArticles( props, cb )
{
var db = getDatabase();
@@ -298,8 +320,8 @@ function syncDeletedArticles( timerSource, props, cb )
}
else {
if ( !working )
- timerSource.setTimeout( _checkNextArticle, 100 );
- timerSource.setTimeout( processArticlesList, 500 );
+ _timerSource.setTimeout( _checkNextArticle, 100 );
+ _timerSource.setTimeout( processArticlesList, 500 );
}
}
@@ -350,7 +372,7 @@ function syncDeletedArticles( timerSource, props, cb )
http.send();
}
- timerSource.setTimeout( processArticlesList, 500 );
+ _timerSource.setTimeout( processArticlesList, 500 );
}
);
}
@@ -527,7 +549,7 @@ function downloadArticles( props, cb )
if ( arts.length )
articles.push.apply( articles, arts );
if ( done )
- cb( articles, null );
+ embedImages( articles, cb );
}
}
);
@@ -589,6 +611,101 @@ function _downloadNextArticles( url, token, page, cb )
http.send();
}
+function embedImages( articles, cb )
+{
+ var ret = new Array;
+ var working = false;
+
+ function _processArticlesList() {
+ if ( !working && articles.length === 0 ) {
+ cb( ret, null );
+ }
+ else {
+ if ( !working )
+ _timerSource.setTimeout( _processNextArticle, 100 );
+ _timerSource.setTimeout( _processArticlesList, 500 );
+ }
+ }
+
+ function _processNextArticle() {
+ working = true;
+ var article = articles.pop();
+ console.debug( "Embedding images for article " + article.id );
+ console.debug( "Length is " + article.content.length + " before" );
+ _embedImages(
+ article,
+ function( content ) {
+ article.content = content;
+ console.debug( "Length is " + article.content.length + " after" );
+ ret.push( article );
+ working = false;
+ }
+ );
+ }
+
+ _timerSource.setTimeout( _processArticlesList, 100 );
+}
+
+function _embedImages( article, cb )
+{
+ var imgRe = /<img[^>]+\bsrc=(["'])(https?:\/\/.+?)\1[^>]+>/g;
+ var match;
+ var targets = new Array;
+ var content = article.content;
+
+ while ( match = imgRe.exec( content ) ) {
+ targets.push(
+ {
+ start: content.indexOf( match[2], match.index ),
+ url: match[2]
+ }
+ )
+ }
+
+ var working = false;
+ var offset = 0;
+
+ function _processImagesList() {
+ if ( !working && targets.length === 0 ) {
+ cb( content );
+ }
+ else {
+ if ( !working )
+ _timerSource.setTimeout( _downloadNextImage, 100 );
+ _timerSource.setTimeout( _processImagesList, 500 );
+ }
+ }
+
+ function _downloadNextImage() {
+ working = true;
+ var target = targets.pop();
+ console.debug( "Downloading image at " + target.url );
+
+ _imageEmbedder.embed(
+ target.url,
+ function( type, binary, err ) {
+ if ( err !== null ) {
+ // No big deal, we'll just leave an external src for this
+ // image.
+ console.error( "Failed to download image at " + target.url + ": " + err );
+ }
+ else if ( type.length && type.substr( 0, 6 ) === "image/" && binary.length ) {
+ console.debug( "Downloaded image at " + target.url + " with type " + type + ", size is " + binary.length );
+ var replacement = "data:" + type + ";base64," + binary;
+ var pre = content.substr( 0, target.start + offset );
+ var post = content.substr( target.start + offset - target.url.length );
+ content = pre + replacement + post;
+ offset += replacement.length - target.url.length;
+ }
+
+ working = false;
+ }
+ );
+ }
+
+ _timerSource.setTimeout( _processImagesList, 100 );
+}
+
function setArticleStar( server, id, star )
{
var db = getDatabase();
@@ -622,7 +739,7 @@ function getDatabase()
{
if ( _db === null ) {
console.debug( "Opening new connection to the database" );
- _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 1000000 );
+ _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 100000000 );
checkDatabaseStatus( _db );
}
diff --git a/qml/types/Server.qml b/qml/types/Server.qml
index 31a98e6..8ba86c8 100644
--- a/qml/types/Server.qml
+++ b/qml/types/Server.qml
@@ -60,14 +60,6 @@ Item {
id: httpRequester
}
- function setTimeout( cb, ms ) {
- var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", server );
- timer.repeat = false
- timer.interval = ms
- timer.triggered.connect( function() { cb(); timer.destroy(); } )
- timer.start()
- }
-
function onServerLoaded( props, err ) {
if ( err !== null ) {
error( qsTr( "Failed to load server information: " ) + err )
@@ -119,7 +111,7 @@ Item {
cb()
}
else {
- WallaBase.syncDeletedArticles( server, { id: serverId, token: accessToken, url: url }, function() { cb(); } )
+ WallaBase.syncDeletedArticles( { id: serverId, token: accessToken, url: url }, function() { cb(); } )
}
}
)
diff --git a/src/harbour-wallaread.cpp b/src/harbour-wallaread.cpp
index 49d9612..726f6e1 100644
--- a/src/harbour-wallaread.cpp
+++ b/src/harbour-wallaread.cpp
@@ -28,6 +28,7 @@
#include <sailfishapp.h>
#include "httprequester.h"
+#include "imageembedder.h"
int main(int argc, char *argv[])
{
@@ -41,6 +42,7 @@ int main(int argc, char *argv[])
// To display the view, call "show()" (will show fullscreen on device).
qmlRegisterType<HttpRequester>( "harbour.wallaread", 1, 0, "HttpRequester" );
+ qmlRegisterType<ImageEmbedder>( "harbour.wallaread", 1, 0, "ImageEmbedder" );
return SailfishApp::main(argc, argv);
}
diff --git a/src/imageembedder.cpp b/src/imageembedder.cpp
new file mode 100644
index 0000000..f41ce78
--- /dev/null
+++ b/src/imageembedder.cpp
@@ -0,0 +1,89 @@
+#include "imageembedder.h"
+
+#include <QDebug>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+
+/*
+ * ImageEmbedRequest
+ */
+
+ImageEmbedRequest::ImageEmbedRequest( QString const& url, QJSValue callback, QObject* parent )
+ : QObject( parent ), mUrl( url ), mReply( NULL ), mCallback( callback )
+{
+}
+
+void ImageEmbedRequest::start()
+{
+ this->doRequest( mUrl );
+}
+
+void ImageEmbedRequest::onFinished()
+{
+ if ( mReply->error() ) {
+ mError = mReply->errorString();
+ requestDone();
+ return;
+ }
+
+ int status = mReply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt();
+
+ if ( status == 301 || status == 302 || status == 303 || status == 307 ) {
+ // Welp, we've been redirected, let's see if we can follow it
+ if ( !mReply->hasRawHeader( QByteArray( "Location" ) ) ) {
+ mError = tr( "Failed to find the image source" );
+ requestDone();
+ return;
+ }
+
+ QString location = mReply->rawHeader( QByteArray( "Location" ) );
+ doRequest( location );
+ }
+ else {
+ mContentType = mReply->rawHeader( "Content-Type" );
+ QByteArray content = mReply->readAll();
+ mEncoded = content.toBase64();
+ requestDone();
+ }
+}
+
+void ImageEmbedRequest::doRequest( QString const& url )
+{
+ if ( mReply != NULL ) {
+ mReply->deleteLater();
+ mReply = NULL;
+ }
+
+ QNetworkRequest rq( url );
+ rq.setRawHeader( QByteArray( "Connection" ), QByteArray( "close" ) );
+
+ mReply = mQnam.get( rq );
+ connect( mReply, &QNetworkReply::finished, this, &ImageEmbedRequest::onFinished );
+}
+
+void ImageEmbedRequest::requestDone()
+{
+ QJSValueList args;
+ args << mContentType;
+ args << mEncoded;
+ if ( mError.isEmpty() )
+ args << QJSValue::NullValue;
+ else
+ args << mError;
+ mCallback.call( args );
+}
+
+/*
+ * ImageEmbedder
+ */
+
+ImageEmbedder::ImageEmbedder( QObject *parent )
+ : QObject( parent )
+{
+}
+
+void ImageEmbedder::embed( QString const& url, QJSValue callback )
+{
+ ImageEmbedRequest *rq = new ImageEmbedRequest( url, callback );
+ rq->start();
+}
diff --git a/src/imageembedder.h b/src/imageembedder.h
new file mode 100644
index 0000000..52b114c
--- /dev/null
+++ b/src/imageembedder.h
@@ -0,0 +1,43 @@
+#ifndef IMAGEEMBEDDER_H
+#define IMAGEEMBEDDER_H
+
+#include <QJSValue>
+#include <QNetworkAccessManager>
+#include <QObject>
+
+class ImageEmbedRequest : public QObject
+{
+ Q_OBJECT
+
+public:
+ ImageEmbedRequest( QString const& url, QJSValue callback, QObject *parent = 0 );
+
+ void start();
+
+private slots:
+ void onFinished();
+
+private:
+ void doRequest( QString const& url );
+ void requestDone();
+
+ QString mUrl;
+ QNetworkAccessManager mQnam;
+ QNetworkReply* mReply;
+ QJSValue mCallback;
+ QString mContentType;
+ QString mEncoded;
+ QString mError;
+};
+
+class ImageEmbedder : public QObject
+{
+ Q_OBJECT
+
+public:
+ explicit ImageEmbedder( QObject *parent = 0 );
+
+ Q_INVOKABLE void embed( QString const& url, QJSValue callback );
+};
+
+#endif // IMAGEEMBEDDER_H
diff --git a/translations/harbour-wallaread.ts b/translations/harbour-wallaread.ts
index 92fe69f..67b57e7 100644
--- a/translations/harbour-wallaread.ts
+++ b/translations/harbour-wallaread.ts
@@ -8,6 +8,13 @@
<translation type="unfinished"></translation>
</message>
</context>
+<context>
+ <name>ImageEmbedRequest</name>
+ <message>
+ <source>Failed to find the image source</source>
+ <translation type="unfinished"></translation>
+ </message>
+</context>
<context>
<name>Server</name>
<message>