From 223e1cfd1b02e84f2de9f274df1dbb506dae4883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Oestreicher?= Date: Mon, 19 Dec 2016 08:29:10 +0100 Subject: [PATCH] Embed images in articles --- harbour-wallaread.pro | 6 +- qml/harbour-wallaread.qml | 26 ++++++ qml/js/WallaBase.js | 129 ++++++++++++++++++++++++++++-- qml/types/Server.qml | 10 +-- src/harbour-wallaread.cpp | 2 + src/imageembedder.cpp | 89 +++++++++++++++++++++ src/imageembedder.h | 43 ++++++++++ translations/harbour-wallaread.ts | 7 ++ 8 files changed, 295 insertions(+), 17 deletions(-) create mode 100644 src/imageembedder.cpp create mode 100644 src/imageembedder.h diff --git a/harbour-wallaread.pro b/harbour-wallaread.pro index 6934099..cd09e33 100644 --- a/harbour-wallaread.pro +++ b/harbour-wallaread.pro @@ -15,7 +15,8 @@ TARGET = harbour-wallaread CONFIG += sailfishapp c++11 SOURCES += src/harbour-wallaread.cpp \ - src/httprequester.cpp + src/httprequester.cpp \ + src/imageembedder.cpp OTHER_FILES += qml/harbour-wallaread.qml \ qml/cover/CoverPage.qml \ @@ -37,7 +38,8 @@ CONFIG += sailfishapp_i18n #TRANSLATIONS += translations/harbour-wallaread-de.ts HEADERS += \ - src/httprequester.h + src/httprequester.h \ + src/imageembedder.h DISTFILES += \ qml/js/WallaBase.js \ diff --git a/qml/harbour-wallaread.qml b/qml/harbour-wallaread.qml index ff964d3..673a37e 100644 --- a/qml/harbour-wallaread.qml +++ b/qml/harbour-wallaread.qml @@ -21,10 +21,36 @@ import QtQuick 2.0 import Sailfish.Silica 1.0 + +import harbour.wallaread 1.0 + import "pages" +import "./js/WallaBase.js" as WallaBase + ApplicationWindow { + QtObject { + id: jsTimerSource + + function setTimeout( cb, ms ) { + var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", jsTimerSource) + timer.repeat = false + timer.interval = ms + timer.triggered.connect( function() { cb(); timer.destroy(); } ) + timer.start() + } + } + + ImageEmbedder { + id: imageEmbedder + } + + Component.onCompleted: { + WallaBase.setTimerSource( jsTimerSource ) + WallaBase.setImageEmbedder( imageEmbedder ) + } + initialPage: Component { ServersPage { } } cover: Qt.resolvedUrl("cover/CoverPage.qml") allowedOrientations: defaultAllowedOrientations diff --git a/qml/js/WallaBase.js b/qml/js/WallaBase.js index f20d490..95b9e2d 100644 --- a/qml/js/WallaBase.js +++ b/qml/js/WallaBase.js @@ -32,6 +32,28 @@ var ArticlesFilter = { Starred: 4 } +/* + Timer management, used for the various setTimeout() calls + */ + +var _timerSource = null; + +function setTimerSource( source ) +{ + _timerSource = source; +} + +/* + Used to embed images in the articles + */ + +var _imageEmbedder = null; + +function setImageEmbedder( embedder ) +{ + _imageEmbedder = embedder; +} + /* Servers management */ @@ -277,7 +299,7 @@ function _sendAuthRequest( props, cb ) Articles management */ -function syncDeletedArticles( timerSource, props, cb ) +function syncDeletedArticles( props, cb ) { var db = getDatabase(); @@ -298,8 +320,8 @@ function syncDeletedArticles( timerSource, props, cb ) } else { if ( !working ) - timerSource.setTimeout( _checkNextArticle, 100 ); - timerSource.setTimeout( processArticlesList, 500 ); + _timerSource.setTimeout( _checkNextArticle, 100 ); + _timerSource.setTimeout( processArticlesList, 500 ); } } @@ -350,7 +372,7 @@ function syncDeletedArticles( timerSource, props, cb ) http.send(); } - timerSource.setTimeout( processArticlesList, 500 ); + _timerSource.setTimeout( processArticlesList, 500 ); } ); } @@ -527,7 +549,7 @@ function downloadArticles( props, cb ) if ( arts.length ) articles.push.apply( articles, arts ); if ( done ) - cb( articles, null ); + embedImages( articles, cb ); } } ); @@ -589,6 +611,101 @@ function _downloadNextArticles( url, token, page, cb ) http.send(); } +function embedImages( articles, cb ) +{ + var ret = new Array; + var working = false; + + function _processArticlesList() { + if ( !working && articles.length === 0 ) { + cb( ret, null ); + } + else { + if ( !working ) + _timerSource.setTimeout( _processNextArticle, 100 ); + _timerSource.setTimeout( _processArticlesList, 500 ); + } + } + + function _processNextArticle() { + working = true; + var article = articles.pop(); + console.debug( "Embedding images for article " + article.id ); + console.debug( "Length is " + article.content.length + " before" ); + _embedImages( + article, + function( content ) { + article.content = content; + console.debug( "Length is " + article.content.length + " after" ); + ret.push( article ); + working = false; + } + ); + } + + _timerSource.setTimeout( _processArticlesList, 100 ); +} + +function _embedImages( article, cb ) +{ + var imgRe = /]+\bsrc=(["'])(https?:\/\/.+?)\1[^>]+>/g; + var match; + var targets = new Array; + var content = article.content; + + while ( match = imgRe.exec( content ) ) { + targets.push( + { + start: content.indexOf( match[2], match.index ), + url: match[2] + } + ) + } + + var working = false; + var offset = 0; + + function _processImagesList() { + if ( !working && targets.length === 0 ) { + cb( content ); + } + else { + if ( !working ) + _timerSource.setTimeout( _downloadNextImage, 100 ); + _timerSource.setTimeout( _processImagesList, 500 ); + } + } + + function _downloadNextImage() { + working = true; + var target = targets.pop(); + console.debug( "Downloading image at " + target.url ); + + _imageEmbedder.embed( + target.url, + function( type, binary, err ) { + if ( err !== null ) { + // No big deal, we'll just leave an external src for this + // image. + console.error( "Failed to download image at " + target.url + ": " + err ); + } + else if ( type.length && type.substr( 0, 6 ) === "image/" && binary.length ) { + console.debug( "Downloaded image at " + target.url + " with type " + type + ", size is " + binary.length ); + var replacement = "data:" + type + ";base64," + binary; + var pre = content.substr( 0, target.start + offset ); + var post = content.substr( target.start + offset - target.url.length ); + content = pre + replacement + post; + offset += replacement.length - target.url.length; + } + + working = false; + } + ); + } + + _timerSource.setTimeout( _processImagesList, 100 ); +} + function setArticleStar( server, id, star ) { var db = getDatabase(); @@ -622,7 +739,7 @@ function getDatabase() { if ( _db === null ) { console.debug( "Opening new connection to the database" ); - _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 1000000 ); + _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 100000000 ); checkDatabaseStatus( _db ); } diff --git a/qml/types/Server.qml b/qml/types/Server.qml index 31a98e6..8ba86c8 100644 --- a/qml/types/Server.qml +++ b/qml/types/Server.qml @@ -60,14 +60,6 @@ Item { id: httpRequester } - function setTimeout( cb, ms ) { - var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", server ); - timer.repeat = false - timer.interval = ms - timer.triggered.connect( function() { cb(); timer.destroy(); } ) - timer.start() - } - function onServerLoaded( props, err ) { if ( err !== null ) { error( qsTr( "Failed to load server information: " ) + err ) @@ -119,7 +111,7 @@ Item { cb() } else { - WallaBase.syncDeletedArticles( server, { id: serverId, token: accessToken, url: url }, function() { cb(); } ) + WallaBase.syncDeletedArticles( { id: serverId, token: accessToken, url: url }, function() { cb(); } ) } } ) diff --git a/src/harbour-wallaread.cpp b/src/harbour-wallaread.cpp index 49d9612..726f6e1 100644 --- a/src/harbour-wallaread.cpp +++ b/src/harbour-wallaread.cpp @@ -28,6 +28,7 @@ #include #include "httprequester.h" +#include "imageembedder.h" int main(int argc, char *argv[]) { @@ -41,6 +42,7 @@ int main(int argc, char *argv[]) // To display the view, call "show()" (will show fullscreen on device). qmlRegisterType( "harbour.wallaread", 1, 0, "HttpRequester" ); + qmlRegisterType( "harbour.wallaread", 1, 0, "ImageEmbedder" ); return SailfishApp::main(argc, argv); } diff --git a/src/imageembedder.cpp b/src/imageembedder.cpp new file mode 100644 index 0000000..f41ce78 --- /dev/null +++ b/src/imageembedder.cpp @@ -0,0 +1,89 @@ +#include "imageembedder.h" + +#include +#include +#include + +/* + * ImageEmbedRequest + */ + +ImageEmbedRequest::ImageEmbedRequest( QString const& url, QJSValue callback, QObject* parent ) + : QObject( parent ), mUrl( url ), mReply( NULL ), mCallback( callback ) +{ +} + +void ImageEmbedRequest::start() +{ + this->doRequest( mUrl ); +} + +void ImageEmbedRequest::onFinished() +{ + if ( mReply->error() ) { + mError = mReply->errorString(); + requestDone(); + return; + } + + int status = mReply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + + if ( status == 301 || status == 302 || status == 303 || status == 307 ) { + // Welp, we've been redirected, let's see if we can follow it + if ( !mReply->hasRawHeader( QByteArray( "Location" ) ) ) { + mError = tr( "Failed to find the image source" ); + requestDone(); + return; + } + + QString location = mReply->rawHeader( QByteArray( "Location" ) ); + doRequest( location ); + } + else { + mContentType = mReply->rawHeader( "Content-Type" ); + QByteArray content = mReply->readAll(); + mEncoded = content.toBase64(); + requestDone(); + } +} + +void ImageEmbedRequest::doRequest( QString const& url ) +{ + if ( mReply != NULL ) { + mReply->deleteLater(); + mReply = NULL; + } + + QNetworkRequest rq( url ); + rq.setRawHeader( QByteArray( "Connection" ), QByteArray( "close" ) ); + + mReply = mQnam.get( rq ); + connect( mReply, &QNetworkReply::finished, this, &ImageEmbedRequest::onFinished ); +} + +void ImageEmbedRequest::requestDone() +{ + QJSValueList args; + args << mContentType; + args << mEncoded; + if ( mError.isEmpty() ) + args << QJSValue::NullValue; + else + args << mError; + mCallback.call( args ); +} + +/* + * ImageEmbedder + */ + +ImageEmbedder::ImageEmbedder( QObject *parent ) + : QObject( parent ) +{ +} + +void ImageEmbedder::embed( QString const& url, QJSValue callback ) +{ + ImageEmbedRequest *rq = new ImageEmbedRequest( url, callback ); + rq->start(); +} diff --git a/src/imageembedder.h b/src/imageembedder.h new file mode 100644 index 0000000..52b114c --- /dev/null +++ b/src/imageembedder.h @@ -0,0 +1,43 @@ +#ifndef IMAGEEMBEDDER_H +#define IMAGEEMBEDDER_H + +#include +#include +#include + +class ImageEmbedRequest : public QObject +{ + Q_OBJECT + +public: + ImageEmbedRequest( QString const& url, QJSValue callback, QObject *parent = 0 ); + + void start(); + +private slots: + void onFinished(); + +private: + void doRequest( QString const& url ); + void requestDone(); + + QString mUrl; + QNetworkAccessManager mQnam; + QNetworkReply* mReply; + QJSValue mCallback; + QString mContentType; + QString mEncoded; + QString mError; +}; + +class ImageEmbedder : public QObject +{ + Q_OBJECT + +public: + explicit ImageEmbedder( QObject *parent = 0 ); + + Q_INVOKABLE void embed( QString const& url, QJSValue callback ); +}; + +#endif // IMAGEEMBEDDER_H diff --git a/translations/harbour-wallaread.ts b/translations/harbour-wallaread.ts index 92fe69f..67b57e7 100644 --- a/translations/harbour-wallaread.ts +++ b/translations/harbour-wallaread.ts @@ -8,6 +8,13 @@ + + ImageEmbedRequest + + Failed to find the image source + + + Server