Embed images in articles

harbour-wallaread.pro

@@ -15,7 +15,8 @@ TARGET = harbour-wallaread
 CONFIG += sailfishapp c++11
 
 SOURCES += src/harbour-wallaread.cpp \
-    src/httprequester.cpp
+    src/httprequester.cpp \
+    src/imageembedder.cpp
 
 OTHER_FILES += qml/harbour-wallaread.qml \
     qml/cover/CoverPage.qml \
@@ -37,7 +38,8 @@ CONFIG += sailfishapp_i18n
 #TRANSLATIONS += translations/harbour-wallaread-de.ts
 
 HEADERS += \
-    src/httprequester.h
+    src/httprequester.h \
+    src/imageembedder.h
 
 DISTFILES += \
     qml/js/WallaBase.js \

qml/harbour-wallaread.qml

@@ -21,10 +21,36 @@
 
 import QtQuick 2.0
 import Sailfish.Silica 1.0
+
+import harbour.wallaread 1.0
+
 import "pages"
 
+import "./js/WallaBase.js" as WallaBase
+
 ApplicationWindow
 {
+    QtObject {
+        id: jsTimerSource
+
+        function setTimeout( cb, ms ) {
+            var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", jsTimerSource)
+            timer.repeat = false
+            timer.interval = ms
+            timer.triggered.connect( function() { cb(); timer.destroy(); } )
+            timer.start()
+        }
+    }
+
+    ImageEmbedder {
+        id: imageEmbedder
+    }
+
+    Component.onCompleted: {
+        WallaBase.setTimerSource( jsTimerSource )
+        WallaBase.setImageEmbedder( imageEmbedder )
+    }
+
     initialPage: Component { ServersPage { } }
     cover: Qt.resolvedUrl("cover/CoverPage.qml")
     allowedOrientations: defaultAllowedOrientations

qml/js/WallaBase.js

@@ -32,6 +32,28 @@ var ArticlesFilter = {
     Starred: 4
 }
 
+/*
+  Timer management, used for the various setTimeout() calls
+ */
+
+var _timerSource = null;
+
+function setTimerSource( source )
+{
+    _timerSource = source;
+}
+
+/*
+  Used to embed images in the articles
+ */
+
+var _imageEmbedder = null;
+
+function setImageEmbedder( embedder )
+{
+    _imageEmbedder = embedder;
+}
+
 /*
   Servers management
  */
@@ -277,7 +299,7 @@ function _sendAuthRequest( props, cb )
   Articles management
  */
 
-function syncDeletedArticles( timerSource, props, cb )
+function syncDeletedArticles( props, cb )
 {
     var db = getDatabase();
 
@@ -298,8 +320,8 @@ function syncDeletedArticles( timerSource, props, cb )
                 }
                 else {
                     if ( !working )
-                        timerSource.setTimeout( _checkNextArticle, 100 );
-                    timerSource.setTimeout( processArticlesList, 500 );
+                        _timerSource.setTimeout( _checkNextArticle, 100 );
+                    _timerSource.setTimeout( processArticlesList, 500 );
                 }
             }
 
@@ -350,7 +372,7 @@ function syncDeletedArticles( timerSource, props, cb )
                 http.send();
             }
 
-            timerSource.setTimeout( processArticlesList, 500 );
+            _timerSource.setTimeout( processArticlesList, 500 );
         }
     );
 }
@@ -527,7 +549,7 @@ function downloadArticles( props, cb )
                 if ( arts.length )
                     articles.push.apply( articles, arts );
                 if ( done )
-                    cb( articles, null );
+                    embedImages( articles, cb );
             }
         }
     );
@@ -589,6 +611,101 @@ function _downloadNextArticles( url, token, page, cb )
     http.send();
 }
 
+function embedImages( articles, cb )
+{
+    var ret = new Array;
+    var working = false;
+
+    function _processArticlesList() {
+        if ( !working && articles.length === 0 ) {
+            cb( ret, null );
+        }
+        else {
+            if ( !working )
+                _timerSource.setTimeout( _processNextArticle, 100 );
+            _timerSource.setTimeout( _processArticlesList, 500 );
+        }
+    }
+
+    function _processNextArticle() {
+        working = true;
+        var article = articles.pop();
+        console.debug( "Embedding images for article " + article.id );
+        console.debug( "Length is " + article.content.length + " before" );
+        _embedImages(
+            article,
+            function( content ) {
+                article.content = content;
+                console.debug( "Length is " + article.content.length + " after" );
+                ret.push( article );
+                working = false;
+            }
+        );
+    }
+
+    _timerSource.setTimeout( _processArticlesList, 100 );
+}
+
+function _embedImages( article, cb )
+{
+    var imgRe = /<img[^>]+\bsrc=(["'])(https?:\/\/.+?)\1[^>]+>/g;
+    var match;
+    var targets = new Array;
+    var content = article.content;
+
+    while ( match = imgRe.exec( content ) ) {
+        targets.push(
+            {
+                start: content.indexOf( match[2], match.index ),
+                url: match[2]
+            }
+        )
+    }
+
+    var working = false;
+    var offset = 0;
+
+    function _processImagesList() {
+        if ( !working && targets.length === 0 ) {
+            cb( content );
+        }
+        else {
+            if ( !working )
+                _timerSource.setTimeout( _downloadNextImage, 100 );
+            _timerSource.setTimeout( _processImagesList, 500 );
+        }
+    }
+
+    function _downloadNextImage() {
+        working = true;
+        var target = targets.pop();
+        console.debug( "Downloading image at " + target.url );
+
+        _imageEmbedder.embed(
+            target.url,
+            function( type, binary, err ) {
+                if ( err !== null ) {
+                    // No big deal, we'll just leave an external src for this
+                    // image.
+                    console.error( "Failed to download image at " + target.url + ": " + err );
+                }
+                else if ( type.length && type.substr( 0, 6 ) === "image/" && binary.length ) {
+                    console.debug( "Downloaded image at " + target.url + " with type " + type + ", size is " + binary.length );
+                    var replacement = "data:" + type + ";base64," + binary;
+                    var pre = content.substr( 0, target.start + offset );
+                    var post = content.substr( target.start + offset - target.url.length );
+                    content = pre + replacement + post;
+                    offset += replacement.length - target.url.length;
+                }
+
+                working = false;
+            }
+        );
+    }
+
+    _timerSource.setTimeout( _processImagesList, 100 );
+}
+
 function setArticleStar( server, id, star )
 {
     var db = getDatabase();
@@ -622,7 +739,7 @@ function getDatabase()
 {
     if ( _db === null ) {
         console.debug( "Opening new connection to the database" );
-        _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 1000000 );
+        _db = Storage.LocalStorage.openDatabaseSync( "WallaRead", "", "WallaRead", 100000000 );
         checkDatabaseStatus( _db );
     }
 

qml/types/Server.qml

@@ -60,14 +60,6 @@ Item {
         id: httpRequester
     }
 
-    function setTimeout( cb, ms ) {
-        var timer = Qt.createQmlObject( "import QtQuick 2.0; Timer {}", server );
-        timer.repeat = false
-        timer.interval = ms
-        timer.triggered.connect( function() { cb(); timer.destroy(); } )
-        timer.start()
-    }
-
     function onServerLoaded( props, err ) {
         if ( err !== null ) {
             error( qsTr( "Failed to load server information: " ) + err )
@@ -119,7 +111,7 @@ Item {
                     cb()
                 }
                 else {
-                    WallaBase.syncDeletedArticles( server, { id: serverId, token: accessToken, url: url }, function() { cb(); } )
+                    WallaBase.syncDeletedArticles( { id: serverId, token: accessToken, url: url }, function() { cb(); } )
                 }
             }
         )

src/harbour-wallaread.cpp

@@ -28,6 +28,7 @@
 #include <sailfishapp.h>
 
 #include "httprequester.h"
+#include "imageembedder.h"
 
 int main(int argc, char *argv[])
 {
@@ -41,6 +42,7 @@ int main(int argc, char *argv[])
     // To display the view, call "show()" (will show fullscreen on device).
 
     qmlRegisterType<HttpRequester>( "harbour.wallaread", 1, 0, "HttpRequester" );
+    qmlRegisterType<ImageEmbedder>( "harbour.wallaread", 1, 0, "ImageEmbedder" );
 
     return SailfishApp::main(argc, argv);
 }

src/imageembedder.cpp

@@ -0,0 +1,89 @@
+#include "imageembedder.h"
+
+#include <QDebug>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+
+/*
+ * ImageEmbedRequest
+ */
+
+ImageEmbedRequest::ImageEmbedRequest( QString const& url, QJSValue callback, QObject* parent )
+    : QObject( parent ), mUrl( url ), mReply( NULL ), mCallback( callback )
+{
+}
+
+void ImageEmbedRequest::start()
+{
+    this->doRequest( mUrl );
+}
+
+void ImageEmbedRequest::onFinished()
+{
+    if ( mReply->error() ) {
+        mError = mReply->errorString();
+        requestDone();
+        return;
+    }
+
+    int status = mReply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt();
+
+    if ( status == 301 || status == 302 || status == 303 || status == 307 ) {
+        // Welp, we've been redirected, let's see if we can follow it
+        if ( !mReply->hasRawHeader( QByteArray( "Location" ) ) ) {
+            mError = tr( "Failed to find the image source" );
+            requestDone();
+            return;
+        }
+
+        QString location = mReply->rawHeader( QByteArray( "Location" ) );
+        doRequest( location );
+    }
+    else {
+        mContentType = mReply->rawHeader( "Content-Type" );
+        QByteArray content = mReply->readAll();
+        mEncoded = content.toBase64();
+        requestDone();
+    }
+}
+
+void ImageEmbedRequest::doRequest( QString const& url )
+{
+    if ( mReply != NULL ) {
+        mReply->deleteLater();
+        mReply = NULL;
+    }
+
+    QNetworkRequest rq( url );
+    rq.setRawHeader( QByteArray( "Connection" ), QByteArray( "close" ) );
+
+    mReply = mQnam.get( rq );
+    connect( mReply, &QNetworkReply::finished, this, &ImageEmbedRequest::onFinished );
+}
+
+void ImageEmbedRequest::requestDone()
+{
+    QJSValueList args;
+    args << mContentType;
+    args << mEncoded;
+    if ( mError.isEmpty() )
+        args << QJSValue::NullValue;
+    else
+        args << mError;
+    mCallback.call( args );
+}
+
+/*
+ * ImageEmbedder
+ */
+
+ImageEmbedder::ImageEmbedder( QObject *parent )
+    : QObject( parent )
+{
+}
+
+void ImageEmbedder::embed( QString const& url, QJSValue callback )
+{
+    ImageEmbedRequest *rq = new ImageEmbedRequest( url, callback );
+    rq->start();
+}

src/imageembedder.h

@@ -0,0 +1,43 @@
+#ifndef IMAGEEMBEDDER_H
+#define IMAGEEMBEDDER_H
+
+#include <QJSValue>
+#include <QNetworkAccessManager>
+#include <QObject>
+
+class ImageEmbedRequest : public QObject
+{
+    Q_OBJECT
+
+public:
+    ImageEmbedRequest( QString const& url, QJSValue callback, QObject *parent = 0 );
+
+    void start();
+
+private slots:
+    void onFinished();
+
+private:
+    void doRequest( QString const& url );
+    void requestDone();
+
+    QString mUrl;
+    QNetworkAccessManager mQnam;
+    QNetworkReply* mReply;
+    QJSValue mCallback;
+    QString mContentType;
+    QString mEncoded;
+    QString mError;
+};
+
+class ImageEmbedder : public QObject
+{
+    Q_OBJECT
+
+public:
+    explicit ImageEmbedder( QObject *parent = 0 );
+
+    Q_INVOKABLE void embed( QString const& url, QJSValue callback );
+};
+
+#endif // IMAGEEMBEDDER_H

translations/harbour-wallaread.ts

@@ -8,6 +8,13 @@
         <translation type="unfinished"></translation>
     </message>
 </context>
+<context>
+    <name>ImageEmbedRequest</name>
+    <message>
+        <source>Failed to find the image source</source>
+        <translation type="unfinished"></translation>
+    </message>
+</context>
 <context>
     <name>Server</name>
     <message>