Changeset 1056 for XmlTools2

Timestamp:

Oct 30, 2016, 12:52:03 AM (8 years ago)

Author:

s10k

Message:

XmlTools 2.0c - small fixes and dropped openmp in favor of Qt threads

Location:

XmlTools2/trunk

Files:

• XmlTools.pro (modified) (1 diff)
• readme.txt (modified) (1 diff)
• xmlcustomcode.cpp (modified) (6 diffs)
• xmlcustomcode.h (modified) (3 diffs)

XmlTools2/trunk/XmlTools.pro

@@ -7 +7 @@
 #QT += qml #for use new google v8 qtscript engine
 QT += script #for use old qtscript engine
-QMAKE_CXXFLAGS+= -fopenmp
-QMAKE_LFLAGS +=  -fopenmp #OpenMP (multithreading) support
 
 # More optimization
 QMAKE_CFLAGS_RELEASE += -O3
 QMAKE_CXXFLAGS_RELEASE += -O3
-
-
 
 

XmlTools2/trunk/readme.txt

@@ -62 +62 @@
 Change Log:
 ----------------------------------
-2.0c, 28-10-2016
+2.0c, 30-10-2016
 -Fixed bug in --update-elements operation
 -Added some exception handling
 -Started migrating some of the source code to C++14
 -Upgraded pugixml to latest 1.7 version
+-Dropped openmp since apple clang doesn't support it, using Qt threads mechanisms now
 ----------------------------------
 2.0b, 13-06-2014

XmlTools2/trunk/xmlcustomcode.cpp

@@ -10 +10 @@
 }
 
-XmlCustomCode::XmlCustomCode(): numThreads(omp_get_num_procs()*2)
+XmlCustomCode::XmlCustomCode(): numThreads(QThread::idealThreadCount())
 {
+    myThreadPool.setMaxThreadCount(numThreads);
+    myThreadPool.setExpiryTimeout(-1); // don't let threads expire
+
     // create individual thread script engines
-    this->scriptEngines.reserve(this->numThreads);
-    this->jsFunctions.reserve(this->numThreads);
-    this->getXmlDataFunctions.reserve(this->numThreads);
-    this->setXmlDataFunctions.reserve(this->numThreads);
+    this->jsScriptEngines.reserve(this->numThreads);
 
     QString jsxmlString;
@@ -26 +26 @@
 
     for(int i=0; i<this->numThreads; i++){
-        this->scriptEngines.append(new QScriptEngine());
-        this->jsFunctions.append(new QScriptValue());
+
+        jsCustomCodeEngine e;
+
+        e.scriptEngine = new QScriptEngine();
+        e.jsFunction = new QScriptValue();
 
         // main needs to be called so the user code is evaluated
         // alternatively you can do: myFunc=engine.evaluate('(function main(){})'); myFunc.call();
         // Note the () around the function
-        this->getXmlDataFunctions.append(new QScriptValue(this->scriptEngines.at(i)->evaluate("(function getXmlData() { return $xmlData; })")));
-        this->setXmlDataFunctions.append(new QScriptValue(this->scriptEngines.at(i)->evaluate("(function setXmlData(newXmlData) { $xmlData=newXmlData; })")));
+        e.getXmlDataFunction = new QScriptValue(e.scriptEngine->evaluate("(function getXmlData() { return $xmlData; })"));
+        e.setXmlDataFunction = new QScriptValue(e.scriptEngine->evaluate("(function setXmlData(newXmlData) { $xmlData=newXmlData; })"));
+        e.isAvailable = true;
 
         // Add echo function so user can debug the code
-        QScriptValue echoFunction = this->scriptEngines.at(i)->newFunction(echo);
-        this->scriptEngines.at(i)->globalObject().setProperty("echo", echoFunction);
+        QScriptValue echoFunction = e.scriptEngine->newFunction(echo);
+        e.scriptEngine->globalObject().setProperty("echo", echoFunction);
 
         // Add the js library for XmlEditing
-        this->scriptEngines.at(i)->evaluate(jsxmlString);
+        e.scriptEngine->evaluate(jsxmlString);
+
+        this->jsScriptEngines.append(e);
     }
 }
@@ -57 +63 @@
     // Reconstruct script functions
     for(int i=0; i<this->numThreads; i++){
-        *this->jsFunctions[i]=this->scriptEngines.at(i)->evaluate("(function main() {"+jsString+"})");
+        *(jsScriptEngines[i].jsFunction) = jsScriptEngines.at(i).scriptEngine->evaluate("(function main() {"+jsString+"})");
     }
 
@@ -66 +72 @@
     clock_t begin; // seconds that a script started
 
-    // Single tread if small number of files
-    if(filesToProcess.size()<CUSTOM_FILES_PER_THREAD){
+    // Single tread if small number of files or number of threads = 1
+    if(filesToProcess.size()<=CUSTOM_FILES_PER_THREAD || numThreads == 1){
+
+        jsCustomCodeEngine &jsEngine = getAvailableJsEngine();
+
         // Process all XmlFiles
         for(int i=0; i<filesToProcess.size(); i++){
-
-            customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*this->scriptEngines.at(0),begin,elapsed_secs,engineResult,
-                                *this->jsFunctions.at(0),*this->getXmlDataFunctions.at(0),*this->setXmlDataFunctions.at(0),backupsEnabled,verboseEnabled);
+            customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*jsEngine.scriptEngine,begin,elapsed_secs,engineResult,
+                                *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
         }
     }
     else{ // Multithread if there are many files
         // Process all XmlFiles
-#pragma omp parallel for num_threads(this->numThreads) schedule(dynamic)
+
+        QVector<QFuture<void>> executingThreads;
+
         for(int i=0; i<filesToProcess.size()-CUSTOM_FILES_PER_THREAD; i+=CUSTOM_FILES_PER_THREAD){
 
-            const int tid=omp_get_thread_num();
+            executingThreads <<
+            QtConcurrent::run(&this->myThreadPool, [=]()
+            {
+                mutexIsAvailable.lock();
+                jsCustomCodeEngine &jsEngine = getAvailableJsEngine();
+                jsEngine.isAvailable = false;
+                mutexIsAvailable.unlock();
 
-            QString currXmlFileStringThread;
+                QString currXmlFileStringThread;
 
-            QScriptValue engineResultThread; // variable to check for js_errors
-            double elapsedSecsThread=0; // elapsed seconds that a user script took
-            clock_t beginThread; // seconds that a script started
+                QScriptValue engineResultThread; // variable to check for js_errors
+                double elapsedSecsThread=0; // elapsed seconds that a user script took
+                clock_t beginThread; // seconds that a script started
 
-            customCodeUnwinding(filesToProcess.at(i),currXmlFileStringThread,*this->scriptEngines.at(tid),beginThread,elapsedSecsThread,engineResultThread,
-                                *this->jsFunctions.at(tid),*this->getXmlDataFunctions.at(tid),*this->setXmlDataFunctions.at(tid),backupsEnabled,verboseEnabled);
+                customCodeUnwinding(filesToProcess.at(i),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread,
+                                    *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
 
-            customCodeUnwinding(filesToProcess.at(i+1),currXmlFileStringThread,*this->scriptEngines.at(tid),beginThread,elapsedSecsThread,engineResultThread,
-                                *this->jsFunctions.at(tid),*this->getXmlDataFunctions.at(tid),*this->setXmlDataFunctions.at(tid),backupsEnabled,verboseEnabled);
+                customCodeUnwinding(filesToProcess.at(i+1),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread,
+                                    *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
+
+                customCodeUnwinding(filesToProcess.at(i+2),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread,
+                                    *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
+
+                customCodeUnwinding(filesToProcess.at(i+3),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread,
+                                    *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
+
+                mutexIsAvailable.lock();
+                jsEngine.isAvailable = true;
+                mutexIsAvailable.unlock();
+            });
+        }
+
+        // Wait for all threads to finish
+        for(QFuture<void> &f :executingThreads){
+            f.waitForFinished();
         }
 
@@ -99 +131 @@
             int alreadyProcessedFiles=(filesToProcess.size()/CUSTOM_FILES_PER_THREAD)*CUSTOM_FILES_PER_THREAD;
 
+            jsCustomCodeEngine &jsEngine = getAvailableJsEngine();
+
             for(int i=alreadyProcessedFiles; i<filesToProcess.size(); i++){
 
-                customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*this->scriptEngines.at(0),begin,elapsed_secs,engineResult,
-                                    *this->jsFunctions.at(0),*this->getXmlDataFunctions.at(0),*this->setXmlDataFunctions.at(0),backupsEnabled,verboseEnabled);
+                customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*jsEngine.scriptEngine,begin,elapsed_secs,engineResult,
+                                    *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled);
             }
         }
@@ -113 +147 @@
     }
 }
+
+XmlCustomCode::jsCustomCodeEngine& XmlCustomCode::getAvailableJsEngine(){
+    for(jsCustomCodeEngine &e : this->jsScriptEngines){
+        if(e.isAvailable){
+            return e;
+        }
+    }
+
+    throw std::runtime_error("Could't find an available js engine for custom command.");
+}

XmlTools2/trunk/xmlcustomcode.h

@@ -10 +10 @@
 #include <QCoreApplication>
 
+#include <QThreadPool>
+#include <QtConcurrent/QtConcurrent>
+
 #define SLOW_SCRIPT_TIME 0.1 // if a user script takes more than 0.1 seconds to execute give a warning.
-#define CUSTOM_FILES_PER_THREAD 2
+#define CUSTOM_FILES_PER_THREAD 4
 
 // Uses a singleton implementation (based on here: http://www.yolinux.com/TUTORIALS/C++Singleton.html)
@@ -22 +25 @@
 private:
     static XmlCustomCode* uniqueInstance;
+
     const int numThreads;
-    QVector<QScriptEngine*> scriptEngines;
-    QVector<QScriptValue*> jsFunctions;
-    QVector<QScriptValue*> getXmlDataFunctions;
-    QVector<QScriptValue*> setXmlDataFunctions;
+    QThreadPool myThreadPool;
+    QMutex mutexIsAvailable;
+
+    struct jsCustomCodeEngine{
+        QScriptEngine* scriptEngine;
+        QScriptValue* jsFunction;
+        QScriptValue* getXmlDataFunction;
+        QScriptValue* setXmlDataFunction;
+        bool isAvailable;
+    };
+
+    QVector<jsCustomCodeEngine> jsScriptEngines;
 
     XmlCustomCode(); // constructor is private (use getInstance)
@@ -33 +45 @@
 
     void displayJsException(QScriptEngine &engine, QScriptValue &engineResult);
+    jsCustomCodeEngine& getAvailableJsEngine();
 
     __attribute__((always_inline)) inline void customCodeUnwinding(const QString &fileName, QString &currXmlFileString,