Index: /XmlTools2/trunk/XmlTools.pro =================================================================== --- /XmlTools2/trunk/XmlTools.pro (revision 1055) +++ /XmlTools2/trunk/XmlTools.pro (revision 1056) @@ -7,12 +7,8 @@ #QT += qml #for use new google v8 qtscript engine QT += script #for use old qtscript engine -QMAKE_CXXFLAGS+= -fopenmp -QMAKE_LFLAGS += -fopenmp #OpenMP (multithreading) support # More optimization QMAKE_CFLAGS_RELEASE += -O3 QMAKE_CXXFLAGS_RELEASE += -O3 - - Index: /XmlTools2/trunk/readme.txt =================================================================== --- /XmlTools2/trunk/readme.txt (revision 1055) +++ /XmlTools2/trunk/readme.txt (revision 1056) @@ -62,9 +62,10 @@ Change Log: ---------------------------------- -2.0c, 28-10-2016 +2.0c, 30-10-2016 -Fixed bug in --update-elements operation -Added some exception handling -Started migrating some of the source code to C++14 -Upgraded pugixml to latest 1.7 version +-Dropped openmp since apple clang doesn't support it, using Qt threads mechanisms now ---------------------------------- 2.0b, 13-06-2014 Index: /XmlTools2/trunk/xmlcustomcode.cpp =================================================================== --- /XmlTools2/trunk/xmlcustomcode.cpp (revision 1055) +++ /XmlTools2/trunk/xmlcustomcode.cpp (revision 1056) @@ -10,11 +10,11 @@ } -XmlCustomCode::XmlCustomCode(): numThreads(omp_get_num_procs()*2) +XmlCustomCode::XmlCustomCode(): numThreads(QThread::idealThreadCount()) { + myThreadPool.setMaxThreadCount(numThreads); + myThreadPool.setExpiryTimeout(-1); // don't let threads expire + // create individual thread script engines - this->scriptEngines.reserve(this->numThreads); - this->jsFunctions.reserve(this->numThreads); - this->getXmlDataFunctions.reserve(this->numThreads); - this->setXmlDataFunctions.reserve(this->numThreads); + this->jsScriptEngines.reserve(this->numThreads); QString jsxmlString; @@ -26,19 +26,25 @@ for(int i=0; inumThreads; i++){ - this->scriptEngines.append(new QScriptEngine()); - this->jsFunctions.append(new QScriptValue()); + + jsCustomCodeEngine e; + + e.scriptEngine = new QScriptEngine(); + e.jsFunction = new QScriptValue(); // main needs to be called so the user code is evaluated // alternatively you can do: myFunc=engine.evaluate('(function main(){})'); myFunc.call(); // Note the () around the function - this->getXmlDataFunctions.append(new QScriptValue(this->scriptEngines.at(i)->evaluate("(function getXmlData() { return $xmlData; })"))); - this->setXmlDataFunctions.append(new QScriptValue(this->scriptEngines.at(i)->evaluate("(function setXmlData(newXmlData) { $xmlData=newXmlData; })"))); + e.getXmlDataFunction = new QScriptValue(e.scriptEngine->evaluate("(function getXmlData() { return $xmlData; })")); + e.setXmlDataFunction = new QScriptValue(e.scriptEngine->evaluate("(function setXmlData(newXmlData) { $xmlData=newXmlData; })")); + e.isAvailable = true; // Add echo function so user can debug the code - QScriptValue echoFunction = this->scriptEngines.at(i)->newFunction(echo); - this->scriptEngines.at(i)->globalObject().setProperty("echo", echoFunction); + QScriptValue echoFunction = e.scriptEngine->newFunction(echo); + e.scriptEngine->globalObject().setProperty("echo", echoFunction); // Add the js library for XmlEditing - this->scriptEngines.at(i)->evaluate(jsxmlString); + e.scriptEngine->evaluate(jsxmlString); + + this->jsScriptEngines.append(e); } } @@ -57,5 +63,5 @@ // Reconstruct script functions for(int i=0; inumThreads; i++){ - *this->jsFunctions[i]=this->scriptEngines.at(i)->evaluate("(function main() {"+jsString+"})"); + *(jsScriptEngines[i].jsFunction) = jsScriptEngines.at(i).scriptEngine->evaluate("(function main() {"+jsString+"})"); } @@ -66,31 +72,57 @@ clock_t begin; // seconds that a script started - // Single tread if small number of files - if(filesToProcess.size()scriptEngines.at(0),begin,elapsed_secs,engineResult, - *this->jsFunctions.at(0),*this->getXmlDataFunctions.at(0),*this->setXmlDataFunctions.at(0),backupsEnabled,verboseEnabled); + customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*jsEngine.scriptEngine,begin,elapsed_secs,engineResult, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); } } else{ // Multithread if there are many files // Process all XmlFiles -#pragma omp parallel for num_threads(this->numThreads) schedule(dynamic) + + QVector> executingThreads; + for(int i=0; imyThreadPool, [=]() + { + mutexIsAvailable.lock(); + jsCustomCodeEngine &jsEngine = getAvailableJsEngine(); + jsEngine.isAvailable = false; + mutexIsAvailable.unlock(); - QString currXmlFileStringThread; + QString currXmlFileStringThread; - QScriptValue engineResultThread; // variable to check for js_errors - double elapsedSecsThread=0; // elapsed seconds that a user script took - clock_t beginThread; // seconds that a script started + QScriptValue engineResultThread; // variable to check for js_errors + double elapsedSecsThread=0; // elapsed seconds that a user script took + clock_t beginThread; // seconds that a script started - customCodeUnwinding(filesToProcess.at(i),currXmlFileStringThread,*this->scriptEngines.at(tid),beginThread,elapsedSecsThread,engineResultThread, - *this->jsFunctions.at(tid),*this->getXmlDataFunctions.at(tid),*this->setXmlDataFunctions.at(tid),backupsEnabled,verboseEnabled); + customCodeUnwinding(filesToProcess.at(i),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); - customCodeUnwinding(filesToProcess.at(i+1),currXmlFileStringThread,*this->scriptEngines.at(tid),beginThread,elapsedSecsThread,engineResultThread, - *this->jsFunctions.at(tid),*this->getXmlDataFunctions.at(tid),*this->setXmlDataFunctions.at(tid),backupsEnabled,verboseEnabled); + customCodeUnwinding(filesToProcess.at(i+1),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); + + customCodeUnwinding(filesToProcess.at(i+2),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); + + customCodeUnwinding(filesToProcess.at(i+3),currXmlFileStringThread,*jsEngine.scriptEngine,beginThread,elapsedSecsThread,engineResultThread, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); + + mutexIsAvailable.lock(); + jsEngine.isAvailable = true; + mutexIsAvailable.unlock(); + }); + } + + // Wait for all threads to finish + for(QFuture &f :executingThreads){ + f.waitForFinished(); } @@ -99,8 +131,10 @@ int alreadyProcessedFiles=(filesToProcess.size()/CUSTOM_FILES_PER_THREAD)*CUSTOM_FILES_PER_THREAD; + jsCustomCodeEngine &jsEngine = getAvailableJsEngine(); + for(int i=alreadyProcessedFiles; iscriptEngines.at(0),begin,elapsed_secs,engineResult, - *this->jsFunctions.at(0),*this->getXmlDataFunctions.at(0),*this->setXmlDataFunctions.at(0),backupsEnabled,verboseEnabled); + customCodeUnwinding(filesToProcess.at(i),currXmlFileString,*jsEngine.scriptEngine,begin,elapsed_secs,engineResult, + *jsEngine.jsFunction,*jsEngine.getXmlDataFunction,*jsEngine.setXmlDataFunction,backupsEnabled,verboseEnabled); } } @@ -113,2 +147,12 @@ } } + +XmlCustomCode::jsCustomCodeEngine& XmlCustomCode::getAvailableJsEngine(){ + for(jsCustomCodeEngine &e : this->jsScriptEngines){ + if(e.isAvailable){ + return e; + } + } + + throw std::runtime_error("Could't find an available js engine for custom command."); +} Index: /XmlTools2/trunk/xmlcustomcode.h =================================================================== --- /XmlTools2/trunk/xmlcustomcode.h (revision 1055) +++ /XmlTools2/trunk/xmlcustomcode.h (revision 1056) @@ -10,6 +10,9 @@ #include +#include +#include + #define SLOW_SCRIPT_TIME 0.1 // if a user script takes more than 0.1 seconds to execute give a warning. -#define CUSTOM_FILES_PER_THREAD 2 +#define CUSTOM_FILES_PER_THREAD 4 // Uses a singleton implementation (based on here: http://www.yolinux.com/TUTORIALS/C++Singleton.html) @@ -22,9 +25,18 @@ private: static XmlCustomCode* uniqueInstance; + const int numThreads; - QVector scriptEngines; - QVector jsFunctions; - QVector getXmlDataFunctions; - QVector setXmlDataFunctions; + QThreadPool myThreadPool; + QMutex mutexIsAvailable; + + struct jsCustomCodeEngine{ + QScriptEngine* scriptEngine; + QScriptValue* jsFunction; + QScriptValue* getXmlDataFunction; + QScriptValue* setXmlDataFunction; + bool isAvailable; + }; + + QVector jsScriptEngines; XmlCustomCode(); // constructor is private (use getInstance) @@ -33,4 +45,5 @@ void displayJsException(QScriptEngine &engine, QScriptValue &engineResult); + jsCustomCodeEngine& getAvailableJsEngine(); __attribute__((always_inline)) inline void customCodeUnwinding(const QString &fileName, QString &currXmlFileString,