squirrel export

build.sh

@@ -88,7 +88,7 @@ echo -e "\nBuilding squirrel library\n"
 echo $QMAKEBIN -o $BUILDDIR/squirrel/Makefile $SRCDIR/squirrel/squirrellib.pro -spec linux-g++
 $QMAKEBIN -o $BUILDDIR/squirrel/Makefile $SRCDIR/squirrel/squirrellib.pro -spec linux-g++
 cd $BUILDDIR/squirrel
-make -B -j 16
+make -j 16
 
 
 # ----- build NiDB core -----

src/nidb/archiveio.cpp

@@ -2372,6 +2372,8 @@ bool archiveIO::WriteSquirrel(qint64 exportid, QString name, QString desc, QStri
 
     n->WriteLog(QString("Entering %1() exportid [%2]").arg(__FUNCTION__).arg(exportid));
 
+    n->WriteLog("Squirrel flags [" + downloadflags.join(", ") + "]");
+
     QStringList msgs;
     QString exportstatus = "complete";
     subjectStudySeriesContainer s;
@@ -2393,8 +2395,12 @@ bool archiveIO::WriteSquirrel(qint64 exportid, QString name, QString desc, QStri
     if (zipfilepath.endsWith("/"))
         zipfilepath.chop(1);
 
-    //if (!zipfilepath.endsWith(".zip", Qt::CaseInsensitive))
-    //    zipfilepath = zipfilepath + ".zip";
+    /* create a local temp directory for exporting files from database */
+    QString localTempDir = n->cfg["tmpdir"] + "/squirrel-" + GenerateRandomString(20);
+    QString m;
+    if (!MakePath(localTempDir, m)) {
+        n->WriteLog(QString("Error create temp directory [%1] - message [%2]").arg(localTempDir).arg(m));
+    }
 
     /* create squirrel object with default settings... */
     squirrel sqrl;
@@ -2711,11 +2717,19 @@ bool archiveIO::WriteSquirrel(qint64 exportid, QString name, QString desc, QStri
 
     /* add experiments to the JSON object */
     if (downloadflags.contains("DOWNLOAD_EXPERIMENTS", Qt::CaseInsensitive)) {
+        n->WriteLog(QString("Processing [%1] experiments").arg(experimentIDs.size()));
         if (experimentIDs.size() > 0) {
             for (int i=0; i<experimentIDs.size(); i++) {
                 /* create and add each squirrelPipeline object */
                 experiment e(experimentIDs[i], n);
+                QStringList stagedFileList;
+                if (e.WriteFiles(localTempDir, stagedFileList, m))
+                    n->WriteLog(QString("Successfully wrote experiment files [%1]").arg(m));
+                else
+                    n->WriteLog(QString("Error writing experiment files [%1]").arg(m));
+
                 squirrelExperiment sqrlExperiment = e.GetSquirrelObject();
+                sqrlExperiment.stagedFiles = stagedFileList;
                 sqrlExperiment.Store();
             }
         }
@@ -2738,6 +2752,14 @@ bool archiveIO::WriteSquirrel(qint64 exportid, QString name, QString desc, QStri
     sqrl.Write(false);
     msgs << n->WriteLog(QString("%1() - squirrel.write() returned [\n" + sqrl.GetLog() + "\n]").arg(__FUNCTION__));
 
+    if (FileDirectoryExists(localTempDir))
+        if (RemoveDir(localTempDir, m))
+            n->WriteLog(QString("Successfully removed localTempDir [%1]").arg(localTempDir));
+        else
+            n->WriteLog(QString("Error removing localTempDir [%1] - message [%2]").arg(localTempDir).arg(m));
+    else
+        n->WriteLog(QString("localTempDir [%1] does not exist").arg(localTempDir));
+
     msg = msgs.join("\n");
     n->WriteLog("Leaving WriteSquirrel()...");
     return true;
@@ -2771,6 +2793,13 @@ bool archiveIO::WritePackage(qint64 exportid, QString zipfilepath, QString &msg)
     if (!zipfilepath.endsWith(".sqrl", Qt::CaseInsensitive))
         zipfilepath = zipfilepath + ".sqrl";
 
+    /* create a local temp directory for exporting files from database */
+    QString localTempDir = n->cfg["tmpdir"] + "/squirrel-" + GenerateRandomString(20);
+    QString m;
+    if (!MakePath(localTempDir, m)) {
+        n->WriteLog(QString("Error create temp directory [%1] - message [%2]").arg(localTempDir).arg(m));
+    }
+
     /* PACKAGE - get the details, and create package object */
     squirrel sqrl(true);
     q.prepare("select * from packages where package_id = :packageid");
@@ -2977,15 +3006,35 @@ bool archiveIO::WritePackage(qint64 exportid, QString zipfilepath, QString &msg)
         experiment e(experimentRowID, n);
         if (!e.isValid) continue;
 
-        squirrelExperiment sqrlExperiment;
-        sqrlExperiment = e.GetSquirrelObject();
+        QString m;
+        if (!MakePath(localTempDir + "/" + e.name, m)) {
+            n->WriteLog(QString("Error create temp directory [%1] - message [%2]").arg(localTempDir + "/" + e.name).arg(m));
+        }
+
+        QStringList stagedFileList;
+        if (e.WriteFiles(localTempDir + "/" + e.name, stagedFileList, m))
+            n->WriteLog(QString("Successfully wrote experiment files [%1]").arg(m));
+        else
+            n->WriteLog(QString("Error writing experiment files [%1]").arg(m));
+
+        squirrelExperiment sqrlExperiment = e.GetSquirrelObject();
+        sqrlExperiment.stagedFiles = stagedFileList;
         sqrlExperiment.Store();
     }
 
     //sqrl.Print();
 
     if (sqrl.Write(false)) {
         msg = sqrl.GetLog();
+
+        // if (FileDirectoryExists(localTempDir))
+        //     if (RemoveDir(localTempDir, m))
+        //         n->WriteLog(QString("Successfully removed localTempDir [%1]").arg(localTempDir));
+        //     else
+        //         n->WriteLog(QString("Error removing localTempDir [%1] - message [%2]").arg(localTempDir).arg(m));
+        // else
+        //     n->WriteLog(QString("localTempDir [%1] does not exist").arg(localTempDir));
+
         return true;
     }
     else {

src/nidb/experiment.cpp

@@ -92,7 +92,7 @@ void experiment::LoadExperimentInfo() {
 /* ---------------------------------------------------------- */
 /* --------- WriteFiles ------------------------------------- */
 /* ---------------------------------------------------------- */
-bool experiment::WriteFiles(QString dir, QString &m) {
+bool experiment::WriteFiles(QString dir, QStringList &fileList, QString &m) {
 
     QDir d(dir);
     if (!d.exists()) {
@@ -120,6 +120,7 @@ bool experiment::WriteFiles(QString dir, QString &m) {
             m = QString("Created file size [%1] does not match database file size [%2]").arg(fi.size()).arg(s.filesize);
             return false;
         }
+        fileList.append(filename);
     }
 
     return true;

src/nidb/experiment.h

@@ -42,7 +42,7 @@ class experiment
     experiment(int id, nidb *a);
     nidb *n;
 
-    bool WriteFiles(QString dir, QString &m);
+    bool WriteFiles(QString dir, QStringList &fileList, QString &m);
 
     /* object variables */
     QString msg;

src/nidb/nidb.pro.user

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE QtCreatorProject>
-<!-- Written by QtCreator 12.0.2, 2024-05-06T10:25:23. -->
+<!-- Written by QtCreator 13.0.0, 2024-05-06T16:04:59. -->
 <qtcreator>
  <data>
   <variable>EnvironmentId</variable>
@@ -240,6 +240,7 @@
     <value type="int" key="PE.EnvironmentAspect.Base">2</value>
     <valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
     <value type="bool" key="PE.EnvironmentAspect.PrintOnRun">false</value>
+    <value type="QString" key="PerfRecordArgsId">-e cpu-cycles --call-graph &quot;dwarf,4096&quot; -F 250</value>
     <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value>
     <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.Qt4RunConfiguration:M:/nidb/src/nidb/nidb.pro</value>
     <value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">M:/nidb/src/nidb/nidb.pro</value>

src/squirrel/squirrel.cpp

@@ -58,6 +58,14 @@ squirrel::squirrel(bool dbg, bool q)
     InitializeDatabase();
 
     Log(QString("Created squirrel object."), __FUNCTION__);
+    if (debug)
+        Log("'debug' is set to true", __FUNCTION__);
+    else
+        Log("'debug' is set to false", __FUNCTION__);
+
+    Log("Next line should be a Debug() line", __FUNCTION__);
+    Debug("This line should be a Debug() line", __FUNCTION__);
+    Log("Previous line should be a Debug() line", __FUNCTION__);
 }
 
 
@@ -535,16 +543,25 @@ bool squirrel::Write(bool writeLog) {
 
                 /* orig vs other formats */
                 if (DataFormat == "orig") {
+                    Log("Export data format is 'orig'. Copying files...", __FUNCTION__);
                     /* copy all of the series files to the temp directory */
                     foreach (QString f, series.stagedFiles) {
                         QString systemstring = QString("cp -uv %1 %2").arg(f).arg(seriesPath);
-                        utils::SystemCommand(systemstring);
+                        Log(utils::SystemCommand(systemstring), __FUNCTION__);
+                    }
+                }
+                else if (study.Modality.toUpper() != "MR") {
+                    Log(QString("Study modality is [%1]. Copying files...").arg(study.Modality.toUpper()), __FUNCTION__);
+                    /* copy all of the series files to the temp directory */
+                    foreach (QString f, series.stagedFiles) {
+                        QString systemstring = QString("cp -uv %1 %2").arg(f).arg(seriesPath);
+                        Log(utils::SystemCommand(systemstring), __FUNCTION__);
                     }
                 }
                 else if ((DataFormat == "anon") || (DataFormat == "anonfull")) {
                     /* create temp directory for the anonymization */
                     QString td;
-                    MakeTempDir(td);
+                    //MakeTempDir(td);
 
                     /* copy all files to temp directory */
                     QString systemstring;
@@ -646,9 +663,9 @@ bool squirrel::Write(bool writeLog) {
     }
 
     /* add staged files to list */
-    foreach (squirrelSubject subject, subjectses) {
-        stagedFiles += subject.GetStagedFileList();
-    }
+    //foreach (squirrelSubject subject, subjectses) {
+    //    stagedFiles += subject.GetStagedFileList();
+    //}
 
     /* add group-analyses */
     QList <squirrelGroupAnalysis> groupAnalyses = GetAllGroupAnalyses();
@@ -726,14 +743,29 @@ bool squirrel::Write(bool writeLog) {
     if (fileMode == NewPackage) {
 
         /* copy in all files from the staged files list */
+        Debug(QString("stagedFiles size is [%1]").arg(stagedFiles.size()), __FUNCTION__);
+        for (int i=0; i<stagedFiles.size(); i++) {
+            Log(QString("[%1] , [%2]").arg(stagedFiles.at(i).first).arg(stagedFiles.at(i).second), __FUNCTION__);
+        }
+
         for (int i=0; i<stagedFiles.size(); i++) {
             QStringPair file = stagedFiles.at(i);
-            QString source = file.first;
+
+            QString sourcePath = file.first;
             QFileInfo fi(file.first);
             QString fname = fi.fileName();
-            QString dest = workingDir + "/" + fname;
-            if (!QFile::copy(source, dest))
-                Log(QString("Error copying [%1] to [%2]").arg(source).arg(dest), __FUNCTION__);
+
+            QString destPath = workingDir + "/" + file.second + "/" + fname;
+            QString destDir = workingDir + "/" + file.second;
+            QString m;
+            if (!utils::MakePath(destDir,m))
+                Log(QString("Error creating directory [%1] - message [%2]").arg(destDir).arg(m), __FUNCTION__);
+            else
+                Log(QString("Successfully created directory [%1] - message [%2]").arg(destDir).arg(m), __FUNCTION__);
+
+            Log(QString("Copying [%1] to [%2]").arg(sourcePath).arg(destPath), __FUNCTION__);
+            if (!QFile::copy(sourcePath, destPath))
+                Log(QString("Error copying [%1] to [%2]").arg(sourcePath).arg(destPath), __FUNCTION__);
         }
 
         Log("Zipping the archive from a temp directory", __FUNCTION__);
@@ -753,8 +785,8 @@ bool squirrel::Write(bool writeLog) {
             if (utils::DirectoryExists(workingDir)) {
                 Log("Temporary export dir [" + workingDir + "] exists and will be deleted", __FUNCTION__);
                 QString m;
-                if (!utils::RemoveDir(workingDir, m))
-                    Log("Error [" + m + "] removing directory [" + workingDir + "]", __FUNCTION__);
+                //if (!utils::RemoveDir(workingDir, m))
+                //    Log("Error [" + m + "] removing directory [" + workingDir + "]", __FUNCTION__);
             }
         }
         else {

src/squirrel/squirrelExperiment.cpp

@@ -152,7 +152,7 @@ void squirrelExperiment::PrintExperiment() {
 /* ----- VirtualPath ------------------------------------------ */
 /* ------------------------------------------------------------ */
 QString squirrelExperiment::VirtualPath() {
-    QString vPath = QString("experiment/%1").arg(utils::CleanString(ExperimentName));
+    QString vPath = QString("experiments/%1").arg(utils::CleanString(ExperimentName));
 
     return vPath;
 }
@@ -164,14 +164,14 @@ QString squirrelExperiment::VirtualPath() {
 QList<QPair<QString,QString>> squirrelExperiment::GetStagedFileList() {
 
     QList<QPair<QString,QString>> stagedList;
-    QString virtualPath = VirtualPath();
 
     QString path;
     foreach (path, stagedFiles) {
         QPair<QString, QString> pair;
         pair.first = path;
-        pair.second = virtualPath;
+        pair.second = VirtualPath();
         stagedList.append(pair);
+        utils::Print(QString("Inside GetStagedFileList() - stagedList [%1] -- [%2]").arg(pair.first).arg(pair.second));
     }
 
     return stagedList;

src/squirrel/squirrelPipeline.cpp

@@ -353,7 +353,7 @@ void squirrelPipeline::PrintPipeline() {
 /* ----- VirtualPath ------------------------------------------ */
 /* ------------------------------------------------------------ */
 QString squirrelPipeline::VirtualPath() {
-    QString vPath = QString("pipeline/%1").arg(PipelineName);
+    QString vPath = QString("pipelines/%1").arg(PipelineName);
 
     return vPath;
 }