New Database Layer

.github/workflows/mac.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-    
+
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
@@ -32,7 +32,13 @@ jobs:
           version: 5.9.5
 
     - name: Install dependencies
-      run: brew install xalan-c boost
+      # Installing Xalan-C will cause Xerces-C to be installed too (as the former depends on the latter)
+      # .:TBD:. Installing Boost here doesn't seem to give us libboost_stacktrace_backtrace
+      #         Also, trying to use the "--cc=clang" option to install boost gives an error ("Error: boost: no bottle
+      #         available!")  For the moment, we're just using Boost header files on Mac though, so this should be OK.
+      run: |
+        brew install xalan-c
+        brew install boost
 
     - name: Build
       env:
@@ -41,14 +47,40 @@ jobs:
         mkdir build
         cd build
         cmake ..
-        make && (CTEST_OUTPUT_ON_FAILURE=TRUE make test)
+        make
+
+    - name: Prep for tests
+      # If a test fails and we get a core, we'd like to analyse it.  This will be easier if we have access to the
+      # relevant directories and there aren't any other files cluttering up the place.
+      #
+      # Running the commands inside an echo statement is a bit of a trick to ensure failure of the rm command (eg
+      # because there are no files to delete) does not count as a build script failure (because the echo command will
+      # return 0 = success).
+      run: |
+        sudo chmod -R +rwx /cores
+        sudo chmod -R +rwx /Library/Logs/DiagnosticReports
+        echo "Clearing contents of /cores directory: $(ls -ltr /cores) $(rm -rf /cores/*)"
+        echo "Clearing contents of /Library/Logs/DiagnosticReports directory: $(ls -ltr /Library/Logs/DiagnosticReports) $(rm -rf /Library/Logs/DiagnosticReports/*)"
+
+    - name: Automated tests
+      # If something does crash we'd like to capture the core, so we need to enable core dumps - hence the call to
+      # ulimit.
+      #
+      # Running "make test" boils down to running ctest (because the invocation of make in the Build step above will
+      # have done all the necessary prep.  Running ctest directly allows us to pass in extra parameters to try to get as
+      # much diagnostics as possible out of a remote build such as this.
+      run: |
+        ulimit -c unlimited
+        echo "Core size limit is $(ulimit -c)"
+        cd build
+        ctest --extra-verbose --output-on-failure 2>&1
 
     - name: Make package
       run: |
         cd build
         make package
 
-    - name: 'Upload mac'
+    - name: Upload Mac
       if: ${{ success() }}
       uses: actions/upload-artifact@v2
       with:
@@ -57,10 +89,38 @@ jobs:
           ${{github.workspace}}/build/brewtarget*.sh
         retention-days: 7
 
-    - name: Recover Debris Artifacts
+    - name: Post-processing on cores
+      if: ${{ failure() }}
+      # It's all very well capturing core files, but if you don't have a Mac to analyse them on they are not a fat lot
+      # of use.  So, if we did get a core, let's at least get a stack trace out of it.
+      run: |
+        sudo chmod -R +rwx /cores
+        sudo chmod -R +rwx /Library/Logs/DiagnosticReports
+        echo "Contents of /cores directory: $(ls -ltr /cores)"
+        echo "Contents of /Library/Logs/DiagnosticReports directory: $(ls -ltr /Library/Logs/DiagnosticReports)"
+        cd /cores
+        echo "bt" | lldb -c $(ls -1)
+
+    - name: Recover Debris Artifacts (aka build output)
       if: ${{ failure() }}
       uses: actions/upload-artifact@v2
       with:
         name: build-results
         path: ${{github.workspace}}/build
         retention-days: 1
+
+    - name: Recover DiagnosticReports (if any)
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: DiagnosticReports
+        path: /Library/Logs/DiagnosticReports
+        retention-days: 1
+
+    - name: Recover Cores (if any)
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: cores
+        path: /cores
+        retention-days: 1

.github/workflows/windows.yml

@@ -45,6 +45,7 @@ jobs:
             mingw-w64-${{ matrix.arch }}-xalan-c
             mingw-w64-${{ matrix.arch }}-nsis
             mingw-w64-${{ matrix.arch }}-boost
+            mingw-w64-${{ matrix.arch }}-libbacktrace
           update: true
           release: true
           path-type: strict
@@ -102,4 +103,4 @@ jobs:
         with:
           name: ${{ matrix.msystem }}-build
           path: C:/_/build/
-          retention-days: 1
+          retention-days: 1

CMakeLists.txt

@@ -99,10 +99,21 @@ IF( ${NO_QTMULTIMEDIA} )
    ADD_DEFINITIONS( -DNO_QTMULTIMEDIA )
 ENDIF()
 
+#====================================================Set build type=====================================================
+# We might always to tell the compiler to include debugging information (eg via the -g option on gcc).  It makes the
+# binaries slightly bigger on Linux, but helps greatly in analysing core dumps etc.  (In closed-source projects people
+# sometimes turn it off for release builds to make it harder to reverse engineer the software, but obviously that's not
+# an issue for us.)
+#
+# However, setting CMAKE_BUILD_TYPE to "Debug", also changes other things, such as the default location for connfig
+# files, which we don't want on a release build, so we would probably need to set compiler flags directly.
+#
+# .:TBD:. Investigate whether setting CMAKE_BUILD_TYPE to "RelWithDebInfo" does what we want.
+#
 IF( ${DO_RELEASE_BUILD} )
-   SET( CMAKE_BUILD_TYPE "Release" )
+   SET(CMAKE_BUILD_TYPE "Release")
 ELSE()
-   SET( CMAKE_BUILD_TYPE "Debug" )
+   SET(CMAKE_BUILD_TYPE "Debug")
 ENDIF()
 
 IF( APPLE AND NOT CMAKE_OSX_ARCHITECTURES )
@@ -140,6 +151,10 @@ ENDIF()
 #==================================Find Qt5====================================
 
 # The minimum QT5 version we need. Keep as low as necessary.
+# Note that if you change this, you need to make corresponding changes to the .github/workflows/*.yml files so that
+# GitHub uses the appropriate version of Qt for the automated builds.
+#
+# For the moment, max version we can have here is 5.9.5, because that's what Ubuntu 18.04 topped out at
 SET(QT5_MIN_VERSION 5.9.5)
 
 # Automatically run moc on source files when necessary
@@ -186,6 +201,9 @@ IF(Qt5_POSITION_INDEPENDENT_CODE)
    SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
 ENDIF()
 
+# Uncomment the following line to show what version of Qt is actually being used
+message(STATUS "Using Qt version " ${QT_VERSION})
+
 MESSAGE( STATUS "CMAKE_SYSTEM_INCLUDE_PATH: ${CMAKE_SYSTEM_INCLUDE_PATH}" )
 
 # Some extra files for the "make clean" target.
@@ -198,6 +216,35 @@ SET_PROPERTY(
    "CPackSourceConfig.cmake"
 )
 
+#======================================================Find Boost=======================================================
+# Boost is a collection of separate libraries, so we need to specify here which ones we are using
+# On Windows, using MSYS2, the mingw-w64-boost packages do not include libboost_stacktrace_backtrace, but
+# https://www.boost.org/doc/libs/1_76_0/doc/html/stacktrace/configuration_and_build.html suggests it is not required
+# (because on Windows, if you have libbacktrace installed, you can set BOOST_STACKTRACE_USE_BACKTRACE in header-only
+# mode).
+#
+# .:TODO:. Not sure how to get libboost_stacktrace_backtrace installed on Mac.  It doesn't seem to be findable by CMake
+# after installing Boost via Homebrew (https://brew.sh/).  For the moment, skip trying to use
+# libboost_stacktrace_backtrace on Mac
+#
+# .:TODO:. So far don't have stacktraces working properly on Windows (everything shows as register_frame_ctor), so that
+# needs some more investigation.  (It could be that it's a bug in Boost, at least according to
+# https://stackoverflow.com/questions/54333608/boost-stacktrace-not-demangling-names-when-cross-compiled)
+IF(WIN32)
+FIND_PACKAGE(Boost REQUIRED)
+ELSEIF(APPLE)
+FIND_PACKAGE(Boost REQUIRED)
+ELSE()
+# TBD Some users report problems getting CMake to find libboost_stacktrace_backtrace on Ubuntu and Gentoo, so disable it
+# for now and fallback to the header-only version
+#FIND_PACKAGE(Boost REQUIRED COMPONENTS stacktrace_backtrace)
+FIND_PACKAGE(Boost REQUIRED)
+ENDIF()
+INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
+# Uncomment the next two lines if you want to find where Boost headers and DLLs are on your system
+message( "Boost include directories: " ${Boost_INCLUDE_DIRS} )
+message( "Boost libraries: " ${Boost_LIBRARIES} )
+
 #===============================Find Xerces-C++================================
 # CMake already knows how to find and configure Xerces-C++, see
 # https://cmake.org/cmake/help/latest/module/FindXercesC.html
@@ -211,8 +258,9 @@ INCLUDE_DIRECTORIES(${XercesC_INCLUDE_DIRS})
 # Same comments apply here as for Xerces above
 FIND_PACKAGE(XalanC REQUIRED)
 INCLUDE_DIRECTORIES(${XalanC_INCLUDE_DIRS})
-message( "Xalan-C++ include directories: " ${XalanC_INCLUDE_DIRS} )
-message( "Xalan-C++ libraries: " ${XalanC_LIBRARIES} )
+# Uncomment the next two lines if you want to find where Xalan headers and DLLs are on your system
+#message( "Xalan-C++ include directories: " ${XalanC_INCLUDE_DIRS} )
+#message( "Xalan-C++ libraries: " ${XalanC_LIBRARIES} )
 
 #=========================Configure brewtarget.qrc.in==========================
 
@@ -317,6 +365,7 @@ SET( brewtarget_DOCS
 # List of data files to be installed.
 SET( brewtarget_DATA
      ${DATADIR}/default_db.sqlite
+     ${DATADIR}/DefaultData.xml
 # Yes, I know this is 'documentation', but Debian policy suggests it should be
 # with the data (see section 12.3 of the policy manual).
      ${ROOTDIR}/doc/manual-en.pdf
@@ -397,6 +446,9 @@ SET( brewtarget_INFO_PLIST "${ROOTDIR}/mac/Info.plist" )
 SET( brewtarget_MAC_ICNS "${ROOTDIR}/mac/BrewtargetIcon.icns" )
 
 #==========================Find dlls for Win32=================================
+# Note that WIN32 is true "when the target system is Windows, including Win64" (see
+# https://cmake.org/cmake/help/latest/variable/WIN32.html), so this is ALL versions of Windows, not just 32-bit.
+#
 # .:TBD:. MY 2020-12-29 What do we need to do for Xerces and Xalan here?
 IF( WIN32 )
    IF( NOT ${NO_QTMULTIMEDIA} )
@@ -487,11 +539,11 @@ IF( WIN32 )
       IF( NOT MINGW_BIN_DIR )
          FIND_PATH(MINGW_BIN_DIR "mingw32-make.exe")
       ENDIF()
-     IF( NOT EXISTS ${MINGW_BIN_DIR} )
-        MESSAGE( FATAL_ERROR "MinGW bin dir not found. Run cmake again with the option -DMINGW_BIN_DIR=c:/path/to/mingw/bin" )
-     ELSE()
-        get_filename_component( Mingw_Path ${CMAKE_CXX_COMPILER} PATH )
-     ENDIF()
+      IF( NOT EXISTS ${MINGW_BIN_DIR} )
+         MESSAGE( FATAL_ERROR "MinGW bin dir not found. Run cmake again with the option -DMINGW_BIN_DIR=c:/path/to/mingw/bin" )
+      ELSE()
+         get_filename_component( Mingw_Path ${CMAKE_CXX_COMPILER} PATH )
+      ENDIF()
    ENDIF()
 ENDIF()