build-system/MXE: build with more up to date MXE tools.

- use hidapi grantlee and mdbtools from MXE
- update MXE version to use QT 5.15, and pull in libzstd and  CMake 3.17.3
- fix linking of winmm on windows build with new mxe
- add some instructions on building the container
- add some new dependancies from QT 5.15 to the packaging
- add a patch to MXE to Build qtconnectivity with native-win32-bluetooth

[Dirk Hohndel: small refactor]

Signed-off-by: Paul Buxton <paulbuxton.mail@googlemail.com>
Signed-off-by: Dirk Hohndel <dirk@hohndel.org>

12 files changed, 118 insertions, 90 deletions

diff --git a/.github/workflows/scripts/windows-container-prep.sh b/.github/workflows/scripts/windows-container-prep.sh
new file mode 100644
index 000000000..e6f447d91
--- /dev/null
+++ b/.github/workflows/scripts/windows-container-prep.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# abstract the prepare commands for the windows build into a script that can be reused
+# instead of a yaml file
+cd /win
+ln -s /__w/subsurface/subsurface .
+echo "downloading sources for fresh build"
+bash subsurface/scripts/get-dep-lib.sh single . libzip
+bash subsurface/scripts/get-dep-lib.sh single . googlemaps
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index d83b8735a..be38ae37e 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -19,20 +19,8 @@ jobs:
 
     - name: get other dependencies
       run: |
-        echo "creating the link from /win/subsurface"
-        cd /win
-        ln -s /__w/subsurface/subsurface .
-        ls -l
-        ls -l subsurface/scripts
-        echo "installing missing container components"
-        apt-get install -y ca-certificates libtool
-        echo "downloading sources for fresh build"
-        bash subsurface/scripts/get-dep-lib.sh single . libzip
-        bash subsurface/scripts/get-dep-lib.sh single . hidapi
-        bash subsurface/scripts/get-dep-lib.sh single . googlemaps
-        bash subsurface/scripts/get-dep-lib.sh single . grantlee
-        bash subsurface/scripts/get-dep-lib.sh single . mdbtools
-
+        echo "Running script to install additional dependancies into container"
+        bash -x subsurface/.github/workflows/scripts/windows-container-prep.sh 2>&1 | tee pre-build.log
     - name: run build
       run: |
         cd /win
diff --git a/INSTALL b/INSTALL
index cbda0b9f1..11c1730bf 100644
--- a/INSTALL
+++ b/INSTALL
@@ -320,6 +320,9 @@ Please read through the explanations and instructions in
 packaging/windows/mxe-based-build.sh if you want to build the Windows
 version on your Linux system.
 
+In addition you can use the Docker container the same way that CI builds do modify/test 
+ modifications to the container build environment following instructions in 
+/scripts/docker/mxe-build-container/instructions.md
 
 Building Subsurface on Windows
 ------------------------------
diff --git a/packaging/windows/mxe-based-build.sh b/packaging/windows/mxe-based-build.sh
index b5e99926c..f78c3d0d8 100755
--- a/packaging/windows/mxe-based-build.sh
+++ b/packaging/windows/mxe-based-build.sh
@@ -138,41 +138,6 @@ else
 	touch Release
 fi
 
-# grantlee
-
-cd "$BUILDDIR"
-if [[ ! -d grantlee || -f build.grantlee ]] ; then
-	rm -f build.grantlee
-	mkdir -p grantlee
-	cd grantlee
-	"$MXEBUILDTYPE"-cmake \
-		-DCMAKE_BUILD_TYPE=$RELEASE \
-		-DBUILD_TESTS=OFF \
-		"$BASEDIR"/grantlee
-
-	make $JOBS
-	make install
-fi
-
-# hidapi for libdivecomputer (if available)
-
-if [[ -d "$BASEDIR"/hidapi ]] ; then
-	cd "$BUILDDIR"
-	if [[ ! -d hidapi || -f build.hidapi ]] ; then
-		rm -f build.hidapi
-		mkdir -p hidapi
-		pushd "$BASEDIR"/hidapi
-		bash ./bootstrap
-		popd
-		cd hidapi
-		"$BASEDIR"/hidapi/configure \
-			CC="$MXEBUILDTYPE"-gcc \
-			--host="$MXEBUILDTYPE" \
-			--prefix="$BASEDIR"/"$MXEDIR"/usr/"$MXEBUILDTYPE"
-		make $JOBS
-		make install
-	fi
-fi
 
 
 # libdivecomputer
@@ -284,7 +249,8 @@ done
 # for some reason we aren't installing Qt5Xml.dll and Qt5Location.dll
 # I need to figure out why and fix that, but for now just manually copy that as well
 EXTRA_MANUAL_DEPENDENCIES="$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Xml$DLL_SUFFIX.dll \
-$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Location$DLL_SUFFIX.dll"
+$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Location$DLL_SUFFIX.dll \
+$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5QmlWorkerScript$DLL_SUFFIX.dll"
 
 for f in $EXTRA_MANUAL_DEPENDENCIES
 do
diff --git a/packaging/windows/smtk2ssrf-mxe-build.sh b/packaging/windows/smtk2ssrf-mxe-build.sh
index c3d8f7ac0..170394493 100755
--- a/packaging/windows/smtk2ssrf-mxe-build.sh
+++ b/packaging/windows/smtk2ssrf-mxe-build.sh
@@ -142,32 +142,6 @@ export PKG_CONFIG_PATH_i686_w64_mingw32_shared="$BASEDIR/mxe/usr/i686-w64-mingw3
 export PKG_CONFIG_PATH="$PKG_CONFIG_PATH_i686_w64_mingw32_static":"$PKG_CONFIG_PATH_i686_w64_mingw32_shared"
 
 #
-# mdbtools
-# build from sources. If build fails, fallback to prebuilt mxe binaries.
-#
-echo -e "$BLUE---> Building mdbtools ... $DEFAULT "
-mkdir -p --verbose "$BASEDIR"/mxe/usr/i686-w64-mingw32.static/include
-mkdir -p --verbose "$BASEDIR"/mxe/usr/i686-w64-mingw32.static/lib
-cd "$BUILDDIR"
-[[ -d mdbtools ]] && rm -rf mdbtools
-mkdir -p mdbtools
-cd mdbtools
-if [ ! -f "$BASEDIR"/mdbtools/configure ] ; then
-	( cd "$BASEDIR"/mdbtools
-	autoreconf -v -f -i )
-fi
-"$BASEDIR"/mdbtools/configure CC=i686-w64-mingw32.static-gcc \
-		     --host=i686-w64-mingw32.static \
-		     --prefix="$BASEDIR"/mxe/usr/i686-w64-mingw32.static \
-		     --enable-shared=no \
-		     --disable-man \
-		     --disable-gmdb2
-# hack to make mdbtools build outsource
-ln -vs "$BUILDDIR"/mdbtools/include/mdbver.h "$BASEDIR"/mdbtools/include/mdbver.h
-
-make $JOBS >/dev/null && make install || \
-	echo -e "$RED---> Building mdbtools failed ...$LIGHT_GRAY Trying to build with precompiled mxe binaries$DEFAULT"
-
 # Subsurface
 #
 if [ "$AUTO" = "false" ]; then
diff --git a/scripts/docker/mxe-build-container/Dockerfile-stage1 b/scripts/docker/mxe-build-container/Dockerfile-stage1
index 501bfe6ce..056e6fa8e 100644
--- a/scripts/docker/mxe-build-container/Dockerfile-stage1
+++ b/scripts/docker/mxe-build-container/Dockerfile-stage1
@@ -8,8 +8,8 @@ ARG mxe_sha=master
 ENV _ver=${mxe_sha}
 
 # update and set up the packages we need for this cross build
-RUN apt-get update  &&  apt-get upgrade -y
-RUN apt-get install -y \
+RUN apt-get update  &&  apt-get upgrade -y && \
+apt-get install -y \
     autoconf \
     automake \
     autopoint \
@@ -17,6 +17,7 @@ RUN apt-get install -y \
     binutils \
     bison \
     bzip2 \
+    ca-certificates \
     flex \
     g++ \
     g++-multilib \
@@ -28,6 +29,7 @@ RUN apt-get install -y \
     libgdk-pixbuf2.0-dev \
     libltdl-dev \
     libssl-dev \
+    libtool \
     libtool-bin \
     libxml-parser-perl \
     make \
@@ -51,6 +53,10 @@ ADD settings-stage1.mk /win/settings.mk
 RUN cd /win ; git clone git://github.com/mxe/mxe ; \
     cd mxe ; \
     git checkout ${_ver} ;
+# Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another
+# backend is not picked
+ADD qtconnectivity-1.patch /win/qtconnectivity-1.patch
+RUN mv /win/qtconnectivity-1.patch /win/mxe/src
 RUN mv /win/settings.mk /win/mxe
 RUN cd /win/mxe ; \
     make -j 6 2>&1 | tee build.log ;
diff --git a/scripts/docker/mxe-build-container/Dockerfile-stage2 b/scripts/docker/mxe-build-container/Dockerfile-stage2
index 55fa3480c..d8760f723 100644
--- a/scripts/docker/mxe-build-container/Dockerfile-stage2
+++ b/scripts/docker/mxe-build-container/Dockerfile-stage2
@@ -9,14 +9,5 @@ ADD settings-stage2.mk /win/mxe/settings.mk
 RUN cd /win/mxe ; \
     make -j 2 2>&1 | tee build.log ;
 RUN cd /win/mxe ; \
-    make MXE_TARGETS=i686-w64-mingw32.static glib -j 6 2>&1 | tee -a build.log ;
+    make MXE_TARGETS=i686-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a build.log ;
 
-# manually build the Win BLE version of QtConnectivity (we can drop this with Qt 5.14)
-RUN cd /win/mxe ; \
-    mkdir -p neolit ; cd neolit ; git clone -b wip/win git://github.com/qt/qtconnectivity ; \
-    sed -i 's/SetupAPI.h/setupapi.h/' qtconnectivity/src/bluetooth/qlowenergycontroller_win.cpp
-RUN cd /win/mxe/neolit/qtconnectivity ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin /win/mxe/usr/i686-w64-mingw32.shared/qt5/bin/qmake qtconnectivity.pro ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin make -j 6 ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin make install ;
-RUN rm -rf /win/mxe/pkg
diff --git a/scripts/docker/mxe-build-container/build-container.sh b/scripts/docker/mxe-build-container/build-container.sh
new file mode 100644
index 000000000..1f5e1eddc
--- /dev/null
+++ b/scripts/docker/mxe-build-container/build-container.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+set -e
+
+SCRIPTPATH=$(dirname $0)
+
+export VERSION=1.1
+pushd $SCRIPTPATH
+docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=1ee37f8 -f Dockerfile-stage1 .
+docker build -t subsurface/mxe-build-container:$VERSION --build-arg=VERSION=$VERSION -f Dockerfile-stage2 .
+popd
diff --git a/scripts/docker/mxe-build-container/instructions.md b/scripts/docker/mxe-build-container/instructions.md
new file mode 100644
index 000000000..2a860ac33
--- /dev/null
+++ b/scripts/docker/mxe-build-container/instructions.md
@@ -0,0 +1,38 @@
+# Instructions for building the container environment and using it to build/package subsurface.
+
+This document assumes you have alreay installed docker and have checked out subsurface according to the instructions in the INSTALL document.
+
+If you are just wantint to build with the current mxe build container then starting from the folder above subsurface run
+
+```bash
+docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.x /bin/sleep 60m
+```
+
+replacing the x in the mxe-build-container tag with the current version e.g.
+```bash
+docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.0 /bin/sleep 60m
+```
+Next you need to prep the container by installing some prerequisites
+
+```bash
+docker exec -t mybuilder bash subsurface/.github/workflows/scripts/windows-container-prep.sh 2>&1 | tee pre-build.log
+```
+
+Finaly the actual build is done with
+```bash
+docker exec -t mybuilder bash subsurface/.github/workflows/scripts/windows-in-container-build.sh 2>&1 | tee build.log
+```
+
+To get the built binary out of the container
+```
+docker exec -t mybuilder bash cp /subsurface-installer.exe /win/win32
+```
+Which will copy the installer into the win32 folder which will be a sibling of the subsurface folder.
+
+## Modifying the container
+If you want to make changes to the build environment used in the conatiner
+The script scripts/docker/mxe-build-container.sh will build the Docker image itself.
+The sha of the version of MXE we are using is built into this, so you can update that to whatever version is required, and modify dockerfiles and settings-stage1.mk and settings-stage2.mk to pull in any other prerequisites as required.
+
+If you are working on updating the container then you should incrment the minor version of the variable VERSION in the script as otherwise it will clash with the version used in production
+
diff --git a/scripts/docker/mxe-build-container/qtconnectivity-1.patch b/scripts/docker/mxe-build-container/qtconnectivity-1.patch
new file mode 100644
index 000000000..fa43ff724
--- /dev/null
+++ b/scripts/docker/mxe-build-container/qtconnectivity-1.patch
@@ -0,0 +1,12 @@
+diff --git a/src/bluetooth/configure.json b/src/bluetooth/configure.json
+index 3bd95903..0d41d88f 100644
+--- a/src/bluetooth/configure.json
++++ b/src/bluetooth/configure.json
+@@ -63,6 +63,7 @@
+             "purpose": "Uses the native Win32 Bluetooth backend.",
+             "section": "Qt Bluetooth",
+             "autoDetect": false,
++            "enable": true,
+             "output": [ "publicFeature", "feature" ]
+         },
+         "winrt_bt": {
diff --git a/scripts/docker/mxe-build-container/settings-stage2.mk b/scripts/docker/mxe-build-container/settings-stage2.mk
index 452961338..f06b5a94a 100644
--- a/scripts/docker/mxe-build-container/settings-stage2.mk
+++ b/scripts/docker/mxe-build-container/settings-stage2.mk
@@ -9,7 +9,32 @@ JOBS := 2
 MXE_TARGETS :=  i686-w64-mingw32.shared
 
 # The three lines below makes `make` build these "local packages" instead of all packages.
-LOCAL_PKG_LIST := qtbase qtconnectivity qtdeclarative qtimageformats qtlocation qtmultimedia qtquickcontrols qtquickcontrols2 qtscript qtsvg qttools qttranslations qtwebview qtwebkit libxml2 libxslt libusb1 libgit2 nsis curl libzip libftdi1
+LOCAL_PKG_LIST := curl \
+                  grantlee \
+                  hidapi \
+                  libftdi1 \
+                  libgit2 \
+                  libusb1 \
+                  libxml2 \
+                  libxslt \
+                  libzip \
+                  mdbtools \
+                  nsis \
+                  qtbase \
+                  qtconnectivity \
+                  qtdeclarative \
+                  qtimageformats \
+                  qtlocation \
+                  qtmultimedia \
+                  qtquickcontrols \
+                  qtquickcontrols2 \
+                  qtscript \
+                  qtsvg \
+                  qttools \
+                  qttranslations \
+                  qtwebkit \
+                  qtwebview \
+                  zstd
 .DEFAULT local-pkg-list:
 local-pkg-list: $(LOCAL_PKG_LIST)
 
diff --git a/smtk-import/CMakeLists.txt b/smtk-import/CMakeLists.txt
index 369dd80a1..19761f56a 100644
--- a/smtk-import/CMakeLists.txt
+++ b/smtk-import/CMakeLists.txt
@@ -92,7 +92,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
         set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} -lssh2 -lz -lpthread)
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
-        set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} -lwsock32 -lws2_32 -lole32 -limm32 -lwinmm)
 	remove_definitions(-DUNICODE)
         add_definitions(-mwindows -D_WIN32)
 endif()
@@ -109,6 +108,13 @@ set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} ${SUBSURFACE_LINK_LIBRARIES} ${SS
 source_group("SmartTrak Import libs" FILES ${SMTK_IMPORT_SRCS})
 set(SMTK_IMPORT_TARGET smtk2ssrf)
 add_library(smtk_import STATIC ${SMTK_IMPORT_SRCS})
+
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+	# Setting the link target this way resolves an issue with not importing timegettime correctly fron winmm
+	target_link_libraries(smtk_import PRIVATE wsock32 ws2_32 ole32 imm32 winmm)
+endif()
+
 add_executable(${SMTK_IMPORT_TARGET} smtk_standalone.cpp ${SUBSURFACE_RESOURCES})
 
 # We just want CLI mode on Linux. Silently drop it if cross building to Windows.