From e71e715cd347d3dc1e503695d7a747f175123e9e Mon Sep 17 00:00:00 2001 From: Dirk Hohndel Date: Wed, 28 Oct 2020 14:19:45 -0700 Subject: build-system: Docker build for 64bit MXE We previously tried to build the MXE Docker container on GitHub using an Action, but that really didn't work well and was a lot more trouble than it was worth. So this goes back to an offline build mechanism where I simply create an updated Docker image when needed and push that to Docker Hub. But this nearly hides the most interesting change here - we are finally switching to using 64bit binaries on Windows. It's 2020 and fewer than 1% of our users use 32bit Windows machines. We'll need to expand this to be able to have both a 32bit and a 64bit version of Subsurface for Windows. But for now, this solves the problem for 99% of our users. Signed-off-by: Dirk Hohndel --- scripts/docker/mxe-build-container/Dockerfile | 74 ++++++++++++++++++++++ .../docker/mxe-build-container/Dockerfile-stage1 | 64 ------------------- .../docker/mxe-build-container/Dockerfile-stage2 | 13 ---- .../docker/mxe-build-container/build-container.sh | 9 ++- scripts/docker/mxe-build-container/instructions.md | 10 ++- .../docker/mxe-build-container/settings-stage1.mk | 15 ----- .../docker/mxe-build-container/settings-stage2.mk | 39 ------------ scripts/docker/mxe-build-container/settings.mk | 44 +++++++++++++ 8 files changed, 128 insertions(+), 140 deletions(-) create mode 100644 scripts/docker/mxe-build-container/Dockerfile delete mode 100644 scripts/docker/mxe-build-container/Dockerfile-stage1 delete mode 100644 scripts/docker/mxe-build-container/Dockerfile-stage2 delete mode 100644 scripts/docker/mxe-build-container/settings-stage1.mk delete mode 100644 scripts/docker/mxe-build-container/settings-stage2.mk create mode 100644 scripts/docker/mxe-build-container/settings.mk (limited to 'scripts/docker') diff --git a/scripts/docker/mxe-build-container/Dockerfile b/scripts/docker/mxe-build-container/Dockerfile new file mode 100644 index 000000000..08bdd619c --- /dev/null +++ b/scripts/docker/mxe-build-container/Dockerfile @@ -0,0 +1,74 @@ +# Build the image using the --build-arg option, e.g.: +# docker build -t boret/myimage:0.1 --build-arg=mxe_sha=123ABC456 . +# + +# Start from Ubuntu +From ubuntu:20.04 + +# very often master is broken, so we pass in a known good SHA +ARG mxe_sha=master +ENV _ver=${mxe_sha} + +# update and set up the packages we need for this cross build +RUN apt-get update && apt-get upgrade -y && \ +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + autoconf \ + automake \ + autopoint \ + bash \ + binutils \ + bison \ + bzip2 \ + ca-certificates \ + flex \ + g++ \ + g++-multilib \ + gettext \ + git \ + gperf \ + intltool \ + libc6-dev-i386 \ + libgdk-pixbuf2.0-dev \ + libltdl-dev \ + libssl-dev \ + libtool \ + libtool-bin \ + libxml-parser-perl \ + make \ + openssl \ + p7zip-full \ + patch \ + perl \ + pkg-config \ + python \ + ruby \ + sed \ + unzip \ + wget \ + xz-utils \ + lzip \ + scons + +# checkout MXE at the right version +RUN mkdir -p /win +RUN cd /win ; git clone git://github.com/mxe/mxe ; \ + cd mxe ; \ + git checkout ${_ver} ; + +# Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another +# backend is not picked +ADD qtconnectivity-1.patch /win/mxe/src/qtconnectivity-1.patch + +# Move the settings into place to build everything that we need +# separate download from build so that we can redo the build +ADD settings.mk /win/mxe/settings.mk +RUN cd /win/mxe ; \ + make -j 6 download 2>&1 | tee mxe-build.log +RUN cd /win/mxe ; \ + make -j 6 2>&1 | tee -a mxe-build.log ; + +# for some reason smtk2ssrf needs a static build of mdbtools +RUN cd /win/mxe ; \ + make MXE_TARGETS=x86_64-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a mxe-build.log ; + +RUN apt-get remove -y bison flex gperf libc6-dev-i386 libgdk-pixbuf2.0-dev libxml-parser-perl python ruby xz-utils scons && rm -rf /var/lib/apt/lists/* diff --git a/scripts/docker/mxe-build-container/Dockerfile-stage1 b/scripts/docker/mxe-build-container/Dockerfile-stage1 deleted file mode 100644 index 056e6fa8e..000000000 --- a/scripts/docker/mxe-build-container/Dockerfile-stage1 +++ /dev/null @@ -1,64 +0,0 @@ -# Build the image using the --build-arg option, e.g.: -# docker build -t boret/myimage:0.1 --build-arg=mxe_sha=123ABC456 . -# - -# Start from Ubuntu -From ubuntu:18.04 -ARG mxe_sha=master -ENV _ver=${mxe_sha} - -# update and set up the packages we need for this cross build -RUN apt-get update && apt-get upgrade -y && \ -apt-get install -y \ - autoconf \ - automake \ - autopoint \ - bash \ - binutils \ - bison \ - bzip2 \ - ca-certificates \ - flex \ - g++ \ - g++-multilib \ - gettext \ - git \ - gperf \ - intltool \ - libc6-dev-i386 \ - libgdk-pixbuf2.0-dev \ - libltdl-dev \ - libssl-dev \ - libtool \ - libtool-bin \ - libxml-parser-perl \ - make \ - openssl \ - p7zip-full \ - patch \ - perl \ - pkg-config \ - python \ - ruby \ - sed \ - unzip \ - wget \ - xz-utils \ - lzip \ - scons - -# copy over the partial MXE settings (simply missing QtWebKit so we don't exceed the 6h limit) -RUN mkdir -p /win -ADD settings-stage1.mk /win/settings.mk -RUN cd /win ; git clone git://github.com/mxe/mxe ; \ - cd mxe ; \ - git checkout ${_ver} ; -# Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another -# backend is not picked -ADD qtconnectivity-1.patch /win/qtconnectivity-1.patch -RUN mv /win/qtconnectivity-1.patch /win/mxe/src -RUN mv /win/settings.mk /win/mxe -RUN cd /win/mxe ; \ - make -j 6 2>&1 | tee build.log ; -RUN cd /win/mxe ; \ - make MXE_TARGETS=i686-w64-mingw32.static glib -j 6 2>&1 | tee -a build.log ; diff --git a/scripts/docker/mxe-build-container/Dockerfile-stage2 b/scripts/docker/mxe-build-container/Dockerfile-stage2 deleted file mode 100644 index d8760f723..000000000 --- a/scripts/docker/mxe-build-container/Dockerfile-stage2 +++ /dev/null @@ -1,13 +0,0 @@ -ARG VERSION - -# Start from the container that we created in stage 1 -From subsurface/mxe-build-container:$VERSION - -# this time the settings include QtWebKit - since everything else was built -# during the first stage, QtWebKit will be the only thing that gets built -ADD settings-stage2.mk /win/mxe/settings.mk -RUN cd /win/mxe ; \ - make -j 2 2>&1 | tee build.log ; -RUN cd /win/mxe ; \ - make MXE_TARGETS=i686-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a build.log ; - diff --git a/scripts/docker/mxe-build-container/build-container.sh b/scripts/docker/mxe-build-container/build-container.sh index 1f5e1eddc..c1ab1c65f 100644 --- a/scripts/docker/mxe-build-container/build-container.sh +++ b/scripts/docker/mxe-build-container/build-container.sh @@ -2,10 +2,13 @@ set -x set -e +# known good MXE sha +MXE_SHA="8966a64" SCRIPTPATH=$(dirname $0) -export VERSION=1.1 +# version of the docker image +VERSION=2.0 + pushd $SCRIPTPATH -docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=1ee37f8 -f Dockerfile-stage1 . -docker build -t subsurface/mxe-build-container:$VERSION --build-arg=VERSION=$VERSION -f Dockerfile-stage2 . +docker build --squash -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=$MXE_SHA -f Dockerfile . popd diff --git a/scripts/docker/mxe-build-container/instructions.md b/scripts/docker/mxe-build-container/instructions.md index 2a860ac33..c78edb790 100644 --- a/scripts/docker/mxe-build-container/instructions.md +++ b/scripts/docker/mxe-build-container/instructions.md @@ -2,16 +2,14 @@ This document assumes you have alreay installed docker and have checked out subsurface according to the instructions in the INSTALL document. -If you are just wantint to build with the current mxe build container then starting from the folder above subsurface run +If you just want to build with the current mxe build container then starting from the folder above subsurface run ```bash -docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.x /bin/sleep 60m +docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:x.y /bin/sleep 60m ``` -replacing the x in the mxe-build-container tag with the current version e.g. -```bash -docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.0 /bin/sleep 60m -``` +replacing the x.y in the mxe-build-container tag with the current version e.g. 2.0 + Next you need to prep the container by installing some prerequisites ```bash diff --git a/scripts/docker/mxe-build-container/settings-stage1.mk b/scripts/docker/mxe-build-container/settings-stage1.mk deleted file mode 100644 index 5563f259a..000000000 --- a/scripts/docker/mxe-build-container/settings-stage1.mk +++ /dev/null @@ -1,15 +0,0 @@ -# This is a template of configuration file for MXE. See -# index.html for more extensive documentations. - -# This variable controls the number of compilation processes -# within one package ("intra-package parallelism"). -JOBS := 2 - -# This variable controls the targets that will build. -MXE_TARGETS := i686-w64-mingw32.shared - -# The three lines below makes `make` build these "local packages" instead of all packages. -LOCAL_PKG_LIST := qtbase qtconnectivity qtdeclarative qtimageformats qtlocation qtmultimedia qtquickcontrols qtquickcontrols2 qtscript qtsvg qttools qttranslations qtwebview libxml2 libxslt libusb1 libgit2 nsis curl libzip libftdi1 -.DEFAULT local-pkg-list: -local-pkg-list: $(LOCAL_PKG_LIST) - diff --git a/scripts/docker/mxe-build-container/settings-stage2.mk b/scripts/docker/mxe-build-container/settings-stage2.mk deleted file mode 100644 index 3d3ad3cc3..000000000 --- a/scripts/docker/mxe-build-container/settings-stage2.mk +++ /dev/null @@ -1,39 +0,0 @@ -# This is a template of configuration file for MXE. See -# index.html for more extensive documentations. - -# This variable controls the number of compilation processes -# within one package ("intra-package parallelism"). -JOBS := 2 - -# This variable controls the targets that will build. -MXE_TARGETS := i686-w64-mingw32.shared - -# The three lines below makes `make` build these "local packages" instead of all packages. -LOCAL_PKG_LIST := curl \ - hidapi \ - libftdi1 \ - libgit2 \ - libusb1 \ - libxml2 \ - libxslt \ - libzip \ - mdbtools \ - nsis \ - qtbase \ - qtconnectivity \ - qtdeclarative \ - qtimageformats \ - qtlocation \ - qtmultimedia \ - qtquickcontrols \ - qtquickcontrols2 \ - qtscript \ - qtsvg \ - qttools \ - qttranslations \ - qtwebkit \ - qtwebview \ - zstd -.DEFAULT local-pkg-list: -local-pkg-list: $(LOCAL_PKG_LIST) - diff --git a/scripts/docker/mxe-build-container/settings.mk b/scripts/docker/mxe-build-container/settings.mk new file mode 100644 index 000000000..fecac6e06 --- /dev/null +++ b/scripts/docker/mxe-build-container/settings.mk @@ -0,0 +1,44 @@ +# This is a template of configuration file for MXE. See +# index.html for more extensive documentations. + +# This variable controls the number of compilation processes +# within one package ("intra-package parallelism"). +JOBS := 8 + +# This variable controls the targets that will build. +MXE_TARGETS := x86_64-w64-mingw32.shared + +# The three lines below makes `make` build these "local packages" instead of all packages. +# The ordering of the list appears weird, but this seems to help to get the build done +# faster on a massively parallel machine to get some of the bottleneck packages built as +# early as possible +LOCAL_PKG_LIST := gcc \ + openssl \ + libmysqlclient \ + postgresql \ + qtbase \ + qtwebkit \ + nsis \ + curl \ + libxml2 \ + libxslt \ + libzip \ + libusb1 \ + hidapi \ + libgit2 \ + libftdi1 \ + mdbtools \ + qtconnectivity \ + qtdeclarative \ + qtimageformats \ + qtlocation \ + qtmultimedia \ + qtquickcontrols \ + qtquickcontrols2 \ + qtcharts \ + qtsvg \ + qttools \ + qttranslations \ + zstd +.DEFAULT local-pkg-list: +local-pkg-list: $(LOCAL_PKG_LIST) -- cgit v1.2.3-70-g09d2