Noir Electrum 3.3.7

.gitignore

@@ -4,7 +4,6 @@
 build/
 dist/
 *.egg/
-contrib/pyinstaller/
 Electrum.egg-info/
 electrum/locale/
 .devlocaltmp/
@@ -25,3 +24,10 @@ electrum/gui/kivy/theming/light.atlas
 .cache/
 .coverage
 .pytest_cache
+
+# build workspaces
+contrib/build-wine/tmp/
+contrib/build-wine/fresh_clone/
+contrib/build-linux/appimage/build/
+contrib/build-linux/appimage/.cache/
+contrib/android_debug.keystore

README.rst

@@ -75,20 +75,26 @@ Compile the protobuf description file::
 Create translations (optional)::
 
     sudo apt-get install python-requests gettext
-    ./contrib/make_locale
+    ./contrib/pull_locale
 
 
 
 
 Creating Binaries
 =================
 
-Linux
------
+Linux (tarball)
+---------------
 
 See :code:`contrib/build-linux/README.md`.
 
 
+Linux (AppImage)
+----------------
+
+See :code:`contrib/build-linux/appimage/README.md`.
+
+
 Mac OS X / macOS
 ----------------
 

RELEASE-NOTES

@@ -1,3 +1,35 @@
+# Release 3.3.7 - (July 3, 2019)
+
+ * The AppImage Linux x86_64 binary and the Windows setup.exe
+   (so now all Windows binaries) are now built reproducibly.
+ * Bump fee (RBF) improvements:
+   Implemented a new fee-bump strategy that can add new inputs,
+   so now any tx can be fee-bumped (d0a4366). The old strategy
+   was to decrease the value of outputs (starting with change).
+   We will now try the new strategy first, and only use the old
+   as a fallback (needed e.g. when spending "Max").
+ * CoinChooser improvements:
+   - more likely to construct txs without change (when possible)
+   - less likely to construct txs with really small change (e864fa5)
+   - will now only spend negative effective value coins when
+     beneficial for privacy (cb69aa8)
+ * fix long-standing bug that broke wallets with >65k addresses (#5366)
+ * Windows binaries: we now build the PyInstaller boot loader ourselves,
+   as this seems to reduce anti-virus false positives (1d0f679)
+ * Android: (fix) BIP70 payment requests could not be paid (#5376)
+ * Android: allow copy-pasting partial transactions from/to clipboard
+ * Fix a performance regression for large wallets (c6a54f0)
+ * Qt: fix some high DPI issues related to text fields (37809be)
+ * Trezor:
+   - allow bypassing "too old firmware" error (#5391)
+   - use only the Bridge to scan devices if it is available (#5420)
+ * hw wallets: (known issue) on Win10-1903, some hw devices
+   (that also have U2F functionality) can only be detected with
+   Administrator privileges. (see #5420 and #5437)
+   A workaround is to run as Admin, or for Trezor to install the Bridge.
+ * Several other minor bugfixes and usability improvements.
+
+
 # Release 3.3.6 - (May 16, 2019)
 
  * qt: fix crash during 2FA wallet creation (#5334)

contrib/build-linux/README.md

@@ -1,19 +1,15 @@
 Source tarballs
 ===============
 
-1. Build locale files
+✗ _This script does not produce reproducible output (yet!)._
 
-    ```
-    contrib/make_locale
-    ```
-
-2. Prepare python dependencies used by Electrum.
+1. Prepare python dependencies used by Electrum.
 
     ```
     contrib/make_packages
     ```
 
-3. Create source tarball.
+2. Create source tarball.
 
     ```
     contrib/make_tgz

contrib/build-linux/appimage/README.md

@@ -1,9 +1,17 @@
 AppImage binary for Electrum
 ============================
 
+✓ _This binary should be reproducible, meaning you should be able to generate
+   binaries that match the official releases._
+
 This assumes an Ubuntu host, but it should not be too hard to adapt to another
-similar system. The docker commands should be executed in the project's root
-folder.
+similar system. The host architecture should be x86_64 (amd64).
+The docker commands should be executed in the project's root folder.
+
+We currently only build a single AppImage, for x86_64 architecture.
+Help to adapt these scripts to build for (some flavor of) ARM would be welcome,
+see [issue #5159](https://github.com/spesmilo/electrum/issues/5159).
+
 
 1. Install Docker
 

contrib/build-linux/appimage/build.sh

@@ -4,15 +4,17 @@ set -e
 
 PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
 CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage"
 DISTDIR="$PROJECT_ROOT/dist"
-BUILDDIR="$CONTRIB/build-linux/appimage/build/appimage"
+BUILDDIR="$CONTRIB_APPIMAGE/build/appimage"
 APPDIR="$BUILDDIR/electrum.AppDir"
-CACHEDIR="$CONTRIB/build-linux/appimage/.cache/appimage"
+CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 
 # pinned versions
 PYTHON_VERSION=3.6.8
-PKG2APPIMAGE_COMMIT="83483c2971fcaa1cb0c1253acd6c731ef8404381"
+PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
 LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
 
 VERSION=`git describe --tags --dirty --always`
@@ -27,10 +29,10 @@ mkdir -p "$APPDIR" "$CACHEDIR" "$DISTDIR"
 
 info "downloading some dependencies."
 download_if_not_exist "$CACHEDIR/functions.sh" "https://raw.githubusercontent.com/AppImage/pkg2appimage/$PKG2APPIMAGE_COMMIT/functions.sh"
-verify_hash "$CACHEDIR/functions.sh" "a73a21a6c1d1e15c0a9f47f017ae833873d1dc6aa74a4c840c0b901bf1dcf09c"
+verify_hash "$CACHEDIR/functions.sh" "78b7ee5a04ffb84ee1c93f0cb2900123773bc6709e5d1e43c37519f590f86918"
 
-download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/probonopd/AppImageKit/releases/download/11/appimagetool-x86_64.AppImage"
-verify_hash "$CACHEDIR/appimagetool" "c13026b9ebaa20a17e7e0a4c818a901f0faba759801d8ceab3bb6007dde00372"
+download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage"
+verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13"
 
 download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz"
 verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "35446241e995773b1bed7d196f4b624dadcadc8429f26282e756b2fb8a351193"
@@ -42,16 +44,36 @@ tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR"
 (
     cd "$BUILDDIR/Python-$PYTHON_VERSION"
     export SOURCE_DATE_EPOCH=1530212462
-    TZ=UTC faketime -f '2019-01-01 01:01:01' ./configure \
+    LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y")
+    LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S")
+    # Patch taken from Ubuntu python3.6_3.6.8-1~18.04.1.debian.tar.xz
+    patch -p1 < "$CONTRIB_APPIMAGE/patches/python-3.6.8-reproducible-buildinfo.diff"
+    ./configure \
       --cache-file="$CACHEDIR/python.config.cache" \
       --prefix="$APPDIR/usr" \
       --enable-ipv6 \
       --enable-shared \
       --with-threads \
       -q
-    TZ=UTC faketime -f '2019-01-01 01:01:01' make -s
-    make -s install > /dev/null
+    make -j4 -s || fail "Could not build Python"
+    make -s install > /dev/null || fail "Could not install Python"
+    # When building in docker on macOS, python builds with .exe extension because the
+    # case insensitive file system of macOS leaks into docker. This causes the build
+    # to result in a different output on macOS compared to Linux. We simply patch
+    # sysconfigdata to remove the extension.
+    # Some more info: https://bugs.python.org/issue27631
+    sed -i -e 's/\.exe//g' "$APPDIR"/usr/lib/python3.6/_sysconfigdata*
+)
+
+
+info "Building squashfskit"
+git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit"
+(
+    cd "$BUILDDIR/squashfskit"
+    git checkout "$SQUASHFSKIT_COMMIT"
+    make -C squashfs-tools mksquashfs || fail "Could not build squashfskit"
 )
+MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs"
 
 
 info "building libsecp256k1."
@@ -71,8 +93,8 @@ info "building libsecp256k1."
       --enable-module-ecdh \
       --disable-jni \
       -q
-    make -s
-    make -s install > /dev/null
+    make -j4 -s || fail "Could not build libsecp"
+    make -s install > /dev/null || fail "Could not install libsecp"
 )
 
 
@@ -97,8 +119,7 @@ info "preparing electrum-locale."
 
     pushd "$CONTRIB"/deterministic-build/electrum-locale
     if ! which msgfmt > /dev/null 2>&1; then
-        echo "Please install gettext"
-        exit 1
+        fail "Please install gettext"
     fi
     for i in ./locale/*; do
         dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES"
@@ -127,7 +148,7 @@ cp "$PROJECT_ROOT/electrum/gui/icons/electrum.png" "$APPDIR/electrum.png"
 
 
 # add launcher
-cp "$CONTRIB/build-linux/appimage/apprun.sh" "$APPDIR/AppRun"
+cp "$CONTRIB_APPIMAGE/apprun.sh" "$APPDIR/AppRun"
 
 info "finalizing AppDir."
 (
@@ -148,7 +169,7 @@ info "finalizing AppDir."
     mv usr/include usr/include.tmp
     delete_blacklisted
     mv usr/include.tmp usr/include
-)
+) || fail "Could not finalize AppDir"
 
 # copy libusb here because it is on the AppImage excludelist and it can cause problems if we use system libusb
 info "Copying libusb"
@@ -175,23 +196,33 @@ remove_emptydirs
 
 
 info "removing some unneeded stuff to decrease binary size."
-rm -rf "$APPDIR"/usr/lib/python3.6/test
-rm -rf "$APPDIR"/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu
-rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/translations/qtwebengine_locales
-rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/resources/qtwebengine_*
-rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/qml
-for component in Web Designer Qml Quick Location Test Xml ; do
-    rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/lib/libQt5${component}*
-    rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt${component}*
+rm -rf "$APPDIR"/usr/{share,include}
+PYDIR="$APPDIR"/usr/lib/python3.6
+rm -rf "$PYDIR"/{test,ensurepip,lib2to3,idlelib,turtledemo}
+rm -rf "$PYDIR"/{ctypes,sqlite3,tkinter,unittest}/test
+rm -rf "$PYDIR"/distutils/{command,tests}
+rm -rf "$PYDIR"/config-3.6m-x86_64-linux-gnu
+rm -rf "$PYDIR"/site-packages/{opt,pip,setuptools,wheel}
+rm -rf "$PYDIR"/site-packages/Cryptodome/SelfTest
+rm -rf "$PYDIR"/site-packages/{psutil,qrcode,websocket}/tests
+for component in connectivity declarative help location multimedia quickcontrols2 serialport webengine websockets xmlpatterns ; do
+  rm -rf "$PYDIR"/site-packages/PyQt5/Qt/translations/qt${component}_*
+  rm -rf "$PYDIR"/site-packages/PyQt5/Qt/resources/qt${component}_*
 done
-rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt.so
-
+rm -rf "$PYDIR"/site-packages/PyQt5/Qt/{qml,libexec}
+rm -rf "$PYDIR"/site-packages/PyQt5/{pyrcc.so,pylupdate.so,uic}
+rm -rf "$PYDIR"/site-packages/PyQt5/Qt/plugins/{bearer,gamepads,geometryloaders,geoservices,playlistformats,position,renderplugins,sceneparsers,sensors,sqldrivers,texttospeech,webview}
+for component in Bluetooth Concurrent Designer Help Location NetworkAuth Nfc Positioning PositioningQuick Qml Quick Sensors SerialPort Sql Test Web Xml ; do
+    rm -rf "$PYDIR"/site-packages/PyQt5/Qt/lib/libQt5${component}*
+    rm -rf "$PYDIR"/site-packages/PyQt5/Qt${component}*
+done
+rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so
 
 # these are deleted as they were not deterministic; and are not needed anyway
 find "$APPDIR" -path '*/__pycache__*' -delete
 rm "$APPDIR"/usr/lib/libsecp256k1.a
-rm "$APPDIR"/usr/lib/python3.6/site-packages/pyblake2-*.dist-info/RECORD
-rm "$APPDIR"/usr/lib/python3.6/site-packages/hidapi-*.dist-info/RECORD
+rm -rf "$PYDIR"/site-packages/*.dist-info/
+rm -rf "$PYDIR"/site-packages/*.egg-info/
 
 
 find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
@@ -202,7 +233,14 @@ info "creating the AppImage."
     cd "$BUILDDIR"
     chmod +x "$CACHEDIR/appimagetool"
     "$CACHEDIR/appimagetool" --appimage-extract
-    env VERSION="$VERSION" ARCH=x86_64 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE"
+    # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option
+    # that mksquashfs from squashfskit does not support. It is not needed for squashfskit.
+    cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF
+#!/bin/sh
+args=\$(echo "\$@" | sed -e 's/-mkfs-fixed-time 0//')
+"$MKSQUASHFS" \$args
+EOF
+    env VERSION="$VERSION" ARCH=x86_64 SOURCE_DATE_EPOCH=1530212462 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE"
 )
 
 

contrib/build-linux/appimage/patches/python-3.6.8-reproducible-buildinfo.diff

@@ -0,0 +1,13 @@
+# DP: Build getbuildinfo.o with DATE/TIME values when defined
+
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -741,6 +741,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
+ 	      -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
+ 	      -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
+ 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
++	      $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \
++	      $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \
+ 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
+
+ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile

contrib/build-wine/README.md

@@ -1,10 +1,10 @@
-Deterministic Windows binaries with Docker
-==========================================
+Windows binaries
+================
 
-Produced binaries are deterministic, so you should be able to generate
-binaries that match the official releases.
+✓ _These binaries should be reproducible, meaning you should be able to generate
+   binaries that match the official releases._
 
-This assumes an Ubuntu host, but it should not be too hard to adapt to another
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
 similar system. The docker commands should be executed in the project's root
 folder.
 
@@ -54,9 +54,6 @@ folder.
 
 
 
-Note: the `setup` binary (NSIS installer) is not deterministic yet.
-
-
 Code Signing
 ============