diff --git a/.gitignore b/.gitignore index f0e9ad03..f5985cac 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ build/ dist/ *.egg/ -contrib/pyinstaller/ Electrum.egg-info/ electrum/locale/ .devlocaltmp/ @@ -25,3 +24,10 @@ electrum/gui/kivy/theming/light.atlas .cache/ .coverage .pytest_cache + +# build workspaces +contrib/build-wine/tmp/ +contrib/build-wine/fresh_clone/ +contrib/build-linux/appimage/build/ +contrib/build-linux/appimage/.cache/ +contrib/android_debug.keystore diff --git a/README.rst b/README.rst index 17539bd9..789bb36c 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,7 @@ Compile the protobuf description file:: Create translations (optional):: sudo apt-get install python-requests gettext - ./contrib/make_locale + ./contrib/pull_locale @@ -83,12 +83,18 @@ Create translations (optional):: Creating Binaries ================= -Linux ------ +Linux (tarball) +--------------- See :code:`contrib/build-linux/README.md`. +Linux (AppImage) +---------------- + +See :code:`contrib/build-linux/appimage/README.md`. + + Mac OS X / macOS ---------------- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index f5a4ee32..6bbdefd5 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,3 +1,35 @@ +# Release 3.3.7 - (July 3, 2019) + + * The AppImage Linux x86_64 binary and the Windows setup.exe + (so now all Windows binaries) are now built reproducibly. + * Bump fee (RBF) improvements: + Implemented a new fee-bump strategy that can add new inputs, + so now any tx can be fee-bumped (d0a4366). The old strategy + was to decrease the value of outputs (starting with change). + We will now try the new strategy first, and only use the old + as a fallback (needed e.g. when spending "Max"). + * CoinChooser improvements: + - more likely to construct txs without change (when possible) + - less likely to construct txs with really small change (e864fa5) + - will now only spend negative effective value coins when + beneficial for privacy (cb69aa8) + * fix long-standing bug that broke wallets with >65k addresses (#5366) + * Windows binaries: we now build the PyInstaller boot loader ourselves, + as this seems to reduce anti-virus false positives (1d0f679) + * Android: (fix) BIP70 payment requests could not be paid (#5376) + * Android: allow copy-pasting partial transactions from/to clipboard + * Fix a performance regression for large wallets (c6a54f0) + * Qt: fix some high DPI issues related to text fields (37809be) + * Trezor: + - allow bypassing "too old firmware" error (#5391) + - use only the Bridge to scan devices if it is available (#5420) + * hw wallets: (known issue) on Win10-1903, some hw devices + (that also have U2F functionality) can only be detected with + Administrator privileges. (see #5420 and #5437) + A workaround is to run as Admin, or for Trezor to install the Bridge. + * Several other minor bugfixes and usability improvements. + + # Release 3.3.6 - (May 16, 2019) * qt: fix crash during 2FA wallet creation (#5334) diff --git a/contrib/build-linux/README.md b/contrib/build-linux/README.md index 8d45038c..2bbe4c32 100644 --- a/contrib/build-linux/README.md +++ b/contrib/build-linux/README.md @@ -1,19 +1,15 @@ Source tarballs =============== -1. Build locale files +✗ _This script does not produce reproducible output (yet!)._ - ``` - contrib/make_locale - ``` - -2. Prepare python dependencies used by Electrum. +1. Prepare python dependencies used by Electrum. ``` contrib/make_packages ``` -3. Create source tarball. +2. Create source tarball. ``` contrib/make_tgz diff --git a/contrib/build-linux/appimage/README.md b/contrib/build-linux/appimage/README.md index 747c84c5..6f6be280 100644 --- a/contrib/build-linux/appimage/README.md +++ b/contrib/build-linux/appimage/README.md @@ -1,9 +1,17 @@ AppImage binary for Electrum ============================ +✓ _This binary should be reproducible, meaning you should be able to generate + binaries that match the official releases._ + This assumes an Ubuntu host, but it should not be too hard to adapt to another -similar system. The docker commands should be executed in the project's root -folder. +similar system. The host architecture should be x86_64 (amd64). +The docker commands should be executed in the project's root folder. + +We currently only build a single AppImage, for x86_64 architecture. +Help to adapt these scripts to build for (some flavor of) ARM would be welcome, +see [issue #5159](https://github.com/spesmilo/electrum/issues/5159). + 1. Install Docker diff --git a/contrib/build-linux/appimage/build.sh b/contrib/build-linux/appimage/build.sh index 959ef4c9..64ff0938 100755 --- a/contrib/build-linux/appimage/build.sh +++ b/contrib/build-linux/appimage/build.sh @@ -4,15 +4,17 @@ set -e PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage" DISTDIR="$PROJECT_ROOT/dist" -BUILDDIR="$CONTRIB/build-linux/appimage/build/appimage" +BUILDDIR="$CONTRIB_APPIMAGE/build/appimage" APPDIR="$BUILDDIR/electrum.AppDir" -CACHEDIR="$CONTRIB/build-linux/appimage/.cache/appimage" +CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage" # pinned versions PYTHON_VERSION=3.6.8 -PKG2APPIMAGE_COMMIT="83483c2971fcaa1cb0c1253acd6c731ef8404381" +PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15" LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7" +SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386" VERSION=`git describe --tags --dirty --always` @@ -27,10 +29,10 @@ mkdir -p "$APPDIR" "$CACHEDIR" "$DISTDIR" info "downloading some dependencies." download_if_not_exist "$CACHEDIR/functions.sh" "https://raw.githubusercontent.com/AppImage/pkg2appimage/$PKG2APPIMAGE_COMMIT/functions.sh" -verify_hash "$CACHEDIR/functions.sh" "a73a21a6c1d1e15c0a9f47f017ae833873d1dc6aa74a4c840c0b901bf1dcf09c" +verify_hash "$CACHEDIR/functions.sh" "78b7ee5a04ffb84ee1c93f0cb2900123773bc6709e5d1e43c37519f590f86918" -download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/probonopd/AppImageKit/releases/download/11/appimagetool-x86_64.AppImage" -verify_hash "$CACHEDIR/appimagetool" "c13026b9ebaa20a17e7e0a4c818a901f0faba759801d8ceab3bb6007dde00372" +download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppImageKit/releases/download/12/appimagetool-x86_64.AppImage" +verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13" download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz" verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "35446241e995773b1bed7d196f4b624dadcadc8429f26282e756b2fb8a351193" @@ -42,16 +44,36 @@ tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR" ( cd "$BUILDDIR/Python-$PYTHON_VERSION" export SOURCE_DATE_EPOCH=1530212462 - TZ=UTC faketime -f '2019-01-01 01:01:01' ./configure \ + LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y") + LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S") + # Patch taken from Ubuntu python3.6_3.6.8-1~18.04.1.debian.tar.xz + patch -p1 < "$CONTRIB_APPIMAGE/patches/python-3.6.8-reproducible-buildinfo.diff" + ./configure \ --cache-file="$CACHEDIR/python.config.cache" \ --prefix="$APPDIR/usr" \ --enable-ipv6 \ --enable-shared \ --with-threads \ -q - TZ=UTC faketime -f '2019-01-01 01:01:01' make -s - make -s install > /dev/null + make -j4 -s || fail "Could not build Python" + make -s install > /dev/null || fail "Could not install Python" + # When building in docker on macOS, python builds with .exe extension because the + # case insensitive file system of macOS leaks into docker. This causes the build + # to result in a different output on macOS compared to Linux. We simply patch + # sysconfigdata to remove the extension. + # Some more info: https://bugs.python.org/issue27631 + sed -i -e 's/\.exe//g' "$APPDIR"/usr/lib/python3.6/_sysconfigdata* +) + + +info "Building squashfskit" +git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit" +( + cd "$BUILDDIR/squashfskit" + git checkout "$SQUASHFSKIT_COMMIT" + make -C squashfs-tools mksquashfs || fail "Could not build squashfskit" ) +MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs" info "building libsecp256k1." @@ -71,8 +93,8 @@ info "building libsecp256k1." --enable-module-ecdh \ --disable-jni \ -q - make -s - make -s install > /dev/null + make -j4 -s || fail "Could not build libsecp" + make -s install > /dev/null || fail "Could not install libsecp" ) @@ -97,8 +119,7 @@ info "preparing electrum-locale." pushd "$CONTRIB"/deterministic-build/electrum-locale if ! which msgfmt > /dev/null 2>&1; then - echo "Please install gettext" - exit 1 + fail "Please install gettext" fi for i in ./locale/*; do dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES" @@ -127,7 +148,7 @@ cp "$PROJECT_ROOT/electrum/gui/icons/electrum.png" "$APPDIR/electrum.png" # add launcher -cp "$CONTRIB/build-linux/appimage/apprun.sh" "$APPDIR/AppRun" +cp "$CONTRIB_APPIMAGE/apprun.sh" "$APPDIR/AppRun" info "finalizing AppDir." ( @@ -148,7 +169,7 @@ info "finalizing AppDir." mv usr/include usr/include.tmp delete_blacklisted mv usr/include.tmp usr/include -) +) || fail "Could not finalize AppDir" # copy libusb here because it is on the AppImage excludelist and it can cause problems if we use system libusb info "Copying libusb" @@ -175,23 +196,33 @@ remove_emptydirs info "removing some unneeded stuff to decrease binary size." -rm -rf "$APPDIR"/usr/lib/python3.6/test -rm -rf "$APPDIR"/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu -rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/translations/qtwebengine_locales -rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/resources/qtwebengine_* -rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/qml -for component in Web Designer Qml Quick Location Test Xml ; do - rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt/lib/libQt5${component}* - rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt${component}* +rm -rf "$APPDIR"/usr/{share,include} +PYDIR="$APPDIR"/usr/lib/python3.6 +rm -rf "$PYDIR"/{test,ensurepip,lib2to3,idlelib,turtledemo} +rm -rf "$PYDIR"/{ctypes,sqlite3,tkinter,unittest}/test +rm -rf "$PYDIR"/distutils/{command,tests} +rm -rf "$PYDIR"/config-3.6m-x86_64-linux-gnu +rm -rf "$PYDIR"/site-packages/{opt,pip,setuptools,wheel} +rm -rf "$PYDIR"/site-packages/Cryptodome/SelfTest +rm -rf "$PYDIR"/site-packages/{psutil,qrcode,websocket}/tests +for component in connectivity declarative help location multimedia quickcontrols2 serialport webengine websockets xmlpatterns ; do + rm -rf "$PYDIR"/site-packages/PyQt5/Qt/translations/qt${component}_* + rm -rf "$PYDIR"/site-packages/PyQt5/Qt/resources/qt${component}_* done -rm -rf "$APPDIR"/usr/lib/python3.6/site-packages/PyQt5/Qt.so - +rm -rf "$PYDIR"/site-packages/PyQt5/Qt/{qml,libexec} +rm -rf "$PYDIR"/site-packages/PyQt5/{pyrcc.so,pylupdate.so,uic} +rm -rf "$PYDIR"/site-packages/PyQt5/Qt/plugins/{bearer,gamepads,geometryloaders,geoservices,playlistformats,position,renderplugins,sceneparsers,sensors,sqldrivers,texttospeech,webview} +for component in Bluetooth Concurrent Designer Help Location NetworkAuth Nfc Positioning PositioningQuick Qml Quick Sensors SerialPort Sql Test Web Xml ; do + rm -rf "$PYDIR"/site-packages/PyQt5/Qt/lib/libQt5${component}* + rm -rf "$PYDIR"/site-packages/PyQt5/Qt${component}* +done +rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so # these are deleted as they were not deterministic; and are not needed anyway find "$APPDIR" -path '*/__pycache__*' -delete rm "$APPDIR"/usr/lib/libsecp256k1.a -rm "$APPDIR"/usr/lib/python3.6/site-packages/pyblake2-*.dist-info/RECORD -rm "$APPDIR"/usr/lib/python3.6/site-packages/hidapi-*.dist-info/RECORD +rm -rf "$PYDIR"/site-packages/*.dist-info/ +rm -rf "$PYDIR"/site-packages/*.egg-info/ find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + @@ -202,7 +233,14 @@ info "creating the AppImage." cd "$BUILDDIR" chmod +x "$CACHEDIR/appimagetool" "$CACHEDIR/appimagetool" --appimage-extract - env VERSION="$VERSION" ARCH=x86_64 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE" + # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option + # that mksquashfs from squashfskit does not support. It is not needed for squashfskit. + cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF +#!/bin/sh +args=\$(echo "\$@" | sed -e 's/-mkfs-fixed-time 0//') +"$MKSQUASHFS" \$args +EOF + env VERSION="$VERSION" ARCH=x86_64 SOURCE_DATE_EPOCH=1530212462 ./squashfs-root/AppRun --no-appstream --verbose "$APPDIR" "$APPIMAGE" ) diff --git a/contrib/build-linux/appimage/patches/python-3.6.8-reproducible-buildinfo.diff b/contrib/build-linux/appimage/patches/python-3.6.8-reproducible-buildinfo.diff new file mode 100644 index 00000000..38d6fbdc --- /dev/null +++ b/contrib/build-linux/appimage/patches/python-3.6.8-reproducible-buildinfo.diff @@ -0,0 +1,13 @@ +# DP: Build getbuildinfo.o with DATE/TIME values when defined + +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -741,6 +741,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ ++ $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \ ++ $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \ + -o $@ $(srcdir)/Modules/getbuildinfo.c + + Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile diff --git a/contrib/build-wine/README.md b/contrib/build-wine/README.md index a8cc19f1..491d81fb 100644 --- a/contrib/build-wine/README.md +++ b/contrib/build-wine/README.md @@ -1,10 +1,10 @@ -Deterministic Windows binaries with Docker -========================================== +Windows binaries +================ -Produced binaries are deterministic, so you should be able to generate -binaries that match the official releases. +✓ _These binaries should be reproducible, meaning you should be able to generate + binaries that match the official releases._ -This assumes an Ubuntu host, but it should not be too hard to adapt to another +This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another similar system. The docker commands should be executed in the project's root folder. @@ -54,9 +54,6 @@ folder. -Note: the `setup` binary (NSIS installer) is not deterministic yet. - - Code Signing ============ diff --git a/contrib/build-wine/build-electrum-git.sh b/contrib/build-wine/build-electrum-git.sh index 192e885f..a0803e0e 100755 --- a/contrib/build-wine/build-electrum-git.sh +++ b/contrib/build-wine/build-electrum-git.sh @@ -12,25 +12,23 @@ PYTHON="wine $PYHOME/python.exe -OO -B" # Let's begin! -cd `dirname $0` set -e -mkdir -p tmp -cd tmp +here="$(dirname "$(readlink -e "$0")")" -pushd $WINEPREFIX/drive_c/electrum +. "$CONTRIB"/build_tools_util.sh -# Load electrum-locale for this release -git submodule init -git submodule update +pushd $WINEPREFIX/drive_c/electrum VERSION=`git describe --tags --dirty --always` -echo "Last commit: $VERSION" +info "Last commit: $VERSION" + +# Load electrum-locale for this release +git submodule update --init pushd ./contrib/deterministic-build/electrum-locale if ! which msgfmt > /dev/null 2>&1; then - echo "Please install gettext" - exit 1 + fail "Please install gettext" fi for i in ./locale/*; do dir=$WINEPREFIX/drive_c/electrum/electrum/$i/LC_MESSAGES @@ -42,22 +40,23 @@ popd find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd -cp $WINEPREFIX/drive_c/electrum/LICENCE . # Install frozen dependencies -$PYTHON -m pip install -r ../../deterministic-build/requirements.txt +$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -$PYTHON -m pip install -r ../../deterministic-build/requirements-hw.txt +$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-hw.txt pushd $WINEPREFIX/drive_c/electrum +# see https://github.com/pypa/pip/issues/2195 -- pip makes a copy of the entire directory +info "Pip installing Noir Electrum. This might take a long time if the project folder is large." $PYTHON -m pip install . popd -cd .. rm -rf dist/ # build standalone and portable versions +info "Running pyinstaller..." wine "$PYHOME/scripts/pyinstaller.exe" --noconfirm --ascii --clean --name $NAME_ROOT-$VERSION -w deterministic.spec # set timestamps in dist, in order to make the installer reproducible @@ -65,7 +64,7 @@ pushd dist find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd -# build NSIS installer +info "building NSIS installer" # $VERSION could be passed to the electrum.nsi script, but this would require some rewriting in the script itself. wine "$WINEPREFIX/drive_c/Program Files (x86)/NSIS/makensis.exe" /DPRODUCT_VERSION=$VERSION electrum.nsi @@ -73,5 +72,4 @@ cd dist mv noir-electrum-setup.exe $NAME_ROOT-$VERSION-setup.exe cd .. -echo "Done." -sha256sum dist/electrum*exe +sha256sum dist/electrum*.exe diff --git a/contrib/build-wine/build-secp256k1.sh b/contrib/build-wine/build-secp256k1.sh index 4d137564..2ae60b90 100755 --- a/contrib/build-wine/build-secp256k1.sh +++ b/contrib/build-wine/build-secp256k1.sh @@ -3,6 +3,14 @@ set -e +here="$(dirname "$(readlink -e "$0")")" +LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7" + +. "$CONTRIB"/build_tools_util.sh + +info "building libsecp256k1..." + + build_dll() { #sudo apt-get install -y mingw-w64 export SOURCE_DATE_EPOCH=1530212462 @@ -14,28 +22,31 @@ build_dll() { --enable-experimental \ --enable-module-ecdh \ --disable-jni - make + make -j4 ${1}-strip .libs/libsecp256k1-0.dll } -cd /tmp/electrum-build +cd "$CACHEDIR" + +if [ -f "secp256k1/libsecp256k1.dll" ]; then + info "libsecp256k1.dll already built, skipping" + exit 0 +fi + if [ ! -d secp256k1 ]; then git clone https://github.com/bitcoin-core/secp256k1.git - cd secp256k1; -else - cd secp256k1 - git pull fi -LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7" -git reset --hard "$LIBSECP_VERSION" +cd secp256k1 +git reset --hard git clean -f -x -q +git checkout $LIBSECP_VERSION build_dll i686-w64-mingw32 # 64-bit would be: x86_64-w64-mingw32 mv .libs/libsecp256k1-0.dll libsecp256k1.dll find -exec touch -d '2000-11-11T11:11:11+00:00' {} + -echo "building libsecp256k1 finished" +info "building libsecp256k1 finished" diff --git a/contrib/build-wine/build.sh b/contrib/build-wine/build.sh index 01ca071f..8b79402f 100755 --- a/contrib/build-wine/build.sh +++ b/contrib/build-wine/build.sh @@ -1,28 +1,36 @@ #!/bin/bash + +set -e + # Lucky number export PYTHONHASHSEED=22 -here=$(dirname "$0") +here="$(dirname "$(readlink -e "$0")")" test -n "$here" -a -d "$here" || exit -echo "Clearing $here/build and $here/dist..." +export CONTRIB="$here/.." +export CACHEDIR="$here/.cache" +export PIP_CACHE_DIR="$CACHEDIR/pip_cache" + +. "$CONTRIB"/build_tools_util.sh + +info "Clearing $here/build and $here/dist..." rm "$here"/build/* -rf rm "$here"/dist/* -rf -mkdir -p /tmp/electrum-build -mkdir -p /tmp/electrum-build/pip-cache -export PIP_CACHE_DIR="/tmp/electrum-build/pip-cache" +mkdir -p "$CACHEDIR" "$PIP_CACHE_DIR" -$here/build-secp256k1.sh || exit 1 +$here/build-secp256k1.sh || fail "build-secp256k1 failed" -$here/prepare-wine.sh || exit 1 +$here/prepare-wine.sh || fail "prepare-wine failed" -echo "Resetting modification time in C:\Python..." +info "Resetting modification time in C:\Python..." # (Because of some bugs in pyinstaller) pushd /opt/wine64/drive_c/python* find -exec touch -d '2000-11-11T11:11:11+00:00' {} + popd ls -l /opt/wine64/drive_c/python* -$here/build-electrum-git.sh && \ -echo "Done." +$here/build-electrum-git.sh || fail "build-electrum-git failed" + +info "Done." diff --git a/contrib/build-wine/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc b/contrib/build-wine/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc new file mode 100644 index 00000000..a87dbe18 --- /dev/null +++ b/contrib/build-wine/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc @@ -0,0 +1,108 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: User-ID: Steve Dower (Python Release Signing) +Comment: Created: 2015-04-06 02:32 +Comment: Type: 4096-bit RSA +Comment: Usage: Signing, Encryption, Certifying User-IDs +Comment: Fingerprint: 7ED10B6531D7C8E1BC296021FC624643487034E5 + + +mQINBFUh1AUBEACdUPt6PwJVO23zGZqgtgBeA9JsO22dk3CMzrwPJdUmMd6mcRWa +vl4BoAba66fuC17GvOgGXimKI+iaw5Vt9QI3uSjUjFSfc24J8T7NB/yAr/0zEcex +raHD2dxT/JpE/iY0yWHxRlitvwGSw1Qlq3NnY8tDI1DJEJD+gBuCktvVvu1FfQTw +6bd+aEq0c4sWJHAOnKLuLH0pNFOznnynAFGPGBBsm/YwYc5BP2JVvka775LUjA+W +1h2Sgg3FAUPIm64pc4Pq6mUo6Tulw72xsWMpCL1/5atXNPXT6rJUOB8euTcNMr4l +1O6GKSsiLeLAuvq4bmhOKtLzjWzXnY1gDVoOfdgpD6o4ZHk4xiVsdVE8hCa/ylz8 +1ZwRW2gGo2jP8t3hKciR2i+Qs+6lPNZpeFIxa6Uo9ER1IBgCHHapIR/UdcOFyoS0 +MNn7Ui7DLQNM4gI/G17eG9tfvjW2dl4SgFSYWMq/OtXnPDUBGqFUWsn8adOL2PFL +B7kM5ZRTPc5SnY9hoSGa5E20rJZIXcpy1aygRz/xUjoKwNzAySSEyyIorUxZ8KaH +EEBQSsqwe04MXIENqnDozH0/cvP4JXEDSl8EkzMSCWSoavQSIYD5pQppyFQpGHqa +5CuOA25Ja+sgp2xqahtr3fEqZUknPQSoYlnJbaHnzsGSlRAVWMsklsZibQARAQAB +tEBTdGV2ZSBEb3dlciAoUHl0aG9uIFJlbGVhc2UgU2lnbmluZykgPHN0ZXZlLmRv +d2VyQG1pY3Jvc29mdC5jb20+iQEcBBABCAAGBQJYsBphAAoJEEhSKohZ29goZggI +ALKlgyoecD5v3ulh1eoctRqtCOxkAoENEfPt3l5x6N8Wq89yHzf10T1rVioEXOHh +Di1m37DDoQmRJD0sOYQymq10xDGRYAJjyOf3X0pvRkZ+F7T0U4dSV3DasLIHcN26 +kRwv1yCYsf0QvhgT6EJZKyUNHtV9qrb9u3A1Zp6epC/EyT8zMZj+21GzTUrnbnug +3Ak9p7+APCZS4Ahh9ZHFuD38MZ7+OwrUd6ot+6cbb1nnQLSAGQOHSp6EP6ktrnsK +zts0L+tzHurxtJgUkR01imJuSFfYpLoZa/L7qXNyEpEUTC/SWzRWD9y2QkM7DLzX +caReVAyJr9rix1lDQbEFIquJAhwEEAEIAAYFAlW2TwMACgkQKeBHm5nIo5fahg/+ +IQSSE/yH8Cf82PYI7IGqDVNwRw2o7dq8iscB+fhFHfFFhXANwUUFpzPeDMrMrdmq +Bke7Vg1D3bIFocXYOiNwf2J7f4mBO6OL0VAvDX02Vyh/C2ZSc15uZyU6CWFQMCG8 +JOSmgQFs3kMHkL4qtut1Y5reoYesmteIe06UVyRw8yT1R1BkxP2whZ97qwsvUUE9 +cVD08wCvH486efw7EswIzYGa1KcZXji0MvjXfksVtkEQQbxMMI7SVXo0345ZReww +buioGL5gvvAPObgU43skORanFHFxiHEKmqgHBHXK/LKqaFUFMKcb4iFTNs2XKrhE +XsEi5EMI1AFsJzjcXRqT50Wi2cZhXeRc70uF6gzqrdWvowa2oOPiO6zGDiTqZCW1 +AArk/QBzGtPjVh+nKEdHwnvpK9913UAkAN682h8QkoVPYXOvIKDYZRBr5EfpUyQt +y2r9MYewz0YN4zlGP1PFS9FxncdSZiZJqQVif0CkOp1tdSxLynHcujQgATZNtgcu +X9JwUwPp60MurgOcIZiW3nZw/z/5vzBBadSa9/TIFSJAFNBlqeKdIGQuik0UH+Cz +RRtSFb38F7jMPwr0QUSktuntQ0HWuvNqj4N8DFm45/n5rN190eRotrVDXZmjGein +qWPITuICslGIKAp+Q6y3t7JA71MIbeu/ZY6ZcftOka6JAhwEEAEIAAYFAlZRWicA +CgkQxiNM8COVzQq5bRAAktnXceO3GCivMt9yR1Qr0Ov4A4Q+CJSIL45efLFmS30k +cbkHHtaq+0FZNh2ZaMartC16MUja4a2OUejg53VBhaSVkQrVk/6M/HA6/o6CvIhb +FW/5C+nRWBd5gfvwsWvjrtC3cKZco4wg+yYclkDbSH+2EPDZOKIHpBy46YTz9WQ1 +8SJ51WVkNUNiZqRBA6Ny5GFoyd6EpWZYEPelmzNemv3zOrQdVzLV24/mLejcLL2t +KmI6ngX4XViXUCRUU3MH8/V+V2YTQGcTM/6HGaHpN0LTqknf6zEto9q9FiRTaiU2 +kzExhBq8Qf+cVqwm+1kMt0FGOgpT47VBWMeUWq62gQ3h5NfAs4DfriLgNURlTC1d +JYAEquFhB/8oBQD1h/d9CjQyk88iib2pJInRBDsK2FcfQBap9iaeBFYoBWTzMQJx +g+RuWK1wIm2n0oqa5urBYZtRHE5RIdDP8ZLogrBOFkfXGJxlRBQD1Gab77qohdp0 +SnErGw4Ne3gJH/SNhK+zzHkHERIrRZCR95zdYkKfZ2jyOPzSuABVRigEQVQPCDn0 +hbv3cblTCeJYwG2mfRdmfyqSMALKIgXe9yvJ2kl8QgaVOsJjNfQzIKeoHFPIm5Uw +3YB6jgDFc5uzEaH7WSz74A7KhGYjC7huw2TugosHbWxphJKddwxfK1WujYaAeJyJ +AhwEEAEIAAYFAlf2sPIACgkQfb+tds3soNuXEQ//XkWYHmJsKyeDZC8MFU+/vsVq +dhnFs6UXZkvf7MoNFkuMDL+zgVoMpFHftTdyBqNAoEnndakk212jK8YWF8g4kQXI +a9uMRqJLM4mqCl9yco/twJ9z9EMA+JLSXYK0ZbTkLdutSDZEDKgpHbmekx2C1OsW +lRLs9PahF5PAZQs0N+m+LJBnw6bEHOSTv4OE5uVUf9nvdes3OARvkGSEGURNmUaF +chxWtZ/SF1q9Jfj0K/xgs9Gt855oueveRXLIGpjiEVoKH/drsgyKFMJVrpZDDgS4 +GVXG8bq3GTFiMAs7BPPd9bjI+jgvqttgItZcYsW/IQK1BIoG6Fere4cPvu+IshCc +km9T8nOK98tZuov8hLbND9mW2d7LChJI1r/HbzbKIl0k6OigdFMrJlun2zmtDxT9 +Tp3uxOYSaW2YggcpNUjI28tv6AwoA8okVY93LWjO5kdZGkbliRnf/eJy7NJYn0LO +ogsvMUJClRAGnZTHLEr32Whq0MImlXa43kr6oPJT5dwXXyw5ELstEQztczCd1PYB +kbQHUpD5j3PwgNVOinCnbd4pc/qVtYSqpg2g6TJi1XiJ1638jhn2k+i8wop/dyet +iN8lGR76twYGex9AavEAUpVR9r6qfpp4KBibEhdvL6o2O03RQu17GcRzXSAYzmUi +5U5jZ3dBz5MYUjgUZM+JAhwEEAEKAAYFAllTh9gACgkQXLNh5VL7DRAk7Q//X8eU +hwEvl/d9Sv2kBNCZFjAW3QmZp2L/sxhScJZXrOFzKUdmjap9Xlul1qr6/Wif7YLK +bOdNUI7KziEBn+9SEd90XauoVkzU2F0Jn9ILGQfUHAIpocRTKuCwBrncaBozHQwD +O3Dk33AhZ6lqTv/AVLRKHQXwigGTBJxK4cCEZ+VwK9tKk6BrQB48Rm7pg9HF5ey5 +JGPRWgUnn1v0IJN5ysZ5m9ChYbqF8VwvMw0txmgKgvdDKpXbF/S59Bp4TH/7Dr2D +kAeNTcuzTFBaFE+siMgksZIYKZ1VkVoiN2qQA7ZaA5LQbUom0WdrKZGefFfPt9ES +A4wyL3OfxRsmWmd/5Fxrwm1VbzgPoMd1Dc5ExlyqnecdGzDui2bmltNqRJd9ytRq +6YUGYzXp4qQkWO61CoC3mkm2M8Ex7DGbUtXhdg0zoa08w9lXuOtHVhY7XlLWjO1U +p8cp4DVxsN/wOXtyH1pcleGo4aEsgyU/DH57prFLGz7Egp2JhRDHnZmlonWp74G1 +VLfqkOqZlqTU4mPA827C8qPCx6cMsRvFS7OEiDBswkFWBKjkUCw4rLC1tBMBCxJW +tZlc+Y0LNyOryJ3h6EJmRIHO57oLen345e1WOi4ROOC/wQMErFk7B3P41Lqmrwb8 +HGuKn3ca+Aw70hVrZ+7Q3RRFTLlOS/vv107Fqu6JAjkEEwEIACMFAlUh1AUCGwMH +CwkIBwMCAQYVCAIJCgsEFgIDAQIeAQIXgAAKCRD8YkZDSHA05RfdD/97wPXnoe7e +ipP7UXQ942z1buV6pTGv0Lea2aHn20o2BBjHp97YXroF/e/8W6h+Y+Fq8hWoXdYJ +dC9DVgzJhvbXAIG8VrF6/IDGQ62r4ff/AIyQY+kiCOCCVhjwuqOTjVYw2pYRUcI3 +UwXVPeptDSXcIZkHCLtEUnS5YMTdkPuZrAmucCCnfcJtevXbHD2yJYP4vwfXMbal +sNBDKJi6uYAFc4yv+/DyS13rfXJvu2pYGvtRd+fs7mBETvUTubhI440pIss6TX6M +lxWexX6Ty8vI5HCQT281H4zqdbe5GdzGmIx1EiYx1sJbgSBNqCh5sRJY5/BXzVJ3 +dfM/Mv5QYY4ulO/qUNFdC8f1cZm0euOo3maB4jY+Sjaff7t0WIz0GufO4dHARwJg +3s0LO9Wf5+z/fbWOMcfvvcfaHNbhaKWk16kslc/g7NYvMfOuleM06YGyGPz//a9c +baX53OiMupNvLlhyPO5NfGppvRn5xAElcAw1RLhHJcgvTtIs/zVVfHPaK41u8A9c +XKnmIUC39K4BGvOpPzEvCdQ2ZbAqzQLmZ1UICr15w1Nfs6uoERJbnuq+JgOPOcOk +ezAWELi5LdZTElnpJpZPTDQ03+3GvxD4R9sR+l5RT8Ul7kF+3PPPzekfQzF+Nisr +BhPFb2lPt3Hw32FgTTIuXCMRTKEBb/6z77kCDQRVIdQFARAAtmnsZ9A8ovJIJ9Rl +WeIylEhHRyQifqzgc/r50uDZVPBjewOA462LjH3+F6zFGEkU+q2aqSe0A0SJPF/W +hj6MNYXLoibxi5D4mGkoIao9ExnXt4LXAc6ogQpY6vFQBJU5Nr8XCefQbm0loa/o +y5uK8JHLWCZ2jAossnVpzDwNeN27+B8h5+OifnWhQCTun1xz5EJiyc0yoBmf46zf +mU4CMUBsPvrXcLmw4J3wp35qmrHg1tNyPhd7VBlikMrgtrWX9IaPZ40dnrGG/WjO +FYB3CKxGb0pTCj7GC4ubxo2upeWZqHLmdIVc7Nzsfp8EcwJbTj+jZ2Zfq6F8y+je +sbgh8CaxYn4hEs23aPYRq5H4/buVmZhUw3/AAL9ZmyX6AtAQ0HktVtQe7ykP7DLs +EpeLG+vPJFY363QeDsLHwOoxnZSfGziVlB4N/KqIkixNWcFTG8GSE1zKcdJVNoW+ +3MB3+FtMZWUJhH0FyKg5qLaJCtC7Yo5gsddU+QCqTn6gcZBnMX5j4LaAmW4hh1RX +ffwwsbfviK5uhXQCeUnbUaokieetDx4s6Kay6t9ahTRr0r/Z3VWzvr+xATxNWZzi +xTdezCGOB2ycZ0vq4bKXBuN8CAyOy5X1hf7Rc1BiAVQCILHJDtz0Ak/Hax6DAa2A +Hnx9YlugHQf000KroLEY+GaxqYEAEQEAAYkCHwQYAQgACQUCVSHUBQIbDAAKCRD8 +YkZDSHA05RtyEACdOEmGolL1xG6I+lDVdot6oBZqC9e021aLWqCUpWJFDp0m0aTm +CfmOI1gTaFjScxhq1W0GPUoJKUZhk3tlVfdSCtUckI+xuWKEfqJYtvUtTXpK4jDe +aZBovJ3KNpJRIynbr1566zCSQJhHiCGWmE/M5KN3gPsORbCBQXEkONSVsslf1Wm6 +6hU6uqSWUaceD+4fl5LClbck1DPWchAP7+uLKPEOtORyH6KRTgKl73zYo7xU1K4Q +MN/1aMjobPkqNvvkXnUNwO7QMz18Nx+WqPc4ksJgW1O1aPQ2qL/ARY5jatZ6BBd7 +iytfz7d6JOh0FOIlmhBqbWd7fEGrLsSA+EjBGBwW5BnIMmxP1xhjhwrcI18y8kAK +5UzdW2hbbAlc2rlsuxEc+xOYh8kGcc+mZ1j/aMn4gALsTbSO/0T+YJhfODNnL1dC +j7oPbJGmmG6pb/o7P4azBUVC9lHOuV3XlAPjSmJylnNsV7+PxwPlXlvKgh4S4C4Z +PUc/iPetsxXR2djccOoNxVU4CqJBqYKgul/pUphXkh7QfEKyH+42UETbVhstdBVU +azJ6SeUnv9ClVDGsCEhfEZfNOnOoDzJGxDfESoAw7ih91vIhTyHHsK83p2HLDMLP +ptLzx/0AFBfo6MWGGpd2RSnMWNbvh59wiThlDeI+Das3ln5nsAo67dMYdA== +=fjOq +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/build-wine/prepare-wine.sh b/contrib/build-wine/prepare-wine.sh index 84ba2561..4bc14784 100755 --- a/contrib/build-wine/prepare-wine.sh +++ b/contrib/build-wine/prepare-wine.sh @@ -13,6 +13,10 @@ LIBUSB_FILENAME=libusb-1.0.22.7z LIBUSB_URL=https://prdownloads.sourceforge.net/project/libusb/libusb-1.0/libusb-1.0.22/$LIBUSB_FILENAME?download LIBUSB_SHA256=671f1a420757b4480e7fadc8313d6fb3cbb75ca00934c417c1efa6e77fb8779b +PYINSTALLER_REPO="https://github.com/SomberNight/pyinstaller.git" +PYINSTALLER_COMMIT=d1cdd726d6a9edc70150d5302453fb90fdd09bf2 +# ^ tag 3.4, plus a custom commit that fixes cross-compilation with MinGW + PYTHON_VERSION=3.6.8 ## These settings probably don't need change @@ -25,60 +29,84 @@ PYTHON="wine $PYHOME/python.exe -OO -B" # Let's begin! -here="$(dirname "$(readlink -e "$0")")" set -e -. $here/../build_tools_util.sh +here="$(dirname "$(readlink -e "$0")")" + +. "$CONTRIB"/build_tools_util.sh +info "Booting wine." wine 'wineboot' -cd /tmp/electrum-build +cd "$CACHEDIR" -# Install Python +info "Installing Python." # note: you might need "sudo apt-get install dirmngr" for the following # keys from https://www.python.org/downloads/#pubkeys -KEYLIST_PYTHON_DEV="531F072D39700991925FED0C0EDDC5F26A45C816 26DEA9D4613391EF3E25C9FF0A5B101836580288 CBC547978A3964D14B9AB36A6AF053F07D9DC8D2 C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF 12EF3DC38047DA382D18A5B999CDEA9DA4135B38 8417157EDBE73D9EAC1E539B126EB563A74B06BF DBBF2EEBF925FAADCF1F3FFFD9866941EA5BBD71 2BA0DB82515BBB9EFFAC71C5C9BE28DEE6DF025C 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D C9B104B3DD3AA72D7CCB1066FB9921286F5E1540 97FC712E4C024BBEA48A61ED3A5CA953F73C700D 7ED10B6531D7C8E1BC296021FC624643487034E5" KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg" -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80) ; do - retry gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --keyserver "$server" --recv-keys $KEYLIST_PYTHON_DEV \ - && break || : ; -done +gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --import "$here"/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc +PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION" +mkdir -p "$PYTHON_DOWNLOADS" for msifile in core dev exe lib pip tools; do echo "Installing $msifile..." - wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi" - wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi.asc" - verify_signature "${msifile}.msi.asc" $KEYRING_PYTHON_DEV - wine msiexec /i "${msifile}.msi" /qb TARGETDIR=$PYHOME + download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi" + download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi.asc" + verify_signature "$PYTHON_DOWNLOADS/${msifile}.msi.asc" $KEYRING_PYTHON_DEV + wine msiexec /i "$PYTHON_DOWNLOADS/${msifile}.msi" /qb TARGETDIR=$PYHOME done -# Install dependencies specific to binaries +info "Installing dependencies specific to binaries." # note that this also installs pinned versions of both pip and setuptools -$PYTHON -m pip install -r "$here"/../deterministic-build/requirements-binaries.txt - -# Install PyInstaller -$PYTHON -m pip install pyinstaller==3.4 --no-use-pep517 - -# Install ZBar -download_if_not_exist $ZBAR_FILENAME "$ZBAR_URL" -verify_hash $ZBAR_FILENAME "$ZBAR_SHA256" -wine "$PWD/$ZBAR_FILENAME" /S - -# Install NSIS installer -download_if_not_exist $NSIS_FILENAME "$NSIS_URL" -verify_hash $NSIS_FILENAME "$NSIS_SHA256" -wine "$PWD/$NSIS_FILENAME" /S - -download_if_not_exist $LIBUSB_FILENAME "$LIBUSB_URL" -verify_hash $LIBUSB_FILENAME "$LIBUSB_SHA256" -7z x -olibusb $LIBUSB_FILENAME -aoa - +$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-binaries.txt + +info "Installing ZBar." +download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL" +verify_hash "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_SHA256" +wine "$CACHEDIR/$ZBAR_FILENAME" /S + +info "Installing NSIS." +download_if_not_exist "$CACHEDIR/$NSIS_FILENAME" "$NSIS_URL" +verify_hash "$CACHEDIR/$NSIS_FILENAME" "$NSIS_SHA256" +wine "$CACHEDIR/$NSIS_FILENAME" /S + +info "Installing libusb." +download_if_not_exist "$CACHEDIR/$LIBUSB_FILENAME" "$LIBUSB_URL" +verify_hash "$CACHEDIR/$LIBUSB_FILENAME" "$LIBUSB_SHA256" +7z x -olibusb "$CACHEDIR/$LIBUSB_FILENAME" -aoa cp libusb/MS32/dll/libusb-1.0.dll $WINEPREFIX/drive_c/$PYTHON_FOLDER/ mkdir -p $WINEPREFIX/drive_c/tmp -cp secp256k1/libsecp256k1.dll $WINEPREFIX/drive_c/tmp/ - -echo "Wine is configured." +cp "$CACHEDIR/secp256k1/libsecp256k1.dll" $WINEPREFIX/drive_c/tmp/ + + +info "Building PyInstaller." +# we build our own PyInstaller boot loader as the default one has high +# anti-virus false positives +( + cd "$WINEPREFIX/drive_c/electrum" + ELECTRUM_COMMIT_HASH=$(git rev-parse HEAD) + cd "$CACHEDIR" + rm -rf pyinstaller + mkdir pyinstaller + cd pyinstaller + # Shallow clone + git init + git remote add origin $PYINSTALLER_REPO + git fetch --depth 1 origin $PYINSTALLER_COMMIT + git checkout FETCH_HEAD + rm -fv PyInstaller/bootloader/Windows-*/run*.exe || true + # add reproducible randomness. this ensures we build a different bootloader for each commit. + # if we built the same one for all releases, that might also get anti-virus false positives + echo "const char *electrum_tag = \"tagged by Electrum@$ELECTRUM_COMMIT_HASH\";" >> ./bootloader/src/pyi_main.c + pushd bootloader + # cross-compile to Windows using host python + python3 ./waf all CC=i686-w64-mingw32-gcc CFLAGS="-Wno-stringop-overflow -static" + popd + # sanity check bootloader is there: + [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!" +) || fail "PyInstaller build failed" +info "Installing PyInstaller." +$PYTHON -m pip install ./pyinstaller + +info "Wine is configured." diff --git a/contrib/deterministic-build/requirements-binaries.txt b/contrib/deterministic-build/requirements-binaries.txt index 7a6a41bd..5b6be40a 100644 --- a/contrib/deterministic-build/requirements-binaries.txt +++ b/contrib/deterministic-build/requirements-binaries.txt @@ -1,6 +1,6 @@ -pip==19.1 \ - --hash=sha256:8f59b6cf84584d7962d79fd1be7a8ec0eb198aa52ea864896551736b3614eee9 \ - --hash=sha256:d9137cb543d8a4d73140a3282f6d777b2e786bb6abb8add3ac5b6539c82cd624 +pip==19.1.1 \ + --hash=sha256:44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958 \ + --hash=sha256:993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676 pycryptodomex==3.7.3 \ --hash=sha256:0bda549e20db1eb8e29fb365d10acf84b224d813b1131c828fc830b2ce313dcd \ --hash=sha256:1210c0818e5334237b16d99b5785aa0cee815d9997ee258bd5e2936af8e8aa50 \ @@ -51,6 +51,6 @@ PyQt5-sip==4.19.13 \ setuptools==41.0.1 \ --hash=sha256:a222d126f5471598053c9a77f4b5d4f26eaa1f150ad6e01dcf1a42e185d05613 \ --hash=sha256:c7769ce668c7a333d84e17fe8b524b1c45e7ee9f7908ad0a73e1eda7e6a5aebf -wheel==0.33.1 \ - --hash=sha256:66a8fd76f28977bb664b098372daef2b27f60dc4d1688cfab7b37a09448f0e9d \ - --hash=sha256:8eb4a788b3aec8abf5ff68d4165441bc57420c9f64ca5f471f58c3969fe08668 +wheel==0.33.4 \ + --hash=sha256:5e79117472686ac0c4aef5bad5172ea73a1c2d1646b808c35926bd26bdfb0c08 \ + --hash=sha256:62fcfa03d45b5b722539ccbc07b190e4bfff4bb9e3a4d470dd9f6a0981002565 diff --git a/contrib/deterministic-build/requirements-hw.txt b/contrib/deterministic-build/requirements-hw.txt index 0ea85b86..bc7ba0a2 100644 --- a/contrib/deterministic-build/requirements-hw.txt +++ b/contrib/deterministic-build/requirements-hw.txt @@ -6,43 +6,14 @@ certifi==2019.3.9 \ chardet==3.0.4 \ --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 -ckcc-protocol==0.7.4 \ - --hash=sha256:5af1d268a62e03997832b6300453c8f005630591df30a7156b450c80dd74a881 \ - --hash=sha256:fb41a4c2fb22c0bd04356d14b0c6dbf3e708bc3ad080dddbc088bb48cda03699 +ckcc-protocol==0.7.6 \ + --hash=sha256:b2a782aa37b22dd21b5859618b84a69bc19271c279eb89fe63aba378916d07b0 \ + --hash=sha256:f2e8181f9814959e4a6dfa3d1175c11b4e622a32a2ce2b311f64e5bcb3e7b271 click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 construct==2.9.45 \ --hash=sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c -Cython==0.29.7 \ - --hash=sha256:0ce8f6c789c907472c9084a44b625eba76a85d0189513de1497ab102a9d39ef8 \ - --hash=sha256:0d67964b747ac09758ba31fe25da2f66f575437df5f121ff481889a7a4485f56 \ - --hash=sha256:1630823619a87a814e5c1fa9f96544272ce4f94a037a34093fbec74989342328 \ - --hash=sha256:1a4c634bb049c8482b7a4f3121330de1f1c1f66eac3570e1e885b0c392b6a451 \ - --hash=sha256:1ec91cc09e9f9a2c3173606232adccc68f3d14be1a15a8c5dc6ab97b47b31528 \ - --hash=sha256:237a8fdd8333f7248718875d930d1e963ffa519fefeb0756d01d91cbfadab0bc \ - --hash=sha256:28a308cbfdf9b7bb44def918ad4a26b2d25a0095fa2f123addda33a32f308d00 \ - --hash=sha256:2fe3dde34fa125abf29996580d0182c18b8a240d7fa46d10984cc28d27808731 \ - --hash=sha256:30bda294346afa78c49a343e26f3ab2ad701e09f6a6373f579593f0cfcb1235a \ - --hash=sha256:33d27ea23e12bf0d420e40c20308c03ef192d312e187c1f72f385edd9bd6d570 \ - --hash=sha256:34d24d9370a6089cdd5afe56aa3c4af456e6400f8b4abb030491710ee765bafc \ - --hash=sha256:4e4877c2b96fae90f26ee528a87b9347872472b71c6913715ca15c8fe86a68c9 \ - --hash=sha256:50d6f1f26702e5f2a19890c7bc3de00f9b8a0ec131b52edccd56a60d02519649 \ - --hash=sha256:55d081162191b7c11c7bfcb7c68e913827dfd5de6ecdbab1b99dab190586c1e8 \ - --hash=sha256:59d339c7f99920ff7e1d9d162ea309b35775172e4bab9553f1b968cd43b21d6d \ - --hash=sha256:6cf4d10df9edc040c955fca708bbd65234920e44c30fccd057ecf3128efb31ad \ - --hash=sha256:6ec362539e2a6cf2329cd9820dec64868d8f0babe0d8dc5deff6c87a84d13f68 \ - --hash=sha256:7edc61a17c14b6e54d5317b0300d2da23d94a719c466f93cafa3b666b058c43b \ - --hash=sha256:8e37fc4db3f2c4e7e1ed98fe4fe313f1b7202df985de4ee1451d2e331332afae \ - --hash=sha256:b8c996bde5852545507bff45af44328fa48a7b22b5bec2f43083f0b8d1024fd9 \ - --hash=sha256:bf9c16f3d46af82f89fdefc0d64b2fb02f899c20da64548a8ea336beefcf8d23 \ - --hash=sha256:c1038aba898bed34ab1b5ddb0d3f9c9ae33b0649387ab9ffe6d0af677f66bfc1 \ - --hash=sha256:d405649c1bfc42e20d86178257658a859a3217b6e6d950ee8cb76353fcea9c39 \ - --hash=sha256:db6eeb20a3bd60e1cdcf6ce9a784bc82aec6ab891c800dc5d7824d5cfbfe77f2 \ - --hash=sha256:e382f8cb40dca45c3b439359028a4b60e74e22d391dc2deb360c0b8239d6ddc0 \ - --hash=sha256:f3f6c09e2c76f2537d61f907702dd921b04d1c3972f01d5530ef1f748f22bd89 \ - --hash=sha256:f749287087f67957c020e1de26906e88b8b0c4ea588facb7349c115a63346f67 \ - --hash=sha256:f86b96e014732c0d1ded2c1f51444c80176a98c21856d0da533db4e4aef54070 ecdsa==0.13.2 \ --hash=sha256:20c17e527e75acad8f402290e158a6ac178b91b881f941fc6ea305bfdfb9657c \ --hash=sha256:5c034ffa23413ac923541ceb3ac14ec15a0d2530690413bff58c12b80e56d884 @@ -65,35 +36,35 @@ keepkey==6.1.0 \ --hash=sha256:058548e733e1df8d1879ea747eef167c84cb04cdd685240e50d599f48d08e5c6 \ --hash=sha256:2e1623409307c86f709054ad191bc7707c4feeacae2e497bd933f2f0054c6eb0 \ --hash=sha256:54ef1b134657d3d14ef24c0c98e29d0276ad8f0e053d5e50d836ba8a520230e7 -libusb1==1.7 \ - --hash=sha256:9d4f66d2ed699986b06bc3082cd262101cb26af7a76a34bd15b7eb56cba37e0f +libusb1==1.7.1 \ + --hash=sha256:adf64a4f3f5c94643a1286f8153bcf4bc787c348b38934aacd7fe17fbeebc571 mnemonic==0.18 \ --hash=sha256:02a7306a792370f4a0c106c2cf1ce5a0c84b9dbd7e71c6792fdb9ad88a727f1d pbkdf2==1.3 \ --hash=sha256:ac6397369f128212c43064a2b4878038dab78dab41875364554aaf2a684e6979 -pip==19.1 \ - --hash=sha256:8f59b6cf84584d7962d79fd1be7a8ec0eb198aa52ea864896551736b3614eee9 \ - --hash=sha256:d9137cb543d8a4d73140a3282f6d777b2e786bb6abb8add3ac5b6539c82cd624 -protobuf==3.7.1 \ - --hash=sha256:21e395d7959551e759d604940a115c51c6347d90a475c9baf471a1a86b5604a9 \ - --hash=sha256:57e05e16955aee9e6a0389fcbd58d8289dd2420e47df1a1096b3a232c26eb2dd \ - --hash=sha256:67819e8e48a74c68d87f25cad9f40edfe2faf278cdba5ca73173211b9213b8c9 \ - --hash=sha256:75da7d43a2c8a13b0bc7238ab3c8ae217cbfd5979d33b01e98e1f78defb2d060 \ - --hash=sha256:78e08371e236f193ce947712c072542ff19d0043ab5318c2ea46bbc2aaebdca6 \ - --hash=sha256:7ee5b595db5abb0096e8c4755e69c20dfad38b2d0bcc9bc7bafc652d2496b471 \ - --hash=sha256:86260ecfe7a66c0e9d82d2c61f86a14aa974d340d159b829b26f35f710f615db \ - --hash=sha256:92c77db4bd33ea4ee5f15152a835273f2338a5246b2cbb84bab5d0d7f6e9ba94 \ - --hash=sha256:9c7b90943e0e188394b4f068926a759e3b4f63738190d1ab3d500d53b9ce7614 \ - --hash=sha256:a77f217ea50b2542bae5b318f7acee50d9fc8c95dd6d3656eaeff646f7cab5ee \ - --hash=sha256:ad589ed1d1f83db22df867b10e01fe445516a5a4d7cfa37fe3590a5f6cfc508b \ - --hash=sha256:b06a794901bf573f4b2af87e6139e5cd36ac7c91ac85d7ae3fe5b5f6fc317513 \ - --hash=sha256:bd8592cc5f8b4371d0bad92543370d4658dc41a5ccaaf105597eb5524c616291 \ - --hash=sha256:be48e5a6248a928ec43adf2bea037073e5da692c0b3c10b34f9904793bd63138 \ - --hash=sha256:cc5eb13f5ccc4b1b642cc147c2cdd121a34278b341c7a4d79e91182fff425836 \ - --hash=sha256:cd3b0e0ad69b74ee55e7c321f52a98effed2b4f4cc9a10f3683d869de00590d5 \ - --hash=sha256:d6e88c4920660aa75c0c2c4b53407aef5efd9a6e0ca7d2fc84d79aba2ccbda3a \ - --hash=sha256:ec3c49b6d247152e19110c3a53d9bb4cf917747882017f70796460728b02722e \ - --hash=sha256:f1f5d8b8e0bc9651d81b40ad3d9fb7cdd858ea31fc116dd230393465849dbecd +pip==19.1.1 \ + --hash=sha256:44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958 \ + --hash=sha256:993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676 +protobuf==3.8.0 \ + --hash=sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8 \ + --hash=sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538 \ + --hash=sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e \ + --hash=sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a \ + --hash=sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6 \ + --hash=sha256:3761ab21883f1d3add8643413b326a0026776879b13ecf904e1e05fe18532c03 \ + --hash=sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0 \ + --hash=sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc \ + --hash=sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47 \ + --hash=sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01 \ + --hash=sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115 \ + --hash=sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277 \ + --hash=sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c \ + --hash=sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea \ + --hash=sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87 \ + --hash=sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7 \ + --hash=sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126 \ + --hash=sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a \ + --hash=sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f pyblake2==1.1.2 \ @@ -106,9 +77,9 @@ pyblake2==1.1.2 \ --hash=sha256:baa2190bfe549e36163aa44664d4ee3a9080b236fc5d42f50dc6fd36bbdc749e \ --hash=sha256:c53417ee0bbe77db852d5fd1036749f03696ebc2265de359fe17418d800196c4 \ --hash=sha256:fbc9fcde75713930bc2a91b149e97be2401f7c9c56d735b46a109210f58d7358 -requests==2.21.0 \ - --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ - --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b +requests==2.22.0 \ + --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ + --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 safet==0.1.4 \ --hash=sha256:522c257910f9472e9c77c487425ed286f6721c314653e232bc41c6cedece1bb1 \ --hash=sha256:b152874acdc89ff0c8b2d680bfbf020b3e53527c2ad3404489dd61a548aa56a1 @@ -118,19 +89,19 @@ setuptools==41.0.1 \ six==1.12.0 \ --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 -trezor==0.11.2 \ - --hash=sha256:7bdec3d6e35e41666580547674f2652c0c466172964da42b325cab2c30b4eb46 \ - --hash=sha256:a6f4b47b37a21247535fc43411cb70a8c61ef0a5a2dfee668bd05611e2741fb8 +trezor==0.11.3 \ + --hash=sha256:c79a500e90d003073c8060d319dceb042caaba9472f13990c77ed37d04a82108 \ + --hash=sha256:f3a99ec0fe7b28f83f936f87bf6ad89c77fef9f576934efc3a70dd569009ded1 typing-extensions==3.7.2 \ --hash=sha256:07b2c978670896022a43c4b915df8958bec4a6b84add7f2c87b2b728bda3ba64 \ --hash=sha256:f3f0e67e1d42de47b5c67c32c9b26641642e9170fe7e292991793705cd5fef7c \ --hash=sha256:fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71 -urllib3==1.24.3 \ - --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \ - --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb +urllib3==1.25.3 \ + --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \ + --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 websocket_client==0.56.0 \ --hash=sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9 \ --hash=sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a -wheel==0.33.1 \ - --hash=sha256:66a8fd76f28977bb664b098372daef2b27f60dc4d1688cfab7b37a09448f0e9d \ - --hash=sha256:8eb4a788b3aec8abf5ff68d4165441bc57420c9f64ca5f471f58c3969fe08668 +wheel==0.33.4 \ + --hash=sha256:5e79117472686ac0c4aef5bad5172ea73a1c2d1646b808c35926bd26bdfb0c08 \ + --hash=sha256:62fcfa03d45b5b722539ccbc07b190e4bfff4bb9e3a4d470dd9f6a0981002565 diff --git a/contrib/deterministic-build/requirements.txt b/contrib/deterministic-build/requirements.txt index d8a74c00..a804caeb 100644 --- a/contrib/deterministic-build/requirements.txt +++ b/contrib/deterministic-build/requirements.txt @@ -24,9 +24,9 @@ aiohttp==3.5.4 \ aiohttp-socks==0.2.2 \ --hash=sha256:e473ee222b001fe33798957b9ce3352b32c187cf41684f8e2259427925914993 \ --hash=sha256:eebd8939a7c3c1e3e7e1b2552c60039b4c65ef6b8b2351efcbdd98290538e310 -aiorpcX==0.17.0 \ - --hash=sha256:13ccc8361bc3049d649094b69aead6118f6deb5f1b88ad77211be85c4e2ed792 \ - --hash=sha256:b08e7c350c78701ec698c851b405a07d20ac64380c394440c1740b48bb3c5502 +aiorpcX==0.18.3 \ + --hash=sha256:42e354c3e0088cb99a4a46e6f7ca777a08d989519ca1bc46323fef836e25579b \ + --hash=sha256:b7a7ced5df95c79c74f7834e7cc58bb7747dbad9eb37bf7580da507e182ca44c async_timeout==3.0.1 \ --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 @@ -83,29 +83,29 @@ multidict==4.5.2 \ --hash=sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9 \ --hash=sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7 \ --hash=sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b -pip==19.1 \ - --hash=sha256:8f59b6cf84584d7962d79fd1be7a8ec0eb198aa52ea864896551736b3614eee9 \ - --hash=sha256:d9137cb543d8a4d73140a3282f6d777b2e786bb6abb8add3ac5b6539c82cd624 -protobuf==3.7.1 \ - --hash=sha256:21e395d7959551e759d604940a115c51c6347d90a475c9baf471a1a86b5604a9 \ - --hash=sha256:57e05e16955aee9e6a0389fcbd58d8289dd2420e47df1a1096b3a232c26eb2dd \ - --hash=sha256:67819e8e48a74c68d87f25cad9f40edfe2faf278cdba5ca73173211b9213b8c9 \ - --hash=sha256:75da7d43a2c8a13b0bc7238ab3c8ae217cbfd5979d33b01e98e1f78defb2d060 \ - --hash=sha256:78e08371e236f193ce947712c072542ff19d0043ab5318c2ea46bbc2aaebdca6 \ - --hash=sha256:7ee5b595db5abb0096e8c4755e69c20dfad38b2d0bcc9bc7bafc652d2496b471 \ - --hash=sha256:86260ecfe7a66c0e9d82d2c61f86a14aa974d340d159b829b26f35f710f615db \ - --hash=sha256:92c77db4bd33ea4ee5f15152a835273f2338a5246b2cbb84bab5d0d7f6e9ba94 \ - --hash=sha256:9c7b90943e0e188394b4f068926a759e3b4f63738190d1ab3d500d53b9ce7614 \ - --hash=sha256:a77f217ea50b2542bae5b318f7acee50d9fc8c95dd6d3656eaeff646f7cab5ee \ - --hash=sha256:ad589ed1d1f83db22df867b10e01fe445516a5a4d7cfa37fe3590a5f6cfc508b \ - --hash=sha256:b06a794901bf573f4b2af87e6139e5cd36ac7c91ac85d7ae3fe5b5f6fc317513 \ - --hash=sha256:bd8592cc5f8b4371d0bad92543370d4658dc41a5ccaaf105597eb5524c616291 \ - --hash=sha256:be48e5a6248a928ec43adf2bea037073e5da692c0b3c10b34f9904793bd63138 \ - --hash=sha256:cc5eb13f5ccc4b1b642cc147c2cdd121a34278b341c7a4d79e91182fff425836 \ - --hash=sha256:cd3b0e0ad69b74ee55e7c321f52a98effed2b4f4cc9a10f3683d869de00590d5 \ - --hash=sha256:d6e88c4920660aa75c0c2c4b53407aef5efd9a6e0ca7d2fc84d79aba2ccbda3a \ - --hash=sha256:ec3c49b6d247152e19110c3a53d9bb4cf917747882017f70796460728b02722e \ - --hash=sha256:f1f5d8b8e0bc9651d81b40ad3d9fb7cdd858ea31fc116dd230393465849dbecd +pip==19.1.1 \ + --hash=sha256:44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958 \ + --hash=sha256:993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676 +protobuf==3.8.0 \ + --hash=sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8 \ + --hash=sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538 \ + --hash=sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e \ + --hash=sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a \ + --hash=sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6 \ + --hash=sha256:3761ab21883f1d3add8643413b326a0026776879b13ecf904e1e05fe18532c03 \ + --hash=sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0 \ + --hash=sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc \ + --hash=sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47 \ + --hash=sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01 \ + --hash=sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115 \ + --hash=sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277 \ + --hash=sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c \ + --hash=sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea \ + --hash=sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87 \ + --hash=sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7 \ + --hash=sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126 \ + --hash=sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a \ + --hash=sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832 pyaes==1.6.1 \ --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f QDarkStyle==2.6.8 \ @@ -124,9 +124,9 @@ typing-extensions==3.7.2 \ --hash=sha256:07b2c978670896022a43c4b915df8958bec4a6b84add7f2c87b2b728bda3ba64 \ --hash=sha256:f3f0e67e1d42de47b5c67c32c9b26641642e9170fe7e292991793705cd5fef7c \ --hash=sha256:fb2cd053238d33a8ec939190f30cfd736c00653a85a2919415cecf7dc3d9da71 -wheel==0.33.1 \ - --hash=sha256:66a8fd76f28977bb664b098372daef2b27f60dc4d1688cfab7b37a09448f0e9d \ - --hash=sha256:8eb4a788b3aec8abf5ff68d4165441bc57420c9f64ca5f471f58c3969fe08668 +wheel==0.33.4 \ + --hash=sha256:5e79117472686ac0c4aef5bad5172ea73a1c2d1646b808c35926bd26bdfb0c08 \ + --hash=sha256:62fcfa03d45b5b722539ccbc07b190e4bfff4bb9e3a4d470dd9f6a0981002565 yarl==1.3.0 \ --hash=sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9 \ --hash=sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f \ diff --git a/contrib/make_apk b/contrib/make_apk index f6d69187..d6d48d73 100755 --- a/contrib/make_apk +++ b/contrib/make_apk @@ -8,7 +8,7 @@ PACKAGES="$ROOT_FOLDER"/packages/ LOCALE="$ROOT_FOLDER"/electrum/locale/ if [ ! -d "$LOCALE" ]; then - echo "Run make_locale first!" + echo "Run pull_locale first!" exit 1 fi @@ -30,6 +30,17 @@ if [[ -n "$1" && "$1" == "release" ]] ; then export P4A_RELEASE_KEYALIAS=electrum make release else + export P4A_DEBUG_KEYSTORE="$CONTRIB"/android_debug.keystore + export P4A_DEBUG_KEYSTORE_PASSWD=unsafepassword + export P4A_DEBUG_KEYALIAS_PASSWD=unsafepassword + export P4A_DEBUG_KEYALIAS=electrum + if [ ! -f "$P4A_DEBUG_KEYSTORE" ]; then + keytool -genkey -v -keystore "$CONTRIB"/android_debug.keystore \ + -alias "$P4A_DEBUG_KEYALIAS" -keyalg RSA -keysize 2048 -validity 10000 \ + -dname "CN=mqttserver.ibm.com, OU=ID, O=IBM, L=Hursley, S=Hants, C=GB" \ + -storepass "$P4A_DEBUG_KEYSTORE_PASSWD" \ + -keypass "$P4A_DEBUG_KEYALIAS_PASSWD" + fi make apk fi diff --git a/contrib/make_tgz b/contrib/make_tgz index 09c0cea7..4505d2c2 100755 --- a/contrib/make_tgz +++ b/contrib/make_tgz @@ -7,16 +7,28 @@ ROOT_FOLDER="$CONTRIB"/.. PACKAGES="$ROOT_FOLDER"/packages/ LOCALE="$ROOT_FOLDER"/electrum/locale/ -if [ ! -d "$LOCALE" ]; then - echo "Run make_locale first!" - exit 1 -fi - if [ ! -d "$PACKAGES" ]; then echo "Run make_packages first!" exit 1 fi +git submodule update --init + +( + rm -rf "$LOCALE" + cd "$CONTRIB/deterministic-build/electrum-locale/" + if ! which msgfmt > /dev/null 2>&1; then + echo "Please install gettext" + exit 1 + fi + for i in ./locale/*; do + dir="$ROOT_FOLDER"/electrum/$i/LC_MESSAGES + mkdir -p $dir + msgfmt --output-file=$dir/electrum.mo $i/electrum.po || true + cp $i/electrum.po "$ROOT_FOLDER"/electrum/$i/electrum.po + done +) + ( cd "$ROOT_FOLDER" diff --git a/contrib/osx/README.md b/contrib/osx/README.md index ca404ff1..fdba1913 100644 --- a/contrib/osx/README.md +++ b/contrib/osx/README.md @@ -1,6 +1,9 @@ Building Mac OS binaries ======================== +✗ _This script does not produce reproducible output (yet!). + Please help us remedy this._ + This guide explains how to build Electrum binaries for macOS systems. diff --git a/contrib/osx/base.sh b/contrib/osx/base.sh index c2e3527c..c11e270a 100644 --- a/contrib/osx/base.sh +++ b/contrib/osx/base.sh @@ -21,3 +21,7 @@ function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity info "Code signing ${infoName}..." codesign -f -v $deep -s "$identity" "$file" || fail "Could not code sign ${infoName}" } + +function realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx index 4369bfe3..ff695d20 100755 --- a/contrib/osx/make_osx +++ b/contrib/osx/make_osx @@ -9,6 +9,10 @@ LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7" . $(dirname "$0")/base.sh +CONTRIB_OSX="$(dirname "$(realpath "$0")")" +CONTRIB="$CONTRIB_OSX/.." +ROOT_FOLDER="$CONTRIB/.." + src_dir=$(dirname "$0") cd $src_dir/../.. @@ -65,13 +69,24 @@ pyinstaller --version rm -rf ./dist -git submodule init -git submodule update +git submodule update --init rm -rf $BUILDDIR > /dev/null 2>&1 mkdir $BUILDDIR -cp -R ./contrib/deterministic-build/electrum-locale/locale/ ./electrum/locale/ +info "generating locale" +( + if ! which msgfmt > /dev/null 2>&1; then + brew install gettext + brew link --force gettext + fi + cd "$CONTRIB"/deterministic-build/electrum-locale + for i in ./locale/*; do + dir="$ROOT_FOLDER"/electrum/$i/LC_MESSAGES + mkdir -p $dir + msgfmt --output-file=$dir/electrum.mo $i/electrum.po || true + done +) || fail "failed generating locale" info "Downloading libusb..." @@ -89,7 +104,7 @@ git reset --hard $LIBSECP_VERSION git clean -f -x -q ./autogen.sh ./configure --enable-module-recovery --enable-experimental --enable-module-ecdh --disable-jni -make +make -j4 popd cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx diff --git a/contrib/pull_locale b/contrib/pull_locale new file mode 100755 index 00000000..4b187504 --- /dev/null +++ b/contrib/pull_locale @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import os +import subprocess +import io +import zipfile +import sys + +try: + import requests +except ImportError as e: + sys.exit(f"Error: {str(e)}. Try 'sudo python3 -m pip install '") + +os.chdir(os.path.dirname(os.path.realpath(__file__))) +os.chdir('..') + +cmd = "find electrum -type f -name '*.py' -o -name '*.kv'" + +files = subprocess.check_output(cmd, shell=True) + +with open("app.fil", "wb") as f: + f.write(files) + +print("Found {} files to translate".format(len(files.splitlines()))) + +# Generate fresh translation template +if not os.path.exists('electrum/locale'): + os.mkdir('electrum/locale') +cmd = 'xgettext -s --from-code UTF-8 --language Python --no-wrap -f app.fil --output=electrum/locale/messages.pot' +print('Generate template') +os.system(cmd) + +os.chdir('electrum') + +crowdin_identifier = 'electrum' +crowdin_file_name = 'files[electrum-client/messages.pot]' +locale_file_name = 'locale/messages.pot' + +# Download & unzip +print('Download translations') +s = requests.request('GET', 'https://crowdin.com/backend/download/project/' + crowdin_identifier + '.zip').content +zfobj = zipfile.ZipFile(io.BytesIO(s)) + +print('Unzip translations') +for name in zfobj.namelist(): + if not name.startswith('electrum-client/locale'): + continue + if name.endswith('/'): + if not os.path.exists(name[16:]): + os.mkdir(name[16:]) + else: + with open(name[16:], 'wb') as output: + output.write(zfobj.read(name)) + +# Convert .po to .mo +print('Installing') +for lang in os.listdir('locale'): + if lang.startswith('messages'): + continue + # Check LC_MESSAGES folder + mo_dir = 'locale/%s/LC_MESSAGES' % lang + if not os.path.exists(mo_dir): + os.mkdir(mo_dir) + cmd = 'msgfmt --output-file="%s/electrum.mo" "locale/%s/electrum.po"' % (mo_dir,lang) + print('Installing', lang) + os.system(cmd) diff --git a/contrib/push_locale b/contrib/push_locale new file mode 100755 index 00000000..01106cf7 --- /dev/null +++ b/contrib/push_locale @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import os +import subprocess +import io +import zipfile +import sys + +try: + import requests +except ImportError as e: + sys.exit(f"Error: {str(e)}. Try 'sudo python3 -m pip install '") + +os.chdir(os.path.dirname(os.path.realpath(__file__))) +os.chdir('..') + +cmd = "find electrum -type f -name '*.py' -o -name '*.kv'" + +files = subprocess.check_output(cmd, shell=True) + +with open("app.fil", "wb") as f: + f.write(files) + +print("Found {} files to translate".format(len(files.splitlines()))) + +# Generate fresh translation template +if not os.path.exists('electrum/locale'): + os.mkdir('electrum/locale') +cmd = 'xgettext -s --from-code UTF-8 --language Python --no-wrap -f app.fil --output=electrum/locale/messages.pot' +print('Generate template') +os.system(cmd) + +os.chdir('electrum') + +crowdin_identifier = 'electrum' +crowdin_file_name = 'files[electrum-client/messages.pot]' +locale_file_name = 'locale/messages.pot' +crowdin_api_key = None + +filename = os.path.expanduser('~/.crowdin_api_key') +if os.path.exists(filename): + with open(filename) as f: + crowdin_api_key = f.read().strip() + +if "crowdin_api_key" in os.environ: + crowdin_api_key = os.environ["crowdin_api_key"] + +if crowdin_api_key: + # Push to Crowdin + print('Push to Crowdin') + url = ('https://api.crowdin.com/api/project/' + crowdin_identifier + '/update-file?key=' + crowdin_api_key) + with open(locale_file_name, 'rb') as f: + files = {crowdin_file_name: f} + response = requests.request('POST', url, files=files) + print("", "update-file:", "-"*20, response.text, "-"*20, sep="\n") + # Build translations + print('Build translations') + response = requests.request('GET', 'https://api.crowdin.com/api/project/' + crowdin_identifier + '/export?key=' + crowdin_api_key) + print("", "export:", "-" * 20, response.text, "-" * 20, sep="\n") + diff --git a/contrib/requirements/requirements-hw.txt b/contrib/requirements/requirements-hw.txt index 49cebd42..f38092f8 100644 --- a/contrib/requirements/requirements-hw.txt +++ b/contrib/requirements/requirements-hw.txt @@ -1,4 +1,3 @@ -Cython>=0.27 trezor[hidapi]>=0.11.0 safet[hidapi]>=0.1.0 keepkey>=6.0.3 diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 4ca24227..1159a046 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -4,8 +4,8 @@ qrcode protobuf dnspython jsonrpclib-pelix -qdarkstyle<3.0 -aiorpcx>=0.17,<0.18 +qdarkstyle<2.7 +aiorpcx>=0.18,<0.19 aiohttp>=3.3.0 aiohttp_socks certifi diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index 9da7b885..843f3679 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -38,8 +38,8 @@ from .logging import Logger if TYPE_CHECKING: - from .storage import WalletStorage from .network import Network + from .json_db import JsonDB TX_HEIGHT_LOCAL = -2 @@ -60,9 +60,8 @@ class AddressSynchronizer(Logger): inherited by wallet """ - def __init__(self, storage: 'WalletStorage'): - self.storage = storage - self.db = self.storage.db + def __init__(self, db: 'JsonDB'): + self.db = db self.network = None # type: Network Logger.__init__(self) # verifier (SPV) and synchronizer are started in start_network @@ -155,7 +154,7 @@ def start_network(self, network): def on_blockchain_updated(self, event, *args): self._get_addr_balance_cache = {} # invalidate cache - def stop_threads(self, write_to_disk=True): + def stop_threads(self): if self.network: if self.synchronizer: asyncio.run_coroutine_threadsafe(self.synchronizer.stop(), self.network.asyncio_loop) @@ -164,9 +163,7 @@ def stop_threads(self, write_to_disk=True): asyncio.run_coroutine_threadsafe(self.verifier.stop(), self.network.asyncio_loop) self.verifier = None self.network.unregister_callback(self.on_blockchain_updated) - self.storage.put('stored_height', self.get_local_height()) - if write_to_disk: - self.storage.write() + self.db.put('stored_height', self.get_local_height()) def add_address(self, address): if not self.db.get_addr_history(address): @@ -192,7 +189,8 @@ def get_conflicting_transactions(self, tx_hash, tx): if spending_tx_hash is None: continue # this outpoint has already been spent, by spending_tx - assert self.db.get_transaction(spending_tx_hash) + # annoying assert that has revealed several bugs over time: + assert self.db.get_transaction(spending_tx_hash), "spending tx not in wallet db" conflicting_txns |= {spending_tx_hash} if tx_hash in conflicting_txns: # this tx is already in history, so it conflicts with itself @@ -366,12 +364,10 @@ def load_local_history(self): @profiler def check_history(self): - save = False hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.db.get_history())) hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.db.get_history())) for addr in hist_addrs_not_mine: self.db.remove_addr_history(addr) - save = True for addr in hist_addrs_mine: hist = self.db.get_addr_history(addr) for tx_hash, tx_height in hist: @@ -380,9 +376,6 @@ def check_history(self): tx = self.db.get_transaction(tx_hash) if tx is not None: self.add_transaction(tx_hash, tx, allow_unrelated=True) - save = True - if save: - self.storage.write() def remove_local_transactions_we_dont_have(self): for txid in itertools.chain(self.db.list_txi(), self.db.list_txo()): @@ -394,7 +387,6 @@ def clear_history(self): with self.lock: with self.transaction_lock: self.db.clear_history() - self.storage.write() def get_txpos(self, tx_hash): """Returns (height, txpos) tuple, even if the tx is unverified.""" @@ -556,7 +548,7 @@ def get_local_height(self): cached_local_height = getattr(self.threadlocal_cache, 'local_height', None) if cached_local_height is not None: return cached_local_height - return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) + return self.network.get_local_height() if self.network else self.db.get('stored_height', 0) def get_tx_height(self, tx_hash: str) -> TxMinedInfo: with self.lock: @@ -576,8 +568,6 @@ def set_up_to_date(self, up_to_date): self.up_to_date = up_to_date if self.network: self.network.notify('status') - if up_to_date: - self.storage.write() def is_up_to_date(self): with self.lock: return self.up_to_date diff --git a/electrum/base_crash_reporter.py b/electrum/base_crash_reporter.py index e432436a..b6d02d88 100644 --- a/electrum/base_crash_reporter.py +++ b/electrum/base_crash_reporter.py @@ -31,10 +31,10 @@ from . import constants from .i18n import _ from .util import make_aiohttp_session -from .logging import describe_os_version +from .logging import describe_os_version, Logger -class BaseCrashReporter: +class BaseCrashReporter(Logger): report_server = "https://crashhub.electrum.org" config_key = "show_crash_reporter" issue_template = """

Traceback

@@ -59,9 +59,10 @@ class BaseCrashReporter: ASK_CONFIRM_SEND = _("Do you want to send this report?") def __init__(self, exctype, value, tb): + Logger.__init__(self) self.exc_args = (exctype, value, tb) - def send_report(self, asyncio_loop, proxy, endpoint="/crash"): + def send_report(self, asyncio_loop, proxy, endpoint="/crash", *, timeout=None): if constants.net.GENESIS[-4:] not in ["4943", "e26f"] and ".electrum.org" in BaseCrashReporter.report_server: # Gah! Some kind of altcoin wants to send us crash reports. raise Exception(_("Missing report URL.")) @@ -69,7 +70,7 @@ def send_report(self, asyncio_loop, proxy, endpoint="/crash"): report.update(self.get_additional_info()) report = json.dumps(report) coro = self.do_post(proxy, BaseCrashReporter.report_server + endpoint, data=report) - response = asyncio.run_coroutine_threadsafe(coro, asyncio_loop).result(5) + response = asyncio.run_coroutine_threadsafe(coro, asyncio_loop).result(timeout) return response async def do_post(self, proxy, url, data): diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 61a30010..470d40ed 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -42,8 +42,9 @@ from .i18n import _ from .util import UserCancelled, InvalidPassword, WalletFileException from .simple_config import SimpleConfig -from .plugin import Plugins +from .plugin import Plugins, HardwarePluginLibraryUnavailable from .logging import Logger +from .plugins.hw_wallet.plugin import OutdatedHwFirmwareException, HW_PluginBase if TYPE_CHECKING: from .plugin import DeviceInfo @@ -255,7 +256,8 @@ def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage=None): def failed_getting_device_infos(name, e): nonlocal debug_msg - self.logger.info(f'error getting device infos for {name}: {e}') + err_str_oneline = ' // '.join(str(e).splitlines()) + self.logger.warning(f'error getting device infos for {name}: {err_str_oneline}') indented_error_msg = ' '.join([''] + str(e).splitlines(keepends=True)) debug_msg += f' {name}: (error getting device infos)\n{indented_error_msg}\n' @@ -281,6 +283,9 @@ def failed_getting_device_infos(name, e): # FIXME: side-effect: unpaired_device_info sets client.handler device_infos = devmgr.unpaired_device_infos(None, plugin, devices=scanned_devices, include_failing_clients=True) + except HardwarePluginLibraryUnavailable as e: + failed_getting_device_infos(name, e) + continue except BaseException as e: self.logger.exception('') failed_getting_device_infos(name, e) @@ -293,14 +298,16 @@ def failed_getting_device_infos(name, e): if not debug_msg: debug_msg = ' {}'.format(_('No exceptions encountered.')) if not devices: - msg = ''.join([ - _('No hardware device detected.') + '\n', - _('To trigger a rescan, press \'Next\'.') + '\n\n', - _('If your device is not detected on Windows, go to "Settings", "Devices", "Connected devices", and do "Remove device". Then, plug your device again.') + ' ', - _('On Linux, you might have to add a new permission to your udev rules.') + '\n\n', - _('Debug message') + '\n', - debug_msg - ]) + msg = (_('No hardware device detected.') + '\n' + + _('To trigger a rescan, press \'Next\'.') + '\n\n') + if sys.platform == 'win32': + msg += _('If your device is not detected on Windows, go to "Settings", "Devices", "Connected devices", ' + 'and do "Remove device". Then, plug your device again.') + '\n' + msg += _('While this is less than ideal, it might help if you run Electrum as Administrator.') + '\n' + else: + msg += _('On Linux, you might have to add a new permission to your udev rules.') + '\n' + msg += '\n\n' + msg += _('Debug message') + '\n' + debug_msg self.confirm_dialog(title=title, message=msg, run_next=lambda x: self.choose_hw_device(purpose, storage=storage)) return @@ -319,7 +326,7 @@ def failed_getting_device_infos(name, e): run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage)) def on_device(self, name, device_info, *, purpose, storage=None): - self.plugin = self.plugins.get_plugin(name) + self.plugin = self.plugins.get_plugin(name) # type: HW_PluginBase try: self.plugin.setup_device(device_info, self, purpose) except OSError as e: @@ -331,6 +338,14 @@ def on_device(self, name, device_info, *, purpose, storage=None): devmgr.unpair_id(device_info.device.id_) self.choose_hw_device(purpose, storage=storage) return + except OutdatedHwFirmwareException as e: + if self.question(e.text_ignore_old_fw_and_continue(), title=_("Outdated device firmware")): + self.plugin.set_ignore_outdated_fw() + # will need to re-pair + devmgr = self.plugins.device_manager + devmgr.unpair_id(device_info.device.id_) + self.choose_hw_device(purpose, storage=storage) + return except (UserCancelled, GoBack): self.choose_hw_device(purpose, storage=storage) return diff --git a/electrum/bip32.py b/electrum/bip32.py index ba7b4a82..102fb0a9 100644 --- a/electrum/bip32.py +++ b/electrum/bip32.py @@ -200,6 +200,8 @@ def is_private(self) -> bool: return isinstance(self.eckey, ecc.ECPrivkey) def subkey_at_private_derivation(self, path: Union[str, Iterable[int]]) -> 'BIP32Node': + if path is None: + raise Exception("derivation path must not be None") if isinstance(path, str): path = convert_bip32_path_to_list_of_uint32(path) if not self.is_private(): @@ -224,6 +226,8 @@ def subkey_at_private_derivation(self, path: Union[str, Iterable[int]]) -> 'BIP3 child_number=child_number) def subkey_at_public_derivation(self, path: Union[str, Iterable[int]]) -> 'BIP32Node': + if path is None: + raise Exception("derivation path must not be None") if isinstance(path, str): path = convert_bip32_path_to_list_of_uint32(path) if not path: diff --git a/electrum/coinchooser.py b/electrum/coinchooser.py index 5ab633db..f7ecffbe 100644 --- a/electrum/coinchooser.py +++ b/electrum/coinchooser.py @@ -24,7 +24,8 @@ # SOFTWARE. from collections import defaultdict from math import floor, log10 -from typing import NamedTuple, List +from typing import NamedTuple, List, Callable +from decimal import Decimal from .bitcoin import sha256, COIN, TYPE_ADDRESS, is_address from .transaction import Transaction, TxOutput @@ -74,11 +75,18 @@ class Bucket(NamedTuple): desc: str weight: int # as in BIP-141 value: int # in satoshis + effective_value: int # estimate of value left after subtracting fees. in satoshis coins: List[dict] # UTXOs min_height: int # min block height where a coin was confirmed witness: bool # whether any coin uses segwit +class ScoredCandidate(NamedTuple): + penalty: float + tx: Transaction + buckets: List[Bucket] + + def strip_unneeded(bkts, sufficient_funds): '''Remove buckets that are unnecessary in achieving the spend amount''' if sufficient_funds([], bucket_value_sum=0): @@ -103,11 +111,14 @@ def __init__(self): def keys(self, coins): raise NotImplementedError - def bucketize_coins(self, coins): + def bucketize_coins(self, coins, *, fee_estimator_vb): keys = self.keys(coins) buckets = defaultdict(list) for key, coin in zip(keys, coins): buckets[key].append(coin) + # fee_estimator returns fee to be paid, for given vbytes. + # guess whether it is just returning a constant as follows. + constant_fee = fee_estimator_vb(2000) == fee_estimator_vb(200) def make_Bucket(desc, coins): witness = any(Transaction.is_segwit_input(coin, guess_for_address=True) for coin in coins) @@ -117,16 +128,30 @@ def make_Bucket(desc, coins): for coin in coins) value = sum(coin['value'] for coin in coins) min_height = min(coin['height'] for coin in coins) - return Bucket(desc, weight, value, coins, min_height, witness) + # the fee estimator is typically either a constant or a linear function, + # so the "function:" effective_value(bucket) will be homomorphic for addition + # i.e. effective_value(b1) + effective_value(b2) = effective_value(b1 + b2) + if constant_fee: + effective_value = value + else: + # when converting from weight to vBytes, instead of rounding up, + # keep fractional part, to avoid overestimating fee + fee = fee_estimator_vb(Decimal(weight) / 4) + effective_value = value - fee + return Bucket(desc=desc, + weight=weight, + value=value, + effective_value=effective_value, + coins=coins, + min_height=min_height, + witness=witness) return list(map(make_Bucket, buckets.keys(), buckets.values())) - def penalty_func(self, tx): - def penalty(candidate): - return 0 - return penalty + def penalty_func(self, base_tx, *, tx_from_buckets) -> Callable[[List[Bucket]], ScoredCandidate]: + raise NotImplementedError - def change_amounts(self, tx, count, fee_estimator, dust_threshold): + def _change_amounts(self, tx, count, fee_estimator_numchange): # Break change up if bigger than max_change output_amounts = [o.value for o in tx.outputs()] # Don't split change of less than 0.02 BTC @@ -135,7 +160,7 @@ def change_amounts(self, tx, count, fee_estimator, dust_threshold): # Use N change outputs for n in range(1, count + 1): # How much is left if we add this many change outputs? - change_amount = max(0, tx.get_fee() - fee_estimator(n)) + change_amount = max(0, tx.get_fee() - fee_estimator_numchange(n)) if change_amount // n <= max_change: break @@ -180,30 +205,72 @@ def trailing_zeroes(val): return amounts - def change_outputs(self, tx, change_addrs, fee_estimator, dust_threshold): - amounts = self.change_amounts(tx, len(change_addrs), fee_estimator, - dust_threshold) + def _change_outputs(self, tx, change_addrs, fee_estimator_numchange, dust_threshold): + amounts = self._change_amounts(tx, len(change_addrs), fee_estimator_numchange) assert min(amounts) >= 0 assert len(change_addrs) >= len(amounts) # If change is above dust threshold after accounting for the # size of the change output, add it to the transaction. - dust = sum(amount for amount in amounts if amount < dust_threshold) amounts = [amount for amount in amounts if amount >= dust_threshold] change = [TxOutput(TYPE_ADDRESS, addr, amount) for addr, amount in zip(change_addrs, amounts)] - self.logger.info(f'change: {change}') - if dust: - self.logger.info(f'not keeping dust {dust}') return change - def make_tx(self, coins, inputs, outputs, change_addrs, fee_estimator, + def _construct_tx_from_selected_buckets(self, *, buckets, base_tx, change_addrs, + fee_estimator_w, dust_threshold, base_weight): + # make a copy of base_tx so it won't get mutated + tx = Transaction.from_io(base_tx.inputs()[:], base_tx.outputs()[:]) + + tx.add_inputs([coin for b in buckets for coin in b.coins]) + tx_weight = self._get_tx_weight(buckets, base_weight=base_weight) + + # change is sent back to sending address unless specified + if not change_addrs: + change_addrs = [tx.inputs()[0]['address']] + # note: this is not necessarily the final "first input address" + # because the inputs had not been sorted at this point + assert is_address(change_addrs[0]) + + # This takes a count of change outputs and returns a tx fee + output_weight = 4 * Transaction.estimated_output_size(change_addrs[0]) + fee_estimator_numchange = lambda count: fee_estimator_w(tx_weight + count * output_weight) + change = self._change_outputs(tx, change_addrs, fee_estimator_numchange, dust_threshold) + tx.add_outputs(change) + + return tx, change + + def _get_tx_weight(self, buckets, *, base_weight) -> int: + """Given a collection of buckets, return the total weight of the + resulting transaction. + base_weight is the weight of the tx that includes the fixed (non-change) + outputs and potentially some fixed inputs. Note that the change outputs + at this point are not yet known so they are NOT accounted for. + """ + total_weight = base_weight + sum(bucket.weight for bucket in buckets) + is_segwit_tx = any(bucket.witness for bucket in buckets) + if is_segwit_tx: + total_weight += 2 # marker and flag + # non-segwit inputs were previously assumed to have + # a witness of '' instead of '00' (hex) + # note that mixed legacy/segwit buckets are already ok + num_legacy_inputs = sum((not bucket.witness) * len(bucket.coins) + for bucket in buckets) + total_weight += num_legacy_inputs + + return total_weight + + def make_tx(self, coins, inputs, outputs, change_addrs, fee_estimator_vb, dust_threshold): """Select unspent coins to spend to pay outputs. If the change is greater than dust_threshold (after adding the change output to the transaction) it is kept, otherwise none is sent and it is added to the transaction fee. - Note: fee_estimator expects virtual bytes + `inputs` and `outputs` are guaranteed to be a subset of the + inputs and outputs of the resulting transaction. + `coins` are further UTXOs we can choose from. + + Note: fee_estimator_vb expects virtual bytes """ # Deterministic randomness from coins @@ -211,33 +278,19 @@ def make_tx(self, coins, inputs, outputs, change_addrs, fee_estimator, self.p = PRNG(''.join(sorted(utxos))) # Copy the outputs so when adding change we don't modify "outputs" - tx = Transaction.from_io(inputs[:], outputs[:]) - input_value = tx.input_value() + base_tx = Transaction.from_io(inputs[:], outputs[:]) + input_value = base_tx.input_value() # Weight of the transaction with no inputs and no change # Note: this will use legacy tx serialization as the need for "segwit" # would be detected from inputs. The only side effect should be that the # marker and flag are excluded, which is compensated in get_tx_weight() # FIXME calculation will be off by this (2 wu) in case of RBF batching - base_weight = tx.estimated_weight() - spent_amount = tx.output_value() + base_weight = base_tx.estimated_weight() + spent_amount = base_tx.output_value() def fee_estimator_w(weight): - return fee_estimator(Transaction.virtual_size_from_weight(weight)) - - def get_tx_weight(buckets): - total_weight = base_weight + sum(bucket.weight for bucket in buckets) - is_segwit_tx = any(bucket.witness for bucket in buckets) - if is_segwit_tx: - total_weight += 2 # marker and flag - # non-segwit inputs were previously assumed to have - # a witness of '' instead of '00' (hex) - # note that mixed legacy/segwit buckets are already ok - num_legacy_inputs = sum((not bucket.witness) * len(bucket.coins) - for bucket in buckets) - total_weight += num_legacy_inputs - - return total_weight + return fee_estimator_vb(Transaction.virtual_size_from_weight(weight)) def sufficient_funds(buckets, *, bucket_value_sum): '''Given a list of buckets, return True if it has enough @@ -248,36 +301,36 @@ def sufficient_funds(buckets, *, bucket_value_sum): return False # note re performance: so far this was constant time # what follows is linear in len(buckets) - total_weight = get_tx_weight(buckets) + total_weight = self._get_tx_weight(buckets, base_weight=base_weight) return total_input >= spent_amount + fee_estimator_w(total_weight) - # Collect the coins into buckets, choose a subset of the buckets - buckets = self.bucketize_coins(coins) - buckets = self.choose_buckets(buckets, sufficient_funds, - self.penalty_func(tx)) - - tx.add_inputs([coin for b in buckets for coin in b.coins]) - tx_weight = get_tx_weight(buckets) - - # change is sent back to sending address unless specified - if not change_addrs: - change_addrs = [tx.inputs()[0]['address']] - # note: this is not necessarily the final "first input address" - # because the inputs had not been sorted at this point - assert is_address(change_addrs[0]) - - # This takes a count of change outputs and returns a tx fee - output_weight = 4 * Transaction.estimated_output_size(change_addrs[0]) - fee = lambda count: fee_estimator_w(tx_weight + count * output_weight) - change = self.change_outputs(tx, change_addrs, fee, dust_threshold) - tx.add_outputs(change) + def tx_from_buckets(buckets): + return self._construct_tx_from_selected_buckets(buckets=buckets, + base_tx=base_tx, + change_addrs=change_addrs, + fee_estimator_w=fee_estimator_w, + dust_threshold=dust_threshold, + base_weight=base_weight) + + # Collect the coins into buckets + all_buckets = self.bucketize_coins(coins, fee_estimator_vb=fee_estimator_vb) + # Filter some buckets out. Only keep those that have positive effective value. + # Note that this filtering is intentionally done on the bucket level + # instead of per-coin, as each bucket should be either fully spent or not at all. + # (e.g. CoinChooserPrivacy ensures that same-address coins go into one bucket) + all_buckets = list(filter(lambda b: b.effective_value > 0, all_buckets)) + # Choose a subset of the buckets + scored_candidate = self.choose_buckets(all_buckets, sufficient_funds, + self.penalty_func(base_tx, tx_from_buckets=tx_from_buckets)) + tx = scored_candidate.tx self.logger.info(f"using {len(tx.inputs())} inputs") - self.logger.info(f"using buckets: {[bucket.desc for bucket in buckets]}") + self.logger.info(f"using buckets: {[bucket.desc for bucket in scored_candidate.buckets]}") return tx - def choose_buckets(self, buckets, sufficient_funds, penalty_func): + def choose_buckets(self, buckets, sufficient_funds, + penalty_func: Callable[[List[Bucket]], ScoredCandidate]) -> ScoredCandidate: raise NotImplemented('To be subclassed') @@ -312,8 +365,7 @@ def bucket_candidates_any(self, buckets, sufficient_funds): candidates.add(tuple(sorted(permutation[:count + 1]))) break else: - # FIXME this assumes that the effective value of any bkt is >= 0 - # we should make sure not to choose buckets with <= 0 eff. val. + # note: this assumes that the effective value of any bkt is >= 0 raise NotEnoughFunds() candidates = [[buckets[n] for n in c] for c in candidates] @@ -359,12 +411,14 @@ def sfunds(bkts, *, bucket_value_sum): def choose_buckets(self, buckets, sufficient_funds, penalty_func): candidates = self.bucket_candidates_prefer_confirmed(buckets, sufficient_funds) - penalties = [penalty_func(cand) for cand in candidates] - winner = candidates[penalties.index(min(penalties))] - self.logger.info(f"Bucket sets: {len(buckets)}") - self.logger.info(f"Winning penalty: {min(penalties)}") + scored_candidates = [penalty_func(cand) for cand in candidates] + winner = min(scored_candidates, key=lambda x: x.penalty) + self.logger.info(f"Total number of buckets: {len(buckets)}") + self.logger.info(f"Num candidates considered: {len(candidates)}. " + f"Winning penalty: {winner.penalty}") return winner + class CoinChooserPrivacy(CoinChooserRandom): """Attempts to better preserve user privacy. First, if any coin is spent from a user address, all coins are. @@ -379,24 +433,28 @@ class CoinChooserPrivacy(CoinChooserRandom): def keys(self, coins): return [coin['address'] for coin in coins] - def penalty_func(self, tx): - min_change = min(o.value for o in tx.outputs()) * 0.75 - max_change = max(o.value for o in tx.outputs()) * 1.33 - spent_amount = sum(o.value for o in tx.outputs()) + def penalty_func(self, base_tx, *, tx_from_buckets): + min_change = min(o.value for o in base_tx.outputs()) * 0.75 + max_change = max(o.value for o in base_tx.outputs()) * 1.33 - def penalty(buckets): + def penalty(buckets) -> ScoredCandidate: + # Penalize using many buckets (~inputs) badness = len(buckets) - 1 - total_input = sum(bucket.value for bucket in buckets) - # FIXME "change" here also includes fees - change = float(total_input - spent_amount) + tx, change_outputs = tx_from_buckets(buckets) + change = sum(o.value for o in change_outputs) # Penalize change not roughly in output range - if change < min_change: + if change == 0: + pass # no change is great! + elif change < min_change: badness += (min_change - change) / (min_change + 10000) + # Penalize really small change; under 1 mBTC ~= using 1 more input + if change < COIN / 1000: + badness += 1 elif change > max_change: badness += (change - max_change) / (max_change + 10000) # Penalize large change; 5 BTC excess ~= using 1 more input badness += change / (COIN * 5) - return badness + return ScoredCandidate(badness, tx, buckets) return penalty diff --git a/electrum/gui/kivy/Readme.md b/electrum/gui/kivy/Readme.md index 85e54494..29c9678c 100644 --- a/electrum/gui/kivy/Readme.md +++ b/electrum/gui/kivy/Readme.md @@ -5,7 +5,10 @@ To generate an APK file, follow these instructions. ## Android binary with Docker -This assumes an Ubuntu host, but it should not be too hard to adapt to another +✗ _This script does not produce reproducible output (yet!). + Please help us remedy this._ + +This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another similar system. The docker commands should be executed in the project's root folder. @@ -27,7 +30,7 @@ folder. 3. Build locale files ``` - $ ./contrib/make_locale + $ ./contrib/pull_locale ``` 4. Prepare pure python dependencies @@ -78,13 +81,34 @@ $ sudo docker run -it --rm \ ``` -### How do I get more verbose logs? +### How do I get more verbose logs for the build? See `log_level` in `buildozer.spec` +### How can I see logs at runtime? +This should work OK for most scenarios: +``` +adb logcat | grep python +``` +Better `grep` but fragile because of `cut`: +``` +adb logcat | grep -F "`adb shell ps | grep org.electrum.electrum | cut -c14-19`" +``` + + ### Kivy can be run directly on Linux Desktop. How? Install Kivy. Build atlas: `(cd electrum/gui/kivy/; make theming)` Run electrum with the `-g` switch: `electrum -g kivy` + +### debug vs release build +If you just follow the instructions above, you will build the apk +in debug mode. The most notable difference is that the apk will be +signed using a debug keystore. If you are planning to upload +what you build to e.g. the Play Store, you should create your own +keystore, back it up safely, and run `./contrib/make_apk release`. + +See e.g. [kivy wiki](https://github.com/kivy/kivy/wiki/Creating-a-Release-APK) +and [android dev docs](https://developer.android.com/studio/build/building-cmdline#sign_cmdline). diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 80269514..cfbaf9d3 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -6,6 +6,7 @@ import traceback from decimal import Decimal import threading +import asyncio from electrum.bitcoin import TYPE_ADDRESS from electrum.storage import WalletStorage @@ -13,7 +14,7 @@ from electrum.paymentrequest import InvoiceStore from electrum.util import profiler, InvalidPassword, send_exception_to_crash_reporter from electrum.plugin import run_hook -from electrum.util import format_satoshis, format_satoshis_plain +from electrum.util import format_satoshis, format_satoshis_plain, format_fee_satoshis from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED from electrum import blockchain from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed @@ -280,6 +281,7 @@ def __init__(self, **kwargs): self.is_exit = False self.wallet = None self.pause_time = 0 + self.asyncio_loop = asyncio.get_event_loop() App.__init__(self)#, **kwargs) @@ -433,7 +435,8 @@ def on_qr_failure(): msg += '\n' + _('Text copied to clipboard.') self._clipboard.copy(text_for_clipboard) Clock.schedule_once(lambda dt: self.show_info(msg)) - popup = QRDialog(title, data, show_text, on_qr_failure) + popup = QRDialog(title, data, show_text, failure_cb=on_qr_failure, + text_for_clipboard=text_for_clipboard) popup.open() def scan_qr(self, on_complete): @@ -454,6 +457,8 @@ def on_qr_result(requestCode, resultCode, intent): String = autoclass("java.lang.String") contents = intent.getStringExtra(String("text")) on_complete(contents) + except Exception as e: # exc would otherwise get lost + send_exception_to_crash_reporter(e) finally: activity.unbind(on_activity_result=on_qr_result) activity.bind(on_activity_result=on_qr_result) @@ -803,6 +808,10 @@ def format_amount(self, x, is_diff=False, whitespaces=False): def format_amount_and_units(self, x): return format_satoshis_plain(x, self.decimal_point()) + ' ' + self.base_unit + def format_fee_rate(self, fee_rate): + # fee_rate is in sat/kB + return format_fee_satoshis(fee_rate/1000) + ' sat/byte' + #@profiler def update_wallet(self, *dt): self._trigger_update_status() diff --git a/electrum/gui/kivy/tools/Dockerfile b/electrum/gui/kivy/tools/Dockerfile index 31938575..13ec31d4 100644 --- a/electrum/gui/kivy/tools/Dockerfile +++ b/electrum/gui/kivy/tools/Dockerfile @@ -127,6 +127,8 @@ USER ${USER} RUN python3 -m pip install --upgrade cython==0.28.6 +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --user wheel # prepare git RUN git config --global user.name "John Doe" \ @@ -136,8 +138,8 @@ RUN git config --global user.name "John Doe" \ RUN cd /opt \ && git clone https://github.com/kivy/buildozer \ && cd buildozer \ - && git checkout 88e4a4b0c7733eec1d14c00579ec412fb59ad7f2 \ - && python3 -m pip install -e . + && git checkout 678b1bf52cf63daa51b06e86a43ea4e2ea8a0b24 \ + && python3 -m pip install --user -e . # install python-for-android RUN cd /opt \ @@ -145,12 +147,14 @@ RUN cd /opt \ && cd python-for-android \ && git remote add sombernight https://github.com/SomberNight/python-for-android \ && git fetch --all \ - && git checkout dec1badc3bd134a9a1c69275339423a95d63413e \ + && git checkout ccb0f8e1bab36f1b7d1508216b4b4afb076e614f \ # allowBackup="false": && git cherry-pick d7f722e4e5d4b3e6f5b1733c95e6a433f78ee570 \ - # enable IPv6: - && git cherry-pick a607f4a446773ac0b0a5150171092b0617fbe670 \ - && python3 -m pip install -e . + # fix gradle "versionCode" overflow: + && git cherry-pick ed20e196fbcdce718a180f88f23bb2d165c4c5d8 \ + # gradle: persist debug keystore: + && git cherry-pick aaa0d5d0e7a334631df71e0a9bf127817e0ab9ab \ + && python3 -m pip install --user -e . # build env vars ENV USE_SDK_WRAPPER=1 diff --git a/electrum/gui/kivy/tools/buildozer.spec b/electrum/gui/kivy/tools/buildozer.spec index 1d1e25ab..6724ad55 100644 --- a/electrum/gui/kivy/tools/buildozer.spec +++ b/electrum/gui/kivy/tools/buildozer.spec @@ -35,7 +35,14 @@ version.filename = %(source.dir)s/electrum/version.py #version = 1.9.8 # (list) Application requirements -requirements = python3, android, openssl, plyer, kivy==b47f669f44dbda4f463bcb7d2cada639f7fed3bc, libffi, libsecp256k1 +requirements = + python3, + android, + openssl, + plyer, + kivy==82d561d62577757d478df52173610f925c05ecab, + libffi, + libsecp256k1 # (str) Presplash of the application #presplash.filename = %(source.dir)s/gui/kivy/theming/splash.png @@ -64,11 +71,8 @@ android.api = 28 # (int) Minimum API required. You will need to set the android.ndk_api to be as low as this value. android.minapi = 21 -# (int) Android SDK version to use -android.sdk = 24 - # (str) Android NDK version to use -android.ndk = 14b +android.ndk = 17c # (int) Android NDK API to use (optional). This is the minimum API your app will support. android.ndk_api = 21 diff --git a/electrum/gui/kivy/uix/dialogs/bump_fee_dialog.py b/electrum/gui/kivy/uix/dialogs/bump_fee_dialog.py index 854be26b..21f3ca2b 100644 --- a/electrum/gui/kivy/uix/dialogs/bump_fee_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/bump_fee_dialog.py @@ -24,8 +24,8 @@ text: _('Current Fee') value: '' BoxLabel: - id: new_fee - text: _('New Fee') + id: old_feerate + text: _('Current Fee rate') value: '' Label: id: tooltip1 @@ -78,15 +78,14 @@ def __init__(self, app, fee, size, callback): self.mempool = self.config.use_mempool_fees() self.dynfees = self.config.is_dynfee() and bool(self.app.network) and self.config.has_dynamic_fees_ready() self.ids.old_fee.value = self.app.format_amount_and_units(self.init_fee) + self.ids.old_feerate.value = self.app.format_fee_rate(fee / self.tx_size * 1000) self.update_slider() self.update_text() def update_text(self): - fee = self.get_fee() - self.ids.new_fee.value = self.app.format_amount_and_units(fee) pos = int(self.ids.slider.value) - fee_rate = self.get_fee_rate() - text, tooltip = self.config.get_fee_text(pos, self.dynfees, self.mempool, fee_rate) + new_fee_rate = self.get_fee_rate() + text, tooltip = self.config.get_fee_text(pos, self.dynfees, self.mempool, new_fee_rate) self.ids.tooltip1.text = text self.ids.tooltip2.text = tooltip @@ -103,16 +102,12 @@ def get_fee_rate(self): fee_rate = self.config.depth_to_fee(pos) if self.mempool else self.config.eta_to_fee(pos) else: fee_rate = self.config.static_fee(pos) - return fee_rate - - def get_fee(self): - fee_rate = self.get_fee_rate() - return int(fee_rate * self.tx_size // 1000) + return fee_rate # sat/kbyte def on_ok(self): - new_fee = self.get_fee() + new_fee_rate = self.get_fee_rate() / 1000 is_final = self.ids.final_cb.active - self.callback(self.init_fee, new_fee, is_final) + self.callback(new_fee_rate, is_final) def on_slider(self, value): self.update_text() diff --git a/electrum/gui/kivy/uix/dialogs/crash_reporter.py b/electrum/gui/kivy/uix/dialogs/crash_reporter.py index dcbd5ccc..f8c087d9 100644 --- a/electrum/gui/kivy/uix/dialogs/crash_reporter.py +++ b/electrum/gui/kivy/uix/dialogs/crash_reporter.py @@ -119,8 +119,11 @@ def send_report(self): try: loop = self.main_window.network.asyncio_loop proxy = self.main_window.network.proxy - response = json.loads(BaseCrashReporter.send_report(self, loop, proxy, "/crash.json")) + # FIXME network request in GUI thread... + response = json.loads(BaseCrashReporter.send_report(self, loop, proxy, + "/crash.json", timeout=10)) except (ValueError, ClientError): + #self.logger.debug("", exc_info=True) self.show_popup(_('Unable to send report'), _("Please check your network connection.")) else: self.show_popup(_('Report sent'), response["text"]) diff --git a/electrum/gui/kivy/uix/dialogs/qr_dialog.py b/electrum/gui/kivy/uix/dialogs/qr_dialog.py index b12eb6ce..0685dfa9 100644 --- a/electrum/gui/kivy/uix/dialogs/qr_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/qr_dialog.py @@ -1,5 +1,11 @@ from kivy.factory import Factory from kivy.lang import Builder +from kivy.core.clipboard import Clipboard +from kivy.app import App +from kivy.clock import Clock + +from electrum.gui.kivy.i18n import _ + Builder.load_string(''' @@ -24,9 +30,12 @@ BoxLayout: size_hint: 1, None height: '48dp' - Widget: + Button: size_hint: 1, None height: '48dp' + text: _('Copy to clipboard') + on_release: + root.copy_to_clipboard() Button: size_hint: 1, None height: '48dp' @@ -36,12 +45,20 @@ ''') class QRDialog(Factory.Popup): - def __init__(self, title, data, show_text, failure_cb=None): + def __init__(self, title, data, show_text, *, + failure_cb=None, text_for_clipboard=None): Factory.Popup.__init__(self) + self.app = App.get_running_app() self.title = title self.data = data self.show_text = show_text self.failure_cb = failure_cb + self.text_for_clipboard = text_for_clipboard if text_for_clipboard else data def on_open(self): self.ids.qr.set_data(self.data, self.failure_cb) + + def copy_to_clipboard(self): + Clipboard.copy(self.text_for_clipboard) + msg = _('Text copied to clipboard.') + Clock.schedule_once(lambda dt: self.app.show_info(msg)) diff --git a/electrum/gui/kivy/uix/dialogs/tx_dialog.py b/electrum/gui/kivy/uix/dialogs/tx_dialog.py index d4c0e67b..d65571f8 100644 --- a/electrum/gui/kivy/uix/dialogs/tx_dialog.py +++ b/electrum/gui/kivy/uix/dialogs/tx_dialog.py @@ -15,6 +15,7 @@ from electrum.util import InvalidPassword from electrum.address_synchronizer import TX_HEIGHT_LOCAL +from electrum.wallet import CannotBumpFee Builder.load_string(''' @@ -27,6 +28,7 @@ can_broadcast: False can_rbf: False fee_str: '' + feerate_str: '' date_str: '' date_label:'' amount_str: '' @@ -65,6 +67,9 @@ BoxLabel: text: _('Transaction fee') if root.fee_str else '' value: root.fee_str + BoxLabel: + text: _('Transaction fee rate') if root.feerate_str else '' + value: root.feerate_str TopLabel: text: _('Transaction ID') + ':' if root.tx_hash else '' TxHashLabel: @@ -148,7 +153,13 @@ def update(self): else: self.is_mine = True self.amount_str = format_amount(-amount) - self.fee_str = format_amount(fee) if fee is not None else _('unknown') + if fee is not None: + self.fee_str = format_amount(fee) + fee_per_kb = fee / self.tx.estimated_size() * 1000 + self.feerate_str = self.app.format_fee_rate(fee_per_kb) + else: + self.fee_str = _('unknown') + self.feerate_str = _('unknown') self.can_sign = self.wallet.can_sign(self.tx) self.ids.output_list.update(self.tx.get_outputs_for_UI()) self.is_local_tx = tx_mined_status.height == TX_HEIGHT_LOCAL @@ -184,7 +195,7 @@ def update_action_button(self): self._action_button_fn = dropdown.open for option in options: if option.enabled: - btn = Button(text=option.text, size_hint_y=None, height=48) + btn = Button(text=option.text, size_hint_y=None, height='48dp') btn.bind(on_release=option.func) dropdown.add_widget(btn) @@ -202,16 +213,14 @@ def do_rbf(self): d = BumpFeeDialog(self.app, fee, size, self._do_rbf) d.open() - def _do_rbf(self, old_fee, new_fee, is_final): - if new_fee is None: - return - delta = new_fee - old_fee - if delta < 0: - self.app.show_error("fee too low") + def _do_rbf(self, new_fee_rate, is_final): + if new_fee_rate is None: return try: - new_tx = self.wallet.bump_fee(self.tx, delta) - except BaseException as e: + new_tx = self.wallet.bump_fee(tx=self.tx, + new_fee_rate=new_fee_rate, + config=self.app.electrum_config) + except CannotBumpFee as e: self.app.show_error(str(e)) return if is_final: diff --git a/electrum/gui/kivy/uix/screens.py b/electrum/gui/kivy/uix/screens.py index d547ea48..17a70a38 100644 --- a/electrum/gui/kivy/uix/screens.py +++ b/electrum/gui/kivy/uix/screens.py @@ -20,11 +20,12 @@ from electrum.util import profiler, parse_URI, format_time, InvalidPassword, NotEnoughFunds, Fiat from electrum import bitcoin -from electrum.transaction import TxOutput -from electrum.util import send_exception_to_crash_reporter +from electrum.transaction import TxOutput, Transaction, tx_from_str +from electrum.util import send_exception_to_crash_reporter, parse_URI, InvalidBitcoinURI from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED from electrum.plugin import run_hook from electrum.wallet import InternalAddressCorruption +from electrum import simple_config from .context_menu import ContextMenu @@ -173,11 +174,10 @@ def set_URI(self, text): if not self.app.wallet: self.payment_request_queued = text return - import electrum try: - uri = electrum.util.parse_URI(text, self.app.on_pr) - except: - self.app.show_info(_("Not a Noir URI")) + uri = parse_URI(text, self.app.on_pr, loop=self.app.asyncio_loop) + except InvalidBitcoinURI as e: + self.app.show_info(_("Error parsing URI") + f":\n{e}") return amount = uri.get('amount') self.screen.address = uri.get('address', '') @@ -233,11 +233,22 @@ def do_save(self): self.payment_request = None def do_paste(self): - contents = self.app._clipboard.paste() - if not contents: + data = self.app._clipboard.paste() + if not data: self.app.show_info(_("Clipboard is empty")) return - self.set_URI(contents) + # try to decode as transaction + try: + raw_tx = tx_from_str(data) + tx = Transaction(raw_tx) + tx.deserialize() + except: + tx = None + if tx: + self.app.tx_dialog(tx) + return + # try to decode as URI/address + self.set_URI(data) def do_send(self): if self.screen.is_pr: @@ -293,8 +304,9 @@ def _do_send(self, amount, message, outputs, rbf): x_fee_address, x_fee_amount = x_fee msg.append(_("Additional fees") + ": " + self.app.format_amount_and_units(x_fee_amount)) - if fee >= config.get('confirm_fee', 100000): - msg.append(_('Warning')+ ': ' + _("The fee for this transaction seems unusually high.")) + feerate_warning = simple_config.FEERATE_WARNING_HIGH_FEE + if fee > feerate_warning * tx.estimated_size() / 1000: + msg.append(_('Warning') + ': ' + _("The fee for this transaction seems unusually high.")) msg.append(_("Enter your PIN code to proceed")) self.app.protected('\n'.join(msg), self.send_tx, (tx, message)) diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index 4656b277..9d3a3808 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -31,7 +31,7 @@ from PyQt5.QtWidgets import QAbstractItemView, QComboBox, QLabel, QMenu from electrum.i18n import _ -from electrum.util import block_explorer_URL +from electrum.util import block_explorer_URL, profiler from electrum.plugin import run_hook from electrum.bitcoin import is_address from electrum.wallet import InternalAddressCorruption @@ -107,6 +107,7 @@ def toggle_used(self, state): self.show_used = state self.update() + @profiler def update(self): self.wallet = self.parent.wallet current_address = self.current_item_user_role(col=self.Columns.LABEL) @@ -187,6 +188,8 @@ def create_menu(self, position): menu = QMenu() if not multi_select: idx = self.indexAt(position) + if not idx.isValid(): + return col = idx.column() item = self.model().itemFromIndex(idx) if not item: diff --git a/electrum/gui/qt/amountedit.py b/electrum/gui/qt/amountedit.py index c3920b9d..8fc63fa1 100644 --- a/electrum/gui/qt/amountedit.py +++ b/electrum/gui/qt/amountedit.py @@ -3,9 +3,11 @@ from decimal import Decimal from PyQt5.QtCore import pyqtSignal, Qt -from PyQt5.QtGui import QPalette, QPainter +from PyQt5.QtGui import QPalette, QPainter, QFontMetrics from PyQt5.QtWidgets import (QLineEdit, QStyle, QStyleOptionFrame) +from .util import char_width_in_lineedit + from electrum.util import (format_satoshis_plain, decimal_point_to_base_unit_name, FEERATE_PRECISION, quantize_feerate) @@ -24,7 +26,7 @@ class AmountEdit(MyLineEdit): def __init__(self, base_unit, is_int=False, parent=None): QLineEdit.__init__(self, parent) # This seems sufficient for hundred-BTC amounts with 8 decimals - self.setFixedWidth(140) + self.setFixedWidth(16 * char_width_in_lineedit()) self.base_unit = base_unit self.textChanged.connect(self.numbify) self.is_int = is_int diff --git a/electrum/gui/qt/exception_window.py b/electrum/gui/qt/exception_window.py index da05044a..4ea065cd 100644 --- a/electrum/gui/qt/exception_window.py +++ b/electrum/gui/qt/exception_window.py @@ -33,7 +33,7 @@ from electrum.i18n import _ from electrum.base_crash_reporter import BaseCrashReporter from electrum.logging import Logger -from .util import MessageBoxMixin, read_QIcon +from .util import MessageBoxMixin, read_QIcon, WaitingDialog class Exception_Window(BaseCrashReporter, QWidget, MessageBoxMixin, Logger): @@ -69,6 +69,8 @@ def __init__(self, main_window, exctype, value, tb): self.description_textfield = QTextEdit() self.description_textfield.setFixedHeight(50) + self.description_textfield.setPlaceholderText(_("Do not enter sensitive/private information here. " + "The report will be visible on the public issue tracker.")) main_box.addWidget(self.description_textfield) main_box.addWidget(QLabel(BaseCrashReporter.ASK_CONFIRM_SEND)) @@ -94,17 +96,23 @@ def __init__(self, main_window, exctype, value, tb): self.show() def send_report(self): - try: - proxy = self.main_window.network.proxy - response = BaseCrashReporter.send_report(self, self.main_window.network.asyncio_loop, proxy) - except BaseException as e: - self.logger.exception('There was a problem with the automatic reporting') - self.main_window.show_critical(_('There was a problem with the automatic reporting:') + '\n' + - str(e) + '\n' + - _("Please report this issue manually.")) - return - QMessageBox.about(self, _("Crash report"), response) - self.close() + def on_success(response): + self.show_message(parent=self, + title=_("Crash report"), + msg=response) + self.close() + def on_failure(exc_info): + e = exc_info[1] + self.logger.error('There was a problem with the automatic reporting', exc_info=exc_info) + self.show_critical(parent=self, + msg=(_('There was a problem with the automatic reporting:') + '\n' + + str(e) + '\n' + + _("Please report this issue manually."))) + + proxy = self.main_window.network.proxy + task = lambda: BaseCrashReporter.send_report(self, self.main_window.network.asyncio_loop, proxy) + msg = _('Sending crash report...') + WaitingDialog(self, msg, task, on_success, on_failure) def on_close(self): Exception_Window._active_window = None diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 333487a1..5edee2a5 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -149,7 +149,7 @@ def data(self, index: QModelIndex, role: Qt.ItemDataRole) -> QVariant: HistoryColumns.STATUS_ICON: # height breaks ties for unverified txns # txpos breaks ties for verified same block txns - (status, conf, -height, -txpos), + (conf, -status, -height, -txpos), HistoryColumns.STATUS_TEXT: status_str, HistoryColumns.DESCRIPTION: tx_item['label'], HistoryColumns.COIN_VALUE: tx_item['value'].value, diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index 909aa752..72294004 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -23,7 +23,7 @@ from .seed_dialog import SeedLayout, KeysLayout from .network_dialog import NetworkChoiceLayout from .util import (MessageBoxMixin, Buttons, icon_path, ChoicesLayout, WWLabel, - InfoButton) + InfoButton, char_width_in_lineedit) from .password_dialog import PasswordLayout, PasswordLayoutForHW, PW_NEW from electrum.plugin import run_hook @@ -121,8 +121,6 @@ def __init__(self, config, app, plugins): self.setWindowTitle('Electrum - ' + _('Install Wizard')) self.app = app self.config = config - # Set for base base class - self.language_for_seed = config.get('language') self.setMinimumSize(600, 400) self.accept_signal.connect(self.accept) self.title = QLabel() @@ -182,7 +180,7 @@ def select_storage(self, path, get_wallet_from_daemon) -> Tuple[str, Optional[Wa vbox.addWidget(self.msg_label) hbox2 = QHBoxLayout() self.pw_e = QLineEdit('', self) - self.pw_e.setFixedWidth(150) + self.pw_e.setFixedWidth(17 * char_width_in_lineedit()) self.pw_e.setEchoMode(2) self.pw_label = QLabel(_('Password') + ':') hbox2.addWidget(self.pw_label) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index af77fc62..25f63dfa 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -37,6 +37,7 @@ from functools import partial import queue import asyncio +from typing import Optional from PyQt5.QtGui import QPixmap, QKeySequence, QIcon, QCursor from PyQt5.QtCore import Qt, QRect, QStringListModel, QSize, pyqtSignal @@ -60,7 +61,8 @@ base_units, base_units_list, base_unit_name_to_decimal_point, decimal_point_to_base_unit_name, quantize_feerate, UnknownBaseUnit, DECIMAL_POINT_DEFAULT, UserFacingException, - get_new_wallet_name, send_exception_to_crash_reporter) + get_new_wallet_name, send_exception_to_crash_reporter, + InvalidBitcoinURI) from electrum.transaction import Transaction, TxOutput from electrum.address_synchronizer import AddTransactionException from electrum.wallet import (Multisig_Wallet, CannotBumpFee, Abstract_Wallet, @@ -70,6 +72,7 @@ from electrum.exchange_rate import FxThread from electrum.simple_config import SimpleConfig from electrum.logging import Logger +from electrum.paymentrequest import PR_PAID from .exception_window import Exception_Hook from .amountedit import AmountEdit, BTCAmountEdit, MyLineEdit, FeerateEdit @@ -82,7 +85,7 @@ OkButton, InfoButton, WWLabel, TaskThread, CancelButton, CloseButton, HelpButton, MessageBoxMixin, EnterButton, expiration_values, ButtonsLineEdit, CopyCloseButton, import_meta_gui, export_meta_gui, - filename_field, address_field) + filename_field, address_field, char_width_in_lineedit) from .installwizard import WIF_HELP_TEXT from .history_list import HistoryList, HistoryModel from .update_checker import UpdateCheck, UpdateCheckThread @@ -108,9 +111,6 @@ def keyPressEvent(self, e): self.func() -from electrum.paymentrequest import PR_PAID - - class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger): payment_request_ok_signal = pyqtSignal() @@ -140,7 +140,7 @@ def __init__(self, gui_object, wallet: Abstract_Wallet): self.tray = gui_object.tray self.app = gui_object.app self.cleaned_up = False - self.payment_request = None + self.payment_request = None # type: Optional[paymentrequest.PaymentRequest] self.checking_accounts = False self.qr_window = None self.not_enough_funds = False @@ -906,6 +906,7 @@ def create_receive_tab(self): msg = _('Noir address where the payment should be received. Note that each payment request uses a different Noir address.') self.receive_address_label = HelpLabel(_('Receiving address'), msg) self.receive_address_e.textChanged.connect(self.update_receive_qr) + self.receive_address_e.textChanged.connect(self.update_receive_address_styling) self.receive_address_e.setFocusPolicy(Qt.ClickFocus) grid.addWidget(self.receive_address_label, 0, 0) grid.addWidget(self.receive_address_e, 0, 1, 1, -1) @@ -1151,6 +1152,16 @@ def update_receive_qr(self): if self.qr_window and self.qr_window.isVisible(): self.qr_window.qrw.setData(uri) + def update_receive_address_styling(self): + addr = str(self.receive_address_e.text()) + if self.wallet.is_used(addr): + self.receive_address_e.setStyleSheet(ColorScheme.RED.as_stylesheet(True)) + self.receive_address_e.setToolTip(_("This address has already been used. " + "For better privacy, do not reuse it for new payments.")) + else: + self.receive_address_e.setStyleSheet("") + self.receive_address_e.setToolTip("") + def set_feerounding_text(self, num_satoshis_added): self.feerounding_text = (_('Additional {} satoshis are going to be added.') .format(num_satoshis_added)) @@ -1205,7 +1216,7 @@ def create_send_tab(self): lambda: self.fiat_send_e.setFrozen(self.amount_e.isReadOnly())) self.max_button = EnterButton(_("Max"), self.spend_max) - self.max_button.setFixedWidth(140) + self.max_button.setFixedWidth(self.amount_e.width()) self.max_button.setCheckable(True) grid.addWidget(self.max_button, 4, 3) hbox = QHBoxLayout() @@ -1237,7 +1248,7 @@ def fee_cb(dyn, pos, fee_rate): self.spend_max() if self.max_button.isChecked() else self.update_fee() self.fee_slider = FeeSlider(self, self.config, fee_cb) - self.fee_slider.setFixedWidth(140) + self.fee_slider.setFixedWidth(self.amount_e.width()) def on_fee_or_feerate(edit_changed, editing_finished): edit_other = self.feerate_e if edit_changed == self.fee_e else self.fee_e @@ -1260,7 +1271,7 @@ def setAmount(self, byte_size): self.size_e = TxSizeLabel() self.size_e.setAlignment(Qt.AlignCenter) self.size_e.setAmount(0) - self.size_e.setFixedWidth(140) + self.size_e.setFixedWidth(self.amount_e.width()) self.size_e.setStyleSheet(ColorScheme.DEFAULT.as_stylesheet()) self.feerate_e = FeerateEdit(lambda: 0) @@ -1282,7 +1293,7 @@ def feerounding_onclick(): self.show_message(title=_('Fee rounding'), msg=text) self.feerounding_icon = QPushButton(read_QIcon('info.png'), '') - self.feerounding_icon.setFixedWidth(20) + self.feerounding_icon.setFixedWidth(round(2.2 * char_width_in_lineedit())) self.feerounding_icon.setFlat(True) self.feerounding_icon.clicked.connect(feerounding_onclick) self.feerounding_icon.setVisible(False) @@ -1590,11 +1601,13 @@ def check_send_tab_outputs_and_show_errors(self, outputs) -> bool: """Returns whether there are errors with outputs. Also shows error dialog to user if so. """ - if self.payment_request and self.payment_request.has_expired(): - self.show_error(_('Payment request has expired')) - return True + pr = self.payment_request + if pr: + if pr.has_expired(): + self.show_error(_('Payment request has expired')) + return True - if not self.payment_request: + if not pr: errors = self.payto_e.get_errors() if errors: self.show_warning(_("Invalid Lines found:") + "\n\n" + '\n'.join([ _("Line #") + str(x[0]+1) + ": " + x[1] for x in errors])) @@ -1683,8 +1696,8 @@ def do_send(self, preview = False): x_fee_address, x_fee_amount = x_fee msg.append( _("Additional fees") + ": " + self.format_amount_and_units(x_fee_amount) ) - confirm_rate = simple_config.FEERATE_WARNING_HIGH_FEE - if fee > confirm_rate * tx.estimated_size() / 1000: + feerate_warning = simple_config.FEERATE_WARNING_HIGH_FEE + if fee > feerate_warning * tx.estimated_size() / 1000: msg.append(_('Warning') + ': ' + _("The fee for this transaction seems unusually high.")) if self.wallet.has_keystore_encryption(): @@ -1808,6 +1821,8 @@ def delete_invoice(self, key): def payment_request_ok(self): pr = self.payment_request + if not pr: + return key = self.invoices.add(pr) status = self.invoices.get_status(key) self.invoice_list.update() @@ -1828,7 +1843,10 @@ def payment_request_ok(self): self.amount_e.textEdited.emit("") def payment_request_error(self): - self.show_message(self.payment_request.error) + pr = self.payment_request + if not pr: + return + self.show_message(pr.error) self.payment_request = None self.do_clear() @@ -1844,8 +1862,8 @@ def pay_to_URI(self, URI): return try: out = util.parse_URI(URI, self.on_pr) - except BaseException as e: - self.show_error(_('Invalid Noir URI:') + '\n' + str(e)) + except InvalidBitcoinURI as e: + self.show_error(_("Error parsing URI") + f":\n{e}") return self.show_send_tab() r = out.get('r') @@ -2180,9 +2198,9 @@ def new_contact_dialog(self): vbox.addWidget(QLabel(_('New Contact') + ':')) grid = QGridLayout() line1 = QLineEdit() - line1.setFixedWidth(280) + line1.setFixedWidth(32 * char_width_in_lineedit()) line2 = QLineEdit() - line2.setFixedWidth(280) + line2.setFixedWidth(32 * char_width_in_lineedit()) grid.addWidget(QLabel(_("Address")), 1, 0) grid.addWidget(line1, 1, 1) grid.addWidget(QLabel(_("Name")), 2, 0) @@ -2223,6 +2241,7 @@ def show_master_public_keys(self): mpk_text.addCopyButton(self.app) def show_mpk(index): mpk_text.setText(mpk_list[index]) + mpk_text.repaint() # macOS hack for #4777 # only show the combobox in case multiple accounts are available if len(mpk_list) > 1: def label(key): @@ -3017,6 +3036,7 @@ def on_set_updatecheck(v): filelogging_cb.setChecked(bool(self.config.get('log_to_file', False))) def on_set_filelogging(v): self.config.set_key('log_to_file', v == Qt.Checked, save=True) + self.need_restart = True filelogging_cb.stateChanged.connect(on_set_filelogging) filelogging_cb.setToolTip(_('Debug logs can be persisted to disk. These are useful for troubleshooting.')) gui_widgets.append((filelogging_cb, None)) @@ -3406,19 +3426,27 @@ def bump_fee_dialog(self, tx): return tx_label = self.wallet.get_label(tx.txid()) tx_size = tx.estimated_size() + old_fee_rate = fee / tx_size # sat/vbyte d = WindowModalDialog(self, _('Bump Fee')) vbox = QVBoxLayout(d) vbox.addWidget(WWLabel(_("Increase your transaction's fee to improve its position in mempool."))) - vbox.addWidget(QLabel(_('Current fee') + ': %s'% self.format_amount(fee) + ' ' + self.base_unit())) - vbox.addWidget(QLabel(_('New fee' + ':'))) - fee_e = BTCAmountEdit(self.get_decimal_point) - fee_e.setAmount(fee * 1.5) - vbox.addWidget(fee_e) - - def on_rate(dyn, pos, fee_rate): - fee = fee_rate * tx_size / 1000 - fee_e.setAmount(fee) - fee_slider = FeeSlider(self, self.config, on_rate) + vbox.addWidget(QLabel(_('Current Fee') + ': %s'% self.format_amount(fee) + ' ' + self.base_unit())) + vbox.addWidget(QLabel(_('Current Fee rate') + ': %s' % self.format_fee_rate(1000 * old_fee_rate))) + vbox.addWidget(QLabel(_('New Fee rate') + ':')) + + def on_textedit_rate(): + fee_slider.deactivate() + feerate_e = FeerateEdit(lambda: 0) + feerate_e.setAmount(max(old_fee_rate * 1.5, old_fee_rate + 1)) + feerate_e.textEdited.connect(on_textedit_rate) + vbox.addWidget(feerate_e) + + def on_slider_rate(dyn, pos, fee_rate): + fee_slider.activate() + if fee_rate is not None: + feerate_e.setAmount(fee_rate / 1000) + fee_slider = FeeSlider(self, self.config, on_slider_rate) + fee_slider.deactivate() vbox.addWidget(fee_slider) cb = QCheckBox(_('Final')) vbox.addWidget(cb) @@ -3426,13 +3454,9 @@ def on_rate(dyn, pos, fee_rate): if not d.exec_(): return is_final = cb.isChecked() - new_fee = fee_e.get_amount() - delta = new_fee - fee - if delta < 0: - self.show_error("fee too low") - return + new_fee_rate = feerate_e.get_amount() try: - new_tx = self.wallet.bump_fee(tx, delta) + new_tx = self.wallet.bump_fee(tx=tx, new_fee_rate=new_fee_rate, config=self.config) except CannotBumpFee as e: self.show_error(str(e)) return diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 00a97de6..511700c3 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -26,11 +26,13 @@ import socket import time from enum import IntEnum +from typing import Tuple from PyQt5.QtCore import Qt, pyqtSignal, QThread from PyQt5.QtWidgets import (QTreeWidget, QTreeWidgetItem, QMenu, QGridLayout, QComboBox, QLineEdit, QDialog, QVBoxLayout, QHeaderView, QCheckBox, QTabWidget, QWidget, QLabel) +from PyQt5.QtGui import QFontMetrics from electrum.i18n import _ from electrum import constants, blockchain @@ -38,7 +40,7 @@ from electrum.network import Network from electrum.logging import get_logger -from .util import Buttons, CloseButton, HelpButton, read_QIcon +from .util import Buttons, CloseButton, HelpButton, read_QIcon, char_width_in_lineedit _logger = get_logger(__name__) @@ -213,14 +215,17 @@ def __init__(self, network: Network, config, wizard=False): tabs.addTab(server_tab, _('Server')) tabs.addTab(proxy_tab, _('Proxy')) + fixed_width_hostname = 24 * char_width_in_lineedit() + fixed_width_port = 6 * char_width_in_lineedit() + # server tab grid = QGridLayout(server_tab) grid.setSpacing(8) self.server_host = QLineEdit() - self.server_host.setFixedWidth(200) + self.server_host.setFixedWidth(fixed_width_hostname) self.server_port = QLineEdit() - self.server_port.setFixedWidth(60) + self.server_port.setFixedWidth(fixed_width_port) self.autoconnect_cb = QCheckBox(_('Select server automatically')) self.autoconnect_cb.setEnabled(self.config.is_modifiable('auto_connect')) @@ -257,15 +262,15 @@ def __init__(self, network: Network, config, wizard=False): self.proxy_mode = QComboBox() self.proxy_mode.addItems(['SOCKS4', 'SOCKS5']) self.proxy_host = QLineEdit() - self.proxy_host.setFixedWidth(200) + self.proxy_host.setFixedWidth(fixed_width_hostname) self.proxy_port = QLineEdit() - self.proxy_port.setFixedWidth(60) + self.proxy_port.setFixedWidth(fixed_width_port) self.proxy_user = QLineEdit() self.proxy_user.setPlaceholderText(_("Proxy user")) self.proxy_password = QLineEdit() self.proxy_password.setPlaceholderText(_("Password")) self.proxy_password.setEchoMode(QLineEdit.Password) - self.proxy_password.setFixedWidth(60) + self.proxy_password.setFixedWidth(fixed_width_port) self.proxy_mode.currentIndexChanged.connect(self.set_proxy) self.proxy_host.editingFinished.connect(self.set_proxy) @@ -521,19 +526,20 @@ def run(self): ports = [9050, 9150] while True: for p in ports: - if TorDetector.is_tor_port(p): - self.found_proxy.emit(("127.0.0.1", p)) + net_addr = ("127.0.0.1", p) + if TorDetector.is_tor_port(net_addr): + self.found_proxy.emit(net_addr) break else: self.found_proxy.emit(None) time.sleep(10) @staticmethod - def is_tor_port(port): + def is_tor_port(net_addr: Tuple[str, int]) -> bool: try: - s = (socket._socketobject if hasattr(socket, "_socketobject") else socket.socket)(socket.AF_INET, socket.SOCK_STREAM) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(0.1) - s.connect(("127.0.0.1", port)) + s.connect(net_addr) # Tor responds uniquely to HTTP-like requests s.send(b"GET\n") if b"Tor is not an HTTP Proxy" in s.recv(1024): diff --git a/electrum/gui/qt/qrcodewidget.py b/electrum/gui/qt/qrcodewidget.py index 659aafd2..4e08e5db 100644 --- a/electrum/gui/qt/qrcodewidget.py +++ b/electrum/gui/qt/qrcodewidget.py @@ -27,7 +27,11 @@ def setData(self, data): if self.data != data: self.data = data if self.data: - self.qr = qrcode.QRCode() + self.qr = qrcode.QRCode( + error_correction=qrcode.constants.ERROR_CORRECT_L, + box_size=10, + border=0, + ) self.qr.add_data(self.data) if not self.fixedSize: k = len(self.qr.get_matrix()) diff --git a/electrum/gui/qt/request_list.py b/electrum/gui/qt/request_list.py index 602f2091..2c0699be 100644 --- a/electrum/gui/qt/request_list.py +++ b/electrum/gui/qt/request_list.py @@ -105,9 +105,10 @@ def update(self): except InternalAddressCorruption as e: self.parent.show_error(str(e)) addr = '' - if not current_address in domain and addr: + if current_address not in domain and addr: self.parent.set_receive_address(addr) self.parent.new_request_button.setEnabled(addr != current_address) + self.parent.update_receive_address_styling() self.model().clear() self.update_headers(self.__class__.headers) diff --git a/electrum/gui/qt/seed_dialog.py b/electrum/gui/qt/seed_dialog.py index f2b559fa..41372228 100644 --- a/electrum/gui/qt/seed_dialog.py +++ b/electrum/gui/qt/seed_dialog.py @@ -24,16 +24,16 @@ # SOFTWARE. from PyQt5.QtCore import Qt -from PyQt5.QtGui import QPixmap +from PyQt5.QtGui import QPixmap, QPalette from PyQt5.QtWidgets import (QVBoxLayout, QCheckBox, QHBoxLayout, QLineEdit, - QLabel, QCompleter, QDialog) + QLabel, QCompleter, QDialog, QStyledItemDelegate) from electrum.i18n import _ from electrum.mnemonic import Mnemonic, seed_type import electrum.old_mnemonic from .util import (Buttons, OkButton, WWLabel, ButtonsTextEdit, icon_path, - EnterButton, CloseButton, WindowModalDialog) + EnterButton, CloseButton, WindowModalDialog, ColorScheme) from .qrtextedit import ShowQRTextEdit, ScanQRTextEdit from .completion_text_edit import CompletionTextEdit @@ -149,11 +149,26 @@ def __init__(self, seed=None, title=None, icon=True, msg=None, options=None, self.addWidget(self.seed_warning) def initialize_completer(self): - english_list = Mnemonic('en').wordlist + bip39_english_list = Mnemonic('en').wordlist old_list = electrum.old_mnemonic.words - self.wordlist = english_list + list(set(old_list) - set(english_list)) #concat both lists + only_old_list = set(old_list) - set(bip39_english_list) + self.wordlist = bip39_english_list + list(only_old_list) # concat both lists self.wordlist.sort() + + class CompleterDelegate(QStyledItemDelegate): + def initStyleOption(self, option, index): + super().initStyleOption(option, index) + # Some people complained that due to merging the two word lists, + # it is difficult to restore from a metal backup, as they planned + # to rely on the "4 letter prefixes are unique in bip39 word list" property. + # So we color words that are only in old list. + if option.text in only_old_list: + # yellow bg looks ~ok on both light/dark theme, regardless if (un)selected + option.backgroundBrush = ColorScheme.YELLOW.as_color(background=True) + self.completer = QCompleter(self.wordlist) + delegate = CompleterDelegate(self.seed_e) + self.completer.popup().setItemDelegate(delegate) self.seed_e.set_completer(self.completer) def get_seed(self): @@ -174,7 +189,7 @@ def on_edit(self): self.seed_type_label.setText(label) self.parent.next_button.setEnabled(b) - # to account for bip39 seeds + # disable suggestions if user already typed an unknown word for word in self.get_seed().split(" ")[:-1]: if word not in self.wordlist: self.seed_e.disable_suggestions() diff --git a/electrum/gui/qt/transaction_dialog.py b/electrum/gui/qt/transaction_dialog.py index c84c99e7..c3596b86 100644 --- a/electrum/gui/qt/transaction_dialog.py +++ b/electrum/gui/qt/transaction_dialog.py @@ -28,6 +28,7 @@ import datetime import json import traceback +from typing import TYPE_CHECKING from PyQt5.QtCore import QSize, Qt from PyQt5.QtGui import QTextCharFormat, QBrush, QFont @@ -47,6 +48,9 @@ from .util import (MessageBoxMixin, read_QIcon, Buttons, CopyButton, MONOSPACE_FONT, ColorScheme, ButtonsLineEdit) +if TYPE_CHECKING: + from .main_window import ElectrumWindow + SAVE_BUTTON_ENABLED_TOOLTIP = _("Save transaction offline") SAVE_BUTTON_DISABLED_TOOLTIP = _("Please sign this transaction in order to save it") @@ -83,7 +87,7 @@ def __init__(self, tx, parent, desc, prompt_if_unsaved): self.tx.deserialize() except BaseException as e: raise SerializationError(e) - self.main_window = parent + self.main_window = parent # type: ElectrumWindow self.wallet = parent.wallet self.prompt_if_unsaved = prompt_if_unsaved self.saved = False @@ -273,8 +277,8 @@ def update(self): if fee is not None: fee_rate = fee/size*1000 fee_str += ' ( %s ) ' % self.main_window.format_fee_rate(fee_rate) - confirm_rate = simple_config.FEERATE_WARNING_HIGH_FEE - if fee_rate > confirm_rate: + feerate_warning = simple_config.FEERATE_WARNING_HIGH_FEE + if fee_rate > feerate_warning: fee_str += ' - ' + _('Warning') + ': ' + _("high fee") + '!' self.amount_label.setText(amount_str) self.fee_label.setText(fee_str) diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 3279ea17..ba5411b5 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -10,7 +10,7 @@ from typing import NamedTuple, Callable, Optional, TYPE_CHECKING, Union, List, Dict from PyQt5.QtGui import (QFont, QColor, QCursor, QPixmap, QStandardItem, - QPalette, QIcon) + QPalette, QIcon, QFontMetrics) from PyQt5.QtCore import (Qt, QPersistentModelIndex, QModelIndex, pyqtSignal, QCoreApplication, QItemSelectionModel, QThread, QSortFilterProxyModel, QSize, QLocale) @@ -92,6 +92,7 @@ class WWLabel(QLabel): def __init__ (self, text="", parent=None): QLabel.__init__(self, text, parent) self.setWordWrap(True) + self.setTextInteractionFlags(Qt.TextSelectableByMouse) class HelpLabel(QLabel): @@ -126,14 +127,15 @@ def __init__(self, text): QPushButton.__init__(self, '?') self.help_text = text self.setFocusPolicy(Qt.NoFocus) - self.setFixedWidth(20) + self.setFixedWidth(round(2.2 * char_width_in_lineedit())) self.clicked.connect(self.onclick) def onclick(self): custom_message_box(icon=QMessageBox.Information, parent=self, title=_('Help'), - text=self.help_text) + text=self.help_text, + rich_text=True) class InfoButton(QPushButton): @@ -141,14 +143,15 @@ def __init__(self, text): QPushButton.__init__(self, 'Info') self.help_text = text self.setFocusPolicy(Qt.NoFocus) - self.setFixedWidth(60) + self.setFixedWidth(6 * char_width_in_lineedit()) self.clicked.connect(self.onclick) def onclick(self): custom_message_box(icon=QMessageBox.Information, parent=self, title=_('Info'), - text=self.help_text) + text=self.help_text, + rich_text=True) class Buttons(QHBoxLayout): @@ -204,11 +207,15 @@ def top_level_window_recurse(self, window=None, test_func=None): def top_level_window(self, test_func=None): return self.top_level_window_recurse(test_func) - def question(self, msg, parent=None, title=None, icon=None): + def question(self, msg, parent=None, title=None, icon=None, **kwargs) -> bool: Yes, No = QMessageBox.Yes, QMessageBox.No - return self.msg_box(icon or QMessageBox.Question, - parent, title or '', - msg, buttons=Yes|No, defaultButton=No) == Yes + return Yes == self.msg_box(icon=icon or QMessageBox.Question, + parent=parent, + title=title or '', + text=msg, + buttons=Yes|No, + defaultButton=No, + **kwargs) def show_warning(self, msg, parent=None, title=None, **kwargs): return self.msg_box(QMessageBox.Warning, parent, @@ -252,7 +259,11 @@ def custom_message_box(*, icon, parent, title, text, buttons=QMessageBox.Ok, d.setDefaultButton(defaultButton) if rich_text: d.setTextInteractionFlags(Qt.TextSelectableByMouse | Qt.LinksAccessibleByMouse) - d.setTextFormat(Qt.RichText) + # set AutoText instead of RichText + # AutoText lets Qt figure out whether to render as rich text. + # e.g. if text is actually plain text and uses "\n" newlines; + # and we set RichText here, newlines would be swallowed + d.setTextFormat(Qt.AutoText) else: d.setTextInteractionFlags(Qt.TextSelectableByMouse) d.setTextFormat(Qt.PlainText) @@ -861,6 +872,12 @@ def __init__(self, parent, create_menu): self.header().setSectionResizeMode(1, sm) +def char_width_in_lineedit() -> int: + char_width = QFontMetrics(QLineEdit().font()).averageCharWidth() + # 'averageCharWidth' seems to underestimate on Windows, hence 'max()' + return max(9, char_width) + + if __name__ == "__main__": app = QApplication([]) t = WaitingDialog(None, 'testing ...', lambda: [time.sleep(1)], lambda x: QMessageBox.information(None, 'done', "done")) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index c18d4426..f549cc9c 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -124,6 +124,8 @@ def create_menu(self, position): menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx, label)) # "Copy ..." idx = self.indexAt(position) + if not idx.isValid(): + return col = idx.column() column_title = self.model().horizontalHeaderItem(col).text() copy_text = self.model().itemFromIndex(idx).text() if col != self.Columns.OUTPOINT else selected[0] diff --git a/electrum/interface.py b/electrum/interface.py index 8a019afb..b5eac439 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -28,6 +28,7 @@ import sys import traceback import asyncio +import socket from typing import Tuple, Union, List, TYPE_CHECKING, Optional from collections import defaultdict from ipaddress import IPv4Network, IPv6Network, ip_address @@ -37,7 +38,8 @@ import aiorpcx from aiorpcx import RPCSession, Notification, NetAddress from aiorpcx.curio import timeout_after, TaskTimeout -from aiorpcx.jsonrpc import JSONRPC +from aiorpcx.jsonrpc import JSONRPC, CodeMessageError +from aiorpcx.rawsocket import RSClient import certifi from .util import ignore_exceptions, log_exceptions, bfh, SilentTaskGroup @@ -106,11 +108,16 @@ async def send_request(self, *args, timeout=None, **kwargs): msg_id = next(self._msg_counter) self.maybe_log(f"<-- {args} {kwargs} (id: {msg_id})") try: + # note: RPCSession.send_request raises TaskTimeout in case of a timeout. + # TaskTimeout is a subclass of CancelledError, which is *suppressed* in TaskGroups response = await asyncio.wait_for( super().send_request(*args, **kwargs), timeout) except (TaskTimeout, asyncio.TimeoutError) as e: raise RequestTimedOut(f'request timed out: {args} (id: {msg_id})') from e + except CodeMessageError as e: + self.maybe_log(f"--> {repr(e)} (id: {msg_id})") + raise else: self.maybe_log(f"--> {response} (id: {msg_id})") return response @@ -166,6 +173,16 @@ def __str__(self): class ErrorParsingSSLCert(Exception): pass class ErrorGettingSSLCertFromServer(Exception): pass +class ConnectError(Exception): pass + + +class _RSClient(RSClient): + async def create_connection(self): + try: + return await super().create_connection() + except OSError as e: + # note: using "from e" here will set __cause__ of ConnectError + raise ConnectError(e) from e def deserialize_server(server_str: str) -> Tuple[str, str, str]: @@ -242,11 +259,11 @@ async def is_server_ca_signed(self, ca_ssl_context): """ try: await self.open_session(ca_ssl_context, exit_early=True) - except ssl.SSLError as e: - if e.reason == 'CERTIFICATE_VERIFY_FAILED': + except ConnectError as e: + cause = e.__cause__ + if isinstance(cause, ssl.SSLError) and cause.reason == 'CERTIFICATE_VERIFY_FAILED': # failures due to self-signed certs are normal return False - # e.g. too weak crypto raise return True @@ -295,7 +312,7 @@ async def _get_ssl_context(self): if not self._is_saved_ssl_cert_available(): try: await self._try_saving_ssl_cert_for_first_time(ca_sslc) - except (OSError, aiorpcx.socks.SOCKSError) as e: + except (OSError, ConnectError, aiorpcx.socks.SOCKSError) as e: raise ErrorGettingSSLCertFromServer(e) from e # now we have a file saved in our certificate store siz = os.stat(self.cert_path).st_size @@ -314,9 +331,13 @@ async def wrapper_func(self: 'Interface', *args, **kwargs): return await func(self, *args, **kwargs) except GracefulDisconnect as e: self.logger.log(e.log_level, f"disconnecting due to {repr(e)}") + except aiorpcx.jsonrpc.RPCError as e: + self.logger.warning(f"disconnecting due to {repr(e)}") + self.logger.debug(f"(disconnect) trace for {repr(e)}", exc_info=True) finally: await self.network.connection_down(self) - self.got_disconnected.set_result(1) + if not self.got_disconnected.done(): + self.got_disconnected.set_result(1) # if was not 'ready' yet, schedule waiting coroutines: self.ready.cancel() return wrapper_func @@ -332,7 +353,7 @@ async def run(self): return try: await self.open_session(ssl_context) - except (asyncio.CancelledError, OSError, aiorpcx.socks.SOCKSError) as e: + except (asyncio.CancelledError, ConnectError, aiorpcx.socks.SOCKSError) as e: self.logger.info(f'disconnecting due to: {repr(e)}') return @@ -379,10 +400,10 @@ async def save_certificate(self): async def get_certificate(self): sslc = ssl.SSLContext() try: - async with aiorpcx.Connector(RPCSession, - host=self.host, port=self.port, - ssl=sslc, proxy=self.proxy) as session: - return session.transport._ssl_protocol._sslpipe._sslobj.getpeercert(True) + async with _RSClient(session_factory=RPCSession, + host=self.host, port=self.port, + ssl=sslc, proxy=self.proxy) as session: + return session.transport._asyncio_transport._ssl_protocol._sslpipe._sslobj.getpeercert(True) except ValueError: return None @@ -417,9 +438,9 @@ def is_main_server(self) -> bool: return self.network.default_server == self.server async def open_session(self, sslc, exit_early=False): - async with aiorpcx.Connector(NotificationSession, - host=self.host, port=self.port, - ssl=sslc, proxy=self.proxy) as session: + async with _RSClient(session_factory=NotificationSession, + host=self.host, port=self.port, + ssl=sslc, proxy=self.proxy) as session: self.session = session # type: NotificationSession self.session.interface = self self.session.set_default_timeout(self.network.get_network_timeout_seconds(NetworkTimeout.Generic)) @@ -440,8 +461,10 @@ async def open_session(self, sslc, exit_early=False): await group.spawn(self.run_fetch_blocks) await group.spawn(self.monitor_connection) except aiorpcx.jsonrpc.RPCError as e: - if e.code in (JSONRPC.EXCESSIVE_RESOURCE_USAGE, JSONRPC.SERVER_BUSY): - raise GracefulDisconnect(e, log_level=logging.ERROR) from e + if e.code in (JSONRPC.EXCESSIVE_RESOURCE_USAGE, + JSONRPC.SERVER_BUSY, + JSONRPC.METHOD_NOT_FOUND): + raise GracefulDisconnect(e, log_level=logging.WARNING) from e raise async def monitor_connection(self): diff --git a/electrum/json_db.py b/electrum/json_db.py index c83c5387..f2e67918 100644 --- a/electrum/json_db.py +++ b/electrum/json_db.py @@ -59,12 +59,12 @@ def __init__(self, raw, *, manual_upgrades): self.data = {} self._modified = False self.manual_upgrades = manual_upgrades - self._called_load_transactions = False + self._called_after_upgrade_tasks = False if raw: # loading existing db self.load_data(raw) else: # creating new db self.put('seed_version', FINAL_SEED_VERSION) - self.load_transactions() + self._after_upgrade_tasks() def set_modified(self, b): with self.lock: @@ -108,12 +108,6 @@ def put(self, key, value): self.data[key] = copy.deepcopy(value) return True elif key in self.data: - # clear current contents in case of references - cur_val = self.data[key] - clear_method = getattr(cur_val, "clear", None) - if callable(clear_method): - clear_method() - # pop from dict to delete key self.data.pop(key) return True return False @@ -149,9 +143,9 @@ def load_data(self, s): if not self.manual_upgrades and self.requires_split(): raise WalletFileException("This wallet has multiple accounts and must be split") - self.load_transactions() - - if not self.manual_upgrades and self.requires_upgrade(): + if not self.requires_upgrade(): + self._after_upgrade_tasks() + elif not self.manual_upgrades: self.upgrade() def requires_split(self): @@ -204,11 +198,9 @@ def requires_upgrade(self): @profiler def upgrade(self): self.logger.info('upgrading wallet format') - if not self._called_load_transactions: - # note: not sure if this is how we should go about this... - # alternatively, we could make sure load_transactions is always called after upgrade - # still, we need strict ordering between the two. - raise Exception("'load_transactions' must be called before 'upgrade'") + if self._called_after_upgrade_tasks: + # we need strict ordering between upgrade() and after_upgrade_tasks() + raise Exception("'after_upgrade_tasks' must NOT be called before 'upgrade'") self._convert_imported() self._convert_wallet_type() self._convert_account() @@ -220,6 +212,12 @@ def upgrade(self): self._convert_version_18() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure + self._after_upgrade_tasks() + + def _after_upgrade_tasks(self): + self._called_after_upgrade_tasks = True + self._load_transactions() + def _convert_wallet_type(self): if not self._is_upgrade_method_needed(0, 13): return @@ -415,15 +413,16 @@ def _convert_version_17(self): self.put('pruned_txo', None) - transactions = self.get('transactions', {}) # txid -> Transaction + transactions = self.get('transactions', {}) # txid -> raw_tx spent_outpoints = defaultdict(dict) - for txid, tx in transactions.items(): + for txid, raw_tx in transactions.items(): + tx = Transaction(raw_tx) for txin in tx.inputs(): if txin['type'] == 'coinbase': continue prevout_hash = txin['prevout_hash'] prevout_n = txin['prevout_n'] - spent_outpoints[prevout_hash][prevout_n] = txid + spent_outpoints[prevout_hash][str(prevout_n)] = txid self.put('spent_outpoints', spent_outpoints) self.put('seed_version', 17) @@ -475,6 +474,7 @@ def _convert_account(self): self.put('accounts', None) def _is_upgrade_method_needed(self, min_version, max_version): + assert min_version <= max_version cur_version = self.get_seed_version() if cur_version > max_version: return False @@ -582,19 +582,22 @@ def get_spent_outpoints(self, prevout_hash): @locked def get_spent_outpoint(self, prevout_hash, prevout_n): - return self.spent_outpoints.get(prevout_hash, {}).get(str(prevout_n)) + prevout_n = str(prevout_n) + return self.spent_outpoints.get(prevout_hash, {}).get(prevout_n) @modifier def remove_spent_outpoint(self, prevout_hash, prevout_n): - self.spent_outpoints[prevout_hash].pop(prevout_n, None) # FIXME + prevout_n = str(prevout_n) + self.spent_outpoints[prevout_hash].pop(prevout_n, None) if not self.spent_outpoints[prevout_hash]: self.spent_outpoints.pop(prevout_hash) @modifier def set_spent_outpoint(self, prevout_hash, prevout_n, tx_hash): + prevout_n = str(prevout_n) if prevout_hash not in self.spent_outpoints: self.spent_outpoints[prevout_hash] = {} - self.spent_outpoints[prevout_hash][str(prevout_n)] = tx_hash + self.spent_outpoints[prevout_hash][prevout_n] = tx_hash @modifier def add_transaction(self, tx_hash: str, tx: Transaction) -> None: @@ -673,6 +676,8 @@ def remove_tx_fee(self, txid): @locked def get_data_ref(self, name): + # Warning: interacts un-intuitively with 'put': certain parts + # of 'data' will have pointers saved as separate variables. if name not in self.data: self.data[name] = {} return self.data[name] @@ -686,12 +691,14 @@ def num_receiving_addresses(self): return len(self.receiving_addresses) @locked - def get_change_addresses(self): - return list(self.change_addresses) + def get_change_addresses(self, *, slice_start=None, slice_stop=None): + # note: slicing makes a shallow copy + return self.change_addresses[slice_start:slice_stop] @locked - def get_receiving_addresses(self): - return list(self.receiving_addresses) + def get_receiving_addresses(self, *, slice_start=None, slice_stop=None): + # note: slicing makes a shallow copy + return self.receiving_addresses[slice_start:slice_stop] @modifier def add_change_address(self, addr): @@ -745,8 +752,7 @@ def load_addresses(self, wallet_type): self._addr_to_addr_index[addr] = (True, i) @profiler - def load_transactions(self): - self._called_load_transactions = True + def _load_transactions(self): # references in self.data self.txi = self.get_data_ref('txi') # txid -> address -> list of (prev_outpoint, value) self.txo = self.get_data_ref('txo') # txid -> address -> list of (output_index, value, is_coinbase) diff --git a/electrum/keystore.py b/electrum/keystore.py index 30a803f0..49f032dc 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -282,7 +282,13 @@ def get_pubkey_from_xpub(self, xpub, sequence): return node.eckey.get_public_key_hex(compressed=True) def get_xpubkey(self, c, i): - s = ''.join(map(lambda x: bitcoin.int_to_hex(x,2), (c, i))) + def encode_path_int(path_int) -> str: + if path_int < 0xffff: + hex = bitcoin.int_to_hex(path_int, 2) + else: + hex = 'ffff' + bitcoin.int_to_hex(path_int, 4) + return hex + s = ''.join(map(encode_path_int, (c, i))) return 'ff' + bh2u(bitcoin.DecodeBase58Check(self.xpub)) + s @classmethod @@ -296,11 +302,14 @@ def parse_xpubkey(self, pubkey): # derivation: dd = pk[78:] s = [] - # FIXME: due to an oversight, levels in the derivation are only - # allocated 2 bytes, instead of 4 (in bip32) while dd: - n = int(bitcoin.rev_hex(bh2u(dd[0:2])), 16) + # 2 bytes for derivation path index + n = int.from_bytes(dd[0:2], byteorder="little") dd = dd[2:] + # in case of overflow, drop these 2 bytes; and use next 4 bytes instead + if n == 0xffff: + n = int.from_bytes(dd[0:4], byteorder="little") + dd = dd[4:] s.append(n) assert len(s) == 2 return xkey, s diff --git a/electrum/network.py b/electrum/network.py index e936a4f3..02c01bf9 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -476,20 +476,26 @@ def get_interfaces(self) -> List[str]: @with_recent_servers_lock def get_servers(self): - # start with hardcoded servers - out = dict(constants.net.DEFAULT_SERVERS) # copy + # note: order of sources when adding servers here is crucial! + # don't let "server_peers" overwrite anything, + # otherwise main server can eclipse the client + out = dict() + # add servers received from main interface + server_peers = self.server_peers + if server_peers: + out.update(filter_version(server_peers.copy())) + # hardcoded servers + out.update(constants.net.DEFAULT_SERVERS) # add recent servers for s in self.recent_servers: try: host, port, protocol = deserialize_server(s) except: continue - if host not in out: + if host in out: + out[host].update({protocol: port}) + else: out[host] = {protocol: port} - # add servers received from main interface - server_peers = self.server_peers - if server_peers: - out.update(filter_version(server_peers.copy())) # potentially filter out some if self.config.get('noonion'): out = filter_noonion(out) diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index a28ee66a..3aa9e0f2 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -27,6 +27,7 @@ import time import traceback import json +from typing import Optional import certifi import urllib.parse @@ -92,9 +93,19 @@ async def get_payment_request(url: str) -> 'PaymentRequest': data_len = len(data) if data is not None else None _logger.info(f'fetched payment request {url} {data_len}') except aiohttp.ClientError as e: - error = f"Error while contacting payment URL:\n{repr(e)}" - if isinstance(e, aiohttp.ClientResponseError) and e.status == 400 and resp_content: - error += "\n" + resp_content.decode("utf8") + error = f"Error while contacting payment URL: {url}.\nerror type: {type(e)}" + if isinstance(e, aiohttp.ClientResponseError): + error += f"\nGot HTTP status code {e.status}." + if resp_content: + try: + error_text_received = resp_content.decode("utf8") + except UnicodeDecodeError: + error_text_received = "(failed to decode error)" + else: + error_text_received = error_text_received[:400] + error_oneline = ' -- '.join(error.split('\n')) + _logger.info(f"{error_oneline} -- [DO NOT TRUST THIS MESSAGE] " + f"{repr(e)} text: {error_text_received}") data = None elif u.scheme == 'file': try: @@ -106,15 +117,15 @@ async def get_payment_request(url: str) -> 'PaymentRequest': else: data = None error = f"Unknown scheme for payment request. URL: {url}" - pr = PaymentRequest(data, error) + pr = PaymentRequest(data, error=error) return pr class PaymentRequest: - def __init__(self, data, error=None): + def __init__(self, data, *, error=None): self.raw = data - self.error = error + self.error = error # FIXME overloaded and also used when 'verify' succeeds self.parse(data) self.requestor = None # known after verify self.tx = None @@ -123,6 +134,7 @@ def __str__(self): return str(self.raw) def parse(self, r): + self.outputs = [] if self.error: return self.id = bh2u(sha256(r)[0:16]) @@ -134,7 +146,6 @@ def parse(self, r): return self.details = pb2.PaymentDetails() self.details.ParseFromString(self.data.serialized_payment_details) - self.outputs = [] for o in self.details.outputs: type_, addr = transaction.get_address_from_output_script(o.script) if type_ != TYPE_ADDRESS: @@ -235,7 +246,9 @@ def verify_dnssec(self, pr, contacts): self.error = "unknown algo" return False - def has_expired(self): + def has_expired(self) -> Optional[bool]: + if not hasattr(self, 'details'): + return None return self.details.expires and self.details.expires < int(time.time()) def get_expiration_date(self): @@ -302,9 +315,19 @@ async def send_payment_and_receive_paymentack(self, raw_tx, refund_addr): print(f"PaymentACK message received: {paymntack.memo}") return True, paymntack.memo except aiohttp.ClientError as e: - error = f"Payment Message/PaymentACK Failed:\n{repr(e)}" - if isinstance(e, aiohttp.ClientResponseError) and e.status == 400 and resp_content: - error += "\n" + resp_content.decode("utf8") + error = f"Payment Message/PaymentACK Failed:\nerror type: {type(e)}" + if isinstance(e, aiohttp.ClientResponseError): + error += f"\nGot HTTP status code {e.status}." + if resp_content: + try: + error_text_received = resp_content.decode("utf8") + except UnicodeDecodeError: + error_text_received = "(failed to decode error)" + else: + error_text_received = error_text_received[:400] + error_oneline = ' -- '.join(error.split('\n')) + _logger.info(f"{error_oneline} -- [DO NOT TRUST THIS MESSAGE] " + f"{repr(e)} text: {error_text_received}") return False, error diff --git a/electrum/paymentrequest_pb2.py b/electrum/paymentrequest_pb2.py index f596128c..20cb32c2 100644 --- a/electrum/paymentrequest_pb2.py +++ b/electrum/paymentrequest_pb2.py @@ -18,6 +18,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='paymentrequest.proto', package='payments', + syntax='proto2', serialized_pb=_b('\n\x14paymentrequest.proto\x12\x08payments\"+\n\x06Output\x12\x11\n\x06\x61mount\x18\x01 \x01(\x04:\x01\x30\x12\x0e\n\x06script\x18\x02 \x02(\x0c\"\xa3\x01\n\x0ePaymentDetails\x12\x15\n\x07network\x18\x01 \x01(\t:\x04main\x12!\n\x07outputs\x18\x02 \x03(\x0b\x32\x10.payments.Output\x12\x0c\n\x04time\x18\x03 \x02(\x04\x12\x0f\n\x07\x65xpires\x18\x04 \x01(\x04\x12\x0c\n\x04memo\x18\x05 \x01(\t\x12\x13\n\x0bpayment_url\x18\x06 \x01(\t\x12\x15\n\rmerchant_data\x18\x07 \x01(\x0c\"\x95\x01\n\x0ePaymentRequest\x12\"\n\x17payment_details_version\x18\x01 \x01(\r:\x01\x31\x12\x16\n\x08pki_type\x18\x02 \x01(\t:\x04none\x12\x10\n\x08pki_data\x18\x03 \x01(\x0c\x12\"\n\x1aserialized_payment_details\x18\x04 \x02(\x0c\x12\x11\n\tsignature\x18\x05 \x01(\x0c\"\'\n\x10X509Certificates\x12\x13\n\x0b\x63\x65rtificate\x18\x01 \x03(\x0c\"i\n\x07Payment\x12\x15\n\rmerchant_data\x18\x01 \x01(\x0c\x12\x14\n\x0ctransactions\x18\x02 \x03(\x0c\x12#\n\trefund_to\x18\x03 \x03(\x0b\x32\x10.payments.Output\x12\x0c\n\x04memo\x18\x04 \x01(\t\">\n\nPaymentACK\x12\"\n\x07payment\x18\x01 \x02(\x0b\x32\x11.payments.Payment\x12\x0c\n\x04memo\x18\x02 \x01(\tB(\n\x1eorg.bitcoin.protocols.paymentsB\x06Protos') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -54,6 +55,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], @@ -126,6 +128,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], @@ -184,6 +187,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], @@ -214,6 +218,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], @@ -265,6 +270,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], @@ -302,6 +308,7 @@ ], options=None, is_extendable=False, + syntax='proto2', extension_ranges=[], oneofs=[ ], diff --git a/electrum/plugin.py b/electrum/plugin.py index 5007a67e..a7313862 100644 --- a/electrum/plugin.py +++ b/electrum/plugin.py @@ -27,6 +27,7 @@ import importlib.util import time import threading +import sys from typing import NamedTuple, Any, Union, TYPE_CHECKING, Optional from .i18n import _ @@ -73,6 +74,9 @@ def load_plugins(self): raise Exception(f"Error pre-loading {full_name}: no spec") try: module = importlib.util.module_from_spec(spec) + # sys.modules needs to be modified for relative imports to work + # see https://stackoverflow.com/a/50395128 + sys.modules[spec.name] = module spec.loader.exec_module(module) except Exception as e: raise Exception(f"Error pre-loading {full_name}: {repr(e)}") from e @@ -283,6 +287,7 @@ def settings_dialog(self): class DeviceNotFoundError(Exception): pass class DeviceUnpairableError(Exception): pass +class HardwarePluginLibraryUnavailable(Exception): pass class Device(NamedTuple): @@ -402,7 +407,7 @@ def xpub_by_id(self, id_): def unpair_xpub(self, xpub): with self.lock: - if not xpub in self.xpub_ids: + if xpub not in self.xpub_ids: return _id = self.xpub_ids.pop(xpub) self._close_client(_id) @@ -502,7 +507,7 @@ def unpaired_device_infos(self, handler, plugin: 'HW_PluginBase', devices=None, unpaired device accepted by the plugin.''' if not plugin.libraries_available: message = plugin.get_library_not_available_message() - raise Exception(message) + raise HardwarePluginLibraryUnavailable(message) if devices is None: devices = self.scan_devices() devices = [dev for dev in devices if not self.xpub_by_id(dev.id_)] diff --git a/electrum/plugins/hw_wallet/plugin.py b/electrum/plugins/hw_wallet/plugin.py index 9b227028..fd3ed697 100644 --- a/electrum/plugins/hw_wallet/plugin.py +++ b/electrum/plugins/hw_wallet/plugin.py @@ -44,6 +44,7 @@ def __init__(self, parent, config, name): BasePlugin.__init__(self, parent, config, name) self.device = self.keystore_class.device self.keystore_class.plugin = self + self._ignore_outdated_fw = False def is_enabled(self): return True @@ -124,6 +125,12 @@ def get_library_not_available_message(self) -> str: message += '\n' + _("Make sure you install it with python3") return message + def set_ignore_outdated_fw(self): + self._ignore_outdated_fw = True + + def is_outdated_fw_ignored(self) -> bool: + return self._ignore_outdated_fw + def is_any_tx_output_on_change_branch(tx: Transaction): if not tx.output_info: @@ -160,3 +167,16 @@ def wrapper(self, *args, **kwargs): class LibraryFoundButUnusable(Exception): def __init__(self, library_version='unknown'): self.library_version = library_version + + +class OutdatedHwFirmwareException(UserFacingException): + + def text_ignore_old_fw_and_continue(self) -> str: + suffix = (_("The firmware of your hardware device is too old. " + "If possible, you should upgrade it. " + "You can ignore this error and try to continue, however things are likely to break.") + "\n\n" + + _("Ignore and continue?")) + if str(self): + return str(self) + "\n\n" + suffix + else: + return suffix diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 86f9a409..5a8c9dfe 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -32,11 +32,13 @@ from electrum.gui.qt.password_dialog import PasswordLayout, PW_PASSPHRASE from electrum.gui.qt.util import (read_QIcon, WWLabel, OkButton, WindowModalDialog, - Buttons, CancelButton, TaskThread) + Buttons, CancelButton, TaskThread, char_width_in_lineedit) from electrum.i18n import _ from electrum.logging import Logger +from .plugin import OutdatedHwFirmwareException + # The trickiest thing about this handler was getting windows properly # parented on macOS. @@ -147,7 +149,7 @@ def word_dialog(self, msg): hbox = QHBoxLayout(dialog) hbox.addWidget(QLabel(msg)) text = QLineEdit() - text.setMaximumWidth(100) + text.setMaximumWidth(12 * char_width_in_lineedit()) text.returnPressed.connect(dialog.accept) hbox.addWidget(text) hbox.addStretch(1) @@ -212,11 +214,27 @@ def load_wallet(self, wallet, window): handler = self.create_handler(window) handler.button = button keystore.handler = handler - keystore.thread = TaskThread(window, window.on_error) + keystore.thread = TaskThread(window, on_error=partial(self.on_task_thread_error, window, keystore)) self.add_show_address_on_hw_device_button_for_receive_addr(wallet, keystore, window) # Trigger a pairing keystore.thread.add(partial(self.get_client, keystore)) + def on_task_thread_error(self, window, keystore, exc_info): + e = exc_info[1] + if isinstance(e, OutdatedHwFirmwareException): + if window.question(e.text_ignore_old_fw_and_continue(), title=_("Outdated device firmware")): + self.set_ignore_outdated_fw() + # will need to re-pair + devmgr = self.device_manager() + def re_pair_device(): + device_id = self.choose_device(window, keystore) + devmgr.unpair_id(device_id) + self.get_client(keystore) + keystore.thread.add(re_pair_device) + return + else: + window.on_error(exc_info) + def choose_device(self, window, keystore): '''This dialog box should be usable even if the user has forgotten their PIN or it is in bootloader mode.''' diff --git a/electrum/plugins/ledger/qt.py b/electrum/plugins/ledger/qt.py index b2e5bc00..225c5ef4 100644 --- a/electrum/plugins/ledger/qt.py +++ b/electrum/plugins/ledger/qt.py @@ -1,7 +1,5 @@ from functools import partial -#from btchip.btchipPersoWizard import StartBTChipPersoDialog - from PyQt5.QtCore import pyqtSignal from PyQt5.QtWidgets import QInputDialog, QLabel, QVBoxLayout, QLineEdit @@ -83,6 +81,3 @@ def get_setup(self): def setup_dialog(self): self.show_error(_('Initialization of Ledger HW devices is currently disabled.')) - return - dialog = StartBTChipPersoDialog() - dialog.exec_() diff --git a/electrum/plugins/trezor/clientbase.py b/electrum/plugins/trezor/clientbase.py index 9ce2b369..7188c379 100644 --- a/electrum/plugins/trezor/clientbase.py +++ b/electrum/plugins/trezor/clientbase.py @@ -7,6 +7,7 @@ from electrum.keystore import bip39_normalize_passphrase from electrum.bip32 import BIP32Node, convert_bip32_path_to_list_of_uint32 as parse_path from electrum.logging import Logger +from electrum.plugins.hw_wallet.plugin import OutdatedHwFirmwareException from trezorlib.client import TrezorClient from trezorlib.exceptions import TrezorFailure, Cancelled, OutdatedFirmwareError @@ -29,6 +30,8 @@ class TrezorClientBase(Logger): def __init__(self, transport, handler, plugin): + if plugin.is_outdated_fw_ignored(): + TrezorClient.is_outdated = lambda *args, **kwargs: False self.client = TrezorClient(transport, ui=self) self.plugin = plugin self.device = plugin.device @@ -62,15 +65,15 @@ def end_flow(self): def __enter__(self): return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, e, traceback): self.end_flow() - if exc_value is not None: - if issubclass(exc_type, Cancelled): - raise UserCancelled from exc_value - elif issubclass(exc_type, TrezorFailure): - raise RuntimeError(str(exc_value)) from exc_value - elif issubclass(exc_type, OutdatedFirmwareError): - raise UserFacingException(exc_value) from exc_value + if e is not None: + if isinstance(e, Cancelled): + raise UserCancelled from e + elif isinstance(e, TrezorFailure): + raise RuntimeError(str(e)) from e + elif isinstance(e, OutdatedFirmwareError): + raise OutdatedHwFirmwareException(e) from e else: return False return True diff --git a/electrum/plugins/trezor/trezor.py b/electrum/plugins/trezor/trezor.py index 588c18c9..b41bf9ef 100644 --- a/electrum/plugins/trezor/trezor.py +++ b/electrum/plugins/trezor/trezor.py @@ -15,7 +15,7 @@ from ..hw_wallet import HW_PluginBase from ..hw_wallet.plugin import (is_any_tx_output_on_change_branch, trezor_validate_op_return_output_and_get_data, - LibraryFoundButUnusable) + LibraryFoundButUnusable, OutdatedHwFirmwareException) _logger = get_logger(__name__) @@ -23,6 +23,7 @@ try: import trezorlib import trezorlib.transport + from trezorlib.transport.bridge import BridgeTransport, call_bridge from .clientbase import TrezorClientBase @@ -137,7 +138,16 @@ def get_library_version(self): raise LibraryFoundButUnusable(library_version=version) def enumerate(self): - devices = trezorlib.transport.enumerate_devices() + # If there is a bridge, prefer that. + # On Windows, the bridge runs as Admin (and Electrum usually does not), + # so the bridge has better chances of finding devices. see #5420 + # This also avoids duplicate entries. + try: + call_bridge("enumerate") + except Exception: + devices = trezorlib.transport.enumerate_devices() + else: + devices = BridgeTransport.enumerate() return [Device(path=d.get_path(), interface_number=-1, id_=d.get_path(), @@ -275,7 +285,7 @@ def setup_device(self, device_info, wizard, purpose): msg = (_('Outdated {} firmware for device labelled {}. Please ' 'download the updated firmware from {}') .format(self.device, client.label(), self.firmware_URL)) - raise UserFacingException(msg) + raise OutdatedHwFirmwareException(msg) # fixme: we should use: client.handler = wizard client.handler = self.create_handler(wizard) diff --git a/electrum/simple_config.py b/electrum/simple_config.py index 853b5c7c..8a970802 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -533,14 +533,16 @@ def fee_per_byte(self): fee_per_kb = self.fee_per_kb() return fee_per_kb / 1000 if fee_per_kb is not None else None - def estimate_fee(self, size): + def estimate_fee(self, size: Union[int, float, Decimal]) -> int: fee_per_kb = self.fee_per_kb() if fee_per_kb is None: raise NoDynamicFeeEstimates() return self.estimate_fee_for_feerate(fee_per_kb, size) @classmethod - def estimate_fee_for_feerate(cls, fee_per_kb, size): + def estimate_fee_for_feerate(cls, fee_per_kb: Union[int, float, Decimal], + size: Union[int, float, Decimal]) -> int: + size = Decimal(size) fee_per_kb = Decimal(fee_per_kb) fee_per_byte = fee_per_kb / 1000 # to be consistent with what is displayed in the GUI, diff --git a/electrum/storage.py b/electrum/storage.py index 4718c2ee..7a711bd8 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -58,6 +58,7 @@ def __init__(self, path, *, manual_upgrades=False): DB_Class = JsonDB self.logger.info(f"wallet path {self.path}") self.pubkey = None + # TODO we should test r/w permissions here (whether file exists or not) if self.file_exists(): with open(self.path, "r", encoding='utf-8') as f: self.raw = f.read() @@ -225,6 +226,9 @@ def requires_upgrade(self): raise Exception("storage not yet decrypted!") return self.db.requires_upgrade() + def is_ready_to_be_used_by_wallet(self): + return not self.requires_upgrade() and self.db._called_after_upgrade_tasks + def upgrade(self): self.db.upgrade() self.write() @@ -239,6 +243,7 @@ def split_accounts(self): path = self.path + '.' + data['suffix'] storage = WalletStorage(path) storage.db.data = data + storage.db._called_after_upgrade_tasks = False storage.db.upgrade() storage.write() out.append(path) diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py index cb4d6cbe..313467a8 100644 --- a/electrum/synchronizer.py +++ b/electrum/synchronizer.py @@ -147,7 +147,7 @@ def __init__(self, wallet: 'AddressSynchronizer'): def _reset(self): super()._reset() self.requested_tx = {} - self.requested_histories = {} + self.requested_histories = set() def diagnostic_name(self): return self.wallet.diagnostic_name() @@ -161,10 +161,10 @@ async def _on_address_status(self, addr, status): history = self.wallet.db.get_addr_history(addr) if history_status(history) == status: return - if addr in self.requested_histories: + if (addr, status) in self.requested_histories: return # request address history - self.requested_histories[addr] = status + self.requested_histories.add((addr, status)) h = address_to_scripthash(addr) self._requests_sent += 1 result = await self.network.get_history_for_scripthash(h) @@ -188,7 +188,7 @@ async def _on_address_status(self, addr, status): await self._request_missing_txs(hist) # Remove request; this allows up_to_date to be True - self.requested_histories.pop(addr) + self.requested_histories.discard((addr, status)) async def _request_missing_txs(self, hist, *, allow_server_not_finding_tx=False): # "hist" is a list of [tx_hash, tx_height] lists diff --git a/electrum/transaction.py b/electrum/transaction.py index f95f663d..ead65ef3 100644 --- a/electrum/transaction.py +++ b/electrum/transaction.py @@ -1043,7 +1043,7 @@ def input_value(self): return sum(x['value'] for x in self.inputs()) def output_value(self): - return sum(val for tp, addr, val in self.outputs()) + return sum(o.value for o in self.outputs()) def get_fee(self): return self.input_value() - self.output_value() diff --git a/electrum/util.py b/electrum/util.py index 749b4470..72746f3e 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -520,6 +520,14 @@ def is_non_negative_integer(val) -> bool: return False +def chunks(items, size: int): + """Break up items, an iterable, into chunks of length size.""" + if size < 1: + raise ValueError(f"size must be positive, not {repr(size)}") + for i in range(0, len(items), size): + yield items[i: i + size] + + def format_satoshis_plain(x, decimal_point = 8): """Display a satoshi amount scaled. Always uses a '.' as a decimal point and has no thousands separator""" @@ -682,18 +690,25 @@ def block_explorer_URL(config: 'SimpleConfig', kind: str, item: str) -> Optional #_ud = re.compile('%([0-9a-hA-H]{2})', re.MULTILINE) #urldecode = lambda x: _ud.sub(lambda m: chr(int(m.group(1), 16)), x) -def parse_URI(uri: str, on_pr: Callable=None) -> dict: +class InvalidBitcoinURI(Exception): pass + + +def parse_URI(uri: str, on_pr: Callable = None, *, loop=None) -> dict: + """Raises InvalidBitcoinURI on malformed URI.""" from . import bitcoin from .bitcoin import COIN + if not isinstance(uri, str): + raise InvalidBitcoinURI(f"expected string, not {repr(uri)}") + if ':' not in uri: if not bitcoin.is_address(uri): - raise Exception("Not a Noir address") + raise InvalidBitcoinURI("Not a Noir address") return {'address': uri} u = urllib.parse.urlparse(uri) if u.scheme != 'noir': - raise Exception("Not a Noir URI") + raise InvalidBitcoinURI("Not a Noir URI") address = u.path # python for android fails to parse query @@ -704,37 +719,50 @@ def parse_URI(uri: str, on_pr: Callable=None) -> dict: pq = urllib.parse.parse_qs(u.query) for k, v in pq.items(): - if len(v)!=1: - raise Exception('Duplicate Key', k) + if len(v) != 1: + raise InvalidBitcoinURI(f'Duplicate Key: {repr(k)}') out = {k: v[0] for k, v in pq.items()} if address: if not bitcoin.is_address(address): - raise Exception("Invalid Noir address:" + address) + raise InvalidBitcoinURI(f"Invalid Noir address: {address}") out['address'] = address if 'amount' in out: am = out['amount'] - m = re.match(r'([0-9.]+)X([0-9])', am) - if m: - k = int(m.group(2)) - 8 - amount = Decimal(m.group(1)) * pow( Decimal(10) , k) - else: - amount = Decimal(am) * COIN - out['amount'] = int(amount) + try: + m = re.match(r'([0-9.]+)X([0-9])', am) + if m: + k = int(m.group(2)) - 8 + amount = Decimal(m.group(1)) * pow( Decimal(10) , k) + else: + amount = Decimal(am) * COIN + out['amount'] = int(amount) + except Exception as e: + raise InvalidBitcoinURI(f"failed to parse 'amount' field: {repr(e)}") from e if 'message' in out: out['message'] = out['message'] out['memo'] = out['message'] if 'time' in out: - out['time'] = int(out['time']) + try: + out['time'] = int(out['time']) + except Exception as e: + raise InvalidBitcoinURI(f"failed to parse 'time' field: {repr(e)}") from e if 'exp' in out: - out['exp'] = int(out['exp']) + try: + out['exp'] = int(out['exp']) + except Exception as e: + raise InvalidBitcoinURI(f"failed to parse 'exp' field: {repr(e)}") from e if 'sig' in out: - out['sig'] = bh2u(bitcoin.base_decode(out['sig'], None, base=58)) + try: + out['sig'] = bh2u(bitcoin.base_decode(out['sig'], None, base=58)) + except Exception as e: + raise InvalidBitcoinURI(f"failed to parse 'sig' field: {repr(e)}") from e r = out.get('r') sig = out.get('sig') name = out.get('name') if on_pr and (r or (name and sig)): + @log_exceptions async def get_payment_request(): from . import paymentrequest as pr if name and sig: @@ -744,7 +772,7 @@ async def get_payment_request(): request = await pr.get_payment_request(r) if on_pr: on_pr(request) - loop = asyncio.get_event_loop() + loop = loop or asyncio.get_event_loop() asyncio.run_coroutine_threadsafe(get_payment_request(), loop) return out diff --git a/electrum/version.py b/electrum/version.py index 95d75153..19248c70 100644 --- a/electrum/version.py +++ b/electrum/version.py @@ -1,5 +1,5 @@ -ELECTRUM_VERSION = '3.3.6' # version of the client package -APK_VERSION = '3.3.6.0' # read by buildozer.spec +ELECTRUM_VERSION = '3.3.7' # version of the client package +APK_VERSION = '3.3.7.0' # read by buildozer.spec PROTOCOL_VERSION = '1.4' # protocol version requested diff --git a/electrum/wallet.py b/electrum/wallet.py index 0f12a4f1..e2d4dbc5 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -38,14 +38,14 @@ from functools import partial from numbers import Number from decimal import Decimal -from typing import TYPE_CHECKING, List, Optional, Tuple, Union, NamedTuple +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, NamedTuple, Sequence from .i18n import _ from .util import (NotEnoughFunds, UserCancelled, profiler, format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, WalletFileException, BitcoinException, InvalidPassword, format_time, timestamp_to_datetime, Satoshis, - Fiat, bfh, bh2u, TxMinedInfo) + Fiat, bfh, bh2u, TxMinedInfo, quantize_feerate) from .bitcoin import (COIN, TYPE_ADDRESS, is_address, address_to_script, is_minikey, relayfee, dust_threshold) from .crypto import sha256d @@ -206,13 +206,13 @@ class Abstract_Wallet(AddressSynchronizer): gap_limit_for_change = 6 def __init__(self, storage: WalletStorage): - if storage.requires_upgrade(): - raise Exception("storage must be upgraded before constructing wallet") + if not storage.is_ready_to_be_used_by_wallet(): + raise Exception("storage not ready to be used by Abstract_Wallet") + self.storage = storage # load addresses needs to be called before constructor for sanity checks - storage.db.load_addresses(self.wallet_type) - - AddressSynchronizer.__init__(self, storage) + self.storage.db.load_addresses(self.wallet_type) + AddressSynchronizer.__init__(self, storage.db) # saved fields self.use_change = storage.get('use_change', True) @@ -235,6 +235,18 @@ def __init__(self, storage: WalletStorage): self._coin_price_cache = {} + def stop_threads(self): + super().stop_threads() + self.storage.write() + + def set_up_to_date(self, b): + super().set_up_to_date(b) + if b: self.storage.write() + + def clear_history(self): + super().clear_history() + self.storage.write() + def load_and_cleanup(self): self.load_keystore() self.test_addresses_sanity() @@ -346,7 +358,11 @@ def get_redeem_script(self, address): def export_private_key(self, address, password): if self.is_watching_only(): - return [] + raise Exception(_("This is a watching-only wallet")) + if not is_address(address): + raise Exception(f"Invalid bitcoin address: {address}") + if not self.is_mine(address): + raise Exception(_('Address not in wallet.') + f' {address}') index = self.get_address_index(address) pk, compressed = self.keystore.get_private_key(index, password) txin_type = self.get_txin_type(address) @@ -389,6 +405,7 @@ def get_tx_info(self, tx) -> TxWalletDetails: else: status = _('Local') can_broadcast = self.network is not None + can_bump = is_mine and not tx.is_final() else: status = _("Signed") can_broadcast = self.network is not None @@ -429,8 +446,15 @@ def get_spendable_coins(self, domain, config, *, nonlocal_only=False): utxos = [utxo for utxo in utxos if not self.is_frozen_coin(utxo)] return utxos + def get_receiving_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence: + raise NotImplementedError() # implemented by subclasses + + def get_change_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence: + raise NotImplementedError() # implemented by subclasses + def dummy_address(self): - return self.get_receiving_addresses()[0] + # first receiving address + return self.get_receiving_addresses(slice_start=0, slice_stop=1)[0] def get_frozen_balance(self): if not self.frozen_coins: # shortcut @@ -670,6 +694,33 @@ def get_unconfirmed_base_tx_for_batching(self) -> Optional[Transaction]: return tx return candidate + def get_change_addresses_for_new_transaction(self, preferred_change_addr=None) -> List[str]: + change_addrs = [] + if preferred_change_addr: + if isinstance(preferred_change_addr, (list, tuple)): + change_addrs = list(preferred_change_addr) + else: + change_addrs = [preferred_change_addr] + elif self.use_change: + # Recalc and get unused change addresses + addrs = self.calc_unused_change_addresses() + # New change addresses are created only after a few + # confirmations. + if addrs: + # if there are any unused, select all + change_addrs = addrs + else: + # if there are none, take one randomly from the last few + addrs = self.get_change_addresses(slice_start=-self.gap_limit_for_change) + change_addrs = [random.choice(addrs)] if addrs else [] + for addr in change_addrs: + assert is_address(addr), f"not valid bitcoin address: {addr}" + # note that change addresses are not necessarily ismine + # in which case this is a no-op + self.check_address(addr) + max_change = self.max_change_outputs if self.multiple_change else 1 + return change_addrs[:max_change] + def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, change_addr=None, is_sweep=False): # check outputs @@ -689,28 +740,6 @@ def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, for item in coins: self.add_input_info(item) - # change address - # if we leave it empty, coin_chooser will set it - change_addrs = [] - if change_addr: - change_addrs = [change_addr] - elif self.use_change: - # Recalc and get unused change addresses - addrs = self.calc_unused_change_addresses() - # New change addresses are created only after a few - # confirmations. - if addrs: - # if there are any unused, select all - change_addrs = addrs - else: - # if there are none, take one randomly from the last few - addrs = self.get_change_addresses()[-self.gap_limit_for_change:] - change_addrs = [random.choice(addrs)] if addrs else [] - for addr in change_addrs: - # note that change addresses are not necessarily ismine - # in which case this is a no-op - self.check_address(addr) - # Fee estimator if fixed_fee is None: fee_estimator = config.estimate_fee @@ -723,32 +752,44 @@ def make_unsigned_transaction(self, coins, outputs, config, fixed_fee=None, if i_max is None: # Let the coin chooser select the coins to spend - max_change = self.max_change_outputs if self.multiple_change else 1 coin_chooser = coinchooser.get_coin_chooser(config) # If there is an unconfirmed RBF tx, merge with it base_tx = self.get_unconfirmed_base_tx_for_batching() if config.get('batch_rbf', False) and base_tx: + # make sure we don't try to spend change from the tx-to-be-replaced: + coins = [c for c in coins if c['prevout_hash'] != base_tx.txid()] is_local = self.get_tx_height(base_tx.txid()).height == TX_HEIGHT_LOCAL base_tx = Transaction(base_tx.serialize()) base_tx.deserialize(force_full_parse=True) base_tx.remove_signatures() base_tx.add_inputs_info(self) base_tx_fee = base_tx.get_fee() - relayfeerate = self.relayfee() / 1000 + relayfeerate = Decimal(self.relayfee()) / 1000 original_fee_estimator = fee_estimator - def fee_estimator(size: int) -> int: + def fee_estimator(size: Union[int, float, Decimal]) -> int: + size = Decimal(size) lower_bound = base_tx_fee + round(size * relayfeerate) lower_bound = lower_bound if not is_local else 0 - return max(lower_bound, original_fee_estimator(size)) + return int(max(lower_bound, original_fee_estimator(size))) txi = base_tx.inputs() txo = list(filter(lambda o: not self.is_change(o.address), base_tx.outputs())) + old_change_addrs = [o.address for o in base_tx.outputs() if self.is_change(o.address)] else: txi = [] txo = [] - tx = coin_chooser.make_tx(coins, txi, outputs[:] + txo, change_addrs[:max_change], + old_change_addrs = [] + # change address. if empty, coin_chooser will set it + change_addrs = self.get_change_addresses_for_new_transaction(change_addr or old_change_addrs) + tx = coin_chooser.make_tx(coins, txi, outputs[:] + txo, change_addrs, fee_estimator, self.dust_threshold()) else: - # FIXME?? this might spend inputs with negative effective value... + # "spend max" branch + # note: This *will* spend inputs with negative effective value (if there are any). + # Given as the user is spending "max", and so might be abandoning the wallet, + # try to include all UTXOs, otherwise leftover might remain in the UTXO set + # forever. see #5433 + # note: Actually it might be the case that not all UTXOs from the wallet are + # being spent if the user manually selected UTXOs. sendable = sum(map(lambda x:x['value'], coins)) outputs[i_max] = outputs[i_max]._replace(value=0) tx = Transaction.from_io(coins, outputs[:]) @@ -855,31 +896,108 @@ def address_is_old(self, address, age_limit=2): age = tx_age return age > age_limit - def bump_fee(self, tx, delta): + def bump_fee(self, *, tx, new_fee_rate, config) -> Transaction: + """Increase the miner fee of 'tx'. + 'new_fee_rate' is the target min rate in sat/vbyte + """ if tx.is_final(): raise CannotBumpFee(_('Cannot bump fee') + ': ' + _('transaction is final')) + old_tx_size = tx.estimated_size() + old_fee = self.get_tx_fee(tx) + if old_fee is None: + raise CannotBumpFee(_('Cannot bump fee') + ': ' + _('current fee unknown')) + old_fee_rate = old_fee / old_tx_size # sat/vbyte + if new_fee_rate <= old_fee_rate: + raise CannotBumpFee(_('Cannot bump fee') + ': ' + _("The new fee rate needs to be higher than the old fee rate.")) + + try: + # method 1: keep all inputs, keep all not is_mine outputs, + # allow adding new inputs + tx_new = self._bump_fee_through_coinchooser( + tx=tx, new_fee_rate=new_fee_rate, config=config) + method_used = 1 + except CannotBumpFee: + # method 2: keep all inputs, no new inputs are added, + # allow decreasing and removing outputs (change is decreased first) + # This is less "safe" as it might end up decreasing e.g. a payment to a merchant; + # but e.g. if the user has sent "Max" previously, this is the only way to RBF. + tx_new = self._bump_fee_through_decreasing_outputs( + tx=tx, new_fee_rate=new_fee_rate) + method_used = 2 + + actual_new_fee_rate = tx_new.get_fee() / tx_new.estimated_size() + if quantize_feerate(actual_new_fee_rate) < quantize_feerate(new_fee_rate): + raise Exception(f"bump_fee feerate target was not met (method: {method_used}). " + f"got {actual_new_fee_rate}, expected >={new_fee_rate}") + + tx_new.locktime = get_locktime_for_new_transaction(self.network) + return tx_new + + def _bump_fee_through_coinchooser(self, *, tx, new_fee_rate, config): + tx = Transaction(tx.serialize()) + tx.deserialize(force_full_parse=True) # need to parse inputs + tx.remove_signatures() + tx.add_inputs_info(self) + old_inputs = tx.inputs()[:] + old_outputs = tx.outputs()[:] + # change address + old_change_addrs = [o.address for o in old_outputs if self.is_change(o.address)] + change_addrs = self.get_change_addresses_for_new_transaction(old_change_addrs) + # which outputs to keep? + if old_change_addrs: + fixed_outputs = list(filter(lambda o: not self.is_change(o.address), old_outputs)) + else: + if all(self.is_mine(o.address) for o in old_outputs): + # all outputs are is_mine and none of them are change. + # we bail out as it's unclear what the user would want! + # the coinchooser bump fee method is probably not a good idea in this case + raise CannotBumpFee(_('Cannot bump fee') + ': all outputs are non-change is_mine') + old_not_is_mine = list(filter(lambda o: not self.is_mine(o.address), old_outputs)) + if old_not_is_mine: + fixed_outputs = old_not_is_mine + else: + fixed_outputs = old_outputs + + coins = self.get_spendable_coins(None, config) + for item in coins: + self.add_input_info(item) + def fee_estimator(size): + return config.estimate_fee_for_feerate(fee_per_kb=new_fee_rate*1000, size=size) + coin_chooser = coinchooser.get_coin_chooser(config) + try: + return coin_chooser.make_tx(coins, old_inputs, fixed_outputs, change_addrs, + fee_estimator, self.dust_threshold()) + except NotEnoughFunds as e: + raise CannotBumpFee(e) + + def _bump_fee_through_decreasing_outputs(self, *, tx, new_fee_rate): tx = Transaction(tx.serialize()) tx.deserialize(force_full_parse=True) # need to parse inputs tx.remove_signatures() tx.add_inputs_info(self) inputs = tx.inputs() outputs = tx.outputs() + # use own outputs - s = list(filter(lambda x: self.is_mine(x[1]), outputs)) + s = list(filter(lambda o: self.is_mine(o.address), outputs)) # ... unless there is none if not s: s = outputs x_fee = run_hook('get_tx_extra_fee', self, tx) if x_fee: x_fee_address, x_fee_amount = x_fee - s = filter(lambda x: x[1]!=x_fee_address, s) + s = filter(lambda o: o.address != x_fee_address, s) + if not s: + raise CannotBumpFee(_('Cannot bump fee') + ': no outputs at all??') # prioritize low value outputs, to get rid of dust - s = sorted(s, key=lambda x: x[2]) + s = sorted(s, key=lambda o: o.value) for o in s: + target_fee = tx.estimated_size() * new_fee_rate + delta = target_fee - tx.get_fee() i = outputs.index(o) if o.value - delta >= self.dust_threshold(): - outputs[i] = o._replace(value=o.value-delta) + outputs[i] = o._replace(value=o.value - delta) delta = 0 break else: @@ -889,9 +1007,8 @@ def bump_fee(self, tx, delta): continue if delta > 0: raise CannotBumpFee(_('Cannot bump fee') + ': ' + _('could not find suitable outputs')) - locktime = get_locktime_for_new_transaction(self.network) - tx_new = Transaction.from_io(inputs, outputs, locktime=locktime) - return tx_new + + return Transaction.from_io(inputs, outputs) def cpfp(self, tx, fee): txid = tx.txid() @@ -1408,10 +1525,10 @@ def get_addresses(self): # note: overridden so that the history can be cleared return self.db.get_imported_addresses() - def get_receiving_addresses(self): + def get_receiving_addresses(self, **kwargs): return self.get_addresses() - def get_change_addresses(self): + def get_change_addresses(self, **kwargs): return [] def import_addresses(self, addresses: List[str], *, @@ -1485,7 +1602,7 @@ def is_mine(self, address): return self.db.has_imported_address(address) def get_address_index(self, address): - # returns None is address is not mine + # returns None if address is not mine return self.get_public_key(address) def get_public_key(self, address): @@ -1563,16 +1680,15 @@ def has_seed(self): def get_addresses(self): # note: overridden so that the history can be cleared. # addresses are ordered based on derivation - out = [] - out += self.get_receiving_addresses() + out = self.get_receiving_addresses() out += self.get_change_addresses() return out - def get_receiving_addresses(self): - return self.db.get_receiving_addresses() + def get_receiving_addresses(self, *, slice_start=None, slice_stop=None): + return self.db.get_receiving_addresses(slice_start=slice_start, slice_stop=slice_stop) - def get_change_addresses(self): - return self.db.get_change_addresses() + def get_change_addresses(self, *, slice_start=None, slice_stop=None): + return self.db.get_change_addresses(slice_start=slice_start, slice_stop=slice_stop) @profiler def try_detecting_internal_addresses_corruption(self): @@ -1650,11 +1766,15 @@ def create_new_address(self, for_change=False): def synchronize_sequence(self, for_change): limit = self.gap_limit_for_change if for_change else self.gap_limit while True: - addresses = self.get_change_addresses() if for_change else self.get_receiving_addresses() - if len(addresses) < limit: + num_addr = self.db.num_change_addresses() if for_change else self.db.num_receiving_addresses() + if num_addr < limit: self.create_new_address(for_change) continue - if any(map(self.address_is_old, addresses[-limit:])): + if for_change: + last_few_addresses = self.get_change_addresses(slice_start=-limit) + else: + last_few_addresses = self.get_receiving_addresses(slice_start=-limit) + if any(map(self.address_is_old, last_few_addresses)): self.create_new_address(for_change) else: break @@ -1666,11 +1786,15 @@ def synchronize(self): def is_beyond_limit(self, address): is_change, i = self.get_address_index(address) - addr_list = self.get_change_addresses() if is_change else self.get_receiving_addresses() limit = self.gap_limit_for_change if is_change else self.gap_limit if i < limit: return False - prev_addresses = addr_list[max(0, i - limit):max(0, i)] + slice_start = max(0, i - limit) + slice_stop = max(0, i) + if is_change: + prev_addresses = self.get_change_addresses(slice_start=slice_start, slice_stop=slice_stop) + else: + prev_addresses = self.get_receiving_addresses(slice_start=slice_start, slice_stop=slice_stop) for addr in prev_addresses: if self.db.get_addr_history(addr): return False @@ -1875,7 +1999,7 @@ def wallet_class(wallet_type): raise WalletFileException("Unknown wallet type: " + str(wallet_type)) -def create_new_wallet(*, path, passphrase=None, password=None, encrypt_file=True, segwit=True): +def create_new_wallet(*, path, passphrase=None, password=None, encrypt_file=True, segwit=True, gap_limit=None): """Create a new wallet""" storage = WalletStorage(path) if storage.file_exists(): @@ -1886,6 +2010,8 @@ def create_new_wallet(*, path, passphrase=None, password=None, encrypt_file=True k = keystore.from_seed(seed, passphrase) storage.put('keystore', k.dump()) storage.put('wallet_type', 'standard') + if gap_limit is not None: + storage.put('gap_limit', gap_limit) wallet = Wallet(storage) wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file) wallet.synchronize() @@ -1896,7 +2022,8 @@ def create_new_wallet(*, path, passphrase=None, password=None, encrypt_file=True def restore_wallet_from_text(text, *, path, network=None, - passphrase=None, password=None, encrypt_file=True): + passphrase=None, password=None, encrypt_file=True, + gap_limit=None): """Restore a wallet from text. Text can be a seed phrase, a master public key, a master private key, a list of bitcoin addresses or bitcoin private keys.""" @@ -1930,6 +2057,8 @@ def restore_wallet_from_text(text, *, path, network=None, raise Exception("Seed or key not recognized") storage.put('keystore', k.dump()) storage.put('wallet_type', 'standard') + if gap_limit is not None: + storage.put('gap_limit', gap_limit) wallet = Wallet(storage) assert not storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" diff --git a/run_electrum b/run_electrum index 0e277183..f9ed0861 100755 --- a/run_electrum +++ b/run_electrum @@ -35,13 +35,16 @@ _min_python_version_tuple = tuple(map(int, (MIN_PYTHON_VERSION.split(".")))) if sys.version_info[:3] < _min_python_version_tuple: sys.exit("Error: Noir Electrum requires Python version >= %s..." % MIN_PYTHON_VERSION) -warnings.simplefilter('default', DeprecationWarning) script_dir = os.path.dirname(os.path.realpath(__file__)) is_bundle = getattr(sys, 'frozen', False) is_local = not is_bundle and os.path.exists(os.path.join(script_dir, "electrum.desktop")) is_android = 'ANDROID_DATA' in os.environ +if is_local: # running from source + # developers should probably see all deprecation warnings. + warnings.simplefilter('default', DeprecationWarning) + # move this back to gui/kivy/__init.py once plugins are moved os.environ['KIVY_DATA_DIR'] = os.path.abspath(os.path.dirname(__file__)) + '/electrum/gui/kivy/data/' @@ -340,7 +343,6 @@ if __name__ == '__main__': # todo: defer this to gui config = SimpleConfig(config_options) - configure_logging(config) cmdname = config.get('cmd') @@ -352,6 +354,7 @@ if __name__ == '__main__': constants.set_simnet() if cmdname == 'gui': + configure_logging(config) fd, server = daemon.get_fd_or_server(config) if fd is not None: plugins = init_plugins(config, config.get('gui', 'qt')) @@ -367,6 +370,7 @@ if __name__ == '__main__': init_daemon(config_options) if subcommand in [None, 'start']: + configure_logging(config) fd, server = daemon.get_fd_or_server(config) if fd is not None: if subcommand == 'start':