Optimises multiexp operation (navcoin#775)

* multiexp exponentiation

* move init

* add mcl

* fix mcl Makefile

* use path

* mcl include

* include mcl in flags

* include mcl when dist

* include mcl in subdirs

* fix gitian

* mcl makefile

* Added some new ignore entries

* Moved some makefile commands from configure step into makefile

* Ignore src/mcl/obj dir

* Revert "Ignore src/mcl/obj dir"

This reverts commit 7c7513d.

* Revert "Revert "Ignore src/mcl/obj dir""

This reverts commit 186fbd6.

* Added the dir for mcl/obj

* update mcl from https://github.com/herumi/mcl

* make verbose

* Revert "make verbose"

This reverts commit 64efbdd.

* build mcl in configure

* remove echo

* Switched to using cmake to build mcl

* Added gitkeep files

* Updated gitian to use focal instead of bionic for builds

* Updated travis to use 20.04 ubuntu

* Disabled interactive CLI when installing apt packages

* Missed OSX gitian when changing bionic to focal

* Removed 32bit linux gitian build support

* Removed sudo commands from gitian config on windows build

* Updated the patch for bls relic code

* Updated distdir to include working DIR changes via a temp stash

* Updated GLIBC requirement to be 2.29

* Removed  -march=native as it was causing a compile issue on arm builds via gitian

* Fixed deprecation notice with gzip variable

* Removed the relic patch so gitian builds will no longer be 'DIRTY'

* Revert "Updated distdir to include working DIR changes via a temp stash"

This reverts commit 59740c9.

* Revert "Removed sudo commands from gitian config on windows build"

This reverts commit 77cc2f9.

* Updated GMP to 6.2.1

* Revert "Updated travis to use 20.04 ubuntu"

This reverts commit c50442d.

* Revert "Missed OSX gitian when changing bionic to focal"

This reverts commit 2e587e6.

* Revert "Updated gitian to use focal instead of bionic for builds"

This reverts commit 13a3e2c.

* Removed torlibs that was missed

* Copies the needed headers into prefix location instead of the system location so no more sudo

* Small fix

* remove unnecessary w variable

* Fixed linking to mcl libs

* Updated mcl to not compile with ASM for windows builds

* Disabled ASM for all builds

* use c++ interface

* fixes c++ interface

* fixes interface

* Updated fp_* methods to have prefix of mcl_fp_*

* Disabled XBLYAT

* Revert "Updated GLIBC requirement to be 2.29"

This reverts commit 8ff1700.

* set serialization for both fr and fp

* bench

* Removed non static version of libmcl which was causing issue with osx build

* Disabled runbench and runtest for bls lib as it was causing an error on windows build on travis

* Updated to build linux with ASM support

* Revert "Updated to build linux with ASM support"

This reverts commit 1da7b19.

Co-authored-by: Barry Deeney <[email protected]>

.gitignore

@@ -56,6 +56,8 @@ ui_*.h
 *.pyc
 *.o
 *.o-*
+*.so
+*.d
 *.patch
 .navcoin
 *.a
@@ -112,6 +114,7 @@ cache/
 share/NavcoindComparisonTool.jar
 
 !src/leveldb*/Makefile
+!src/mcl/Makefile
 
 /doc/doxygen/
 
@@ -128,6 +131,8 @@ src/secp256k1/build-aux/m4/ltoptions.m4
 src/secp256k1/build-aux/m4/ltsugar.m4
 src/secp256k1/build-aux/m4/ltversion.m4
 src/secp256k1/build-aux/m4/lt~obsolete.m4
+
+
 src/NavCoin4-Qt.creator.user
 
 # header files generated by QT from .ui files
@@ -168,6 +173,9 @@ src/bls-signatures/build
 # Ignore temp conf dirs
 conf*/
 
+# MCL ignores for generated OBJ files
+src/mcl/obj
+
 # Ignore .tmp files from tor build
 *.tmp
 

Makefile.am

@@ -1,5 +1,5 @@
 ACLOCAL_AMFLAGS = -I build-aux/m4
-SUBDIRS = src/bls/build src
+SUBDIRS = src/bls/build src/mcl/build src
 .PHONY: deploy FORCE
 
 # GZIP_ENV="-9n" # Disabled cause it was giving out a warning
@@ -158,10 +158,12 @@ $(NAVCOIN_CLI_BIN): FORCE
 
 check-recursive:
 	$(MAKE) -C src/bls/build
+	$(MAKE) -C src/mcl/build
 	$(MAKE) -C src check
 
 install-recursive:
 	$(MAKE) -C src/bls/build
+	$(MAKE) -C src/mcl/build
 	$(MAKE) -C src install
 
 if USE_LCOV
@@ -229,7 +231,7 @@ endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
-EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/pull-tester/rpc-tests.py test/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) src/bls
+EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/pull-tester/rpc-tests.py test/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) src/bls src/mcl
 
 CLEANFILES = $(OSX_DMG) $(NAVCOIN_WIN_INSTALLER)
 

configure.ac

@@ -1337,6 +1337,41 @@ BLS_CXXFLAGS=`echo $BLS_CXXFLAGS $CXXFLAGS | awk '{ print $0 }'`
 BLS_CFLAGS="'$BLS_CFLAGS $CFLAGS $CPPFLAGS -I$BLS_DEPENDS/include'"
 BLS_CXXFLAGS="'$BLS_CXXFLAGS $CXXFLAGS $CPPFLAGS -I$BLS_DEPENDS/include'"
 
+dnl Run cmake for bls-sigs (This project does not support autotools)
+AC_CONFIG_COMMANDS([src/mcl/build],
+                   [rm -rf src/mcl/build/* || true && \
+                    mkdir -p src/mcl/build || true && \
+                    CC=$CC \
+                    CMAKE_PREFIX_PATH=$prefix \
+                    CXX=$CXX \
+                    CFLAGS=$CFLAGS \
+                    CXXFLAGS=$CXXFLAGS \
+                    LDFLAGS=$LDFLAGS \
+                    cmake -Bsrc/mcl/build -Hsrc/mcl \
+                    -DMCL_USE_GMP=On \
+                    -DMCL_USE_ASM=Off \
+                    -DMCL_USE_XBYAK=Off \
+                    -DMCL_USE_OPENSSL=Off \
+                    -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=$prefix/include \
+                    -DCMAKE_C_STANDARD_INCLUDE_DIRECTORIES=$prefix/include \
+                    -DCMAKE_CXX_COMPILER=$BLS_CXX \
+                    $BLS_CCACHE_CMAKE \
+                    $BLS_AR \
+                    $BLS_RANLIB && \
+                    printf 'check:\n\techo\n\n' >> src/mcl/build/Makefile && \
+                    printf 'distdir:\n\techo\n\n' >> src/mcl/build/Makefile && \
+                    printf 'distclean:\n\techo\n\n' >> src/mcl/build/Makefile],
+                   [prefix="$BLS_DEPENDS"
+                    CC=$BLS_CC_RAW
+                    CXX=$BLS_CXX_RAW
+                    CFLAGS=$BLS_CFLAGS
+                    CXXFLAGS=$BLS_CXXFLAGS
+                    BLS_CC=$BLS_CC
+                    BLS_CXX=$BLS_CXX
+                    BLS_CCACHE_CMAKE=$BLS_CCACHE_CMAKE
+                    BLS_AR=$BLS_AR
+                    BLS_RANLIB=$BLS_RANLIB])
+
 dnl Run cmake for bls-sigs (This project does not support autotools)
 AC_CONFIG_COMMANDS([src/bls/build],
                    [rm -rf src/bls/build/* || true && \

depends/packages/gmp.mk

@@ -1,11 +1,11 @@
 package=gmp
-$(package)_version=6.1.2
+$(package)_version=6.2.1
 $(package)_download_path=https://gmplib.org/download/gmp
 $(package)_file_name=gmp-$($(package)_version).tar.bz2
-$(package)_sha256_hash=5275bb04f4863a13516b2f39392ac5e272f5e1bb8057b18aec1c9b79d73d8fb2
+$(package)_sha256_hash=eae9326beb4158c386e39a356818031bd28f3124cf915f8c5b1dc4c7a36b4d7c
 
 define $(package)_set_vars
-$(package)_config_opts+=--enable-cxx --enable-fat --with-pic --disable-shared
+$(package)_config_opts+=--enable-cxx --enable-fat --with-pic --disable-shared --enable-cxx
 $(package)_cflags_armv7l_linux+=-march=armv7-a
 endef
 

src/Makefile.am

@@ -1,4 +1,4 @@
-DIST_SUBDIRS = secp256k1 univalue
+DIST_SUBDIRS = secp256k1 mcl univalue
 
 AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS)
 AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS)
@@ -18,7 +18,7 @@ endif
 NAVCOIN_CONFIG_INCLUDES=-I$(builddir)/config
 NAVCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) $(SODIUM_CFLAGS)
 
-NAVCOIN_INCLUDES += -I$(srcdir)/secp256k1/include -I$(srcdir)/bls/src -I$(srcdir)/bls/build/_deps/relic-build/include -I$(srcdir)/bls/build/_deps/relic-src/include
+NAVCOIN_INCLUDES += -I$(srcdir)/secp256k1/include -I$(srcdir)/mcl/include -I$(srcdir)/bls/src -I$(srcdir)/bls/build/_deps/relic-build/include -I$(srcdir)/bls/build/_deps/relic-src/include
 NAVCOIN_INCLUDES += $(UNIVALUE_CFLAGS)
 
 LIBNAVCOIN_SERVER=libnavcoin_server.a
@@ -30,6 +30,8 @@ LIBNAVCOIN_CRYPTO=crypto/libnavcoin_crypto.a
 LIBNAVCOINQT=qt/libnavcoinqt.a
 LIBSECP256K1=secp256k1/libsecp256k1.la
 LIBBLS=bls/build/libbls.a
+LIBMCLBN=mcl/build/lib/libmclbn384_256.a
+LIBMCL=mcl/build/lib/libmcl.a
 
 if ENABLE_ZMQ
 LIBNAVCOIN_ZMQ=libnavcoin_zmq.a
@@ -45,7 +47,11 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F)
 
 $(LIBBLS):
-	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D)
+	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C bls/build
+
+$(LIBMCL):
+$(LIBMCLBN):
+	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C mcl/build
 
 # Make is not made aware of per-object dependencies to avoid limiting building parallelization
 # But to build the less dependent modules first, we manually select their order here:
@@ -365,6 +371,8 @@ libnavcoin_consensus_a_SOURCES = \
   uint256.h \
   utilstrencodings.cpp \
   utilstrencodings.h \
+  utiltime.cpp \
+  utiltime.h \
   version.h
 
 # common: shared between navcoind, and navcoin-qt and non-server tools
@@ -468,6 +476,8 @@ navcoind_LDADD += \
                   $(EVENT_PTHREADS_LIBS) \
                   $(EVENT_LIBS) \
                   $(ZMQ_LIBS) \
+                  $(LIBMCLBN) \
+                  $(LIBMCL) \
                   $(LIBBLS) \
                   $(SODIUM_LIBS) \
                   $(LIBEVENT_LDFLAGS) \
@@ -512,7 +522,7 @@ navcoin_tx_LDADD = \
   $(LIBNAVCOIN_CRYPTO) \
   $(LIBSECP256K1)
 
-navcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(LIBBLS) $(SODIUM_LIBS)
+navcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(LIBMCLBN) $(LIBMCL) $(LIBBLS) $(SODIUM_LIBS)
 #
 
 # navcoinconsensus library #
@@ -525,8 +535,8 @@ if GLIBC_BACK_COMPAT
 endif
 
 libnavcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(BOOST_LIBS) $(SODIUM_LIBS)
-libnavcoinconsensus_la_LIBADD = $(LIBSECP256K1) $(LIBBLS) $(SODIUM_LIBS)
-libnavcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -I$(srcdir)/bls/src -I$(srcdir)/bls/build/_deps/relic-build/include -I$(srcdir)/bls/build/_deps/relic-src/include -DBUILD_NAVCOIN_INTERNAL
+libnavcoinconsensus_la_LIBADD = $(LIBSECP256K1) $(LIBMCLBN) $(LIBMCL) $(LIBBLS) $(SODIUM_LIBS)
+libnavcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -I$(srcdir)/mcl/include -I$(srcdir)/bls/src -I$(srcdir)/bls/build/_deps/relic-build/include -I$(srcdir)/bls/build/_deps/relic-src/include -DBUILD_NAVCOIN_INTERNAL
 libnavcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 endif
@@ -559,6 +569,7 @@ EXTRA_DIST = $(CTAES_DIST)
 clean-local:
 	-$(MAKE) -C secp256k1 clean
 	-$(MAKE) -C univalue clean
+	-$(MAKE) -C mcl/build clean
 	-$(MAKE) -C bls/build clean
 	-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
 	-rm -f config.h

src/Makefile.qt.include

@@ -639,6 +639,8 @@ qt_navcoin_qt_LDADD += \
                        $(LIBSECP256K1) \
                        $(EVENT_PTHREADS_LIBS) \
                        $(EVENT_LIBS) \
+                       $(LIBMCLBN) \
+                       $(LIBMCL) \
                        $(LIBBLS) \
                        $(SODIUM_LIBS) \
                        $(CHARTS_LIBS) \

src/Makefile.qttest.include

@@ -50,6 +50,8 @@ qt_test_test_navcoin_qt_LDADD += \
                                  $(EVENT_PTHREADS_LIBS) \
                                  $(EVENT_LIBS) \
                                  $(CURL_LIBS) \
+                                 $(LIBMCLBN) \
+                                 $(LIBMCL) \
                                  $(LIBBLS) \
                                  $(SODIUM_LIBS) \
                                  $(LIBEVENT_LIBS) \

src/Makefile.test.include

@@ -185,6 +185,8 @@ test_test_navcoin_LDADD += \
                            $(LIBNAVCOIN_CRYPTO) \
                            $(LIBNAVCOIN_CONSENSUS) \
                            $(LIBBLS) \
+                           $(LIBMCLBN) \
+                           $(LIBMCL) \
                            $(SODIUM_LIBS) \
                            $(BDB_LIBS) \
                            $(ZLIB_LIBS) \

src/bls/src/CMakeLists.txt

@@ -72,13 +72,13 @@ file(GLOB includes "${CMAKE_CURRENT_SOURCE_DIR}/*.hpp")
 install(FILES ${includes} DESTINATION include/chiabls)
 install(FILES ${C_LIB} DESTINATION lib)
 
-add_executable(runtest test.cpp)
-add_executable(runbench test-bench.cpp)
-
-if (SODIUM_FOUND)
-  target_link_libraries(runtest blstmp relic_s sodium)
-  target_link_libraries(runbench blstmp relic_s sodium)
-else()
-  target_link_libraries(runtest blstmp relic_s)
-  target_link_libraries(runbench blstmp relic_s)
-endif()
+#add_executable(runtest test.cpp)
+#add_executable(runbench test-bench.cpp)
+
+#if (SODIUM_FOUND)
+#  target_link_libraries(runtest blstmp relic_s sodium)
+#  target_link_libraries(runbench blstmp relic_s sodium)
+#else()
+#  target_link_libraries(runtest blstmp relic_s)
+#  target_link_libraries(runbench blstmp relic_s)
+#endif()

src/blsct/bulletproofs.cpp

@@ -10,6 +10,7 @@
 
 #include <blsct/bulletproofs.h>
 #include <tinyformat.h>
+#include <utiltime.h>
 
 bool BLSInitResult = bls::BLS::Init();
 
@@ -61,6 +62,11 @@ bool BulletproofsRangeproof::Init()
     if (fInit)
         return true;
 
+    initPairing(mcl::BLS12_381);
+
+    Fp::setETHserialization(true);
+    Fr::setETHserialization(true);
+
     BulletproofsRangeproof::one = 1;
     BulletproofsRangeproof::two = 2;
 
@@ -87,6 +93,32 @@ bool BulletproofsRangeproof::Init()
 
 // Todo multi-exp optimization
 bls::G1Element MultiExp(std::vector<MultiexpData> multiexp_data)
+{
+    G1 x[multiexp_data.size()], z;
+    Fr y[multiexp_data.size()];
+
+
+    for (size_t i = 0; i < multiexp_data.size(); i++)
+    {
+        std::vector<unsigned char> base = multiexp_data[i].base.Serialize();
+        std::vector<unsigned char> exp = multiexp_data[i].exp.GetVch();
+
+        x[i].deserialize(&base[0], base.size());
+        y[i].deserialize(&exp[0], exp.size());
+    }
+
+    G1::mulVec(z, x, y, multiexp_data.size());
+
+    std::vector<unsigned char> res(48);
+
+    z.serialize(&res[0], 48);
+
+    bls::G1Element result = bls::G1Element::FromByteVector(res);
+
+    return result;
+}
+
+bls::G1Element MultiExpLegacy(std::vector<MultiexpData> multiexp_data)
 {
     bls::G1Element result;
 
@@ -116,6 +148,7 @@ static bls::G1Element VectorCommitment(const std::vector<Scalar> &a, const std::
         multiexp_data.push_back({BulletproofsRangeproof::Gi[i], a[i]});
         multiexp_data.push_back({BulletproofsRangeproof::Hi[i], b[i]});
     }
+
     return MultiExp(multiexp_data);
 }
 

src/blsct/bulletproofs.h

@@ -22,9 +22,16 @@
 #include <streams.h>
 #include <utilstrencodings.h>
 
+#define MCL_DONT_USE_XBYAK
+#define MCL_DONT_USE_OPENSSL
+
+#include <mcl/bls12_381.hpp>
+
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/lock_guard.hpp>
 
+using namespace mcl::bn;
+
 static const size_t maxN = 64;
 static const size_t maxMessageSize = 54;
 static const size_t maxM = 16;

src/blsct/transaction.cpp

@@ -38,9 +38,9 @@ bool CreateBLSCTOutput(bls::PrivateKey blindingKey, bls::G1Element& nonce, CTxOu
     {
         bprp.Prove(value, nonces[0], vMemo);
     }
-    catch(...)
+    catch(std::runtime_error& e)
     {
-        strFailReason = "Range proof failed with exception";
+        strFailReason = strprintf("Range proof failed with exception: %s", e.what());
         return false;
     }
 

src/blsct/verification.cpp

@@ -3,9 +3,11 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "verification.h"
+#include "utiltime.h"
 
 bool VerifyBLSCT(const CTransaction &tx, bls::PrivateKey viewKey, std::vector<RangeproofEncodedData> &vData, const CStateViewCache& view, CValidationState& state, bool fOnlyRecover, CAmount nMixFee)
 {
+    auto nStart = GetTimeMicros();
     std::vector<std::pair<int, BulletproofsRangeproof>> proofs;
     std::vector<bls::G1Element> nonces;
 
@@ -208,6 +210,7 @@ bool VerifyBLSCT(const CTransaction &tx, bls::PrivateKey viewKey, std::vector<Ra
         }
     }
 
+    std::cout << strprintf("%s: took %.2f ms\n", __func__, (GetTimeMicros()-nStart)/1000);
     return true;
 }
 

src/init.cpp

@@ -106,13 +106,13 @@ enum BindFlags {
 
 static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat";
 
-static float fBootstrapProgress = 0.0;
+static int fBootstrapProgress = 0;
 
 static int xferinfo(void *p,
                     curl_off_t dltotal, curl_off_t dlnow,
                     curl_off_t ultotal, curl_off_t ulnow)
 {
-    float fProgress = (float)dlnow/(float)dltotal*100.0f;
+    int fProgress = (float)dlnow/(float)dltotal*100.0f;
     if (fProgress == fBootstrapProgress)
         return 0;
 

src/mcl/.gitignore

@@ -0,0 +1,16 @@
+CVS
+java/*_wrap.cxx
+lib/*.so
+lib/*.a
+*.class
+GPATH
+GRTAGS
+GTAGS
+*.o
+*.d
+*.exe
+*.swp
+.cvsignore
+build/
+external/
+Testing/