Update to upstream v0.14 branch

.golangci.yml

@@ -2,7 +2,11 @@ run:
   deadline: 10m
 
 linters:
-  skip-files: "mobile\\/.*generated\\.go"
+  # Skip autogenerated files for mobile and gRPC.
+  skip-files:
+    - "mobile\\/.*generated\\.go"
+    - "\\.pb\\.go$"
+    - "\\.pb\\.gw\\.go$"
 
   disable-all: true
   enable:

Makefile

@@ -3,26 +3,25 @@ ESCPKG := github.com\/decred\/dcrlnd
 
 DCRD_PKG := github.com/decred/dcrd
 DCRWALLET_PKG := github.com/decred/dcrwallet
-GOVERALLS_PKG := github.com/mattn/goveralls
 LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
 GOACC_PKG := github.com/ory/go-acc
 FALAFEL_PKG := github.com/lightninglabs/falafel
 GOIMPORTS_PKG := golang.org/x/tools/cmd/goimports
 GOFUZZ_BUILD_PKG := github.com/dvyukov/go-fuzz/go-fuzz-build
 GOFUZZ_PKG := github.com/dvyukov/go-fuzz/go-fuzz
+GOFUZZ_DEP_PKG := github.com/dvyukov/go-fuzz/go-fuzz-dep
 
 GO_BIN := ${GOPATH}/bin
 DCRD_BIN := $(GO_BIN)/dcrd
 GOMOBILE_BIN := GO111MODULE=off $(GO_BIN)/gomobile
-GOVERALLS_BIN := $(GO_BIN)/goveralls
 LINT_BIN := $(GO_BIN)/golangci-lint
 GOACC_BIN := $(GO_BIN)/go-acc
 GOFUZZ_BUILD_BIN := $(GO_BIN)/go-fuzz-build
 GOFUZZ_BIN := $(GO_BIN)/go-fuzz
 
 MOBILE_BUILD_DIR :=${GOPATH}/src/$(MOBILE_PKG)/build
 IOS_BUILD_DIR := $(MOBILE_BUILD_DIR)/ios
-IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.framework
+IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.xcframework
 ANDROID_BUILD_DIR := $(MOBILE_BUILD_DIR)/android
 ANDROID_BUILD := $(ANDROID_BUILD_DIR)/Lndmobile.aar
 
@@ -50,15 +49,14 @@ DCRWALLET_TMPDIR := $(shell mktemp -d)
 GOACC_COMMIT := 80342ae2e0fcf265e99e76bcc4efd022c7c3811b
 LINT_COMMIT := v1.18.0
 FALAFEL_COMMIT := v0.7.1
-GOFUZZ_COMMIT := 21309f307f61
+GOFUZZ_COMMIT := b1f3d6f
 
-DEPGET := cd /tmp && GO111MODULE=on go get -v
-GOBUILD := CGO_ENABLED=0 GO111MODULE=on go build -v
-GOINSTALL := CGO_ENABLED=0 GO111MODULE=on go install -v
-GOTEST := GO111MODULE=on go test -v
+GOBUILD := go build -v
+GOINSTALL := go install -v
+GOTEST := go test
 
 GOVERSION := $(shell go version | awk '{print $$3}')
-GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name "*pb.go" -not -name "*pb.gw.go")
+GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name "*pb.go" -not -name "*pb.gw.go" -not -name "*.pb.json.go")
 
 TESTBINPKG := dcrlnd_testbins.tar.gz
 
@@ -72,17 +70,7 @@ include make/fuzz_flags.mk
 
 DEV_TAGS := $(if ${tags},$(DEV_TAGS) ${tags},$(DEV_TAGS))
 
-LINT = $(LINT_BIN) \
-	run \
-	--skip-files="mobile\\/.*generated\\.go" \
-	--disable-all \
-	--enable=gofmt \
-	--enable=vet \
-	--enable=gosimple \
-	--enable=unconvert \
-	--enable=ineffassign \
-	--enable=unused \
-	--deadline=10m
+LINT = $(LINT_BIN) run
 
 GREEN := "\\033[0;32m"
 NC := "\\033[0m"
@@ -97,18 +85,13 @@ all: scratch check install
 # ============
 # DEPENDENCIES
 # ============
-
-$(GOVERALLS_BIN):
-	@$(call print, "Fetching goveralls.")
-	go get -u $(GOVERALLS_PKG)
-
 $(LINT_BIN):
-	@$(call print, "Fetching linter")
-	$(DEPGET) $(LINT_PKG)@$(LINT_COMMIT)
+	@$(call print, "Installing linter.")
+	go install $(LINT_PKG)
 
 $(GOACC_BIN):
-	@$(call print, "Fetching go-acc")
-	$(DEPGET) $(GOACC_PKG)@$(GOACC_COMMIT)
+	@$(call print, "Installing go-acc.")
+	go install $(GOACC_PKG)
 
 dcrd:
 	@$(call print, "Installing dcrd $(DCRD_COMMIT).")
@@ -132,29 +115,40 @@ falafel:
 
 goimports:
 	@$(call print, "Installing goimports.")
-	$(DEPGET) $(GOIMPORTS_PKG)
+	go install $(GOIMPORTS_PKG)
 
 $(GOFUZZ_BIN):
-	@$(call print, "Fetching go-fuzz")
-	$(DEPGET) $(GOFUZZ_PKG)@$(GOFUZZ_COMMIT)
+	@$(call print, "Installing go-fuzz.")
+	go install $(GOFUZZ_PKG)
 
 $(GOFUZZ_BUILD_BIN):
-	@$(call print, "Fetching go-fuzz-build")
-	$(DEPGET) $(GOFUZZ_BUILD_PKG)@$(GOFUZZ_COMMIT)
+	@$(call print, "Installing go-fuzz-build.")
+	go install $(GOFUZZ_BUILD_PKG)
+
+$(GOFUZZ_DEP_BIN):
+	@$(call print, "Installing go-fuzz-dep.")
+	go install $(GOFUZZ_DEP_PKG)
 
 # ============
 # INSTALLATION
 # ============
 
 build:
 	@$(call print, "Building debug dcrlnd and dcrlncli.")
-	$(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlnd-debug $(LDFLAGS) $(PKG)/cmd/dcrlnd
-	$(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlncli-debug $(LDFLAGS) $(PKG)/cmd/dcrlncli
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlnd-debug $(LDFLAGS) $(PKG)/cmd/dcrlnd
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlncli-debug $(LDFLAGS) $(PKG)/cmd/dcrlncli
 
 build-itest:
 	@$(call print, "Building itest dcrlnd and dcrlncli.")
-	$(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlnd-itest$(EXEC_SUFFIX) $(ITEST_LDFLAGS) $(PKG)/cmd/dcrlnd
-	$(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlncli-itest$(EXEC_SUFFIX) $(ITEST_LDFLAGS) $(PKG)/cmd/dcrlncli
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlnd-itest$(EXEC_SUFFIX) $(ITEST_LDFLAGS) $(PKG)/cmd/dcrlnd
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlncli-itest$(EXEC_SUFFIX) $(ITEST_LDFLAGS) $(PKG)/cmd/dcrlncli
+
+build-itest-race:
+	@$(call print, "Building itest with race detector.")
+	CGO_ENABLED=1 $(GOBUILD) -race -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest$(EXEC_SUFFIX) $(ITEST_LDFLAGS) $(PKG)/cmd/dcrlnd
+
+	@$(call print, "Building itest binary for ${backend} backend.")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -c -o lntest/itest/itest.test$(EXEC_SUFFIX)
 
 install:
 	@$(call print, "Installing dcrlnd and dcrlncli.")
@@ -180,7 +174,23 @@ scratch: build
 
 check: unit itest
 
-itest-only:
+db-instance:
+ifeq ($(dbbackend),postgres)
+	# Remove a previous postgres instance if it exists.
+	docker rm lnd-postgres --force || echo "Starting new postgres container"
+
+	# Start a fresh postgres instance. Allow a maximum of 500 connections so
+	# that multiple lnd instances with a maximum number of connections of 50
+	# each can run concurrently.
+	docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres:13-alpine -N 500
+	docker logs -f lnd-postgres &
+
+	# Wait for the instance to be started.
+	sleep $(POSTGRES_START_DELAY)
+endif
+
+
+itest-only: db-instance
 	@$(call print, "Building itest binary for $(backend) backend")
 	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -c -o lntest/itest/itest.test
 
@@ -199,7 +209,14 @@ itest-parallel-run:
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*
 	EXEC_SUFFIX=$(EXEC_SUFFIX) echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
 
-itest-parallel: dcrd dcrwallet build-itest itest-parallel-run
+
+itest-race: build-itest-race itest-only
+
+itest-parallel: dcrd dcrwallet db-instance build-itest itest-parallel-run
+
+itest-clean:
+	@$(call print, "Cleaning old itest processes")
+	killall dcrlnd-itest || echo "no running dcrlnd-itest process found";
 
 unit-only:
 	@$(call print, "Running unit tests.")
@@ -220,14 +237,8 @@ unit-race:
 	@$(call print, "Running unit race tests.")
 	$(UNIT_RACE)
 
-goveralls: $(GOVERALLS_BIN)
-	@$(call print, "Sending coverage report.")
-	$(GOVERALLS_BIN) -coverprofile=coverage.txt -service=travis-ci
-
 ci-race: dcrd dcrwallet unit-race
 
-travis-cover: dcrd dcrwallet unit-cover goveralls
-
 ci-itest: itest
 
 package-test-binaries: dcrd dcrwallet build-itest
@@ -260,7 +271,7 @@ flakehunter-parallel:
 # =============
 # FUZZING
 # =============
-fuzz-build: $(GOFUZZ_BUILD_BIN)
+fuzz-build: $(GOFUZZ_BUILD_BIN) $(GOFUZZ_DEP_BIN)
 	@$(call print, "Creating fuzz harnesses for packages '$(FUZZPKG)'.")
 	scripts/fuzz.sh build "$(FUZZPKG)"
 
@@ -299,7 +310,7 @@ rpc-format:
 
 rpc-check: rpc
 	@$(call print, "Verifying protos.")
-	for rpc in $$(find lnrpc/ -name "*.proto" | $(XARGS) awk '/    rpc /{print $$2}'); do if ! grep -q $$rpc lnrpc/rest-annotations.yaml; then echo "RPC $$rpc not added to lnrpc/rest-annotations.yaml"; exit 1; fi; done
+	cd ./lnrpc; ../scripts/check-rest-annotations.sh
 	if test -n "$$(git describe --dirty | grep dirty)"; then echo "Protos not properly formatted or not compiled with v3.4.0"; git status; git diff; exit 1; fi
 
 sample-conf-check:
@@ -312,7 +323,7 @@ mobile-rpc:
 
 vendor:
 	@$(call print, "Re-creating vendor directory.")
-	rm -r vendor/; GO111MODULE=on go mod vendor
+	rm -r vendor/; go mod vendor
 
 ios: vendor mobile-rpc
 	@$(call print, "Building iOS framework ($(IOS_BUILD)).")
@@ -350,9 +361,7 @@ clean-mobile:
 	unit-cover \
 	unit-race \
 	falafel \
-	goveralls \
 	ci-race \
-	travis-cover \
 	ci-itest \
 	flakehunter \
 	flake-unit \

README.md

@@ -112,7 +112,7 @@ as well as compatible versions of `dcrd` and `dcrwallet`.
 ## Safety
 
 When operating a mainnet `dcrlnd` node, please refer to our [operational safety
-guildelines](docs/safety.md). It is important to note that `dcrlnd` is still
+guidelines](docs/safety.md). It is important to note that `dcrlnd` is still
 **beta** software and that ignoring these operational guidelines can lead to
 loss of funds.
 

automation/automation.go

@@ -64,15 +64,15 @@ func (s *Server) runForceCloseStaleChanReestablish() {
 		log.Debugf("Time to check channels for force close due to stale " +
 			"chan reestablish messages")
 
-		chans, err := s.cfg.DB.FetchAllOpenChannels()
+		chans, err := s.cfg.DB.ChannelStateDB().FetchAllOpenChannels()
 		if err != nil {
 			log.Errorf("Unable to list open channels: %v", err)
 			continue
 		}
 
 		for _, c := range chans {
 			sid := c.ShortChannelID
-			waitTime, err := s.cfg.DB.GetChanReestablishWaitTime(sid)
+			waitTime, err := s.cfg.DB.ChannelStateDB().GetChanReestablishWaitTime(sid)
 			if err != nil {
 				log.Errorf("Unable to get chan reestablish msg "+
 					"times for %s: %v", sid, err)

autopilot/graph.go

@@ -152,7 +152,7 @@ func (d *databaseChannelGraph) addRandChannel(node1, node2 *secp256k1.PublicKey,
 				return nil, err
 			}
 
-			dbNode, err := d.db.FetchLightningNode(nil, vertex)
+			dbNode, err := d.db.FetchLightningNode(vertex)
 			switch {
 			case err == channeldb.ErrGraphNodeNotFound:
 				fallthrough

brontide/bench_test.go

@@ -0,0 +1,66 @@
+package brontide
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkReadHeaderAndBody(t *testing.B) {
+	// Create a test connection, grabbing either side of the connection
+	// into local variables. If the initial crypto handshake fails, then
+	// we'll get a non-nil error here.
+	localConn, remoteConn, cleanUp, err := establishTestConnection()
+	require.NoError(t, err, "unable to establish test connection: %v", err)
+	defer cleanUp()
+
+	rand.Seed(time.Now().Unix())
+
+	noiseRemoteConn := remoteConn.(*Conn)
+	noiseLocalConn := localConn.(*Conn)
+
+	// Now that we have a local and remote side (to set up the initial
+	// handshake state, we'll have the remote side write out something
+	// similar to a large message in the protocol.
+	const pktSize = 60_000
+	msg := bytes.Repeat([]byte("a"), pktSize)
+	err = noiseRemoteConn.WriteMessage(msg)
+	require.NoError(t, err, "unable to write encrypted message: %v", err)
+
+	cipherHeader := noiseRemoteConn.noise.nextHeaderSend
+	cipherMsg := noiseRemoteConn.noise.nextBodySend
+
+	var (
+		benchErr error
+		msgBuf   [math.MaxUint16]byte
+	)
+
+	t.ReportAllocs()
+	t.ResetTimer()
+
+	nonceValue := noiseLocalConn.noise.recvCipher.nonce
+	for i := 0; i < t.N; i++ {
+		pktLen, benchErr := noiseLocalConn.noise.ReadHeader(
+			bytes.NewReader(cipherHeader),
+		)
+		require.NoError(
+			t, benchErr, "#%v: failed decryption: %v", i, benchErr,
+		)
+		_, benchErr = noiseLocalConn.noise.ReadBody(
+			bytes.NewReader(cipherMsg), msgBuf[:pktLen],
+		)
+		require.NoError(
+			t, benchErr, "#%v: failed decryption: %v", i, benchErr,
+		)
+
+		// We reset the internal nonce each time as otherwise, we'd
+		// continue to increment it which would cause a decryption
+		// failure.
+		noiseLocalConn.noise.recvCipher.nonce = nonceValue
+	}
+	require.NoError(t, benchErr)
+}

brontide/noise.go

@@ -856,8 +856,11 @@ func (b *Machine) ReadHeader(r io.Reader) (uint32, error) {
 	}
 
 	// Attempt to decrypt+auth the packet length present in the stream.
+	//
+	// By passing in `nextCipherHeader` as the destination, we avoid making
+	// the library allocate a new buffer to decode the plaintext.
 	pktLenBytes, err := b.recvCipher.Decrypt(
-		nil, nil, b.nextCipherHeader[:],
+		nil, b.nextCipherHeader[:0], b.nextCipherHeader[:],
 	)
 	if err != nil {
 		return 0, err
@@ -882,10 +885,13 @@ func (b *Machine) ReadBody(r io.Reader, buf []byte) ([]byte, error) {
 		return nil, err
 	}
 
-	// Finally, decrypt the message held in the buffer, and return a
-	// new byte slice containing the plaintext.
-	// TODO(roasbeef): modify to let pass in slice
-	return b.recvCipher.Decrypt(nil, nil, buf)
+	// Finally, decrypt the message held in the buffer, and return a new
+	// byte slice containing the plaintext.
+	//
+	// By passing in the buf (the ciphertext) as the first argument, we end
+	// up re-using it as we don't force the library to allocate a new
+	// buffer to decode the plaintext.
+	return b.recvCipher.Decrypt(nil, buf[:0], buf)
 }
 
 // SetCurveToNil sets the 'Curve' parameter to nil on the handshakeState keys.