Skip to content

Commit

Permalink
Merge pull request #9368 from lightningnetwork/yy-waiting-on-merge
Browse files Browse the repository at this point in the history
Fix itest re new behaviors introduced by `blockbeat`
  • Loading branch information
guggero authored Dec 20, 2024
2 parents fe48e65 + 2913f6e commit a388c1f
Show file tree
Hide file tree
Showing 74 changed files with 4,189 additions and 3,660 deletions.
112 changes: 84 additions & 28 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,14 @@ defaults:
env:
BITCOIN_VERSION: "28"

TRANCHES: 8
# TRANCHES defines the number of tranches used in the itests.
TRANCHES: 16

# SMALL_TRANCHES defines the number of tranches used in the less stable itest
# builds
#
# TODO(yy): remove this value and use TRANCHES.
SMALL_TRANCHES: 8

# If you change this please also update GO_VERSION in Makefile (then run
# `make lint` to see where else it needs to be updated as well).
Expand Down Expand Up @@ -262,10 +269,10 @@ jobs:


########################
# run ubuntu integration tests
# run integration tests with TRANCHES
########################
ubuntu-integration-test:
name: run ubuntu itests
basic-integration-test:
name: basic itests
runs-on: ubuntu-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
strategy:
Expand All @@ -279,18 +286,6 @@ jobs:
args: backend=bitcoind cover=1
- name: bitcoind-notxindex
args: backend="bitcoind notxindex"
- name: bitcoind-rpcpolling
args: backend="bitcoind rpcpolling" cover=1
- name: bitcoind-etcd
args: backend=bitcoind dbbackend=etcd
- name: bitcoind-postgres
args: backend=bitcoind dbbackend=postgres
- name: bitcoind-sqlite
args: backend=bitcoind dbbackend=sqlite
- name: bitcoind-postgres-nativesql
args: backend=bitcoind dbbackend=postgres nativesql=true
- name: bitcoind-sqlite-nativesql
args: backend=bitcoind dbbackend=sqlite nativesql=true
- name: neutrino
args: backend=neutrino cover=1
steps:
Expand All @@ -316,7 +311,7 @@ jobs:
run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION

- name: run ${{ matrix.name }}
run: make itest-parallel tranches=${{ env.TRANCHES }} ${{ matrix.args }}
run: make itest-parallel tranches=${{ env.TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }}

- name: Send coverage
if: ${{ contains(matrix.args, 'cover=1') }}
Expand All @@ -339,12 +334,79 @@ jobs:
path: logs-itest-${{ matrix.name }}.zip
retention-days: 5

########################
# run integration tests with SMALL_TRANCHES
########################
integration-test:
name: itests
runs-on: ubuntu-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
strategy:
# Allow other tests in the matrix to continue if one fails.
fail-fast: false
matrix:
include:
- name: bitcoind-rpcpolling
args: backend="bitcoind rpcpolling"
- name: bitcoind-etcd
args: backend=bitcoind dbbackend=etcd
- name: bitcoind-sqlite
args: backend=bitcoind dbbackend=sqlite
- name: bitcoind-sqlite-nativesql
args: backend=bitcoind dbbackend=sqlite nativesql=true
- name: bitcoind-postgres
args: backend=bitcoind dbbackend=postgres
- name: bitcoind-postgres-nativesql
args: backend=bitcoind dbbackend=postgres nativesql=true
steps:
- name: git checkout
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase

- name: setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test

- name: install bitcoind
run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION

- name: run ${{ matrix.name }}
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }}

- name: Send coverage
if: ${{ contains(matrix.args, 'cover=1') }}
uses: shogo82148/actions-goveralls@v1
with:
path-to-profile: coverage.txt
flag-name: 'itest-${{ matrix.name }}'
parallel: true

- name: Zip log files on failure
if: ${{ failure() }}
timeout-minutes: 5 # timeout after 5 minute
run: 7z a logs-itest-${{ matrix.name }}.zip itest/**/*.log

- name: Upload log files on failure
uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: logs-itest-${{ matrix.name }}
path: logs-itest-${{ matrix.name }}.zip
retention-days: 5


########################
# run windows integration test
########################
windows-integration-test:
name: run windows itest
name: windows itest
runs-on: windows-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
steps:
Expand All @@ -364,7 +426,7 @@ jobs:
key-prefix: integration-test

- name: run itest
run: make itest-parallel tranches=${{ env.TRANCHES }} windows=1
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} windows=1 shuffleseed=${{ github.run_id }}

- name: kill any remaining lnd processes
if: ${{ failure() }}
Expand All @@ -388,7 +450,7 @@ jobs:
# run macOS integration test
########################
macos-integration-test:
name: run macOS itest
name: macOS itest
runs-on: macos-14
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
steps:
Expand All @@ -407,14 +469,8 @@ jobs:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test

- name: install bitcoind
run: |
wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}.0/bitcoin-${BITCOIN_VERSION}.0-arm64-apple-darwin.tar.gz
tar zxvf bitcoin-${BITCOIN_VERSION}.0-arm64-apple-darwin.tar.gz
mv bitcoin-${BITCOIN_VERSION}.0 /tmp/bitcoin
- name: run itest
run: PATH=$PATH:/tmp/bitcoin/bin make itest-parallel tranches=${{ env.TRANCHES }} backend=bitcoind
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} shuffleseed=${{ github.run_id }}

- name: Zip log files on failure
if: ${{ failure() }}
Expand Down Expand Up @@ -473,7 +529,7 @@ jobs:
# Notify about the completion of all coverage collecting jobs.
finish:
if: ${{ always() }}
needs: [unit-test, ubuntu-integration-test]
needs: [unit-test, basic-integration-test]
runs-on: ubuntu-latest
steps:
- uses: ziggie1984/actions-goveralls@c440f43938a4032b627d2b03d61d4ae1a2ba2b5c
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ clean-itest-logs:
itest-only: clean-itest-logs db-instance
@$(call print, "Running integration tests with ${backend} backend.")
date
EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
$(COLLECT_ITEST_COVERAGE)

#? itest: Build and run integration tests
Expand All @@ -233,7 +233,7 @@ itest-race: build-itest-race itest-only
itest-parallel: clean-itest-logs build-itest db-instance
@$(call print, "Running tests")
date
EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
$(COLLECT_ITEST_COVERAGE)

#? itest-clean: Kill all running itest processes
Expand Down
4 changes: 4 additions & 0 deletions docs/release-notes/release-notes-0.19.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,10 @@ The underlying functionality between those two options remain the same.
estimator provided by bitcoind or btcd in regtest and simnet modes instead of
static fee estimator if feeurl is not provided.

* The integration tests CI have been optimized to run faster and all flakes are
now documented and
[fixed](https://github.com/lightningnetwork/lnd/pull/9368).

## Database

* [Migrate the mission control
Expand Down
111 changes: 111 additions & 0 deletions itest/list_exclude_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
//go:build integration

package itest

import (
"fmt"

"github.com/lightningnetwork/lnd/fn/v2"
"github.com/lightningnetwork/lnd/lntest"
)

// excludedTestsWindows is a list of tests that are flaky on Windows and should
// be excluded from the test suite atm.
//
// TODO(yy): fix these tests and remove them from this list.
var excludedTestsWindows = []string{
"batch channel funding",
"zero conf channel open",
"open channel with unstable utxos",
"funding flow persistence",

// Gives "channel link not found" error.
"zero conf-channel policy update public zero conf",

"listsweeps",
"sweep htlcs",
"sweep cpfp anchor incoming timeout",
"payment succeeded htlc remote swept",
"3rd party anchor spend",

"send payment amp",
"async payments benchmark",
"async bidirectional payments",

"multihop-htlc aggregation leased",
"multihop-htlc aggregation leased zero conf",
"multihop-htlc aggregation anchor",
"multihop-htlc aggregation anchor zero conf",
"multihop-htlc aggregation simple taproot",
"multihop-htlc aggregation simple taproot zero conf",

"channel force closure anchor",
"channel force closure simple taproot",
"channel backup restore force close",
"wipe forwarding packages",

"coop close with htlcs",
"coop close with external delivery",

"forward interceptor restart",
"forward interceptor dedup htlcs",
"invoice HTLC modifier basic",
"lookup htlc resolution",

"remote signer-taproot",
"remote signer-account import",
"remote signer-bump fee",
"remote signer-funding input types",
"remote signer-funding async payments taproot",
"remote signer-funding async payments",
"remote signer-random seed",
"remote signer-verify msg",
"remote signer-channel open",
"remote signer-shared key",
"remote signer-psbt",
"remote signer-sign output raw",

"on chain to blinded",
"query blinded route",

"data loss protection",
}

// filterWindowsFlakyTests filters out the flaky tests that are excluded from
// the test suite on Windows.
func filterWindowsFlakyTests() []*lntest.TestCase {
// filteredTestCases is a substest of allTestCases that excludes the
// above flaky tests.
filteredTestCases := make([]*lntest.TestCase, 0, len(allTestCases))

// Create a set for the excluded test cases for fast lookup.
excludedSet := fn.NewSet(excludedTestsWindows...)

// Remove the tests from the excludedSet if it's found in the list of
// all test cases. This is done to ensure the excluded tests are not
// pointing to a test case that doesn't exist.
for _, tc := range allTestCases {
if excludedSet.Contains(tc.Name) {
excludedSet.Remove(tc.Name)

continue
}

filteredTestCases = append(filteredTestCases, tc)
}

// Exit early if all the excluded tests are found in allTestCases.
if excludedSet.IsEmpty() {
return filteredTestCases
}

// Otherwise, print out the tests that are not found in allTestCases.
errStr := "\nThe following tests are not found, please make sure the " +
"test names are correct in `excludedTestsWindows`.\n"
for _, name := range excludedSet.ToSlice() {
errStr += fmt.Sprintf("Test not found in test suite: %v\n",
name)
}

panic(errStr)
}
Loading

0 comments on commit a388c1f

Please sign in to comment.