From 98b43045cb4786defc74e21c637489109377ea35 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 25 Nov 2024 08:59:38 +0000 Subject: [PATCH] fix: add retries to certificate provisioning (#2841) The p2p-forge service at libp2p.direct sometimes rejects requests with 401 errors, the only thing to do is to retry. This can probably be reverted in future if it becomes clear why some requests are rejected. --- packages/auto-tls/README.md | 18 ++------ packages/auto-tls/package.json | 3 +- packages/auto-tls/src/auto-tls.ts | 73 ++++++++++++++++++++++++++---- packages/auto-tls/src/constants.ts | 3 +- packages/auto-tls/src/index.ts | 10 +++- 5 files changed, 82 insertions(+), 25 deletions(-) diff --git a/packages/auto-tls/README.md b/packages/auto-tls/README.md index f6bd613536..29f0394f10 100644 --- a/packages/auto-tls/README.md +++ b/packages/auto-tls/README.md @@ -25,7 +25,7 @@ repo and examine the changes made. --> When a publicly dialable address is detected, use the p2p-forge service at - to acquire a valid Let's Encrypted-backed + to acquire a valid Let's Encrypt-backed TLS certificate, which the node can then use with the relevant transports. The node must be configured with a listener for at least one of the following @@ -82,27 +82,19 @@ console.info(node.getMultiaddrs()) # Install ```console -$ npm i @libp2p/plaintext -``` - -## Browser ` +$ npm i @libp2p/auto-tls ``` # API Docs -- +- # License Licensed under either of -- Apache 2.0, ([LICENSE-APACHE](https://github.com/libp2p/js-libp2p/blob/main/packages/connection-encrypter-plaintext/LICENSE-APACHE) / ) -- MIT ([LICENSE-MIT](https://github.com/libp2p/js-libp2p/blob/main/packages/connection-encrypter-plaintext/LICENSE-MIT) / ) +- Apache 2.0, ([LICENSE-APACHE](https://github.com/libp2p/js-libp2p/blob/main/packages/auto-tls/LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](https://github.com/libp2p/js-libp2p/blob/main/packages/auto-tls/LICENSE-MIT) / ) # Contribution diff --git a/packages/auto-tls/package.json b/packages/auto-tls/package.json index e746b33187..583e31b214 100644 --- a/packages/auto-tls/package.json +++ b/packages/auto-tls/package.json @@ -57,6 +57,8 @@ "@multiformats/multiaddr-matcher": "^1.4.0", "@peculiar/x509": "^1.12.3", "acme-client": "^5.4.0", + "any-signal": "^4.1.1", + "delay": "^6.0.0", "interface-datastore": "^8.3.1", "multiformats": "^13.3.1", "uint8arrays": "^5.1.0" @@ -66,7 +68,6 @@ "@libp2p/peer-id": "^5.0.7", "aegir": "^44.0.1", "datastore-core": "^10.0.2", - "delay": "^6.0.0", "p-event": "^6.0.1", "sinon": "^19.0.2", "sinon-ts": "^2.0.0" diff --git a/packages/auto-tls/src/auto-tls.ts b/packages/auto-tls/src/auto-tls.ts index f2e9201364..4ec59d8477 100644 --- a/packages/auto-tls/src/auto-tls.ts +++ b/packages/auto-tls/src/auto-tls.ts @@ -1,14 +1,16 @@ import { ClientAuth } from '@libp2p/http-fetch/auth' -import { serviceCapabilities, serviceDependencies, start, stop } from '@libp2p/interface' +import { serviceCapabilities, serviceDependencies, setMaxListeners, start, stop } from '@libp2p/interface' import { debounce } from '@libp2p/utils/debounce' import { X509Certificate } from '@peculiar/x509' import * as acme from 'acme-client' +import { anySignal } from 'any-signal' +import delay from 'delay' import { Key } from 'interface-datastore' import { base36 } from 'multiformats/bases/base36' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { DEFAULT_ACCOUNT_PRIVATE_KEY_BITS, DEFAULT_ACCOUNT_PRIVATE_KEY_NAME, DEFAULT_ACME_DIRECTORY, DEFAULT_CERTIFICATE_DATASTORE_KEY, DEFAULT_CERTIFICATE_PRIVATE_KEY_BITS, DEFAULT_CERTIFICATE_PRIVATE_KEY_NAME, DEFAULT_FORGE_DOMAIN, DEFAULT_FORGE_ENDPOINT, DEFAULT_PROVISION_DELAY, DEFAULT_PROVISION_TIMEOUT, DEFAULT_RENEWAL_THRESHOLD } from './constants.js' +import { DEFAULT_ACCOUNT_PRIVATE_KEY_BITS, DEFAULT_ACCOUNT_PRIVATE_KEY_NAME, DEFAULT_ACME_DIRECTORY, DEFAULT_CERTIFICATE_DATASTORE_KEY, DEFAULT_CERTIFICATE_PRIVATE_KEY_BITS, DEFAULT_CERTIFICATE_PRIVATE_KEY_NAME, DEFAULT_FORGE_DOMAIN, DEFAULT_FORGE_ENDPOINT, DEFAULT_PROVISION_DELAY, DEFAULT_PROVISION_REQUEST_TIMEOUT, DEFAULT_PROVISION_TIMEOUT, DEFAULT_RENEWAL_THRESHOLD } from './constants.js' import { DomainMapper } from './domain-mapper.js' import { createCsr, importFromPem, loadOrCreateKey, supportedAddressesFilter } from './utils.js' import type { AutoTLSComponents, AutoTLSInit, AutoTLS as AutoTLSInterface } from './index.js' @@ -19,6 +21,8 @@ import type { DebouncedFunction } from '@libp2p/utils/debounce' import type { Multiaddr } from '@multiformats/multiaddr' import type { Datastore } from 'interface-datastore' +const RETRY_DELAY = 5_000 + type CertificateEvent = 'certificate:provision' | 'certificate:renew' interface Certificate { @@ -40,6 +44,7 @@ export class AutoTLS implements AutoTLSInterface { private readonly acmeDirectory: URL private readonly clientAuth: ClientAuth private readonly provisionTimeout: number + private readonly provisionRequestTimeout: number private readonly renewThreshold: number private started: boolean private shutdownController?: AbortController @@ -68,6 +73,7 @@ export class AutoTLS implements AutoTLSInterface { this.forgeDomain = init.forgeDomain ?? DEFAULT_FORGE_DOMAIN this.acmeDirectory = new URL(init.acmeDirectory ?? DEFAULT_ACME_DIRECTORY) this.provisionTimeout = init.provisionTimeout ?? DEFAULT_PROVISION_TIMEOUT + this.provisionRequestTimeout = init.provisionRequestTimeout ?? DEFAULT_PROVISION_REQUEST_TIMEOUT this.renewThreshold = init.renewThreshold ?? DEFAULT_RENEWAL_THRESHOLD this.accountPrivateKeyName = init.accountPrivateKeyName ?? DEFAULT_ACCOUNT_PRIVATE_KEY_NAME this.accountPrivateKeyBits = init.accountPrivateKeyBits ?? DEFAULT_ACCOUNT_PRIVATE_KEY_BITS @@ -108,6 +114,7 @@ export class AutoTLS implements AutoTLSInterface { await start(this.domainMapper) this.events.addEventListener('self:peer:update', this.onSelfPeerUpdate) this.shutdownController = new AbortController() + setMaxListeners(Infinity, this.shutdownController.signal) this.started = true } @@ -120,7 +127,8 @@ export class AutoTLS implements AutoTLSInterface { } private _onSelfPeerUpdate (): void { - const addresses = this.addressManager.getAddresses().filter(supportedAddressesFilter) + const addresses = this.addressManager.getAddresses() + .filter(supportedAddressesFilter) if (addresses.length === 0) { this.log('not fetching certificate as we have no public addresses') @@ -139,11 +147,29 @@ export class AutoTLS implements AutoTLSInterface { this.fetching = true - this.fetchCertificate(addresses, { - signal: AbortSignal.timeout(this.provisionTimeout) + Promise.resolve().then(async () => { + let attempt = 0 + + while (true) { + if (this.shutdownController?.signal.aborted === true) { + throw this.shutdownController.signal.reason + } + + try { + await this.fetchCertificate(addresses, { + signal: AbortSignal.timeout(this.provisionTimeout) + }) + + return + } catch (err) { + this.log.error('provisioning certificate failed on attempt %d - %e', attempt++, err) + } + + await delay(RETRY_DELAY) + } }) .catch(err => { - this.log.error('error fetching certificates - %e', err) + this.log.error('giving up provisioning certificate - %e', err) }) .finally(() => { this.fetching = false @@ -190,7 +216,9 @@ export class AutoTLS implements AutoTLSInterface { // emit a certificate event this.log('dispatching %s', event) this.events.safeDispatchEvent(event, { - detail: this.certificate + detail: { + ...this.certificate + } }) } @@ -271,7 +299,33 @@ export class AutoTLS implements AutoTLSInterface { email: this.email, termsOfServiceAgreed: true, challengeCreateFn: async (authz, challenge, keyAuthorization) => { - await this.configureAcmeChallengeResponse(multiaddrs, keyAuthorization, options) + const signal = anySignal([this.shutdownController?.signal, options?.signal]) + setMaxListeners(Infinity, signal) + + let attempt = 0 + + while (true) { + if (signal.aborted) { + throw signal.reason + } + + try { + const timeout = AbortSignal.timeout(this.provisionRequestTimeout) + const signal = anySignal([timeout, options?.signal]) + setMaxListeners(Infinity, timeout, signal) + + await this.configureAcmeChallengeResponse(multiaddrs, keyAuthorization, { + ...options, + signal + }) + + return + } catch (err: any) { + this.log.error('contacting %s failed on attempt %d - %e', this.forgeEndpoint, attempt++, err.cause ?? err) + } + + await delay(RETRY_DELAY) + } }, challengeRemoveFn: async (authz, challenge, keyAuthorization) => { // no-op @@ -285,8 +339,9 @@ export class AutoTLS implements AutoTLSInterface { const addresses = multiaddrs.map(ma => ma.toString()) const endpoint = `${this.forgeEndpoint}v1/_acme-challenge` - this.log('asking %sv1/_acme-challenge to respond to the acme DNS challenge on our behalf', endpoint) + this.log('asking %s to respond to the acme DNS challenge on our behalf', endpoint) this.log('dialback public addresses: %s', addresses.join(', ')) + const response = await this.clientAuth.authenticatedFetch(endpoint, { method: 'POST', headers: { diff --git a/packages/auto-tls/src/constants.ts b/packages/auto-tls/src/constants.ts index b8f26e82fb..8ef00fe699 100644 --- a/packages/auto-tls/src/constants.ts +++ b/packages/auto-tls/src/constants.ts @@ -1,7 +1,8 @@ export const DEFAULT_FORGE_ENDPOINT = 'https://registration.libp2p.direct' export const DEFAULT_FORGE_DOMAIN = 'libp2p.direct' export const DEFAULT_ACME_DIRECTORY = 'https://acme-v02.api.letsencrypt.org/directory' -export const DEFAULT_PROVISION_TIMEOUT = 10_000 +export const DEFAULT_PROVISION_TIMEOUT = 120_000 +export const DEFAULT_PROVISION_REQUEST_TIMEOUT = 10_000 export const DEFAULT_PROVISION_DELAY = 5_000 export const DEFAULT_RENEWAL_THRESHOLD = 86_400_000 export const DEFAULT_ACCOUNT_PRIVATE_KEY_NAME = 'auto-tls-acme-account-private-key' diff --git a/packages/auto-tls/src/index.ts b/packages/auto-tls/src/index.ts index 5e06311eef..0bf440c27d 100644 --- a/packages/auto-tls/src/index.ts +++ b/packages/auto-tls/src/index.ts @@ -104,10 +104,18 @@ export interface AutoTLSInit { /** * How long to attempt to acquire a certificate before timing out in ms * - * @default 10000 + * @default 120_000 */ provisionTimeout?: number + /** + * How long asking the forge endpoint to answer a DNS challenge can take + * before we retry + * + * @default 10_000 + */ + provisionRequestTimeout?: number + /** * Certificates are acquired when the `self:peer:update` event fires, which * happens when the node's addresses change. To avoid starting to map ports