From c786654f2127f4169142a204985ed71a6b8a3c6e Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Dec 2024 11:57:04 +0100 Subject: [PATCH 01/18] docs: add bus section to openapi spec with accounts routes --- bus/routes.go | 9 +++-- openapi.yml | 106 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/bus/routes.go b/bus/routes.go index be6ba00ba..af76ce304 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -45,13 +45,16 @@ func (b *Bus) accountsFundHandler(jc jape.Context) { return } - // contract metadata + // fetch contract cm, err := b.store.Contract(jc.Request.Context(), req.ContractID) - if jc.Check("failed to fetch contract metadata", err) != nil { + if errors.Is(err, api.ErrContractNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to fetch contract metadata", err) != nil { return } - // host + // fetch host host, err := b.store.Host(jc.Request.Context(), cm.HostKey) if jc.Check("failed to fetch host for contract", err) != nil { return diff --git a/openapi.yml b/openapi.yml index f82156e8b..e64c09824 100644 --- a/openapi.yml +++ b/openapi.yml @@ -180,7 +180,7 @@ paths: ############################# # - # Autopilot routes + # Worker routes # ############################# /worker/account/{hostkey}: @@ -715,6 +715,110 @@ paths: allOf: - $ref: "#/components/schemas/PublicKey" - description: The host's public key + + ############################# + # + # Bus routes + # + ############################# + /bus/accounts: + get: + summary: Get all ephemeral accounts + description: Returns all known ephemeral accounts. + responses: + "200": + description: Successfully retrieved ephemeral accounts + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Account" + post: + summary: Save accounts + description: Saves the provided accounts to the database. + requestBody: + content: + application/json: + schema: + type: object + properties: + accounts: + type: array + items: + $ref: "#/components/schemas/Account" + responses: + "200": + description: Successfully saved accounts + content: + text/plain: + schema: + type: string + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + examples: + missingOwnerField: + summary: Missing owner field example + value: "account is missing a valid 'owner' field" + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + /bus/accounts/fund: + post: + summary: Fund account + description: Funds the specified account with the provided amount. + requestBody: + content: + application/json: + schema: + type: object + properties: + accountID: + allOf: + - $ref: "#/components/schemas/PublicKey" + - description: The ID of the account to fund. + amount: + allOf: + - $ref: "#/components/schemas/Currency" + - description: The amount to fund the account with. + contractID: + allOf: + - $ref: "#/components/schemas/FileContractID" + - description: The ID of the contract to fund the account with. + responses: + "200": + description: Successfully funded account + content: + text/plain: + schema: + type: string + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + "404": + description: Contract not found + content: + text/plain: + schema: + type: string + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + + components: schemas: ############################# From a9995034dc5c7cf8c567432bd14efbd234bfb6b5 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Dec 2024 12:16:57 +0100 Subject: [PATCH 02/18] docs: add alerts routes --- openapi.yml | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/openapi.yml b/openapi.yml index e64c09824..38c0fac6d 100644 --- a/openapi.yml +++ b/openapi.yml @@ -818,6 +818,127 @@ paths: schema: type: string + /bus/alerts: + get: + summary: Get all alerts + description: Returns all currently registered alerts. + parameters: + - name: limit + in: query + description: The maximum number of alerts to return + schema: + type: integer + minimum: -1 + default: -1 + - name: offset + in: query + description: The number of alerts to skip + schema: + type: integer + minimum: 0 + default: 0 + responses: + "200": + description: Successfully retrieved alerts + content: + application/json: + schema: + type: object + properties: + alerts: + type: array + items: + $ref: "#/components/schemas/Alert" + hasMore: + type: boolean + description: Whether there are more alerts to fetch + totals: + type: object + properties: + info: + type: integer + format: uint64 + description: The number of info alerts + warning: + type: integer + format: uint64 + description: The number of warning alerts + error: + type: integer + format: uint64 + description: The number of error alerts + critical: + type: integer + format: uint64 + description: The number of critical alerts + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + examples: + invalidLimit: + summary: Invalid limit example + value: "limit must be greater than or equal to -1" + invalidOffset: + summary: Invalid offset example + value: "offset must be greater than or equal to 0" + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + /bus/alerts/dismiss: + post: + summary: Dismiss alerts + description: Dismisses the specified alerts. + requestBody: + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Hash256" + responses: + "200": + description: Successfully dismissed alerts + content: + text/plain: + schema: + type: string + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + /bus/alerts/register: + post: + summary: Register an alert + description: Registers a new alert. + requestBody: + content: + application/json: + schema: + type: object + properties: + severity: + $ref: "#/components/schemas/Alert" + responses: + "200": + description: Successfully registered alert + content: + text/plain: + schema: + type: string + "500": + description: Internal server error + content: + text/plain: + schema: + type: string components: schemas: @@ -856,6 +977,32 @@ components: type: boolean description: Whether the account requires a sync with the host. This is usually the case when the host reports insufficient balance for an account that the worker still believes to be funded. + Alert: + type: object + properties: + id: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: The alert's ID + severity: + type: string + enum: + - info + - warning + - error + - critical + description: The severity of the alert + message: + type: string + description: The alert's message + date: + type: object + description: Arbitrary data providing additional context for the alert + timestamp: + type: string + format: date-time + description: The time the alert was created + Currency: type: string pattern: "^\\d+$" From 56ce3ff57c1b8effe1ec9e60352b11c915e13d87 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Dec 2024 12:18:00 +0100 Subject: [PATCH 03/18] docs: add changelog --- .changeset/add_bus_routes_to_openapi_spec.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/add_bus_routes_to_openapi_spec.md diff --git a/.changeset/add_bus_routes_to_openapi_spec.md b/.changeset/add_bus_routes_to_openapi_spec.md new file mode 100644 index 000000000..bc800a136 --- /dev/null +++ b/.changeset/add_bus_routes_to_openapi_spec.md @@ -0,0 +1,5 @@ +--- +default: major +--- + +# Add bus routes to openapi spec From a350551b7ce6b7daa34e87743fbee8fdbaf2639d Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Dec 2024 12:50:11 +0100 Subject: [PATCH 04/18] docs: add bucket routes --- bus/routes.go | 28 ++++++-- openapi.yml | 180 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 4 deletions(-) diff --git a/bus/routes.go b/bus/routes.go index af76ce304..5d35b692e 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -279,21 +279,32 @@ func (b *Bus) bucketsHandlerPOST(jc jape.Context) { } else if bucket.Name == "" { jc.Error(errors.New("no name provided"), http.StatusBadRequest) return - } else if jc.Check("failed to create bucket", b.store.CreateBucket(jc.Request.Context(), bucket.Name, bucket.Policy)) != nil { + } + err := b.store.CreateBucket(jc.Request.Context(), bucket.Name, bucket.Policy) + if errors.Is(err, api.ErrBucketExists) { + jc.Error(err, http.StatusBadRequest) return } + jc.Check("failed to create bucket", err) } func (b *Bus) bucketsHandlerPolicyPUT(jc jape.Context) { var req api.BucketUpdatePolicyRequest if jc.Decode(&req) != nil { return - } else if bucket := jc.PathParam("name"); bucket == "" { + } + bucket := jc.PathParam("name") + if bucket == "" { jc.Error(errors.New("no bucket name provided"), http.StatusBadRequest) return - } else if jc.Check("failed to create bucket", b.store.UpdateBucketPolicy(jc.Request.Context(), bucket, req.Policy)) != nil { + } + + err := b.store.UpdateBucketPolicy(jc.Request.Context(), bucket, req.Policy) + if errors.Is(err, api.ErrBucketNotFound) { + jc.Error(err, http.StatusNotFound) return } + jc.Check("failed to create bucket", err) } func (b *Bus) bucketHandlerDELETE(jc jape.Context) { @@ -303,9 +314,18 @@ func (b *Bus) bucketHandlerDELETE(jc jape.Context) { } else if name == "" { jc.Error(errors.New("no name provided"), http.StatusBadRequest) return - } else if jc.Check("failed to delete bucket", b.store.DeleteBucket(jc.Request.Context(), name)) != nil { + } + + err := b.store.DeleteBucket(jc.Request.Context(), name) + if errors.Is(err, api.ErrBucketNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if errors.Is(err, api.ErrBucketNotEmpty) { + jc.Error(err, http.StatusConflict) return } + + jc.Check("failed to delete bucket", err) } func (b *Bus) bucketHandlerGET(jc jape.Context) { diff --git a/openapi.yml b/openapi.yml index 38c0fac6d..9a37ff788 100644 --- a/openapi.yml +++ b/openapi.yml @@ -940,6 +940,186 @@ paths: schema: type: string + /bus/buckets: + get: + summary: Get all buckets + description: Returns all known buckets. + responses: + "200": + description: Successfully retrieved buckets + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Bucket" + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + post: + summary: Create bucket + description: Create a new bucket. + requestBody: + content: + application/json: + schema: + type: object + properties: + name: + type: string + pattern: (?!(^xn--|.+-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ + description: The name of the bucket + policy: + type: object + properties: + publicReadAccess: + type: boolean + description: Whether the bucket is publicly readable + responses: + "200": + description: Successfully saved buckets + content: + text/plain: + schema: + type: string + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + examples: + invalidBucketName: + summary: Invalid bucket name example + value: "bucket name must match pattern '^(?!(^xn--|.+-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$'" + bucketAlreadyExists: + summary: Bucket already exists example + value: "bucket already exists" + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + /bus/bucket/{name}/policy: + put: + summary: Update bucket policy + description: Updates the policy of the specified bucket. + parameters: + - name: name + in: path + required: true + schema: + type: string + description: The name of the bucket + requestBody: + content: + application/json: + schema: + type: object + properties: + policy: + type: object + properties: + publicReadAccess: + type: boolean + description: Whether the bucket is publicly readable + responses: + "200": + description: Successfully updated bucket policy + content: + text/plain: + schema: + type: string + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + examples: + noBucketName: + summary: No bucket name provided + value: "bucket name is required" + "404": + description: Bucket not found + content: + text/plain: + schema: + type: string + /bus/bucket/{name}: + get: + summary: Get bucket + description: Returns the specified bucket. + parameters: + - name: name + in: path + required: true + schema: + type: string + description: The name of the bucket + responses: + "200": + description: Successfully retrieved bucket + content: + application/json: + schema: + $ref: "#/components/schemas/Bucket" + "404": + description: Bucket not found + content: + text/plain: + schema: + type: string + delete: + summary: Delete bucket + description: Deletes the specified bucket. + parameters: + - name: name + in: path + required: true + schema: + type: string + description: The name of the bucket + responses: + "200": + description: Successfully deleted bucket + content: + text/plain: + schema: + type: string + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + examples: + noBucketName: + summary: No bucket name provided + value: "bucket name is required" + "404": + description: Bucket not found + content: + text/plain: + schema: + type: string + "409": + description: Bucket not empty + content: + text/plain: + schema: + type: string + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + components: schemas: ############################# From a474dc88cfbed81d2fd21b5f07c9ef9e6d6c0fb3 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 09:45:49 +0100 Subject: [PATCH 05/18] docs: add bucket name refs --- .changeset/add_bus_routes_to_openapi_spec.md | 7 ++- openapi.yml | 58 +++++++++++--------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/.changeset/add_bus_routes_to_openapi_spec.md b/.changeset/add_bus_routes_to_openapi_spec.md index bc800a136..0adbbb58b 100644 --- a/.changeset/add_bus_routes_to_openapi_spec.md +++ b/.changeset/add_bus_routes_to_openapi_spec.md @@ -2,4 +2,9 @@ default: major --- -# Add bus routes to openapi spec +# Add bus section to openapi spec + +Added routes: +- accounts +- alerts +- buckets diff --git a/openapi.yml b/openapi.yml index 9a37ff788..50ebf2f5d 100644 --- a/openapi.yml +++ b/openapi.yml @@ -271,12 +271,12 @@ paths: example: "myDir/myFile" minLength: 1 - name: bucket - description: The bucket the multipart upload belongs to + description: The name of the bucket the multipart upload belongs to example: "myBucket" in: query required: true schema: - $ref: "#/components/schemas/Bucket" + $ref: "#/components/schemas/BucketName" - name: uploadid description: The ID of the ongoing multipart upload in: query @@ -364,12 +364,12 @@ paths: example: "myDir/myFile" minLength: 1 - name: bucket - description: The bucket the object belongs to + description: The name of the bucket the object belongs to example: "myBucket" in: query required: true schema: - $ref: "#/components/schemas/Bucket" + $ref: "#/components/schemas/BucketName" - name: Range in: header description: The range of bytes to download. If not provided, the entire object will be downloaded. @@ -442,12 +442,12 @@ paths: example: "myDir/myFile" minLength: 1 - name: bucket - description: The bucket the object belongs to + description: The name of the bucket the object belongs to example: "myBucket" in: query required: true schema: - $ref: "#/components/schemas/Bucket" + $ref: "#/components/schemas/BucketName" - name: minshards description: Used to override the minimum number of shards the object should be split into. example: 10 @@ -516,12 +516,12 @@ paths: example: "myDir/myFile" minLength: 1 - name: bucket - description: The bucket the object belongs to + description: The name of the bucket the object belongs to example: "myBucket" in: query required: true schema: - $ref: "#/components/schemas/Bucket" + $ref: "#/components/schemas/BucketName" responses: "200": description: Successfully deleted object @@ -548,8 +548,8 @@ paths: properties: bucket: allOf: - - $ref: "#/components/schemas/Bucket" - - description: The bucket the objects belong to + - $ref: "#/components/schemas/BucketName" + - description: The name of the bucket the objects belong to prefix: type: string example: "myDir/" @@ -969,9 +969,7 @@ paths: type: object properties: name: - type: string - pattern: (?!(^xn--|.+-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ - description: The name of the bucket + $ref: "#/components/schemas/BucketName" policy: type: object properties: @@ -981,10 +979,6 @@ paths: responses: "200": description: Successfully saved buckets - content: - text/plain: - schema: - type: string "400": description: Malformed request content: @@ -1013,7 +1007,7 @@ paths: in: path required: true schema: - type: string + $ref: "#/components/schemas/BucketName" description: The name of the bucket requestBody: content: @@ -1030,10 +1024,6 @@ paths: responses: "200": description: Successfully updated bucket policy - content: - text/plain: - schema: - type: string "400": description: Malformed request content: @@ -1059,7 +1049,7 @@ paths: in: path required: true schema: - type: string + $ref: "#/components/schemas/BucketName" description: The name of the bucket responses: "200": @@ -1067,7 +1057,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Bucket" + $ref: "#/components/schemas/BucketName" "404": description: Bucket not found content: @@ -1082,7 +1072,7 @@ paths: in: path required: true schema: - type: string + $ref: "#/components/schemas/BucketName" description: The name of the bucket responses: "200": @@ -1227,9 +1217,25 @@ components: $ref: "#/components/schemas/HostsConfig" Bucket: + type: object + properties: + name: + $ref: "#/components/schemas/BucketName" + policy: + type: object + properties: + publicReadAccess: + type: boolean + description: Whether the bucket is publicly readable + createdAt: + type: string + format: date-time + description: The time the bucket was created + + BucketName: type: string pattern: (?!(^xn--|.+-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ - description: A bucket logically groups together objects. + description: The name of the bucket. BuildState: type: object From 85cc70d9edfa33b6e91a6cabf8d5b3ec81fd659f Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 09:54:39 +0100 Subject: [PATCH 06/18] docs: fix empty responses --- bus/routes.go | 4 +--- openapi.yml | 33 +++++++-------------------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/bus/routes.go b/bus/routes.go index 5d35b692e..3bede9ea3 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1887,9 +1887,7 @@ func (b *Bus) accountsHandlerPOST(jc jape.Context) { return } } - if b.store.SaveAccounts(jc.Request.Context(), req.Accounts) != nil { - return - } + jc.Check("failed to save accounts", b.store.SaveAccounts(jc.Request.Context(), req.Accounts)) } func (b *Bus) hostsCheckHandlerPUT(jc jape.Context) { diff --git a/openapi.yml b/openapi.yml index 50ebf2f5d..da1e16a60 100644 --- a/openapi.yml +++ b/openapi.yml @@ -525,10 +525,6 @@ paths: responses: "200": description: Successfully deleted object - content: - text/plain: - schema: - type: string "404": description: Object not found content: @@ -558,10 +554,6 @@ paths: responses: "200": description: Successfully removed objects - content: - text/plain: - schema: - type: string "400": description: Missing prefix or bucket content: @@ -750,10 +742,6 @@ paths: responses: "200": description: Successfully saved accounts - content: - text/plain: - schema: - type: string "400": description: Malformed request content: @@ -796,9 +784,14 @@ paths: "200": description: Successfully funded account content: - text/plain: + application/json: schema: - type: string + type: object + properties: + deposit: + allOf: + - $ref: "#/components/schemas/Currency" + - description: The amount that was deposited into the account "400": description: Malformed request content: @@ -904,10 +897,6 @@ paths: responses: "200": description: Successfully dismissed alerts - content: - text/plain: - schema: - type: string "500": description: Internal server error content: @@ -929,10 +918,6 @@ paths: responses: "200": description: Successfully registered alert - content: - text/plain: - schema: - type: string "500": description: Internal server error content: @@ -1077,10 +1062,6 @@ paths: responses: "200": description: Successfully deleted bucket - content: - text/plain: - schema: - type: string "400": description: Malformed request content: From 1ad3b464655807316d5a8ab0e36f98796c0cc7c1 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 10:07:57 +0100 Subject: [PATCH 07/18] docs: add autopilot routes --- .changeset/add_bus_routes_to_openapi_spec.md | 1 + openapi.yml | 50 ++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/.changeset/add_bus_routes_to_openapi_spec.md b/.changeset/add_bus_routes_to_openapi_spec.md index 0adbbb58b..21dfe2510 100644 --- a/.changeset/add_bus_routes_to_openapi_spec.md +++ b/.changeset/add_bus_routes_to_openapi_spec.md @@ -7,4 +7,5 @@ default: major Added routes: - accounts - alerts +- autopilot - buckets diff --git a/openapi.yml b/openapi.yml index da1e16a60..1a06ae070 100644 --- a/openapi.yml +++ b/openapi.yml @@ -925,6 +925,56 @@ paths: schema: type: string + /bus/autopilot: + get: + summary: Get autopilot configuration + description: Returns the current autopilot configuration. + responses: + "200": + description: Successfully retrieved autopilot configuration + content: + application/json: + schema: + $ref: "#/components/schemas/AutopilotConfig" + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + put: + summary: Update autopilot configuration + description: Updates the autopilot configuration. + requestBody: + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + description: Whether the autopilot is enabled + contracts: + $ref: "#/components/schemas/ContractsConfig" + hosts: + $ref: "#/components/schemas/HostsConfig" + required: [] + responses: + "200": + description: Successfully updated autopilot configuration + "400": + description: Malformed request + content: + text/plain: + schema: + type: string + "500": + description: Internal server error + content: + text/plain: + schema: + type: string + /bus/buckets: get: summary: Get all buckets From 532aa4924db9b4330e1d2cacd9d103a8324b9a0f Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 10:11:20 +0100 Subject: [PATCH 08/18] docs: remove required --- openapi.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/openapi.yml b/openapi.yml index 1a06ae070..24aa16d03 100644 --- a/openapi.yml +++ b/openapi.yml @@ -958,7 +958,6 @@ paths: $ref: "#/components/schemas/ContractsConfig" hosts: $ref: "#/components/schemas/HostsConfig" - required: [] responses: "200": description: Successfully updated autopilot configuration From 5ffa74c5df020a28266d544b3d54d2e33d1a52e6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 12:57:15 +0100 Subject: [PATCH 09/18] sql: add missing v2 migration --- internal/sql/migrations.go | 6 ++++ stores/sql/mysql/main.go | 4 +-- .../migrations/main/migration_00034_v2.sql | 32 +++++++++++++++++++ stores/sql/sqlite/main.go | 4 +-- .../migrations/main/migration_00034_v2.sql | 32 +++++++++++++++++++ 5 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 stores/sql/mysql/migrations/main/migration_00034_v2.sql create mode 100644 stores/sql/sqlite/migrations/main/migration_00034_v2.sql diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index e5daba57c..22f45793d 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -537,6 +537,12 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00033_remove_contract_sets", log) }, }, + { + ID: "00034_v2", + Migrate: func(tx Tx) error { + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00034_v2", log) + }, + }, } } MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration { diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index cd9ef5acc..8d856fe8c 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -912,10 +912,8 @@ func (tx MainDatabaseTx) SaveAccounts(ctx context.Context, accounts []api.Accoun res, err := stmt.Exec(ctx, time.Now(), (ssql.PublicKey)(acc.ID), acc.CleanShutdown, (ssql.PublicKey)(acc.HostKey), (*ssql.BigInt)(acc.Balance), (*ssql.BigInt)(acc.Drift), acc.RequiresSync, acc.Owner) if err != nil { return fmt.Errorf("failed to insert account %v: %w", acc.ID, err) - } else if n, err := res.RowsAffected(); err != nil { + } else if _, err := res.RowsAffected(); err != nil { return fmt.Errorf("failed to get rows affected: %w", err) - } else if n != 1 && n != 2 { // 1 for insert, 2 for update - return fmt.Errorf("expected 1 row affected, got %v", n) } } return nil diff --git a/stores/sql/mysql/migrations/main/migration_00034_v2.sql b/stores/sql/mysql/migrations/main/migration_00034_v2.sql new file mode 100644 index 000000000..096e91b1d --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00034_v2.sql @@ -0,0 +1,32 @@ +-- add v2 settings to host +ALTER TABLE hosts +ADD COLUMN v2_settings JSON; + +UPDATE hosts +SET + hosts.v2_settings = '{}'; + +-- drop resolved addresses +ALTER TABLE hosts +DROP COLUMN resolved_addresses; + +-- add column to host_checks +ALTER TABLE host_checks +ADD COLUMN `usability_low_max_duration` boolean NOT NULL DEFAULT false; + +CREATE INDEX `idx_host_checks_usability_low_max_duration` ON `host_checks` (`usability_low_max_duration`); + +-- drop host announcements +DROP TABLE host_announcements; + +-- add new table host_addresses +CREATE TABLE `host_addresses` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime (3) DEFAULT NULL, + `db_host_id` bigint unsigned NOT NULL, + `net_address` longtext NOT NULL, + `protocol` tinyint unsigned NOT NULL, + PRIMARY KEY (`id`), + KEY `ìdx_host_addresses_db_host_id` (`db_host_id`), + CONSTRAINT `fk_host_addresses_db_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci; diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index b22494fda..51fcd5592 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -919,10 +919,8 @@ func (tx *MainDatabaseTx) SaveAccounts(ctx context.Context, accounts []api.Accou res, err := stmt.Exec(ctx, time.Now(), (ssql.PublicKey)(acc.ID), acc.CleanShutdown, (ssql.PublicKey)(acc.HostKey), (*ssql.BigInt)(acc.Balance), (*ssql.BigInt)(acc.Drift), acc.RequiresSync, acc.Owner) if err != nil { return fmt.Errorf("failed to insert account %v: %w", acc.ID, err) - } else if n, err := res.RowsAffected(); err != nil { + } else if _, err := res.RowsAffected(); err != nil { return fmt.Errorf("failed to get rows affected: %w", err) - } else if n != 1 { - return fmt.Errorf("expected 1 row affected, got %v", n) } } return nil diff --git a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql new file mode 100644 index 000000000..b391b8e6f --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql @@ -0,0 +1,32 @@ +-- add v2 settings to host +ALTER TABLE hosts +ADD COLUMN v2_settings text; + +UPDATE hosts +SET + hosts.v2_settings = '{}'; + +-- drop resolved addresses +ALTER TABLE hosts +DROP COLUMN resolved_addresses; + +-- add column to host_checks +ALTER TABLE host_checks +ADD COLUMN usability_low_max_duration INTEGER NOT NULL DEFAULT 0; + +CREATE INDEX `idx_host_checks_usability_low_max_duration` ON `host_checks` (`usability_low_max_duration`); + +-- drop host announcements +DROP TABLE host_announcements; + +-- add new table host_addresses +CREATE TABLE `host_addresses` ( + `id` integer PRIMARY KEY AUTOINCREMENT, + `created_at` datetime NOT NULL, + `db_host_id` integer NOT NULL, + `net_address` text NOT NULL, + `protocol` integer NOT NULL, + CONSTRAINT `fk_host_addresses_db_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE +); + +CREATE INDEX `idx_host_addresses_db_host_id` ON `host_addresses` (`db_host_id`); From 9b05f245be2355e17e51162e730511facaa0ca0a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 13:46:51 +0100 Subject: [PATCH 10/18] add migration for minAccountExpiry and minPriceTableValidity --- internal/gouging/gouging.go | 4 +-- .../migrations/main/migration_00034_v2.sql | 35 +++++++++++++++++++ .../migrations/main/migration_00034_v2.sql | 35 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/internal/gouging/gouging.go b/internal/gouging/gouging.go index de7e6d051..80b15bdf4 100644 --- a/internal/gouging/gouging.go +++ b/internal/gouging/gouging.go @@ -210,7 +210,7 @@ func checkPriceGougingHS(gs api.GougingSettings, hs *rhpv2.HostSettings) error { // check EA expiry if hs.EphemeralAccountExpiry < time.Duration(gs.MinAccountExpiry) { - return fmt.Errorf("'EphemeralAccountExpiry' is less than the allowed minimum value, %v < %v", hs.EphemeralAccountExpiry, gs.MinAccountExpiry) + return fmt.Errorf("'EphemeralAccountExpiry' is less than the allowed minimum value, %v < %v", hs.EphemeralAccountExpiry, time.Duration(gs.MinAccountExpiry)) } return nil @@ -280,7 +280,7 @@ func checkPriceGougingPT(gs api.GougingSettings, cs api.ConsensusState, pt *rhpv // check Validity if pt.Validity < time.Duration(gs.MinPriceTableValidity) { - return fmt.Errorf("'Validity' is less than the allowed minimum value, %v < %v", pt.Validity, gs.MinPriceTableValidity) + return fmt.Errorf("'Validity' is less than the allowed minimum value, %v < %v", pt.Validity, time.Duration(gs.MinPriceTableValidity)) } return nil diff --git a/stores/sql/mysql/migrations/main/migration_00034_v2.sql b/stores/sql/mysql/migrations/main/migration_00034_v2.sql index 096e91b1d..18694ca64 100644 --- a/stores/sql/mysql/migrations/main/migration_00034_v2.sql +++ b/stores/sql/mysql/migrations/main/migration_00034_v2.sql @@ -30,3 +30,38 @@ CREATE TABLE `host_addresses` ( KEY `ìdx_host_addresses_db_host_id` (`db_host_id`), CONSTRAINT `fk_host_addresses_db_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci; + +-- update gouging setting durations from ns to ms +UPDATE settings +SET + value = ( + -- Update settings to new values + SELECT + JSON_REPLACE ( + value, + '$.minAccountExpiry', + newMinAccountExpiry, + '$.minPriceTableValidity', + newMinPriceTableValidity + ) + FROM + ( + -- Convert ns to ms by trimming the last 3 digits + SELECT + SUBSTR (minAccountExpiry, 1, LENGTH (minAccountExpiry) -3) AS newMinAccountExpiry, + SUBSTR ( + minPriceTableValidity, + 1, + LENGTH (minPriceTableValidity) -3 + ) AS newMinPriceTableValidity + FROM + ( + -- SELECT previous settings + SELECT + JSON_UNQUOTE (JSON_EXTRACT (value, '$.minAccountExpiry')) AS minAccountExpiry, + JSON_UNQUOTE (JSON_EXTRACT (value, '$.minPriceTableValidity')) AS minPriceTableValidity + ) AS _ + ) AS _ + ) +WHERE + settings.key = "gouging"; diff --git a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql index b391b8e6f..74ed4ed5c 100644 --- a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql +++ b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql @@ -30,3 +30,38 @@ CREATE TABLE `host_addresses` ( ); CREATE INDEX `idx_host_addresses_db_host_id` ON `host_addresses` (`db_host_id`); + +-- update gouging setting durations from ns to ms +UPDATE settings +SET + value = ( + -- Update settings to new values + SELECT + JSON_REPLACE ( + value, + '$.minAccountExpiry', + newMinAccountExpiry, + '$.minPriceTableValidity', + newMinPriceTableValidity + ) + FROM + ( + -- Convert ns to ms by trimming the last 3 digits + SELECT + SUBSTR (minAccountExpiry, 1, LENGTH (minAccountExpiry) -3) AS newMinAccountExpiry, + SUBSTR ( + minPriceTableValidity, + 1, + LENGTH (minPriceTableValidity) -3 + ) AS newMinPriceTableValidity + FROM + ( + -- SELECT previous settings + SELECT + JSON_EXTRACT (value, '$.minAccountExpiry') AS minAccountExpiry, + JSON_EXTRACT (value, '$.minPriceTableValidity') AS minPriceTableValidity + ) AS _ + ) AS _ + ) +WHERE + settings.key = "gouging"; From 22348fad5d9874bbb227032412db7de199494d57 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 09:59:24 +0100 Subject: [PATCH 11/18] fix syntax error --- stores/sql/sqlite/migrations/main/migration_00034_v2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql index 74ed4ed5c..c1fe11262 100644 --- a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql +++ b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql @@ -4,7 +4,7 @@ ADD COLUMN v2_settings text; UPDATE hosts SET - hosts.v2_settings = '{}'; + v2_settings = '{}'; -- drop resolved addresses ALTER TABLE hosts From 8baf5015e31571d8597c391b5d443488354f125c Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 10:22:44 +0100 Subject: [PATCH 12/18] integer cast --- stores/sql/sqlite/migrations/main/migration_00034_v2.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql index c1fe11262..b7f1be826 100644 --- a/stores/sql/sqlite/migrations/main/migration_00034_v2.sql +++ b/stores/sql/sqlite/migrations/main/migration_00034_v2.sql @@ -40,9 +40,9 @@ SET JSON_REPLACE ( value, '$.minAccountExpiry', - newMinAccountExpiry, + CAST(newMinAccountExpiry AS INTEGER), '$.minPriceTableValidity', - newMinPriceTableValidity + CAST(newMinPriceTableValidity AS INTEGER) ) FROM ( From c071508b6bd944ec2bc0928ac37ac18059446392 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 11:21:47 +0100 Subject: [PATCH 13/18] chore(go.mod): update hostd dependency to latest master --- go.mod | 4 ++-- go.sum | 12 ++++++------ internal/test/e2e/cluster.go | 2 +- internal/test/e2e/host.go | 20 ++++++++------------ 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 3727fd900..a2820e7f2 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806 go.sia.tech/gofakes3 v0.0.5 - go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85 + go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db go.sia.tech/jape v0.12.1 go.sia.tech/mux v1.3.0 go.sia.tech/web/renterd v0.69.0 @@ -32,7 +32,7 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect - github.com/cloudflare/cloudflare-go v0.109.0 // indirect + github.com/cloudflare/cloudflare-go v0.111.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/go.sum b/go.sum index 70dfb004b..c5157f74d 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/cloudflare/cloudflare-go v0.109.0 h1:Wjp+RfJD1lidIFUlrTBqUQnCBrUnmVsLxgzWYiURueg= -github.com/cloudflare/cloudflare-go v0.109.0/go.mod h1:m492eNahT/9MsN7Ppnoge8AaI7QhVFtEgVm3I9HJFeU= +github.com/cloudflare/cloudflare-go v0.111.0 h1:bFgl5OyR7iaV9DkTaoI2jU8X4rXDzEaFDaPfMTp+Ewo= +github.com/cloudflare/cloudflare-go v0.111.0/go.mod h1:w5c4Vm00JjZM+W0mPi6QOC+eWLncGQPURtgDck3z5xU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -51,8 +51,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dc github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb h1:JHX+qWKS9sAXmEroICAu2jPQkr3CYUF7iWd/zlATsBM= @@ -61,8 +61,8 @@ go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806 h1:zmLtpmFQPKMukYMiQB go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806/go.mod h1:6z3oHrQqcLoFEAT/l6XnvOivEGXgIfWBKcq0OqsouWA= go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw= go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54= -go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85 h1:iWitZVJDsazpQHjvmSUHMP6u4p5gySvogkAIQNaaVBU= -go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85/go.mod h1:7REKFrGbO6/Nv49K7p1G8ZwYhZWgIgWyD7iscEMRsEY= +go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db h1:ey3ezMYHPzY+FZ4yL8xsAWnCJWI2J9z4rtpmRa8dj0A= +go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 8c2341d6b..eca33da30 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -629,7 +629,7 @@ func addStorageFolderToHost(ctx context.Context, hosts []*Host) error { func announceHosts(hosts []*Host) error { for _, host := range hosts { settings := defaultHostSettings - settings.NetAddress = host.RHPv2Addr() + settings.NetAddress = host.rhp4Listener.Addr().(*net.TCPAddr).IP.String() if err := host.settings.UpdateSettings(settings); err != nil { return err } diff --git a/internal/test/e2e/host.go b/internal/test/e2e/host.go index a3d0aeef5..3d5899544 100644 --- a/internal/test/e2e/host.go +++ b/internal/test/e2e/host.go @@ -196,12 +196,12 @@ func (h *Host) UpdateSettings(settings settings.Settings) error { // RHPv2Settings returns the host's current RHPv2 settings func (h *Host) RHPv2Settings() (crhpv2.HostSettings, error) { - return h.rhpv2.Settings() + return h.settings.RHP2Settings() } // RHPv3PriceTable returns the host's current RHPv3 price table func (h *Host) RHPv3PriceTable() (crhpv3.HostPriceTable, error) { - return h.rhpv3.PriceTable() + return h.settings.RHP3PriceTable() } // WalletAddress returns the host's wallet address @@ -283,9 +283,11 @@ func NewHost(privKey types.PrivateKey, cm *chain.Manager, dir string, network *c return nil, fmt.Errorf("failed to create rhp3 listener: %w", err) } - settings, err := settings.NewConfigManager(privKey, db, cm, s, wallet, storage, + settings, err := settings.NewConfigManager(privKey, db, cm, s, storage, wallet, settings.WithValidateNetAddress(false), - settings.WithRHP4AnnounceAddresses([]chain.NetAddress{{Protocol: rhp4.ProtocolTCPSiaMux, Address: rhp4Listener.Addr().String()}}), + settings.WithRHP2Port(uint16(rhp2Listener.Addr().(*net.TCPAddr).Port)), + settings.WithRHP3Port(uint16(rhp3Listener.Addr().(*net.TCPAddr).Port)), + settings.WithRHP4Port(uint16(rhp4Listener.Addr().(*net.TCPAddr).Port)), ) if err != nil { return nil, fmt.Errorf("failed to create settings manager: %w", err) @@ -299,16 +301,10 @@ func NewHost(privKey types.PrivateKey, cm *chain.Manager, dir string, network *c registry := registry.NewManager(privKey, db, zap.NewNop()) accounts := accounts.NewManager(db, settings) - rhpv2, err := rhpv2.NewSessionHandler(rhp2Listener, privKey, rhp3Listener.Addr().String(), cm, s, wallet, contracts, settings, storage, log.Named("rhpv2")) - if err != nil { - return nil, fmt.Errorf("failed to create rhpv2 session handler: %w", err) - } + rhpv2 := rhpv2.NewSessionHandler(rhp2Listener, privKey, cm, s, wallet, contracts, settings, storage, log.Named("rhpv2")) go rhpv2.Serve() - rhpv3, err := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv2")) - if err != nil { - return nil, fmt.Errorf("failed to create rhpv3 session handler: %w", err) - } + rhpv3 := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv2")) go rhpv3.Serve() rhpv4 := rhp4.NewServer(privKey, cm, s, contracts, wallet, settings, storage, rhp4.WithPriceTableValidity(30*time.Minute), rhp4.WithContractProofWindowBuffer(1)) From 21efa03f2aeb6b1e0c1394ffbf68bd9254276cca Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 11:43:56 +0100 Subject: [PATCH 14/18] fix TestDownloadAllHosts --- autopilot/migrator.go | 3 ++- internal/rhp/v4/rhp.go | 6 ------ internal/test/e2e/cluster_test.go | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/autopilot/migrator.go b/autopilot/migrator.go index e13076111..43d15499e 100644 --- a/autopilot/migrator.go +++ b/autopilot/migrator.go @@ -154,7 +154,8 @@ func (m *migrator) performMigrations(p *workerPool) { } else if err != nil { m.logger.Errorw("migration failed", zap.Float64("health", j.Health), - zap.Stringer("slab", j.EncryptionKey)) + zap.Stringer("slab", j.EncryptionKey), + zap.Error(err)) } } }(w) diff --git a/internal/rhp/v4/rhp.go b/internal/rhp/v4/rhp.go index eb656e7d3..4bb9471df 100644 --- a/internal/rhp/v4/rhp.go +++ b/internal/rhp/v4/rhp.go @@ -5,7 +5,6 @@ import ( "errors" "io" "net" - "strings" "time" "go.sia.tech/core/consensus" @@ -140,11 +139,6 @@ func (c *Client) AccountBalance(ctx context.Context, hk types.PublicKey, hostIP err := c.tpool.withTransport(ctx, hk, hostIP, func(c rhp.TransportClient) (err error) { balance, err = rhp.RPCAccountBalance(ctx, c, account) if err != nil { - // TODO: remove this hack once the host is fixed - if strings.Contains(err.Error(), "internal error") { - err = nil - balance = types.ZeroCurrency - } return err } return err diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index f70e130b2..88cfa5cdd 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2564,7 +2564,7 @@ func TestDownloadAllHosts(t *testing.T) { var randomHost []string for _, host := range cluster.hosts { if _, used := usedHosts[host.PublicKey()]; used { - randomHost = []string{host.settings.Settings().NetAddress, host.RHPv4Addr()} + randomHost = []string{host.RHPv2Addr(), host.RHPv4Addr()} break } } From 8695ad2b59132e253ca79e536d442a639c9258a8 Mon Sep 17 00:00:00 2001 From: Peter-Jan Brone Date: Fri, 13 Dec 2024 11:04:49 +0100 Subject: [PATCH 15/18] host: fix logger name --- internal/test/e2e/host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test/e2e/host.go b/internal/test/e2e/host.go index 3d5899544..b683ab270 100644 --- a/internal/test/e2e/host.go +++ b/internal/test/e2e/host.go @@ -304,7 +304,7 @@ func NewHost(privKey types.PrivateKey, cm *chain.Manager, dir string, network *c rhpv2 := rhpv2.NewSessionHandler(rhp2Listener, privKey, cm, s, wallet, contracts, settings, storage, log.Named("rhpv2")) go rhpv2.Serve() - rhpv3 := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv2")) + rhpv3 := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv3")) go rhpv3.Serve() rhpv4 := rhp4.NewServer(privKey, cm, s, contracts, wallet, settings, storage, rhp4.WithPriceTableValidity(30*time.Minute), rhp4.WithContractProofWindowBuffer(1)) From d801a0162009e4792179a985a785f3754ed3c932 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 17:49:58 +0100 Subject: [PATCH 16/18] db: improve FK check output --- stores/sql/sqlite/common.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/stores/sql/sqlite/common.go b/stores/sql/sqlite/common.go index 6a061e84f..9e586c699 100644 --- a/stores/sql/sqlite/common.go +++ b/stores/sql/sqlite/common.go @@ -44,9 +44,26 @@ func applyMigration(ctx context.Context, db *sql.DB, fn func(tx sql.Tx) (bool, e } else if !migrated { return nil } + // perform foreign key integrity check - if err := tx.QueryRow(ctx, "PRAGMA foreign_key_check").Scan(); !errors.Is(err, dsql.ErrNoRows) { - return fmt.Errorf("foreign key constraints are not satisfied") + rows, err := tx.Query(ctx, "PRAGMA foreign_key_check") + if err != nil { + return err + } + defer rows.Close() + + // check if there are any foreign key constraint violations + var tableName, foreignKey string + var rowID int + for rows.Next() { + if err := rows.Scan(&tableName, &rowID, &foreignKey); err != nil { + return fmt.Errorf("failed to scan foreign key check result: %w", err) + } + return fmt.Errorf("foreign key constraint violation in table '%s': row %d, foreign key %s", tableName, rowID, foreignKey) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("error iterating foreign key check results: %w", err) } return nil }) From a7815d9fcd64ed7dd53e050aebcc57fb10aae4fe Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Dec 2024 18:05:11 +0100 Subject: [PATCH 17/18] docs: add changelog --- .../improve_migration_out_after_foreignkey_check_fails.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/improve_migration_out_after_foreignkey_check_fails.md diff --git a/.changeset/improve_migration_out_after_foreignkey_check_fails.md b/.changeset/improve_migration_out_after_foreignkey_check_fails.md new file mode 100644 index 000000000..f5e073904 --- /dev/null +++ b/.changeset/improve_migration_out_after_foreignkey_check_fails.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Improve migration out after foreignkey check fails From c5c89a7760d25c3ac7e101c7c8dbdcdb7b9a22cd Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 13 Dec 2024 11:38:35 +0100 Subject: [PATCH 18/18] db: return all fk errors --- stores/sql/sqlite/common.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stores/sql/sqlite/common.go b/stores/sql/sqlite/common.go index 9e586c699..01cc71cec 100644 --- a/stores/sql/sqlite/common.go +++ b/stores/sql/sqlite/common.go @@ -53,19 +53,20 @@ func applyMigration(ctx context.Context, db *sql.DB, fn func(tx sql.Tx) (bool, e defer rows.Close() // check if there are any foreign key constraint violations + var errs []error var tableName, foreignKey string var rowID int for rows.Next() { if err := rows.Scan(&tableName, &rowID, &foreignKey); err != nil { return fmt.Errorf("failed to scan foreign key check result: %w", err) } - return fmt.Errorf("foreign key constraint violation in table '%s': row %d, foreign key %s", tableName, rowID, foreignKey) + errs = append(errs, fmt.Errorf("foreign key constraint violation in table '%s': row %d, foreign key %s", tableName, rowID, foreignKey)) } if err := rows.Err(); err != nil { return fmt.Errorf("error iterating foreign key check results: %w", err) } - return nil + return errors.Join(errs...) }) }