Merge branch 'dev' of github.com:SiaFoundation/renterd into its-happe…

…ning

.github/workflows/project-add.yml

@@ -10,12 +10,5 @@ on:
 
 jobs:
   add-to-project:
-    name: Add issue to project
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/[email protected]
-        with:
-          # You can target a project in a different organization
-          # to the issue
-          project-url: https://github.com/orgs/SiaFoundation/projects/5
-          github-token: ${{ secrets.PAT_ADD_TO_PROJECT }}
+    uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master
+    secrets: inherit

.github/workflows/publish.yml

@@ -1,13 +1,13 @@
 name: Publish
 
-# Controls when the action will run. 
+# Controls when the action will run.
 on:
   # Triggers the workflow on new SemVer tags
   push:
     branches:
       - master
       - dev
-    tags: 
+    tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-**'
 
@@ -28,7 +28,7 @@ jobs:
     permissions:
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
       - uses: docker/login-action@v2
@@ -71,8 +71,8 @@ jobs:
       matrix:
         network: ["mainnet" , "zen"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: 'stable'
       - name: Setup
@@ -116,14 +116,14 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-mac: 
+  build-mac:
     runs-on: macos-latest
     strategy:
       matrix:
         network: ["mainnet" , "zen"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: 'stable'
       - name: Setup Notarization
@@ -212,14 +212,14 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-windows: 
+  build-windows:
     runs-on: windows-latest
     strategy:
       matrix:
         network: ["mainnet" , "zen"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: 'stable'
       - name: Set build tag environment variable
@@ -253,28 +253,62 @@ jobs:
         with:
           name: renterd
           path: release/
-  dispatch:
+
+  dispatch-homebrew: # only runs on full releases
     if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
     needs: [docker, build-linux, build-mac, build-windows]
-    strategy:
-        matrix:
-            repo: ['siafoundation/homebrew-sia', 'siafoundation/linux']
     runs-on: ubuntu-latest
     steps:
       - name: Extract Tag Name
         id: get_tag
         run: echo "tag_name=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
-      - name: Repository Dispatch
+      - name: Dispatch
         uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
-          repository: ${{ matrix.repo }}
+          repository: siafoundation/homebrew-sia
           event-type: release-tagged
           client-payload: >
             {
               "description": "Renterd: The Next-Gen Sia Renter",
               "tag": "${{ env.tag_name }}",
               "project": "renterd",
               "workflow_id": "${{ github.run_id }}"
-            }
+            }
+  dispatch-linux: # run on full releases, release candidates, and master branch
+    if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, 'master')
+    needs: [docker, build-linux, build-mac, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build Dispatch Payload
+        id: get_payload
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const isRelease = context.ref.startsWith('refs/tags/v'),
+              isBeta = isRelease && context.ref.includes('-beta'),
+              tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master';
+
+            let component = 'nightly';
+            if (isBeta) {
+              component = 'beta';
+            } else if (isRelease) {
+              component = 'main';
+            }
+
+            return {
+              description: "renterd: The Next-Gen Sia Renter",
+              tag: tag,
+              project: "renterd",
+              workflow_id: context.runId,
+              component: component
+            };
+
+      - name: Dispatch
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
+          repository: siafoundation/linux
+          event-type: release-tagged
+          client-payload: ${{ steps.get_payload.outputs.result }}

.github/workflows/test.yml

@@ -18,9 +18,9 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: git config --global core.autocrlf false # fixes go lint fmt error
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
       - name: Lint

.github/workflows/ui.yml

@@ -1,71 +1,18 @@
-name: Update UI
+name: Update UI and open PR
 
 on:
-  # Run daily
-  schedule:
-    - cron: '0 0 * * *'
+  repository_dispatch:
+    types: [update-ui]
   # Enable manual trigger
   workflow_dispatch:
 
 jobs:
   update-ui:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.21.0'
-
-      - name: Check for new renterd tag in SiaFoundation/web
-        id: check-tag
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          # Fetch tags with pagination
-          TAGS_JSON=$(gh api --paginate repos/SiaFoundation/web/tags)
-
-          # Extract tags that start with "renterd/", sort them in version order, and pick the highest version
-          LATEST_RENTERD_GO_TAG=$(echo "$TAGS_JSON" | jq -r '.[] | select(.name | startswith("renterd/")).name' | sort -Vr | head -n 1)
-          LATEST_RENTERD_VERSION=$(echo "$LATEST_RENTERD_GO_TAG" | sed 's/renterd\///')
-
-          echo "Latest renterd tag is $LATEST_RENTERD_GO_TAG"
-          echo "GO_TAG=$LATEST_RENTERD_GO_TAG" >> $GITHUB_ENV
-          echo "VERSION=$LATEST_RENTERD_VERSION" >> $GITHUB_ENV
-
-      - name: Fetch release notes for the release
-        id: release-notes
-        env:
-          GH_TOKEN: ${{ github.token }}
-        if: env.GO_TAG != 'null'
-        run: |
-          RELEASE_TAG_FORMATTED=$(echo "$GO_TAG" | sed 's/\/v/@/')
-          RELEASES_JSON=$(gh api --paginate repos/SiaFoundation/web/releases)
-
-          RELEASE_NOTES=$(echo "$RELEASES_JSON" | jq -r --arg TAG_NAME "$RELEASE_TAG_FORMATTED" '.[] | select(.name == $TAG_NAME).body')
-          echo "Release notes for $RELEASE_TAG_FORMATTED: $RELEASE_NOTES"
-          echo "RELEASE_NOTES<<EOF" >> $GITHUB_ENV
-          echo "$RELEASE_NOTES" >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-
-      - name: Update go.mod with latest module
-        if: env.GO_TAG != 'null'
-        run: |
-          GO_MODULE_FORMATTED=$(echo "$GO_TAG" | sed 's/\//@/')
-          echo "Updating go.mod to use $GO_MODULE_FORMATTED"
-          go clean -modcache
-          go get go.sia.tech/web/$GO_MODULE_FORMATTED
-          go mod tidy
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
-        if: env.GO_TAG != 'null'
+      - name: Update UI and open PR
+        uses: SiaFoundation/workflows/.github/actions/ui-update@master
         with:
+          moduleName: 'renterd'
+          goVersion: '1.21'
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "ui: ${{ env.VERSION }}"
-          title: "ui: ${{ env.VERSION }}"
-          body: ${{ env.RELEASE_NOTES }}
-          branch: "ui/update"
-          delete-branch: true

README.md

@@ -23,6 +23,147 @@ API documentation can be found [here](https://api.sia.tech/renterd).<br>
 Setup guides are available on our [website](https://docs.sia.tech/renting/setting-up-renterd).<br>
 A project roadmap is available on [GitHub](https://github.com/orgs/SiaFoundation/projects/5).
 
+## Backups
+
+This section provides a step-by-step guide covering the procedures for creating
+and restoring backups for `renterd`. Regularly backing up your renter's metadata
+and verifying its restorability are essential practices to ensure your data's
+integrity and availability in the event of a disaster scenario. Ensure backups
+are stored on a different machine than the one running `renterd` to avoid data
+loss in case of hardware failure.
+
+---
+**IMPORTANT NOTE**
+
+It is important to note that having a backup is not enough to ensure the data
+can be recovered from the network. The renter has to be online enough of the
+time to ensure data gets migrated away from hosts that went offline.
+
+---
+
+### Creating a backup
+
+#### Step 1: shut down renter
+
+It's strongly recommended to shut down the renter before creating a backup to
+ensure data consistency. This precaution addresses the "upload packing" feature
+(enabled by default), which relies on on-disk buffers. To capture a consistent
+state between the database and on-disk data, shut down `renterd` first. Even if
+the feature is not enabled, it is best to shut down the renter before taking a
+backup to be on the safe side and ensure consistency.
+
+#### Step 2: backing up the database
+
+`renterd` uses two SQL databases: a main database for contracts, host metadata
+and object metadata essential for data retrieval, and a metrics database for UI
+features. Both are critical but prioritize the main database for file recovery.
+
+Depending on how the renter is configured, the databases are either SQLite
+(default) or MySQL databases. By default, the SQLite databases are called
+`db.sqlite` and `metrics.sqlite` and are located in a folder called `db`, right
+in the renter's root directory.
+
+**SQLite**
+
+Use the `.backup` command to create a backup of the SQLite databases.
+
+```bash
+sqlite3 db.sqlite ".backup 'db.bkp'"
+sqlite3 metrics.sqlite ".backup 'metrics.bkp'"
+```
+
+There should only be two files in the `db` folder, if you encounter write-ahead
+log files or index files (usually named `-wal` or `-shm`) it indicates the
+renter was not shut down gracefully. In that case it's best to restart the
+renter and shut it down again.
+
+**MySQL**
+
+Use the `mysqldump` command to create a backup of the MySQL databases. It's a
+utility provided by MySQL to backup or transfer a MySQL database and it's
+usually installed alongside the MySQL cient tools. Replace placeholders with
+actual user and password.
+
+```bash
+mysqldump -u [RENTERD_DB_USER] --password=[RENTERD_DB_PASSWORD] renterd > renterd_bkp.sql
+mysqldump -u [RENTERD_DB_USER] --password=[RENTERD_DB_PASSWORD] renterd_metrics > renterd_metrics_bkp.sql
+```
+
+#### Step 3: backing up partial slabs
+
+If "upload packing" is enabled, back up the `partial_slabs` folder located in
+the renter's root directory. These files contain data that has not been uploaded
+to the network yet, losing these files means an immediate loss of your data.
+
+```bash
+tar -cvf partial_slabs.tar partial_slabs/
+```
+
+### Restoring from a backup
+
+If the goal is to install `renterd` from a backup on a new machine, the easiest
+way is to do a fresh `renterd` install and then overwrite the empty database
+with the backup. Use the same `RENTERD_SEED` as the original installation.
+
+#### Step 1: shutdown renter
+
+Same as before we advise to shut down the renter before reinstating a backup to
+ensure consistency. It's a good idea to backup the database right before trying
+to restore a backup to be safe and have a way out in case overwriting the
+database renders it corrupt somehow.
+
+#### Step 2: restore the database backup
+
+**SQLite**
+
+For SQLite we can reinstate the database by replacing both `.sqlite` files with
+our backups. Make sure to rename them to their original filename `db.sqlite` and
+`metrics.sqlite`. These filenames are configurable, so make sure you match the
+configured values.
+
+**MySQL**
+
+Depending on when the backup was taken its schema might be out of date. To
+maximize the chance the schema migrations go smoothly, it's advisable to
+recreate the databases before importing the backup. Take a backup before doing
+this.
+
+```bash
+# log in to MySQL shell
+mysql -u [RENTERD_DB_USER] -p
+
+# recreate renterd database
+DROP DATABASE IF EXISTS renterd;
+CREATE DATABASE renterd;
+
+# recreate renterd_metrics database
+DROP DATABASE IF EXISTS renterd_metrics;
+CREATE DATABASE renterd_metrics;
+```
+
+The backups can then be imported using the following commands:
+
+```
+cat renterd_bkp.sql | mysql -u [RENTERD_DB_USER] --password=[RENTERD_DB_PASSWORD] renterd
+cat renterd_metrics_bkp.sql | mysql -u [RENTERD_DB_USER] --password=[RENTERD_DB_PASSWORD] renterd_metrics
+```
+
+#### Step 3: restore the partial slabs
+
+If applicable, remove the contents of the `partial_slabs` directory and replace it with your backup.
+
+```bash
+rm -rf partial_slabs
+tar -xvf partial_slabs.tar
+```
+
+#### Step 4: start the renter
+
+After starting the renter it is possible it has to run through migrations to its
+database schema. Depending on when the backup was taken, this might take some
+time. If we restored the backup on a fresh `renterd` install, it will take some
+time for consensus to sync.
+
 ## Docker Support
 
 `renterd` includes a `Dockerfile` which can be used for building and running