diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 00000000..ee251986 --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,60 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages dependencies preinstalled + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main", "docs_site"] + paths: + - "site/**" + - ".github/workflows/jekyll-gh-pages.yml" + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + cache-version: 0 # Increment this number if you need to re-download cached gems + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Build with Jekyll + # Outputs to the './_site' directory by default + run: bundle exec jekyll build --source ./site --baseurl "${{ steps.pages.outputs.base_path }}" + env: + JEKYLL_ENV: production + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 + diff --git a/.gitignore b/.gitignore index a896d862..5afc0108 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /yellowstone-faithful /bin/ *.car +_site diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..37c2961c --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.7.2 diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..c08eefbe --- /dev/null +++ b/Gemfile @@ -0,0 +1,39 @@ +source "https://rubygems.org" +# Hello! This is where you manage which Jekyll version is used to run. +# When you want to use a different version, change it below, save the +# file and run `bundle install`. Run Jekyll with `bundle exec`, like so: +# +# bundle exec jekyll serve +# +# This will help ensure the proper Jekyll version is running. +# Happy Jekylling! +gem "jekyll", "~> 4.3.2" +# This is the default theme for new Jekyll sites. You may change this to anything you like. +gem "minima", "~> 2.5" +# If you want to use GitHub Pages, remove the "gem "jekyll"" above and +# uncomment the line below. To upgrade, run `bundle update github-pages`. +#gem "github-pages", "~> 228", group: :jekyll_plugins +# If you have any plugins, put them here! +group :jekyll_plugins do + gem "jekyll-feed", "~> 0.12" +end + +# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem +# and associated library. +platforms :mingw, :x64_mingw, :mswin, :jruby do + gem "tzinfo", ">= 1", "< 3" + gem "tzinfo-data" +end + +# Performance-booster for watching directories on Windows +gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] + +# Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem +# do not have a Java counterpart. +gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] + +# theme +gem "just-the-docs" + +gem 'jemoji' + diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..39cc5978 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,109 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.4) + public_suffix (>= 2.0.2, < 6.0) + colorator (1.1.0) + concurrent-ruby (1.2.2) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + ffi (1.15.5) + forwardable-extended (2.6.0) + gemoji (3.0.1) + html-pipeline (2.14.3) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + jekyll (4.3.2) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (>= 2.0, < 4.0) + jekyll-watch (~> 2.0) + kramdown (~> 2.3, >= 2.3.1) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (>= 0.3.6, < 0.5) + pathutil (~> 0.9) + rouge (>= 3.0, < 5.0) + safe_yaml (~> 1.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (2.1.0) + sassc (> 2.0.1, < 3.0) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + just-the-docs (0.5.4) + jekyll (>= 3.8.5) + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.4.0) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.18.1) + nokogiri (1.15.3-x86_64-linux) + racc (~> 1.4) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.7.1) + rake (13.0.6) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.5) + rouge (3.26.0) + safe_yaml (1.0.5) + sassc (2.4.0) + ffi (~> 1.9) + terminal-table (2.0.0) + unicode-display_width (~> 1.1, >= 1.1.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (1.8.0) + webrick (1.8.1) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + http_parser.rb (~> 0.6.0) + jekyll (~> 4.3.2) + jekyll-feed (~> 0.12) + jemoji + just-the-docs + minima (~> 2.5) + tzinfo (>= 1, < 3) + tzinfo-data + wdm (~> 0.1.1) + +BUNDLED WITH + 2.2.16 diff --git a/README.md b/README.md index ea6444ed..341b1c8c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Project Yellowstone: Old Faithful +# :hotsprings: Project Yellowstone: Old Faithful -Project Yellowstone: Old Faithful is the project to make all of Solana's history accessible, content addressable and available via a variety of means. The goal of this project is to completely replace the Bigtable dependency for Solana history access with a self-hostable, decentralized history archive that is usable for infrastructure providers, individual Solana users, explorers, indexers, and anyone else in need of historical access. +:hotsprings: Project Yellowstone: Old Faithful is the project to make all of Solana's history accessible, content addressable and available via a variety of means. The goal of this project is to completely replace the Bigtable dependency for Solana history access with a self-hostable, decentralized history archive that is usable for infrastructure providers, individual Solana users, explorers, indexers, and anyone else in need of historical access. This is currently in RFC stage, which means that it is not intended for production use and that there may be breaking changes to the format, the CLI utilities or any other details related to the project. diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 00000000..f40fbd8b --- /dev/null +++ b/site/.gitignore @@ -0,0 +1,5 @@ +_site +.sass-cache +.jekyll-cache +.jekyll-metadata +vendor diff --git a/site/404.html b/site/404.html new file mode 100644 index 00000000..086a5c9e --- /dev/null +++ b/site/404.html @@ -0,0 +1,25 @@ +--- +permalink: /404.html +layout: default +--- + + + +
+

404

+ +

Page not found :(

+

The requested page could not be found.

+
diff --git a/site/Gemfile.lock b/site/Gemfile.lock new file mode 100644 index 00000000..31d6ca05 --- /dev/null +++ b/site/Gemfile.lock @@ -0,0 +1,271 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.4) + public_suffix (>= 2.0.2, < 6.0) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.23.9) + concurrent-ruby (1.2.2) + dnsruby (1.70.0) + simpleidn (~> 0.2.1) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.16.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.8.1) + faraday (2.7.10) + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.2) + ffi (1.15.5) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (228) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.3) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.4.0) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) + mercenary (~> 0.3) + minima (= 2.5.1) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) + terminal-table (~> 1.4) + github-pages-health-check (1.17.9) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (~> 4.0) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + jekyll (3.9.3) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.4.0) + commonmarker (~> 0.23.7) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) + octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) + html-pipeline (~> 2.3) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + just-the-docs (0.5.4) + jekyll (>= 3.8.5) + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.18.1) + nokogiri (1.15.3-x86_64-linux) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.7.1) + rake (13.0.6) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.5) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + simpleidn (0.2.1) + unf (~> 0.1.4) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + unicode-display_width (1.8.0) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + github-pages (~> 228) + http_parser.rb (~> 0.6.0) + jekyll-feed (~> 0.12) + jemoji + just-the-docs + minima (~> 2.5) + tzinfo (>= 1, < 3) + tzinfo-data + wdm (~> 0.1.1) + +BUNDLED WITH + 2.2.16 diff --git a/site/_config.yml b/site/_config.yml new file mode 100644 index 00000000..d096168e --- /dev/null +++ b/site/_config.yml @@ -0,0 +1,67 @@ +# Welcome to Jekyll! +# +# This config file is meant for settings that affect your whole blog, values +# which you are expected to set up once and rarely edit after that. If you find +# yourself editing this file very often, consider using Jekyll's data files +# feature for the data you need to update frequently. +# +# For technical reasons, this file is *NOT* reloaded automatically when you use +# 'bundle exec jekyll serve'. If you change this file, please restart the server process. +# +# If you need help with YAML syntax, here are some quick references for you: +# https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml +# https://learnxinyminutes.com/docs/yaml/ +# +# Site settings +# These are used to personalize your new site. If you look in the HTML files, +# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. +# You can create any custom variable you would like, and they will be accessible +# in the templates via {{ site.myvariable }}. + +title: ♨️ Old Faithful +email: help@triton.one +description: >- # this means to ignore newlines until "baseurl:" + Old Faithful is part of Project Yellowstone, reinventing the Solana RPC stack. +baseurl: "" # the subpath of your site, e.g. /blog +url: "https://old-faithful.net" # the base hostname & protocol for your site, e.g. http://example.com +twitter_username: triton_one +github_username: rpcpool + +# Build settings +theme: just-the-docs +plugins: + - jekyll-feed + - jemoji + +callouts: + warning: + title: Warning + color: red + important: + title: Important + color: blue + + +aux_links: + "Github": + - "https://github.com/rpcpool/yellowstone-faithful" + +# Exclude from processing. +# The following items will not be processed, by default. +# Any item listed under the `exclude:` key here will be automatically added to +# the internal "default list". +# +# Excluded items can be processed by explicitly listing the directories or +# their entries' file path in the `include:` list. +# +# exclude: +# - .sass-cache/ +# - .jekyll-cache/ +# - gemfiles/ +# - Gemfile +# - Gemfile.lock +# - node_modules/ +# - vendor/bundle/ +# - vendor/cache/ +# - vendor/gems/ +# - vendor/ruby/ diff --git a/site/_includes/nav_footer_custom.html b/site/_includes/nav_footer_custom.html new file mode 100644 index 00000000..c0c0ffe1 --- /dev/null +++ b/site/_includes/nav_footer_custom.html @@ -0,0 +1 @@ +  diff --git a/site/about.markdown b/site/about.markdown new file mode 100644 index 00000000..633f6228 --- /dev/null +++ b/site/about.markdown @@ -0,0 +1,16 @@ +--- +layout: page +title: About +permalink: /about/ +nav_order: 7 +--- + +This project is currently managed by [Triton One](https://triton.one/). If you want more information contact us via [Telegram](https://t.me/+K0ONdq7fE4s0Mjdl). + +Old Faithful is a part of Project Yellowstone. Project Yellowstone is an initiative to reinvent the Solana RPC stack. + +The originator of this project was [Richard Patel](https://github.com/terorie) ([Twitter](https://twitter.com/fd_ripatel)). + +[@immaterial.ink](https://github.com/gagliardetto) ([Twitter](https://twitter.com/immaterial_ink)) is currently the lead dev on this project at Triton One. + +This work has been supported greatly by Protocol Labs (special shout out to [anjor](https://github.com/anjor) ([Twitter](https://twitter.com/__anjor)) for all the guidance in Filecoin land to us Solana locals). diff --git a/site/archives.markdown b/site/archives.markdown new file mode 100644 index 00000000..d81fad2c --- /dev/null +++ b/site/archives.markdown @@ -0,0 +1,13 @@ +--- +layout: page +title: Archives +permalink: /archives/ +has_children: true +nav_order: 4 +--- + +# Archives + +The core of the project is history archives in Content Addressable format ([overview](https://web3.storage/docs/how-tos/work-with-car-files/), [specs](https://ipld.io/specs/transport/car/carv1/)). These represent a verifiable, immutable view of the Solana history. The CAR files that this project generates follows a [schema](https://github.com/rpcpool/yellowstone-faithful/blob/main/ledger.ipldsch) specifically developed for Solana's historical archives. + +The content addressable nature means that each epoch, block, transaction and shredding is uniquely identified by a content hash. By knowing this content hash a user will be able to retreive a specific object of interest in a trustless manner, i.e. retrieve an object verifiably from a non-trusted source. Retrievals can be made via IPFS, the Filecoin network, or even by hosting the CAR files yourself on disk, a ceph cluster, S3, you name it. diff --git a/site/cid.markdown b/site/cid.markdown new file mode 100644 index 00000000..8308e580 --- /dev/null +++ b/site/cid.markdown @@ -0,0 +1,25 @@ +--- +layout: page +title: Content Identifiers +permalink: /archives/cid/ +parent: Archives +nav_order: 1 +--- + +# Content Identifiers + +In the archives each Epoch, Block and Transaction gets a computed content identifier. These content identifiers can be used to uniquely identify any of these elements when retrieving either from a locally stored CAR file or from IPFS or Filecoin. + +In order to retrieve objects using Solana slot numbers, transaction signatures or epoch numbers, indexes are used to map these Solana specific identifiers onto CIDs. + +To learn more about CIDs, you can deep dive into the IPFS documentation [here](https://docs.ipfs.tech/concepts/content-addressing/). + +From [https://docs.ipfs.tech/concepts/content-addressing/#what-is-a-cid](https://docs.ipfs.tech/concepts/content-addressing/#what-is-a-cid): + + > A content identifier, or CID, is a label used to point to material in IPFS. It doesn't indicate where the content is stored, but it forms a kind of address based on the content itself. CIDs are short, regardless of the size of their underlying content. + > CIDs are based on the content’s cryptographic hash. That means: + > - Any difference in the content will produce a different CID + > - The same content added to two different IPFS nodes using the same settings will produce the same CID. + + +The CAR files and CIDs that are generated by Old Faithful are reproducible and will be the same regardless from which Rocksdb ledger archive they are generated from. This means that entities that are running archive nodes and store Rocksdb ledger archives can easily generated their own CIDs for the same blocks and transactions and compare to the ones being generated by the Old Faithful project. In theory this means that you can also choose to run a node that /just/ generates and stores the CIDs but not the durable data and then compare those CIDs to the ones provided by Faithful. Using these CIDs you can retrieve the actual data any time without trusting anyone but your own data generating nodes. \ No newline at end of file diff --git a/site/data-preparation.markdown b/site/data-preparation.markdown new file mode 100644 index 00000000..ab63809e --- /dev/null +++ b/site/data-preparation.markdown @@ -0,0 +1,42 @@ +--- +layout: page +title: Data Preparation +permalink: /archives/dataprep/ +parent: Archives +nav_order: 2 +--- + +# Data preparation + +The primary data preparation tooling used in this project is based in the `radiance` tool developed by Jump's Firedancer team. It is rapidely developing, and active development for this project is currently based out of this repository and branch: [Radiance Triton](https://github.com/gagliardetto/radiance-triton/). + +The radiance tool utilises the rocksdb snapshots that have been generated by [Warehouse](https://github.com/solana-labs/solana-bigtable) nodes. From these snapshots a CAR file per epoch is generated. This CAR file then needs to be processed by Filecoin tools such as [split-and-commp](https://github.com/anjor/go-fil-dataprep/) which generates the details needed for making a Filecoin deal. + +Currently, this tool is being tested from the following warehouse archives: + - Solana Foundation (public) + - gs://mainnet-beta-ledger-us-ny5 + - gs://mainnet-beta-ledger-europe-fr2 + - gs://mainnet-beta-ledger-asia-sg1 + - Triton One (private) + +If you have warehouse nodes generating rocksdb archive snapshots, please contact lk@triton.one (even if they can't be made publicly available). We would like to have you generate CAR files for verification purposes. + +## CAR files + +CAR file generation produces a CAR containing a DAG. This DAG is reproducible and follows the structure of Epoch -> Block -> Transaction see [schema](https://github.com/rpcpool/yellowstone-faithful/blob/main/ledger.ipldsch). The CAR file generation is deterministic, so even if you use different rocksdb source snapshots you should end up with the same CAR output. This allows comparison between different providers. + +The data generation flow is illustrated below: + +![radiance datagen flow](https://github.com/rpcpool/yellowstone-faithful/assets/5172293/65f6dd75-189b-4253-968a-e81bfe6c130f) + +## Generating an epoch car file + +Once you have downloaded rocksdb ledger archives you can run the Radiance tool to generate a car file for an epoch. Make sure you have all the slots available in rocksdb ledger archive for the epoch. You may need to download multiple ledger snapshots in order to have a full set of slots available. Once you know you have a rocksdb that covers all the slots for the epoch run the radiance tool like follows: + +``` +radiance car create2 107 --db=46223992/rocksdb --out=/storage/car/epoch-107.car +``` + +This will produce a car file called epoch-107.car containing all the blocks and transactions for that epoch. + + diff --git a/site/filecoin-rpc.markdown b/site/filecoin-rpc.markdown new file mode 100644 index 00000000..03ca5537 --- /dev/null +++ b/site/filecoin-rpc.markdown @@ -0,0 +1,38 @@ +--- +layout: page +title: Filecoin RPC +permalink: /filecoin/rpc/ +parent: Filecoin +--- + +# Filecoin RPC + +The filecoin RPC server allows provide getBlock, getTransaction and getSignaturesForAddress powered by Filecoin. This requires access to indexes. The indexes allow you to lookup transaction signatures, block numbers and addresses and map them to Filecoin CIDs. + +You can run it in the following way: + +``` +faithful-cli rpc-server-filecoin -config 455.yml +``` + +The config file points faithful to the location of the required indexes (`455.yaml`): +``` +indexes: + slot_to_cid: './epoch-455.car.bafyreibkequ55hyrhyk6f24ctsofzri6bjykh76jxl3zju4oazu3u3ru7y.slot-to-cid.index' + sig_to_cid: './epoch-455.car.bafyreibkequ55hyrhyk6f24ctsofzri6bjykh76jxl3zju4oazu3u3ru7y.sig-to-cid.index' + gsfa: './epoch-455.car.gsfa.index' +``` + +Due to latency in fetching signatures, typically the getSignaturesForAddress index needs to be stored in a local directory, but the other indexes can be fetched via HTTP or via local file system access. If you provide a URL, you need to make sure that the url supports HTTP Range requests. S3 or similar APIs will support this. + +You can enter URLs from old-faithful.net in these config files. + +There is a mode in which you can use a remote gSFA index, which limits it to only return signatures and not additional transaction meta data. In this mode, you can use a remote gSFA index. To enable this mode run faithful-cli in the following way: + +``` +faithful-cli rpc-server-filecoin -config 455.yml -gsfa-only-signatures=true +``` + +## Configuration files + +Configuration files for filecoin accesses are still under development. \ No newline at end of file diff --git a/site/filecoin.markdown b/site/filecoin.markdown new file mode 100644 index 00000000..f10bd925 --- /dev/null +++ b/site/filecoin.markdown @@ -0,0 +1,28 @@ +--- +layout: page +title: Filecoin +permalink: /filecoin/ +nav_order: 5 +has_children: true +--- + +# Filecoin + +As part of the Old Faithful project we are placing all of Solana's archive data on Filecoin. This will allow secure, decentralized access to Solana's history data. + +## CIDs + +To retrieve data from Filecoin you will need knowledge about the [content identifiers](/archives/cid/) for the epochs, blocks or transactions you are interested in. In theory, you can clone an entire epoch via Filecoin, but you can also retrieve an individual block or transaction. When fetching blocs or transaction, you can fetch only the metdata or the full data of the transaction/block. + +## Fetching CIDs + +If you already know the CID of the data you are looking for you can fetch it via `faithful-cli fetch `. This requires no further indexes and can also be used to recursively fetch data for example for an epoch. To avoid fetching the full dataset for an epoch (100s of GB) you probably want to pass the parameter `--dag-scope=block` to fetch only the particular CID entity that you are interested in. + +## Retrieving Epoch CIDs + +You can fetch the latest Epoch CIDs uploaded from old-faithful.net: + +``` +curl https://files.old-faithful.net/100/epoch-100.cid +``` + diff --git a/site/hosting.markdown b/site/hosting.markdown new file mode 100644 index 00000000..761f8b55 --- /dev/null +++ b/site/hosting.markdown @@ -0,0 +1,11 @@ +--- +layout: page +title: Hosting +permalink: /archives/hosting/ +parent: Archives +nav_order: 4 +--- + +# Hosting Faithful archives + +You can download and host the faithful archives using any file system data source or HTTP endpoint that supports range requests. To host archives or indexes all you need to do is [generate](/archives/data-prep/) or [download](/rpc-server/old-faithful-net/) Epoch car files or indexes. You can then use the config file syntax or command line to serve these car files. \ No newline at end of file diff --git a/site/index.markdown b/site/index.markdown new file mode 100644 index 00000000..103ba59a --- /dev/null +++ b/site/index.markdown @@ -0,0 +1,17 @@ +--- +# Feel free to add content and custom Front Matter to this file. +# To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults + +layout: home +nav_order: 1 +--- +# :hotsprings: Project Yellowstone: Old Faithful + +Old Faithful is the project to make all of Solana's history accessible, content addressable and available via a variety of means. The goal of this project is to completely replace the Bigtable dependency for Solana history access with a self-hostable, decentralized history archive that is usable for infrastructure providers, individual Solana users, explorers, indexers, and anyone else in need of historical access. + +{: .important-title } +> Requests for comments release +> +> This is currently in RFC stage, which means that it is not intended for production use and that there may be breaking changes to the format, the CLI utilities or any other details related to the project. +> +> We appreciate all and any testing, questions, feedback, code submissions as issues via [github.com/rpcpool/yellowstone-faithful](https://github.com/rpcpool/yellowstone-faithful/issues). We can also be found on the [Solana tech discord](https://discord.com/invite/kBbATFA7PW) (e.g. @linuskendall). \ No newline at end of file diff --git a/site/indexes.markdown b/site/indexes.markdown new file mode 100644 index 00000000..fd052042 --- /dev/null +++ b/site/indexes.markdown @@ -0,0 +1,51 @@ +--- +layout: page +title: Indexes +permalink: /archives/indexes/ +parent: Archives +nav_order: 3 +--- + +# Indexes + +Indexes will be needed to map Solana's block numbers, transaction signatures and addresses to their respective Content Identifiers. These indexes will be developed as part of this project. There are three content identifier indexes kinds that the Old Faithful index generation can provide: + + - slot-to-cid: Lookup a CID based on a slot number + - tx-to-cid: Lookup a CID based on a transaction signature + - cid-to-offset: Index for a specific CAR file, used by the local rpc server (see above) to find CIDs in a car file + +In addition to these Old Faithful supports an index called `gsfa` that maps Solana addresses to a list of transaction signatures. + +## Index generation + +Once the radiance tooling has been used to prepare a car file (or if you have downloaded a car file externally) you can generate indexes from this car file by using the `faithful-cli`: + +``` +NAME: + faithful index + +USAGE: + faithful index command [command options] [arguments...] + +DESCRIPTION: + Create various kinds of indexes for CAR files. + +COMMANDS: + cid-to-offset + slot-to-cid + sig-to-cid + all + gsfa + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +For example, to generate the three indexes cid-to-offset, slot-to-cid, sig-to-cid you would run: + +``` +faithful-cli index all epoch-107.car . +``` + +This would generate the indexes in the current dir for epoch-107. \ No newline at end of file diff --git a/site/installation.markdown b/site/installation.markdown new file mode 100644 index 00000000..b59e579e --- /dev/null +++ b/site/installation.markdown @@ -0,0 +1,14 @@ +--- +layout: page +title: Installation +permalink: /installation/ +nav_order: 2 +--- + +# Installing faithful-cli + +The easiest way to install faithful-cli is to download the pre-built binaries for Linux, Windows or Mac OS provided at [github.com/rpcpool/yellowstone-faithful](https://github.com/rpcpool/yellowstone-faithful/releases). + +## Building from source + +You can also build from source by cloning [the github repo](https://github.com/rpcpool/yellowstone-faithful/releases) and then running `make compile-linux` / `make compile-windows` / `make compile-mac`. \ No newline at end of file diff --git a/site/old-faithful-net.markdown b/site/old-faithful-net.markdown new file mode 100644 index 00000000..05815c27 --- /dev/null +++ b/site/old-faithful-net.markdown @@ -0,0 +1,69 @@ +--- +layout: page +title: old-faithful.net +permalink: /rpc-server/old-faithful-net/ +parent: RPC server +nav_order: 1 +--- + +# Local CAR files and old-faithful.net + +old-faithful.net provides archives that have been generated by Triton One as part of the Old Faithful project. You can use this data either by cloning it locally or by partially/fully serving it from old-faithful.net. A simple mode of operation can be to clone the index files locally but source the data from old-faithful.net. A similar mode of operation can be used with [Filecoin](/filecoin/). + +For running a local server, it is usually easiest to have access to the [tools folder](https://github.com/rpcpool/yellowstone-faithful/tree/main/tools) from the repository. It provides some useful scripts to interface with the archives and indexes hosted on old-faithful.net as well as running the faithful-cli rpc server. + +## Fully remote + +You can use the RPC server with absolutely nothing downloaded locally. A helper script is available from the repository: + +``` +$ ./tools/run-rpc-server-remote.sh 0 +``` + +## RPC server with local indexes + +For ongoing testing, we strongly recommend that you download at least the indexes for best performance. If you have local indexes downloaded you can use the following helper script: + +``` +$ ./tools/run-rpc-server-local-indexes.sh 0 ./epoch0 +``` + +There is a utility script in the `tools` folder that will download the indexes hosted on old-faithful.net. The indexes will also be available on Filecoin. + +``` +$ mkdir epoch0 +$ cd epoch0 +$ ../tools/download-indexes.sh 0 ./epoch0 +$ ../tools/download-gsfa.sh 0 ./epoch0 +``` + +## RPC server running fully locally + +If you have a local copy of a CAR archive and the indexes and run a RPC server servicing data from them. For example: + +``` +/usr/local/bin/faithful-cli rpc-server-car \ + --listen $PORT \ + epoch-455.car \ + epoch-455.car.*.cid-to-offset.index \ + epoch-455.car.*.slot-to-cid.index \ + epoch-455.car.*.sig-to-cid.index \ + epoch-455.car-*-gsfa-index +``` + +You can download the CAR files either via Filecoin or via the bucket provided by Triton. There are helper scripts in the `tools` folder. To download the full epoch data: + +``` +$ mkdir epoch0 +$ cd epoch0 +$ ../tools/download-epoch.sh 0 +$ ../tools/download-indexes.sh 0 +$ ../tools/download-gsfa.sh 0 +``` + +Once files are downloaded there are also utility scripts to run the server: +``` +$ ./tools/run-rpc-server-local.sh 0 ./epoch0 +``` + +This will host epoch 0 from the data available in the folder epoch0. diff --git a/site/roadmap.markdown b/site/roadmap.markdown new file mode 100644 index 00000000..0a592c8e --- /dev/null +++ b/site/roadmap.markdown @@ -0,0 +1,13 @@ +--- +layout: page +title: Roadmap +permalink: /roadmap/ +nav_order: 6 +--- + +# Roadmap + +- Multi-epoch indexing and retrievals +- Full archives on filecoin and old-faithful.net +- Continuous uploads of new epochs to Filecoin and old-faithful.net +- Root index/IPNS for all solana history diff --git a/site/rpc-server.markdown b/site/rpc-server.markdown new file mode 100644 index 00000000..e762d345 --- /dev/null +++ b/site/rpc-server.markdown @@ -0,0 +1,41 @@ +--- +layout: page +title: RPC server +permalink: /rpc-server/ +nav_order: 3 +has_children: true +--- + +# RPC Server + +The Old Faithful RPC server allows you to run a Solana compatible RPC server to service data from various sources. The RPC server currently supports: + + - getBlock + - getTransaction + - getSignaturesForAddress + +It can query data from multiple locations: + + - [old-faithful.net](/rpc-server/old-faithful-net/) archives hosted by Triton One + - Local files either generated by yourself or downloaded from old-faithful.net, see [old-faithful.net for more details](/rpc-server/old-faithful-net/) + - [Filecoin](/filecoin/), you can use the faithful-cli to directly interface with the Filecoin network for data access + - Any HTTP source that supports HTTP RANGE requests, to host your own archives see details about [hosting](/archives/hosting/) + +## Quickstart: RPC server from old-faithful.net + +We are hosting data on old-faithful.net for testing and cloning purposes. This allows you to run a sample test server without downloading any data. There is a simple tool that you can run for this available from [https://github.com/rpcpool/yellowstone-faithful/tree/main/tools](https://github.com/rpcpool/yellowstone-faithful/tree/main/tools). You can run a fully remote server like this: + +``` +$ ./tools/run-rpc-server-remote.sh 0 +``` + +This will create a server that hosts epoch 0. + +If you want to test a recent epoch you can run: +``` +$ ./tools/run-rpc-server-remote.sh 475 +``` + +While this is a possible runtime scenario, for blocks this is very, very slow. We recommend downloading indexes locally but keeping the data remote for good performance. See this more as a demonstration that a fully remote use scenario is theoretically possible. More details about how to run with local indexes can be found [here](rpc-server/old-faithful-net/). + +New epochs are being added continuously, but the upload cadence is currently a little irregular due to pipeline development work.