Setting up tide-disco bindings

.github/workflows/lint.yml

@@ -20,12 +20,6 @@ jobs:
         features:
           # No optional features
           - ''
-          # Each optional feature on its own
-          - sql-data-source
-          - file-system-data-source
-          - metrics-data-source
-          # All optional features together
-          - sql-data-source,file-system-data-source,metrics-data-source
     env:
         RUSTFLAGS: "--cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\""
         RUST_LOG: info

Cargo.toml

@@ -7,7 +7,15 @@ edition = "2021"
 
 [dependencies]
 async-trait = "0.1"
+clap = { version = "4.4", features = ["derive", "env"] }
+commit = { git = "https://github.com/EspressoSystems/commit.git" }
+derive_more = "0.99"
+futures = "0.3"
 hotshot-types = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.7.1" }
 serde = { version = "1.0", features = ["derive"] }
 sha2 = "0.10"
+snafu = { version = "0.7", features = ["backtraces"] }
+tagged-base64 = { git = "https://github.com/EspressoSystems/tagged-base64", tag = "0.3.4" }
+tide-disco = { git = "https://github.com/EspressoSystems/tide-disco.git", tag = "v0.4.6" }
+toml = "0.8"
 

flake.nix

@@ -0,0 +1,192 @@
+# Copyright (c) 2022 Espresso Systems (espressosys.com)
+# This file is part of the HotShot Query Service library.
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+# You should have received a copy of the GNU General Public License along with this program. If not,
+# see <https://www.gnu.org/licenses/>.
+
+{
+  description = "Generic query service for HotShot applications";
+
+  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  inputs.flake-compat.url = "github:edolstra/flake-compat";
+  inputs.flake-compat.flake = false;
+
+  inputs.rust-overlay.url = "github:oxalica/rust-overlay";
+
+  inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
+  inputs.pre-commit-hooks.inputs.flake-utils.follows = "flake-utils";
+  inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs";
+
+  inputs.poetry2nixFlake = {
+    url = "github:nix-community/poetry2nix";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, rust-overlay, pre-commit-hooks, poetry2nixFlake, ... }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        overlays = [ (import rust-overlay) ];
+        pkgs = import nixpkgs { inherit system overlays; };
+        poetry2nix = poetry2nixFlake.lib.mkPoetry2Nix { inherit pkgs; };
+        rustToolchain = pkgs.rust-bin.stable.latest.minimal.override {
+          extensions = [ "rustfmt" "clippy" "llvm-tools-preview" "rust-src" ];
+        };
+        rustDeps = with pkgs;
+          [
+            pkg-config
+            openssl
+            bash
+
+            curl
+            docker
+
+            cargo-audit
+            cargo-edit
+            cargo-udeps
+            cargo-sort
+            cmake
+
+            # `postgresql` defaults to an older version (15), so we select the latest version (16)
+            # explicitly.
+            postgresql_16
+          ] ++ lib.optionals stdenv.isDarwin [
+            darwin.apple_sdk.frameworks.Security
+            darwin.apple_sdk.frameworks.CoreFoundation
+            darwin.apple_sdk.frameworks.SystemConfiguration
+
+            # https://github.com/NixOS/nixpkgs/issues/126182
+            libiconv
+          ] ++ lib.optionals (!stdenv.isDarwin) [
+            cargo-watch # broken: https://github.com/NixOS/nixpkgs/issues/146349
+          ];
+        # nixWithFlakes allows pre v2.4 nix installations to use
+        # flake commands (like `nix flake update`)
+        nixWithFlakes = pkgs.writeShellScriptBin "nix" ''
+          exec ${pkgs.nixFlakes}/bin/nix --experimental-features "nix-command flakes" "$@"
+        '';
+        cargo-llvm-cov = pkgs.rustPlatform.buildRustPackage rec {
+          pname = "cargo-llvm-cov";
+          version = "0.5.0";
+
+          doCheck = false;
+
+          buildInputs = [ pkgs.libllvm ];
+
+          src = builtins.fetchTarball {
+            url =
+              "https://crates.io/api/v1/crates/${pname}/${version}/download";
+            sha256 =
+              "sha256:1a0grmpcjnqrz5c9jjbk07705py4573pmq0jcgr9m7l5xf4g1yc9";
+          };
+
+          cargoSha256 = "sha256-11xNgiOw0qysTWpoKAXQ5gx1uJSAsp+aDDir0zpkpeQ=";
+          meta = with pkgs.lib; {
+            description = "Cargo llvm cov generates code coverage via llvm.";
+            homepage = "https://github.com/taiki-e/cargo-llvm-cov";
+
+            license = with licenses; [ mit asl20 ];
+          };
+        };
+        pythonEnv = poetry2nix.mkPoetryEnv { projectDir = ./.; };
+        myPython = with pkgs; [ poetry pythonEnv ];
+        shellHook  = ''
+          # Prevent cargo aliases from using programs in `~/.cargo` to avoid conflicts with rustup
+          # installations.
+          export CARGO_HOME=$HOME/.cargo-nix
+          export PATH="$PWD/$CARGO_TARGET_DIR/release:$PATH"
+        '';
+
+        RUST_SRC_PATH = "${rustToolchain}/lib/rustlib/src/rust/library";
+        RUST_BACKTRACE = 1;
+        RUST_LOG = "info";
+        RUSTFLAGS=" --cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\" --cfg hotshot_example";
+      in {
+      	checks = {
+          pre-commit-check = pre-commit-hooks.lib.${system}.run {
+            src = ./.;
+            hooks = {
+              cargo-fmt = {
+                enable = true;
+                description = "Enforce rustfmt";
+                entry = "cargo fmt --all -- --check";
+                pass_filenames = false;
+              };
+              cargo-sort = {
+                enable = true;
+                description = "Ensure Cargo.toml are sorted";
+                entry = "cargo sort -g -w";
+                pass_filenames = false;
+              };
+              cargo-clippy = {
+                enable = true;
+                description = "Run clippy";
+                entry = "cargo clippy --workspace --all-features --all-targets -- -D clippy::dbg-macro";
+                pass_filenames = false;
+              };
+              license-header-c-style = {
+                enable = true;
+                description =
+                  "Ensure files with c-style comments have license header";
+                entry = ''
+                  insert_license --license-filepath .license-header.txt  --comment-style "//"'';
+                types_or = [ "rust" ];
+                pass_filenames = true;
+              };
+              license-header-hash = {
+                enable = true;
+                description =
+                  "Ensure files with hash style comments have license header";
+                entry = ''
+                  insert_license --license-filepath .license-header.txt --comment-style "#"'';
+                types_or = [ "bash" "python" "toml" "nix" ];
+                excludes = [ "poetry.lock" ];
+                pass_filenames = true;
+              };
+              license-header-html = {
+                enable = true;
+                description = "Ensure markdown files have license header";
+                entry = ''
+                  insert_license --license-filepath .license-header.txt --comment-style "<!--| ~| -->"'';
+                types_or = [ "markdown" ];
+                pass_filenames = true;
+              };
+            };
+          };
+        };
+        devShell = pkgs.mkShell {
+          shellHook = shellHook
+          	# install pre-commit hooks
+            + self.checks.${system}.pre-commit-check.shellHook;
+          buildInputs = with pkgs;
+            [
+              rust-bin.nightly.latest.rust-analyzer
+              nixWithFlakes
+              nixpkgs-fmt
+              git
+              mdbook # make-doc, documentation generation
+              protobuf
+              rustToolchain
+            ] ++ myPython ++ rustDeps;
+
+          inherit RUST_SRC_PATH RUST_BACKTRACE RUST_LOG RUSTFLAGS;
+        };
+        devShells = {
+          perfShell = pkgs.mkShell {
+            shellHook = shellHook;
+            buildInputs = with pkgs;
+              [ nixWithFlakes cargo-llvm-cov rustToolchain protobuf ] ++ rustDeps;
+
+            inherit RUST_SRC_PATH RUST_BACKTRACE RUST_LOG RUSTFLAGS;
+          };
+        };
+      });
+}

src/api.rs

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 Espresso Systems (espressosys.com)
+// This file is part of the HotShot Query Service library.
+//
+// This program is free software: you can redistribute it and/or modify it under the terms of the GNU
+// General Public License as published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+// even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+// You should have received a copy of the GNU General Public License along with this program. If not,
+// see <https://www.gnu.org/licenses/>.
+
+use std::fs;
+use std::path::Path;
+use tide_disco::api::{Api, ApiError};
+use toml::{map::Entry, Value};
+
+pub(crate) fn load_api<State, Error>(
+    path: Option<impl AsRef<Path>>,
+    default: &str,
+    extensions: impl IntoIterator<Item = Value>,
+) -> Result<Api<State, Error>, ApiError> {
+    let mut toml = match path {
+        Some(path) => load_toml(path.as_ref())?,
+        None => toml::from_str(default).map_err(|err| ApiError::CannotReadToml {
+            reason: err.to_string(),
+        })?,
+    };
+    for extension in extensions {
+        merge_toml(&mut toml, extension);
+    }
+    Api::new(toml)
+}
+
+fn merge_toml(into: &mut Value, from: Value) {
+    if let (Value::Table(into), Value::Table(from)) = (into, from) {
+        for (key, value) in from {
+            match into.entry(key) {
+                Entry::Occupied(mut entry) => merge_toml(entry.get_mut(), value),
+                Entry::Vacant(entry) => {
+                    entry.insert(value);
+                }
+            }
+        }
+    }
+}
+
+fn load_toml(path: &Path) -> Result<Value, ApiError> {
+    let bytes = fs::read(path).map_err(|err| ApiError::CannotReadToml {
+        reason: err.to_string(),
+    })?;
+    let string = std::str::from_utf8(&bytes).map_err(|err| ApiError::CannotReadToml {
+        reason: err.to_string(),
+    })?;
+    toml::from_str(string).map_err(|err| ApiError::CannotReadToml {
+        reason: err.to_string(),
+    })
+}

src/block_metadata.rs

@@ -1,17 +1,43 @@
-use std::marker::PhantomData;
+use std::{hash::Hash, marker::PhantomData};
 
-use hotshot_types::traits::node_implementation::NodeType;
+use commit::{Commitment, Committable};
+use hotshot_types::traits::{node_implementation::NodeType, BlockPayload};
 use serde::{Deserialize, Serialize};
-use sha2::digest::{generic_array::GenericArray, typenum};
+use sha2::{Digest, Sha256};
 
-pub type BlockHash = GenericArray<u8, typenum::consts::U32>;
+pub struct HashableBlock<I: NodeType>(
+    <I as NodeType>::BlockPayload,
+    <<I as NodeType>::BlockPayload as BlockPayload>::Metadata,
+);
+pub type BlockHash<I> = Commitment<HashableBlock<I>>;
+impl<I: NodeType> Default for HashableBlock<I> {
+    fn default() -> Self {
+        let (bp, bm) = <I as NodeType>::BlockPayload::from_transactions(Vec::new())
+            .unwrap_or_else(|_| <I as NodeType>::BlockPayload::genesis());
+        Self(bp, bm)
+    }
+}
+
+impl<I: NodeType> Committable for HashableBlock<I> {
+    fn commit(&self) -> Commitment<Self> {
+        let builder = commit::RawCommitmentBuilder::new("Hashable Block Payload");
+        let mut hasher = Sha256::new();
+        let encoded = if let Ok(encoder) = self.0.encode() {
+            encoder.collect()
+        } else {
+            Vec::new()
+        };
+        hasher.update(&encoded);
+        let generic_array = hasher.finalize();
+        builder.generic_byte_array(&generic_array).finalize()
+    }
+}
 
-#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[serde(bound = "")]
 pub struct BlockMetadata<I: NodeType> {
-    block_hash: BlockHash,
+    block_hash: BlockHash<I>,
     block_size: u64,
     offered_fee: u64,
     _phantom: PhantomData<I>,
 }
-