From 46a072af64fcf02c50c2b02cc79bcf0de25eafd4 Mon Sep 17 00:00:00 2001 From: hopeyen Date: Mon, 11 Sep 2023 13:56:08 -0500 Subject: [PATCH] refactor: repo meta-documentations --- .auto-changelog | 7 + .github/ISSUE_TEMPLATE/bug_report.md | 33 +++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++ .github/workflows/docker.yml | 55 +++++ .github/workflows/gen-binaries.yml | 61 +++++ .github/workflows/release.yml | 33 +++ CODE_OF_CONDUCT.md | 128 ++++++++++ contributing.md => CONTRIBUTORS.md | 0 Cargo.lock | 30 ++- Cargo.toml | 1 + README.md | 287 ++++++++++++++-------- service/Cargo.toml | 1 + 12 files changed, 544 insertions(+), 112 deletions(-) create mode 100644 .auto-changelog create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/gen-binaries.yml create mode 100644 .github/workflows/release.yml create mode 100644 CODE_OF_CONDUCT.md rename contributing.md => CONTRIBUTORS.md (100%) diff --git a/.auto-changelog b/.auto-changelog new file mode 100644 index 00000000..5642db51 --- /dev/null +++ b/.auto-changelog @@ -0,0 +1,7 @@ +{ + "output": "changelog.md", + "template": "keepachangelog", + "unreleased": true, + "commitLimit": false, + "commitMessageFormat": "{{type}}: {{message}}" + } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..fe25cfa2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,33 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Version [e.g. 22] + - Rust Version [e.g. 22] + - Other Versionings [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..1f876925 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[Feat.Req]" +labels: '' +assignees: '' + +--- + +**Problem statement** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Expectation proposal** +A clear and concise description of what you want to happen. + +**Alternative considerations** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..63debca0 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,55 @@ +name: Build and upload Docker image + +on: + release: + types: [published] + push: + tags: + - "*.*.*" + +env: + REGISTRY: ghcr.io + BASE_NAME: graphops/indexer-service-rs + +jobs: + build-linux: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + # list of Docker images to use as base name for tags + images: | + ${{ env.REGISTRY }}/${{ env.BASE_NAME }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}}.{{minor}}.{{patch}} + type=semver,pattern={{major}} + type=sha + latest:main + latest:dev + + - name: Log in to the Container registry + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: ./ + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + dockerfile: Dockerfile diff --git a/.github/workflows/gen-binaries.yml b/.github/workflows/gen-binaries.yml new file mode 100644 index 00000000..8f2fdbd1 --- /dev/null +++ b/.github/workflows/gen-binaries.yml @@ -0,0 +1,61 @@ +name: Build and upload release binaries + +on: + release: + types: [published] + +jobs: + build-linux: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Install Go + run: | + if [ $(uname -m) = "aarch64" ]; then + wget https://golang.org/dl/go1.19.5.linux-arm64.tar.gz && + sudo tar -C /usr/local -xzf go1.19.5.linux-arm64.tar.gz + else + wget https://golang.org/dl/go1.19.5.linux-amd64.tar.gz && + sudo tar -C /usr/local -xzf go1.19.5.linux-amd64.tar.gz + fi + export PATH=$PATH:/usr/local/go/bin + - name: Build release binary + run: | + cargo build --release + - name: Upload Ubuntu binary + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./target/release/indexer-service-rs + asset_name: indexer-service-rs-${{ github.event.release.tag_name }}-ubuntu + asset_content_type: binary/octet-stream + build-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - uses: actions/setup-go@v4 + with: + go-version: "1.19.5" + - name: Build release binary + run: | + cargo build --release + - name: Upload MacOS binary + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./target/release/indexer-service-rs + asset_name: indexer-service-rs-${{ github.event.release.tag_name }}-macos + asset_content_type: binary/octet-stream diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..a74ec840 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +changelog: + exclude: + labels: + - ignore-for-release + authors: + - octocat + categories: + - title: Breaking Changes 🛠 + labels: + - Semver-Major + - breaking-change + - title: Features 🏕 + labels: + - Semver-Minor + - feat + - enhancement + exclude: + labels: + - dependencies + - chore + - title: Dependencies 👒 + labels: + - dependencies + - chore + - title: Bug fixes 🐛 + labels: + - bug + - title: Refactor 🌱 + labels: + - refactor + - title: Other Changes 🗳️ + labels: + - "*" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..14bca0cc --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +conduct@graphops.xyz. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/contributing.md b/CONTRIBUTORS.md similarity index 100% rename from contributing.md rename to CONTRIBUTORS.md diff --git a/Cargo.lock b/Cargo.lock index b1736ac7..a98e7914 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1875,6 +1875,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "fixed-hash" version = "0.6.1" @@ -2672,6 +2678,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -2750,7 +2765,7 @@ dependencies = [ "diff", "ena", "is-terminal", - "itertools", + "itertools 0.10.5", "lalrpop-util", "petgraph", "regex", @@ -4659,7 +4674,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a94494913728908efa7a25a2dd2e4f037e714897985c24273c40596638ed909" dependencies = [ - "itertools", + "itertools 0.10.5", "lalrpop", "lalrpop-util", "phf", @@ -4694,11 +4709,11 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" dependencies = [ - "itertools", + "itertools 0.11.0", "nom", "unicode_categories", ] @@ -4930,10 +4945,11 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" dependencies = [ + "finl_unicode", "unicode-bidi", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index fec25c17..7b1de4d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "service", # "common", ] +resolver = "2" [profile.dev.package."*"] opt-level = 3 diff --git a/README.md b/README.md index e01e2c30..a0a911a7 100644 --- a/README.md +++ b/README.md @@ -1,108 +1,18 @@ # indexer-service-rs -Experimental rust impl for The Graph [indexer service](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-service) +## Introduction -## Dependency choices - -- switching from actix-web to `axum` for the service server -- App profiling should utilize `perf`, flamegraphs or cpu profilers, and benches to track and collect performance data. The typescript implementation uses `gcloud-profile` -- Consider replacing and adding parts from TAP manager -- `postgres` database connection required to indexer management server database, shared with the indexer agent -- No migration in indexer service as it might introduce conflicts to the database; indexer agent is solely responsible for database management. - -> Don't know if receipt validation is actually correct, need testing - -## Components checklist (basic, not extensive) - -- [x] Server path routing - - [x] basic structure - - [x] CORS - - [x] timeouts - - [x] Rate limiting levels - - [x] Logger stream -- [ ] Query processor - - [x] graph node query endpoint at specific subgraph path - - [x] wrap request to and response from graph node - - [x] extract receipt header - - [x] Free query - - [x] Query struct - - [x] Free query auth token check - - [x] Query routes + responses - - [x] set `graph-attestable` in response header to `true` - - [x] Network subgraph query - - [x] Query struct - - [x] serve network subgraph boolean + auth token check - - [x] Query routes + responses - - [x] set `graph-attestable` in response header to `false` - - [ ] Paid query - - [ ] receipts graphQL schema - - [ ] [TAP](https://github.com/semiotic-ai/timeline-aggregation-protocol/) manager to handle receipts logic - - [ ] derive, cache, and look up attestation signers - - [ ] contracts - connect by network chain id - - [ ] network provider - - [x] validate receipt format (need unit tests) - - [x] parse receipt (need unit tests) - - [x] validate signature (need unit tests) - - [ ] store - - [ ] extract graph-attestable from graph node response header - - [ ] monitor eligible allocations - - [ ] network subgraph - - [ ] operator wallet -> indexer address - - [ ] subgraph health check - - [ ] query timing logs -- [x] Deployment health server - - [x] query status endpoint and process result -- [ ] Status server - - [x] indexing status resolver - to query indexingStatuses - - [ ] Filter for unsupported queries -- [x] Cost server - - [x] Simple indexer management client to track postgres connection and network subgraph endpoint. - - [x] serve queries with defined graphQL schema and psql resolvers to database: `costModel(deployment)` and `costModels(deployments)`. If deployments is empty, all cost models are returned. - - [x] Global cost model fallback used when specific deployments are queried -- [x] Constant service paths - - [x] health - - [x] ready to roll - - [x] versions - - [x] operator public key - - [x] validate mnemonics to public key -- [x] Import indexer native -- [ ] Metrics - - [x] Metrics setup - - [x] serve basic indexer service metrics - - [ ] Add cost model metrics -- [x] CLI args -- [ ] App profiling - - [ ] No gcloud profiling, can use `perf` to collect performance data. +A Rust impl for The Graph [indexer service](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-service) to provide data services as an Indexer, integrated with [TAP](https://github.com/semiotic-ai/timeline-aggregation-protocol) which is a fast, efficient, and trustless unidirectional micro-payments system. -### Indexer common components +## Features -Temporarily live inside the indexer-service package under `src/common`. +- Receive paid or free query requests and route to graph node +- Route "meta" queries on indexing statuses and deployment health +- Serve indexer information such as health, indexer version, and operator address +- Monitor allocations, attestation signers, and manage receipts using TAP, store receipts in the indexer database +- Record performance and service metrics -Simple indexer management client to track NetworkSubgraph and postgres connection. -- NetworkSubgraph instance track both remote API endpoint and local deployment query endpoint. - - TODO: query indexing status of local deployment, only use remote API as fallback. -- Keeps cost model schema and resolvers with postgres and graphQL types: `costModel(deployment)` and `costModels(deployments)`. If deployments is empty, all cost models are returned. - - Global cost model fallback used when specific deployments are queried -- No database migration in indexer service as it might introduce schema conflicts; indexer agent is solely responsible for database management. - -### Indexer native dependency - -Linked dependency could not be linked directly with git url "https://github.com/graphprotocol/indexer" and path "packages/indexer-native/native" at the same time, and could not access it on crates.io. So copid the folder to local repo with the version at https://github.com/graphprotocol/indexer/blob/972658b3ce8c512ad7b4dc575d29cd9d5377e3fe/packages/indexer-native/native. - -Since indexer-service will be written in Rust and no need for typescript, indexer-native's neon build and util has been removed. - -Component `NativeSignatureVerifier` renamed to `SignatureVerifier`. - -Separate package in the workspace under 'native'. - -### common-ts components - -Temporarily live inside the indexer-service package under `src/types` - -- Address -- readNumber - -### Quick attempts +## Quick start Configure required start-up args, check description by ``` @@ -137,13 +47,81 @@ cargo run -- --ethereum \ ``` -After service start up, try with command + +## Configuring + ``` -curl -X POST \ - -H 'Content-Type: application/json' \ - --data '{"query": "{_meta{block{number}}}"}' \ - http://127.0.0.1:7300/network +✗ cargo run -p service -- --help + +Usage: service [OPTIONS] <--ethereum |--ethereum-polling-interval |--mnemonic |--indexer-address > <--port |--metrics-port |--graph-node-query-endpoint |--graph-node-status-endpoint |--log-level |--gcloud-profiling|--free-query-auth-token > <--postgres-host |--postgres-port |--postgres-database |--postgres-username |--postgres-password > <--network-subgraph-deployment |--network-subgraph-endpoint |--network-subgraph-auth-token |--serve-network-subgraph|--allocation-syncing-interval |--client-signer-address > + +Options: + --ethereum + Ethereum node or provider URL [env: ETH_NODE=] + --ethereum-polling-interval + Polling interval for the Ethereum provider (ms) [env: ETHEREUM_POLLING_INTERVAL=] [default: 4000] + --mnemonic + Mnemonic for the operator wallet [env: MNEMONIC=] + --indexer-address + Ethereum address of the indexer [env: INDEXER_ADDRESS=] + --port + Port to serve queries at [env: PORT=] [default: 7600] + --metrics-port + Port to serve Prometheus metrics at [env: METRICS_PORT=] [default: 7300] + --graph-node-query-endpoint + Graph node GraphQL HTTP service endpoint [env: GRAPH_NODE_QUERY_ENDPOINT=] [default: http://0.0.0.0:8000] + --graph-node-status-endpoint + Graph node endpoint for the index node server [env: GRAPH_NODE_STATUS_ENDPOINT=] [default: http://0.0.0.0:8030] + --log-level + Log level in RUST_LOG format [env: LOG_LEVEL=] + --gcloud-profiling + Whether to enable Google Cloud profiling [env: GCLOUD_PROFILING=] + --free-query-auth-token + Auth token that clients can use to query for free [env: FREE_QUERY_AUTH_TOKEN=] + --postgres-host + Postgres host [env: POSTGRES_HOST=] [default: http://0.0.0.0/] + --postgres-port + Postgres port [env: POSTGRES_PORT=] [default: 5432] + --postgres-database + Postgres database name [env: POSTGRES_DATABASE=] + --postgres-username + Postgres username [env: POSTGRES_USERNAME=] [default: postgres] + --postgres-password + Postgres password [env: POSTGRES_PASSWORD=] [default: ] + --network-subgraph-deployment + Network subgraph deployment [env: NETWORK_SUBGRAPH_DEPLOYMENT=] + --network-subgraph-endpoint + Endpoint to query the network subgraph from [env: NETWORK_SUBGRAPH_ENDPOINT=] [default: https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-goerli] + --network-subgraph-auth-token + Bearer token to require for /network queries [env: NETWORK_SUBGRAPH_AUTH_TOKEN=] + --serve-network-subgraph + Whether to serve the network subgraph at /network [env: SERVE_NETWORK_SUBGRAPH=] + --allocation-syncing-interval + Interval (in ms) for syncing indexer allocations from the network [env: ALLOCATION_SYNCING_INTERVAL=] [default: 120000] + --client-signer-address + Address that signs query fee receipts from a known client [env: CLIENT_SIGNER_ADDRESS=] + -c + Indexer service configuration file (YAML format) [env: CONFIG=] + -h, --help + Print help + -V, --version + Print version ``` +## Upgrading + +We follow conventional semantics for package versioning. An indexer may set a minor version specification for automatic patch updates while preventing breaking changes. To safely upgrading the package, we recommend the following steps: + +1. **Review Release Notes**: Before upgrading, check the release notes for the new version to understand what changes, fixes, or new features are included. +2. **Review Documentation**: Check the up-to-date documentation for an accurate reflection of the changes made during the upgrade. +3. **Backup Configuration**: Save your current configuration files and any local modifications you've made to the existing codebase. +4. **Deploy**: Replace the old executable or docker image with the new one and restart the service to apply the upgrade. +5. **Monitor and Validate**: After the upgrade, monitor system behavior and performance metrics to validate that the service is running as expected. + +These steps should ensure a smooth transition to the latest version of `indexer-service-rs`, harnessing new capabilities while maintaining system integrity. + +## Contributing + +[Contributions guide](/contributing.md) ### Supported request and response format examples @@ -214,4 +192,103 @@ curl -X GET -H 'Content-Type: application/json' --data '{"query": "{ costModel(d curl -X GET -H 'Content-Type: application/json' --data '{"query": "{ costModels(deployments: [\"Qmb5Ysp5oCUXhLA8NmxmYKDAX2nCMnh7Vvb5uffb9n5vss\"]) { deployment model variables }} "}' http://localhost:7300/cost {"data":{"costModels":[{"deployment":"0xbd499f7673ca32ef4a642207a8bebdd0fb03888cf2678b298438e3a1ae5206ea","model":"default => 0.00025;","variables":null}]}}% -``` \ No newline at end of file +``` + +## Dependency choices + +- switching from actix-web to `axum` for the service server +- App profiling should utilize `perf`, flamegraphs or cpu profilers, and benches to track and collect performance data. The typescript implementation uses `gcloud-profile` +- Consider replacing and adding parts from TAP manager +- `postgres` database connection required to indexer management server database, shared with the indexer agent +- No migration in indexer service as it might introduce conflicts to the database; indexer agent is solely responsible for database management. + + +### Indexer common components + +Temporarily live inside the indexer-service package under `src/common`. + +Simple indexer management client to track NetworkSubgraph and postgres connection. +- NetworkSubgraph instance track both remote API endpoint and local deployment query endpoint. + - TODO: query indexing status of local deployment, only use remote API as fallback. +- Keeps cost model schema and resolvers with postgres and graphQL types: `costModel(deployment)` and `costModels(deployments)`. If deployments is empty, all cost models are returned. + - Global cost model fallback used when specific deployments are queried +- No database migration in indexer service as it might introduce schema conflicts; indexer agent is solely responsible for database management. + +### Indexer native dependency + +Linked dependency could not be linked directly with git url "https://github.com/graphprotocol/indexer" and path "packages/indexer-native/native" at the same time, and could not access it on crates.io. So copid the folder to local repo with the version at https://github.com/graphprotocol/indexer/blob/972658b3ce8c512ad7b4dc575d29cd9d5377e3fe/packages/indexer-native/native. + +Since indexer-service will be written in Rust and no need for typescript, indexer-native's neon build and util has been removed. + +Component `NativeSignatureVerifier` renamed to `SignatureVerifier`. + +Separate package in the workspace under 'native'. + +### common-ts components + +Temporarily live inside the indexer-service package under `src/types` + +- Address +- readNumber + +## Components checklist (basic, not extensive) + +- [x] Server path routing + - [x] basic structure + - [x] CORS + - [x] timeouts + - [x] Rate limiting levels + - [x] Logger stream +- [ ] Query processor + - [x] graph node query endpoint at specific subgraph path + - [x] wrap request to and response from graph node + - [x] extract receipt header + - [x] Free query + - [x] Query struct + - [x] Free query auth token check + - [x] Query routes + responses + - [x] set `graph-attestable` in response header to `true` + - [x] Network subgraph query + - [x] Query struct + - [x] serve network subgraph boolean + auth token check + - [x] Query routes + responses + - [x] set `graph-attestable` in response header to `false` + - [ ] Paid query + - [ ] receipts graphQL schema + - [ ] [TAP](https://github.com/semiotic-ai/timeline-aggregation-protocol/) manager to handle receipts logic + - [ ] derive, cache, and look up attestation signers + - [ ] contracts - connect by network chain id + - [ ] network provider + - [x] validate receipt format (need unit tests) + - [x] parse receipt (need unit tests) + - [x] validate signature (need unit tests) + - [ ] store + - [ ] extract graph-attestable from graph node response header + - [ ] monitor eligible allocations + - [ ] network subgraph + - [ ] operator wallet -> indexer address + - [ ] subgraph health check + - [ ] query timing logs +- [x] Deployment health server + - [x] query status endpoint and process result +- [ ] Status server + - [x] indexing status resolver - to query indexingStatuses + - [ ] Filter for unsupported queries +- [x] Cost server + - [x] Simple indexer management client to track postgres connection and network subgraph endpoint. + - [x] serve queries with defined graphQL schema and psql resolvers to database: `costModel(deployment)` and `costModels(deployments)`. If deployments is empty, all cost models are returned. + - [x] Global cost model fallback used when specific deployments are queried +- [x] Constant service paths + - [x] health + - [x] ready to roll + - [x] versions + - [x] operator public key + - [x] validate mnemonics to public key +- [x] Import indexer native +- [ ] Metrics + - [x] Metrics setup + - [x] serve basic indexer service metrics + - [ ] Add cost model metrics +- [x] CLI args +- [ ] App profiling + - [ ] No gcloud profiling, can use `perf` to collect performance data. diff --git a/service/Cargo.toml b/service/Cargo.toml index ba375052..5225b338 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -3,6 +3,7 @@ name = "service" version = "0.1.0" edition = "2021" description="Could not find crate on crates.io and could not import with git and path at the same time, so copied a version directly at https://github.com/graphprotocol/indexer/blob/972658b3ce8c512ad7b4dc575d29cd9d5377e3fe/packages/indexer-native/native" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html