Merge branch 'main' into data-test-haproxy

.github/workflows/main.yml

@@ -0,0 +1,24 @@
+name: Update Repository
+
+on: [ push, pull_request ]
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.GIT_CC }}
+    - name: Clone repository
+      run: |
+        git clone https://github.com/cardano-community/koios-artifacts.git
+        cd koios-artifacts
+        git config --global user.name 'cardano-bot'
+        git config --global user.email '${{ secrets.GIT_EMAIL }}'
+    - name: Sync repositories
+      run: |
+        git remote add new https://github.com/koios-official/koios-artifacts.git
+        git fetch --unshallow origin
+        git fetch --all
+        git remote set-url --add --push new https://github.com/koios-official/koios-artifacts.git
+        git push -f --all new

.gitignore

@@ -1,3 +1,4 @@
 tests/**/__pycache__
 tests/**/.*
 .vscode
+.swp

README.md

@@ -1,26 +1,30 @@
 # Koios Artifacts Repository
 
-Various Artifacts related to [Koios project](https://www.koios.rest) management and assets that are used for website, monitoring services as well as public-facing topology files.
+Various Artifacts related to [Koios project](https://www.koios.rest) management and assets that are used for website - look at repository map below for further info.
+Provisioning scripts to run an instance are part of [guild-operators repo](https://cardano-community.github.io/guild-operators/Build/grest/) alongwith Koios SPO tools
 
 ## Repository Map
+
 ```
 .
 ├── grafana-dashboards/ # Grafana dashboards used for monitoring Koios nodes
 ├── html/               # HTML Page(s) used for https://api.koios.rest website
-├── images/             # Images (logos, Design diagram, etc) used for Koios
+├── images/             # Images used for website (incl. projects building on Koios)
 ├── specs/              # Files used for creation of API Specs with examples
 ├── tests/              # Test suites for Koios nodes
 ├── topology/           # Topology files of public Koios nodes
     ├── topology-guild.json
     ├── topology-mainnet.json
-    └── topology-testnet.json
-├── LICENSE                 # License for use of artifacts within this repository
-├── README.md               # This file
-└── projects.json           # List of projects (in JSON format) that are using Koios
+    └── topology-preprod.json
+    └── topology-preview.json
+├── LICENSE             # License for use of artifacts within this repository
+├── README.md           # This file
+└── projects.json       # List of projects (in JSON format) that are using Koios
 ```
+
 ## Project Management
 
-Koios team operates gRest layer in a transparent manner and progress/millestones can be accessed at any time (accessible [here](https://github.com/orgs/cardano-community/projects/1))
+Koios team operates gRest layer in a transparent manner and progress/millestones can be accessed at any time (accessible [here](https://github.com/orgs/cardano-community/projects/1/views/2))
 
 ## API Specs
 
@@ -29,9 +33,10 @@ The specs can be browsed for each network using below:
 | Network | Link                       |
 |:--------|:--------------------------:|
 | Mainnet | https://api.koios.rest     |
-| Testnet | https://testnet.koios.rest |
 | Guild   | https://guild.koios.rest   |
+| PreProd | https://preprod.koios.rest |
+| Preview | https://preview.koios.rest |
 
 ## Further discussions
 
-You can connect and discuss with Koios teams on [Telegram](https://t.me/+zE4Lce_QUepiY2U1), or feel free to contribute to any of the repositories.
+You can connect and discuss with Koios teams on [Telegram](https://t.me/CardanoKoios/1), or feel free to contribute to any of the repositories.

files/grest/.sqlfluff

@@ -0,0 +1,18 @@
+[sqlfluff]
+dialect = postgres
+exclude_rules = structure.column_order, references.keywords
+max_line_length=260
+recurse = 0
+capitalisation_policy = upper
+extended_capitalisation_policy = upper
+idented_joins = True
+indented_using_on = False
+tab_space_size = 2
+large_file_skip_byte_limit=35000
+
+[sqlfluff:indentation]
+tab_space_size = 2
+allow_implicit_indents = True
+
+[sqlfluff:rules:convention.count_rows]
+prefer_count_1 = True

files/grest/cron/jobs/active-stake-cache-update.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running active stake cache update..."
+
+# High level check in db to see if update needed at all (should be updated only once on epoch transition)
+[[ $(psql ${DB_NAME} -qbt -c "SELECT grest.active_stake_cache_update_check();" | tail -2 | tr -cd '[:alnum:]') != 't' ]] &&
+  echo "No update needed, exiting..." &&
+  exit 0
+
+# This could break due to upstream changes on db-sync (based on log format)
+last_epoch_stakes_log=$(grep -r 'Inserted.*.EpochStake for EpochNo ' "$(dirname "$0")"/../../logs/dbsync-*.json "$(dirname "$0")"/../../logs/archive/dbsync-*.json 2>/dev/null | sed -e 's#.*.Inserted ##' -e 's#EpochStake for EpochNo##' -e 's#\"}.*.$##' | sort -k2 -n | tail -1)
+[[ -z ${last_epoch_stakes_log} ]] &&
+  echo "Could not find any 'Handling stakes' log entries, exiting..." &&
+  exit 1
+
+logs_last_epoch_stakes_count=$(echo "${last_epoch_stakes_log}" | cut -d\  -f1)
+logs_last_epoch_no=$(echo "${last_epoch_stakes_log}" | cut -d\  -f3)
+
+db_last_epoch_no=$(psql ${DB_NAME} -qbt -c "SELECT MAX(NO) from EPOCH;" | tr -cd '[:alnum:]')
+[[ "${db_last_epoch_no}" != "${logs_last_epoch_no}" ]] &&
+  echo "Mismatch between last epoch in logs and database, exiting..." &&
+  exit 1
+
+# Count current epoch entries processed by db-sync
+db_epoch_stakes_count=$(psql ${DB_NAME} -qbt -c "SELECT COUNT(1) FROM EPOCH_STAKE WHERE epoch_no = ${db_last_epoch_no};" | tr -cd '[:alnum:]')
+
+# Check if db-sync completed handling stakes
+[[ "${db_epoch_stakes_count}" != "${logs_last_epoch_stakes_count}" ]] &&
+  echo "Logs last epoch stakes count: ${logs_last_epoch_stakes_count}" &&
+  echo "DB last epoch stakes count: ${db_epoch_stakes_count}" &&
+  echo "db-sync stakes handling still incomplete, exiting..." &&
+  exit 0
+
+# Stakes have been validated, run the cache update
+psql ${DB_NAME} -qbt -c "SELECT GREST.active_stake_cache_update(${db_last_epoch_no});" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/asset-info-cache-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running asset info cache update..."
+psql ${DB_NAME} -qbt -c "SELECT grest.asset_info_cache_update();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/asset-registry-update.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+CNODE_VNAME=cnode
+DB_NAME=cexplorer
+TR_URL=https://github.com/cardano-foundation/cardano-token-registry
+TR_SUBDIR=mappings
+TR_DIR=${HOME}/git
+TR_NAME=${CNODE_VNAME}-token-registry
+
+echo "$(date +%F_%H:%M:%S) - START - Asset Registry Update"
+
+if [[ ! -d "${TR_DIR}/${TR_NAME}" ]]; then
+  [[ -z ${HOME} ]] && echo "HOME variable not set, aborting..." && exit 1
+  mkdir -p "${TR_DIR}"
+  cd "${TR_DIR}" >/dev/null || exit 1
+  git clone ${TR_URL} ${TR_NAME} >/dev/null || exit 1
+fi
+pushd "${TR_DIR}/${TR_NAME}" >/dev/null || exit 1
+git pull >/dev/null || exit 1
+
+last_commit="$(psql ${DB_NAME} -c "select last_value from grest.control_table where key='asset_registry_commit'" -t | xargs)"
+[[ -z "${last_commit}" ]] && last_commit="$(git rev-list HEAD | tail -n 1)"
+latest_commit="$(git rev-list HEAD | head -n 1)"
+
+[[ "${last_commit}" == "${latest_commit}" ]] && echo "$(date +%F_%H:%M:%S) - END - Asset Registry Update, no updates necessary." && exit 0
+
+asset_cnt=0
+
+[[ -f '.assetregistry.csv' ]] && rm -f .assetregistry.csv
+while IFS= read -re assetfile; do
+  if ! asset_data_csv=$(jq -er '[
+      .subject[0:56],
+      .subject[56:],
+      .name.value,
+      .description.value // "",
+      .ticker.value // "",
+      .url.value // "",
+      .logo.value // "",
+      .decimals.value // 0
+      ] | @csv' "${assetfile}"); then
+    echo "Failure parsing '${assetfile}', skipping..."
+    continue
+  fi
+  echo "${asset_data_csv}" >> .assetregistry.csv
+  ((asset_cnt++))
+done < <(git diff --name-only "${last_commit}" "${latest_commit}" | grep ^${TR_SUBDIR})
+cat << EOF > .assetregistry.sql
+CREATE TEMP TABLE tmparc (like grest.asset_registry_cache);
+\COPY tmparc FROM '.assetregistry.csv' DELIMITER ',' CSV;
+INSERT INTO grest.asset_registry_cache SELECT DISTINCT ON (asset_policy,asset_name) * FROM tmparc ON CONFLICT(asset_policy,asset_name) DO UPDATE SET asset_policy=excluded.asset_policy, asset_name=excluded.asset_name, name=excluded.name, description=excluded.description, ticker=excluded.ticker, url=excluded.url, logo=excluded.logo,decimals=excluded.decimals;
+UPDATE grest.asset_info_cache SET decimals=x.decimals FROM
+  (SELECT ma.id, t.decimals FROM tmparc t LEFT JOIN multi_asset ma ON decode(t.asset_name,'hex')=ma.name AND decode(t.asset_policy,'hex')=ma.policy WHERE t.decimals != 0) as x
+  WHERE asset_id = x.id;
+EOF
+
+psql ${DB_NAME} -qb -f .assetregistry.sql >/dev/null && rm -f .assetregistry.sql
+psql ${DB_NAME} -qb -c "INSERT INTO grest.control_table (key, last_value) VALUES ('asset_registry_commit','${latest_commit}') ON CONFLICT(key) DO UPDATE SET last_value='${latest_commit}'"
+echo "$(date +%F_%H:%M:%S) - END - Asset Registry Update, ${asset_cnt} assets added/updated for commits ${last_commit} to ${latest_commit}."

files/grest/cron/jobs/epoch-info-cache-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "SELECT EXTRACT(EPOCH FROM time)::integer FROM block ORDER BY id DESC LIMIT 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running epoch info cache update..."
+psql ${DB_NAME} -qbt -c "SELECT grest.epoch_info_cache_update();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/epoch-summary-corrections-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "SELECT EXTRACT(EPOCH FROM time)::integer FROM block ORDER BY id DESC LIMIT 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running epoch summary corrections update..."
+psql ${DB_NAME} -qbt -c "SELECT GREST.EPOCH_SUMMARY_CORRECTIONS_UPDATE();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/pool-history-cache-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running pool history cache update..."
+psql ${DB_NAME} -qbt -c "SELECT GREST.pool_history_cache_update();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/populate-next-epoch-nonce.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+DB_NAME=cexplorer
+NWMAGIC=
+EPOCH_LENGTH=
+PROM_URL=
+CCLI=
+export CARDANO_NODE_SOCKET_PATH=
+
+echo "$(date +%F_%H:%M:%S) Running next epoch nonce calculation..."
+
+min_slot=$((EPOCH_LENGTH * 7 / 10))
+current_epoch=$(curl -s "${PROM_URL}" | grep epoch | awk '{print $2}')
+current_slot_in_epoch=$(curl -s "${PROM_URL}" | grep slotInEpoch | awk '{print $2}')
+next_epoch=$((current_epoch + 1))
+
+[[ ${current_slot_in_epoch} -ge ${min_slot} ]] &&
+  next_epoch_nonce=$(echo "$(${CCLI} query protocol-state --testnet-magic "${NWMAGIC}" | jq -r .candidateNonce.contents)$(${CCLI} query protocol-state --testnet-magic "${NWMAGIC}" | jq -r .lastEpochBlockNonce.contents)" | xxd -r -p | b2sum -b -l 256 | awk '{print $1}') &&
+  psql ${DB_NAME} -c "INSERT INTO grest.epoch_info_cache (epoch_no, p_nonce) VALUES (${next_epoch}, '${next_epoch_nonce}') ON CONFLICT DO NOTHING;"
+
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/stake-distribution-new-accounts-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running stake distribution update for new accounts..."
+psql ${DB_NAME} -qbt -c "CALL GREST.UPDATE_NEWLY_REGISTERED_ACCOUNTS_STAKE_DISTRIBUTION_CACHE();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/stake-distribution-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running stake distribution update..."
+psql ${DB_NAME} -qbt -c "SELECT GREST.STAKE_DISTRIBUTION_CACHE_UPDATE_CHECK();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/cron/jobs/stake-snapshot-cache.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "select extract(epoch from time)::integer from block order by id desc limit 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Capturing last epochs' snapshot..."
+psql ${DB_NAME} -qbt -c "CALL GREST.CAPTURE_LAST_EPOCH_SNAPSHOT();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/rpc/00_blockchain/genesis.sql

@@ -0,0 +1,36 @@
+CREATE OR REPLACE FUNCTION grest.genesis()
+RETURNS TABLE (
+  networkmagic varchar,
+  networkid varchar,
+  activeslotcoeff varchar,
+  updatequorum varchar,
+  maxlovelacesupply varchar,
+  epochlength varchar,
+  systemstart integer,
+  slotsperkesperiod varchar,
+  slotlength varchar,
+  maxkesrevolutions varchar,
+  securityparam varchar,
+  alonzogenesis varchar
+)
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    g.networkmagic,
+    g.networkid,
+    g.activeslotcoeff,
+    g.updatequorum,
+    g.maxlovelacesupply,
+    g.epochlength,
+    EXTRACT(EPOCH FROM g.systemstart::timestamp)::integer,
+    g.slotsperkesperiod,
+    g.slotlength,
+    g.maxkesrevolutions,
+    g.securityparam,
+    g.alonzogenesis
+  FROM
+    grest.genesis AS g;
+END;
+$$;