From 268a7159cf4a2ba228fabb31d934c920273e3128 Mon Sep 17 00:00:00 2001 From: Bruno Wu Date: Mon, 23 Oct 2023 10:39:49 +0100 Subject: [PATCH 1/2] ZIL-5434: PDT - Fix block reconcile --- products/pdt/pdt/src/bqimport.rs | 2 +- products/pdt/pdt/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/products/pdt/pdt/src/bqimport.rs b/products/pdt/pdt/src/bqimport.rs index ebf6624b5..7444ad474 100644 --- a/products/pdt/pdt/src/bqimport.rs +++ b/products/pdt/pdt/src/bqimport.rs @@ -117,7 +117,7 @@ pub async fn reconcile_blocks( // We should have coverage for every block, extant or not, up to the // last batch, which will be short. let mut blk: i64 = 0; - while blk < max_block { + while (blk + scale) < max_block { let span = std::cmp::min(scale, max_block - (blk + scale)); println!("blk {} span {}", blk, span); match project.is_txn_range_covered_by_entry(blk, span).await? { diff --git a/products/pdt/pdt/src/main.rs b/products/pdt/pdt/src/main.rs index 9fcf8f639..07eefef85 100644 --- a/products/pdt/pdt/src/main.rs +++ b/products/pdt/pdt/src/main.rs @@ -264,8 +264,8 @@ async fn bigquery_reconcile_blocks(unpack_dir: &str, opts: &ReconcileOptions) -> bqimport::reconcile_blocks( unpack_dir, opts.batch_blocks, - &opts.dataset_id, &opts.project_id, + &opts.dataset_id, ) .await } From fa69d6d51a515b7edf8b8cd531653ed07aba66a7 Mon Sep 17 00:00:00 2001 From: Bruno Wu Date: Mon, 23 Oct 2023 14:44:32 +0100 Subject: [PATCH 2/2] ZIL-5441: PDT add mainnet deployments --- products/pdt/cd/base/configmap.yaml | 13 +-- products/pdt/cd/base/cronjob.yaml | 86 +++++++++++++++++-- products/pdt/cd/base/deployment.yaml | 57 ++++++++++-- products/pdt/cd/base/pvc.yaml | 17 +++- .../pdt/cd/overlays/staging/configmap.yaml | 13 +-- .../cd/overlays/staging/kustomization.yaml | 18 +++- 6 files changed, 176 insertions(+), 28 deletions(-) diff --git a/products/pdt/cd/base/configmap.yaml b/products/pdt/cd/base/configmap.yaml index e4b5a9b2a..64a21b808 100644 --- a/products/pdt/cd/base/configmap.yaml +++ b/products/pdt/cd/base/configmap.yaml @@ -5,10 +5,13 @@ metadata: namespace: pdt data: PROJECT_ID: "prj-c-data-analytics-3xs14wez" - DATASET_ID: "ds_zq1_testnet" - NETWORK_TYPE: "testnet" - NETWORK: "testnet-v925" - DOWNLOAD_DIR: "/data/testnet" - UNPACK_DIR: "/data/testnet" + DATASET_ID_TESTNET: "ds_zq1_testnet" + DATASET_ID_MAINNET: "ds_zq1_mainnet" + NETWORK_TYPE_TESTNET: "testnet" + NETWORK_TYPE_MAINNET: "mainnet" + NETWORK_TESTNET: "testnet-v925" + NETWORK_MAINNET: "mainnet-v901" + DOWNLOAD_DIR: "/data/download" + UNPACK_DIR: "/data/download" NR_THREADS: "1" BATCH_BLOCKS: "1000" diff --git a/products/pdt/cd/base/cronjob.yaml b/products/pdt/cd/base/cronjob.yaml index 570e03ab6..eb80b2cf1 100644 --- a/products/pdt/cd/base/cronjob.yaml +++ b/products/pdt/cd/base/cronjob.yaml @@ -1,10 +1,10 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: pdt-cronjob + name: pdt-cronjob-testnet namespace: pdt labels: - app.kubernetes.io/name: "pdt-cronjob" + app.kubernetes.io/name: "pdt-cronjob-testnet" spec: schedule: "0 0 * * *" jobTemplate: @@ -12,13 +12,13 @@ spec: template: metadata: labels: - app.kubernetes.io/name: "pdt-cronjob" + app.kubernetes.io/name: "pdt-cronjob-testnet" spec: restartPolicy: OnFailure serviceAccountName: default containers: - image: pdt - name: pdt-cronjob + name: pdt-cronjob-testnet volumeMounts: - mountPath: /data name: data @@ -32,12 +32,12 @@ spec: valueFrom: configMapKeyRef: name: pdt - key: DATASET_ID + key: DATASET_ID_TESTNET - name: NETWORK valueFrom: configMapKeyRef: name: pdt - key: NETWORK + key: NETWORK_TESTNET - name: DOWNLOAD_DIR valueFrom: configMapKeyRef: @@ -62,8 +62,78 @@ spec: args: - | ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR --network=$NETWORK download - ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS + ./pdt --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS volumes: - name: data persistentVolumeClaim: - claimName: pdt-pvc + claimName: pdt-pvc-testnet +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pdt-cronjob-mainnet + namespace: pdt + labels: + app.kubernetes.io/name: "pdt-cronjob-mainnet" +spec: + schedule: "0 0 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app.kubernetes.io/name: "pdt-cronjob-mainnet" + spec: + restartPolicy: OnFailure + serviceAccountName: default + containers: + - image: pdt + name: pdt-cronjob-mainnet + volumeMounts: + - mountPath: /data + name: data + env: + - name: PROJECT_ID + valueFrom: + configMapKeyRef: + name: pdt + key: PROJECT_ID + - name: DATASET_ID + valueFrom: + configMapKeyRef: + name: pdt + key: DATASET_ID_MAINNET + - name: NETWORK + valueFrom: + configMapKeyRef: + name: pdt + key: NETWORK_MAINNET + - name: DOWNLOAD_DIR + valueFrom: + configMapKeyRef: + name: pdt + key: DOWNLOAD_DIR + - name: UNPACK_DIR + valueFrom: + configMapKeyRef: + name: pdt + key: UNPACK_DIR + - name: NR_THREADS + valueFrom: + configMapKeyRef: + name: pdt + key: NR_THREADS + - name: BATCH_BLOCKS + valueFrom: + configMapKeyRef: + name: pdt + key: BATCH_BLOCKS + command: ["/bin/bash", "-c"] + args: + - | + ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR --network=$NETWORK download + ./pdt --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS + volumes: + - name: data + persistentVolumeClaim: + claimName: pdt-pvc-mainnet diff --git a/products/pdt/cd/base/deployment.yaml b/products/pdt/cd/base/deployment.yaml index 70f84ed02..4059a26f7 100644 --- a/products/pdt/cd/base/deployment.yaml +++ b/products/pdt/cd/base/deployment.yaml @@ -1,26 +1,26 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: pdt-listen + name: pdt-listen-testnet namespace: pdt labels: - "app.kubernetes.io/name": "pdt-listen" + "app.kubernetes.io/name": "pdt-listen-testnet" spec: replicas: 1 selector: matchLabels: - "app.kubernetes.io/name": "pdt-listen" + "app.kubernetes.io/name": "pdt-listen-testnet" strategy: type: RollingUpdate template: metadata: labels: - "app.kubernetes.io/name": "pdt-listen" + "app.kubernetes.io/name": "pdt-listen-testnet" spec: serviceAccountName: default containers: - image: pdt - name: pdt-listen + name: pdt-listen-testnet env: - name: PROJECT_ID valueFrom: @@ -31,12 +31,55 @@ spec: valueFrom: configMapKeyRef: name: pdt - key: DATASET_ID + key: DATASET_ID_TESTNET - name: NETWORK_TYPE valueFrom: configMapKeyRef: name: pdt - key: NETWORK_TYPE + key: NETWORK_TYPE_TESTNET + command: ["/bin/bash", "-c"] + args: + - ./pdt --network-type $NETWORK_TYPE bqlisten --project-id $PROJECT_ID --dataset-id $DATASET_ID +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pdt-listen-mainnet + namespace: pdt + labels: + "app.kubernetes.io/name": "pdt-listen-mainnet" +spec: + replicas: 1 + selector: + matchLabels: + "app.kubernetes.io/name": "pdt-listen-mainnet" + strategy: + type: RollingUpdate + template: + metadata: + labels: + "app.kubernetes.io/name": "pdt-listen-mainnet" + spec: + serviceAccountName: default + containers: + - image: pdt + name: pdt-listen-mainnet + env: + - name: PROJECT_ID + valueFrom: + configMapKeyRef: + name: pdt + key: PROJECT_ID + - name: DATASET_ID + valueFrom: + configMapKeyRef: + name: pdt + key: DATASET_ID_MAINNET + - name: NETWORK_TYPE + valueFrom: + configMapKeyRef: + name: pdt + key: NETWORK_TYPE_MAINNET command: ["/bin/bash", "-c"] args: - ./pdt --network-type $NETWORK_TYPE bqlisten --project-id $PROJECT_ID --dataset-id $DATASET_ID diff --git a/products/pdt/cd/base/pvc.yaml b/products/pdt/cd/base/pvc.yaml index 0c06881c4..ccd23c337 100644 --- a/products/pdt/cd/base/pvc.yaml +++ b/products/pdt/cd/base/pvc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: pdt-pvc + name: pdt-pvc-testnet namespace: pdt labels: "app.kubernetes.io/name": "pdt" @@ -12,3 +12,18 @@ spec: requests: storage: 128Gi storageClassName: standard +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pdt-pvc-mainnet + namespace: pdt + labels: + "app.kubernetes.io/name": "pdt" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 512Gi + storageClassName: standard diff --git a/products/pdt/cd/overlays/staging/configmap.yaml b/products/pdt/cd/overlays/staging/configmap.yaml index e4b5a9b2a..64a21b808 100644 --- a/products/pdt/cd/overlays/staging/configmap.yaml +++ b/products/pdt/cd/overlays/staging/configmap.yaml @@ -5,10 +5,13 @@ metadata: namespace: pdt data: PROJECT_ID: "prj-c-data-analytics-3xs14wez" - DATASET_ID: "ds_zq1_testnet" - NETWORK_TYPE: "testnet" - NETWORK: "testnet-v925" - DOWNLOAD_DIR: "/data/testnet" - UNPACK_DIR: "/data/testnet" + DATASET_ID_TESTNET: "ds_zq1_testnet" + DATASET_ID_MAINNET: "ds_zq1_mainnet" + NETWORK_TYPE_TESTNET: "testnet" + NETWORK_TYPE_MAINNET: "mainnet" + NETWORK_TESTNET: "testnet-v925" + NETWORK_MAINNET: "mainnet-v901" + DOWNLOAD_DIR: "/data/download" + UNPACK_DIR: "/data/download" NR_THREADS: "1" BATCH_BLOCKS: "1000" diff --git a/products/pdt/cd/overlays/staging/kustomization.yaml b/products/pdt/cd/overlays/staging/kustomization.yaml index ed46750c4..e11c9d3ef 100644 --- a/products/pdt/cd/overlays/staging/kustomization.yaml +++ b/products/pdt/cd/overlays/staging/kustomization.yaml @@ -14,12 +14,26 @@ patches: value: pdt target: kind: CronJob - name: pdt-cronjob + name: pdt-cronjob-testnet + - patch: |- + - op: replace + path: /spec/jobTemplate/spec/template/spec/serviceAccountName + value: pdt + target: + kind: CronJob + name: pdt-cronjob-mainnet + - patch: |- + - op: replace + path: /spec/template/spec/serviceAccountName + value: pdt + target: + kind: Deployment + name: pdt-listen-testnet - patch: |- - op: replace path: /spec/template/spec/serviceAccountName value: pdt target: kind: Deployment - name: pdt-listen + name: pdt-listen-mainnet - path: configmap.yaml