ZIL-5441: PDT deploy mainnet staging

products/pdt/cd/base/configmap.yaml

@@ -5,10 +5,13 @@ metadata:
   namespace: pdt
 data:
   PROJECT_ID: "prj-c-data-analytics-3xs14wez"
-  DATASET_ID: "ds_zq1_testnet"
-  NETWORK_TYPE: "testnet"
-  NETWORK: "testnet-v925"
-  DOWNLOAD_DIR: "/data/testnet"
-  UNPACK_DIR: "/data/testnet"
+  DATASET_ID_TESTNET: "ds_zq1_testnet"
+  DATASET_ID_MAINNET: "ds_zq1_mainnet"
+  NETWORK_TYPE_TESTNET: "testnet"
+  NETWORK_TYPE_MAINNET: "mainnet"
+  NETWORK_TESTNET: "testnet-v925"
+  NETWORK_MAINNET: "mainnet-v901"
+  DOWNLOAD_DIR: "/data/download"
+  UNPACK_DIR: "/data/download"
   NR_THREADS: "1"
   BATCH_BLOCKS: "1000"

products/pdt/cd/base/cronjob.yaml

@@ -1,24 +1,24 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: pdt-cronjob
+  name: pdt-cronjob-testnet
   namespace: pdt
   labels:
-    app.kubernetes.io/name: "pdt-cronjob"
+    app.kubernetes.io/name: "pdt-cronjob-testnet"
 spec:
   schedule: "0 0 * * *"
   jobTemplate:
     spec:
       template:
         metadata:
           labels:
-            app.kubernetes.io/name: "pdt-cronjob"
+            app.kubernetes.io/name: "pdt-cronjob-testnet"
         spec:
           restartPolicy: OnFailure
           serviceAccountName: default
           containers:
             - image: pdt
-              name: pdt-cronjob
+              name: pdt-cronjob-testnet
               volumeMounts:
                 - mountPath: /data
                   name: data
@@ -32,12 +32,12 @@ spec:
                   valueFrom:
                     configMapKeyRef:
                       name: pdt
-                      key: DATASET_ID
+                      key: DATASET_ID_TESTNET
                 - name: NETWORK
                   valueFrom:
                     configMapKeyRef:
                       name: pdt
-                      key: NETWORK
+                      key: NETWORK_TESTNET
                 - name: DOWNLOAD_DIR
                   valueFrom:
                     configMapKeyRef:
@@ -62,8 +62,78 @@ spec:
               args:
                 - |
                   ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR --network=$NETWORK download
-                  ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS
+                  ./pdt --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS
           volumes:
             - name: data
               persistentVolumeClaim:
-                claimName: pdt-pvc
+                claimName: pdt-pvc-testnet
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: pdt-cronjob-mainnet
+  namespace: pdt
+  labels:
+    app.kubernetes.io/name: "pdt-cronjob-mainnet"
+spec:
+  schedule: "0 0 * * *"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: "pdt-cronjob-mainnet"
+        spec:
+          restartPolicy: OnFailure
+          serviceAccountName: default
+          containers:
+            - image: pdt
+              name: pdt-cronjob-mainnet
+              volumeMounts:
+                - mountPath: /data
+                  name: data
+              env:
+                - name: PROJECT_ID
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: PROJECT_ID
+                - name: DATASET_ID
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: DATASET_ID_MAINNET
+                - name: NETWORK
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: NETWORK_MAINNET
+                - name: DOWNLOAD_DIR
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: DOWNLOAD_DIR
+                - name: UNPACK_DIR
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: UNPACK_DIR
+                - name: NR_THREADS
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: NR_THREADS
+                - name: BATCH_BLOCKS
+                  valueFrom:
+                    configMapKeyRef:
+                      name: pdt
+                      key: BATCH_BLOCKS
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  ./pdt --download-dir=$DOWNLOAD_DIR --unpack-dir=$UNPACK_DIR --network=$NETWORK download
+                  ./pdt --unpack-dir=$UNPACK_DIR bqmulti --project-id=$PROJECT_ID --dataset-id=$DATASET_ID --nr-threads=$NR_THREADS --batch-blocks=$BATCH_BLOCKS
+          volumes:
+            - name: data
+              persistentVolumeClaim:
+                claimName: pdt-pvc-mainnet

products/pdt/cd/base/deployment.yaml

@@ -1,26 +1,26 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: pdt-listen
+  name: pdt-listen-testnet
   namespace: pdt
   labels:
-    "app.kubernetes.io/name": "pdt-listen"
+    "app.kubernetes.io/name": "pdt-listen-testnet"
 spec:
   replicas: 1
   selector:
     matchLabels:
-      "app.kubernetes.io/name": "pdt-listen"
+      "app.kubernetes.io/name": "pdt-listen-testnet"
   strategy:
     type: RollingUpdate
   template:
     metadata:
       labels:
-        "app.kubernetes.io/name": "pdt-listen"
+        "app.kubernetes.io/name": "pdt-listen-testnet"
     spec:
       serviceAccountName: default
       containers:
         - image: pdt
-          name: pdt-listen
+          name: pdt-listen-testnet
           env:
             - name: PROJECT_ID
               valueFrom:
@@ -31,12 +31,55 @@ spec:
               valueFrom:
                 configMapKeyRef:
                   name: pdt
-                  key: DATASET_ID
+                  key: DATASET_ID_TESTNET
             - name: NETWORK_TYPE
               valueFrom:
                 configMapKeyRef:
                   name: pdt
-                  key: NETWORK_TYPE
+                  key: NETWORK_TYPE_TESTNET
+          command: ["/bin/bash", "-c"]
+          args:
+            - ./pdt --network-type $NETWORK_TYPE bqlisten --project-id $PROJECT_ID --dataset-id $DATASET_ID
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pdt-listen-mainnet
+  namespace: pdt
+  labels:
+    "app.kubernetes.io/name": "pdt-listen-mainnet"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      "app.kubernetes.io/name": "pdt-listen-mainnet"
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        "app.kubernetes.io/name": "pdt-listen-mainnet"
+    spec:
+      serviceAccountName: default
+      containers:
+        - image: pdt
+          name: pdt-listen-mainnet
+          env:
+            - name: PROJECT_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: pdt
+                  key: PROJECT_ID
+            - name: DATASET_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: pdt
+                  key: DATASET_ID_MAINNET
+            - name: NETWORK_TYPE
+              valueFrom:
+                configMapKeyRef:
+                  name: pdt
+                  key: NETWORK_TYPE_MAINNET
           command: ["/bin/bash", "-c"]
           args:
             - ./pdt --network-type $NETWORK_TYPE bqlisten --project-id $PROJECT_ID --dataset-id $DATASET_ID

products/pdt/cd/base/pvc.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: pdt-pvc
+  name: pdt-pvc-testnet
   namespace: pdt
   labels:
     "app.kubernetes.io/name": "pdt"
@@ -12,3 +12,18 @@ spec:
     requests:
       storage: 128Gi
   storageClassName: standard
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pdt-pvc-mainnet
+  namespace: pdt
+  labels:
+    "app.kubernetes.io/name": "pdt"
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 512Gi
+  storageClassName: standard

products/pdt/cd/overlays/staging/configmap.yaml

@@ -5,10 +5,13 @@ metadata:
   namespace: pdt
 data:
   PROJECT_ID: "prj-c-data-analytics-3xs14wez"
-  DATASET_ID: "ds_zq1_testnet"
-  NETWORK_TYPE: "testnet"
-  NETWORK: "testnet-v925"
-  DOWNLOAD_DIR: "/data/testnet"
-  UNPACK_DIR: "/data/testnet"
+  DATASET_ID_TESTNET: "ds_zq1_testnet"
+  DATASET_ID_MAINNET: "ds_zq1_mainnet"
+  NETWORK_TYPE_TESTNET: "testnet"
+  NETWORK_TYPE_MAINNET: "mainnet"
+  NETWORK_TESTNET: "testnet-v925"
+  NETWORK_MAINNET: "mainnet-v901"
+  DOWNLOAD_DIR: "/data/download"
+  UNPACK_DIR: "/data/download"
   NR_THREADS: "1"
   BATCH_BLOCKS: "1000"

products/pdt/cd/overlays/staging/kustomization.yaml

@@ -14,12 +14,26 @@ patches:
         value: pdt
     target:
       kind: CronJob
-      name: pdt-cronjob
+      name: pdt-cronjob-testnet
+  - patch: |-
+      - op: replace
+        path: /spec/jobTemplate/spec/template/spec/serviceAccountName
+        value: pdt
+    target:
+      kind: CronJob
+      name: pdt-cronjob-mainnet
+  - patch: |-
+      - op: replace
+        path: /spec/template/spec/serviceAccountName
+        value: pdt
+    target:
+      kind: Deployment
+      name: pdt-listen-testnet
   - patch: |-
       - op: replace
         path: /spec/template/spec/serviceAccountName
         value: pdt
     target:
       kind: Deployment
-      name: pdt-listen
+      name: pdt-listen-mainnet
   - path: configmap.yaml

products/pdt/pdt/src/bqimport.rs

@@ -117,7 +117,7 @@ pub async fn reconcile_blocks(
     // We should have coverage for every block, extant or not, up to the
     // last batch, which will be short.
     let mut blk: i64 = 0;
-    while blk < max_block {
+    while (blk + scale) < max_block {
         let span = std::cmp::min(scale, max_block - (blk + scale));
         println!("blk {} span {}", blk, span);
         match project.is_txn_range_covered_by_entry(blk, span).await? {

products/pdt/pdt/src/main.rs

@@ -264,8 +264,8 @@ async fn bigquery_reconcile_blocks(unpack_dir: &str, opts: &ReconcileOptions) ->
     bqimport::reconcile_blocks(
         unpack_dir,
         opts.batch_blocks,
-        &opts.dataset_id,
         &opts.project_id,
+        &opts.dataset_id,
     )
     .await
 }