[added] adapter container, supporting helm chart, pvc

.github/workflows/action.yml

@@ -52,14 +52,23 @@ jobs:
       #     username: ${{ secrets.JFROG_USERNAME }}
       #     password: ${{ secrets.JFROG_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push (pewpew)
         uses: docker/build-push-action@v4
         with:
           context: .
           file: ./docker/Dockerfile
           push: true
           tags: |
             sjultra/pewpew:latest
+      
+      - name: Build and push (adapter)
+        uses: docker/build-push-action@v4
+        with:
+          context: ./docker/datasource
+          file: ./docker/datasource/Dockerfile
+          push: true
+          tags: |
+            sjultra/pewpew-adapter:latest
 
     # ghcr.io/${{ github.repository_owner }}/pewpew:latest
       # install and authenticate jfrog/artifactory CLI

docker/Dockerfile

@@ -9,5 +9,9 @@ RUN mkdir -p /opt/pewpew
 
 VOLUME ./data
 
+# init external data file
+RUN touch ./data/data.json
+RUN echo "{\"data\": []}" > ./data/data.json
+
 CMD ["http-server","-a","0.0.0.0","-p","8080","/opt/pewpew"]
 

docker/datasource/package.json

@@ -8,7 +8,7 @@
   },
   "author": "",
   "license": "ISC",
-  "devDependencies": {
+  "dependencies": {
     "@types/node": "^20.8.7",
     "ts-node": "^10.9.1",
     "typescript": "^5.2.2"

docker/datasource/src/index.ts

@@ -29,8 +29,7 @@ async function main() {
     const signIns = await getSignIns(accessToken)
 
 
-    const datapoints = await traceSignIns(signIns)
-    console.log(datapoints);
+    const datapoints = await traceSignIns(signIns, false)
     await writeToDisk({data: datapoints})
 
 }
@@ -111,18 +110,24 @@ async function traceSignIns(signIns: SignIn[], smoke: boolean = true): Promise<D
     let iteration = 1
     const origins: DataPoint[] = []
     for (const {ip, occurredAt} of signIns) {
-        if (counter === 0) {
-            console.log(`Fetching IP info for ${LOOKUP_IP_LIMIT} IP addresses (batch ${iteration}/${Math.ceil(signIns.length/LOOKUP_IP_LIMIT)})`);
-
-        }
-        console.log(`Processing ${counter+1}/${LOOKUP_IP_LIMIT}`);
-
         if (counter >= LOOKUP_IP_LIMIT) {
-            if (smoke) break
+            if (smoke) {
+                console.log("[SMOKE] smoke run completed.");
+                break
+            }
+            console.log("[SLEEP] 62500ms/62.5s");
             await sleep(62500)
             counter = 0
             iteration += 1
         }
+
+        if (counter === 0) {
+            console.log(`Fetching IP info for ${LOOKUP_IP_LIMIT} IP addresses (batch ${iteration}/${Math.ceil(signIns.length/LOOKUP_IP_LIMIT)})`);
+
+        }
+
+        console.log(`Processing ${counter+1}/${LOOKUP_IP_LIMIT}`);
+
         const origin = await lookupIP(ip)
         const datapoint = {
             hit: {

helm/service/templates/datasource.yaml

@@ -0,0 +1,47 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: datasource
+spec:
+  schedule: {{ .Values.datasource.schedule }}
+  successfulJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      selector:
+        matchLabels:
+          app: datasource
+      template:
+        metadata:
+          labels:
+            app: datasource
+        spec:       
+          containers:
+            - name: datasource
+              image: "{{ .Values.datasource.image }}:{{ .Values.datasource.tag }}"
+
+              volumeMounts:
+                - name: {{ .Values.adapter.name }}
+                  mountPath: /datasource/data
+              # args:
+              #   - sh
+              #   - -c
+              #   - |
+              #     sleep 2 
+              #     "/opt/myapp/myscript.sh"
+              #     sleep 30
+            # env: 
+            #   - name: APP_SECRET
+            #     valueFrom:
+            #       secretKeyRef:
+            #         name: {{ .Values.appsecret }}
+            #         key: app_secret
+            #   - name: APP_CLIENT_ID
+            #     valueFrom:
+            #       secretKeyRef:
+            #         name: {{ .Values.appsecret }}
+            #         key: app_client_id 
+            # ports:
+            # - containerPort: {{ .Values.service.internalPort }} 
+          # restartPolicy: {{ .Values.restartPolicy }}
+          # imagePullSecrets:
+            # - name: artifactory