diff --git a/.github/workflows/docker-multi-arch.yml b/.github/workflows/docker-multi-arch.yml index 5a2c88b..11ff036 100644 --- a/.github/workflows/docker-multi-arch.yml +++ b/.github/workflows/docker-multi-arch.yml @@ -21,7 +21,7 @@ jobs: flavor: | suffix=${{ matrix.tag }},onlatest=true tags: | - type=ref,branch=dev + type=edge,branch=dev,suffix=-dev type=edge,branch=main,suffix=${{ matrix.tag }} type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}} @@ -32,12 +32,11 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 - id: buildx - uses: docker/build-push-action@v3 with: context: . file: Dockerfile - platforms: linux/arm64, linux/amd64, linux/arm/v7, linux/386, linux/arm/v6 + platforms: linux/arm64, linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index c4a81de..c782544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,21 @@ -FROM python:3.9 +FROM python:3.9-slim -WORKDIR /usr/src/app +RUN apt-get update \ + && apt-get install gcc python3-dev libpq-dev -y \ + && apt-get clean -COPY requirements.txt . +RUN addgroup --gid 1001 --system crypto && \ + adduser --shell /bin/false --disabled-password --uid 1001 --system --group crypto +RUN mkdir -p /app +RUN chown crypto:crypto /app -RUN pip install -r requirements.txt +WORKDIR /app +USER crypto -COPY ./src . +COPY --chown=crypto:crypto requirements.txt /tmp/pip-tmp/ +RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ + && rm -rf /tmp/pip-tmp -CMD [ "python","-u","runner.py" ] +COPY --chown=crypto:crypto ./src . + +CMD [ "python", "-u", "runner.py" ] diff --git a/src/runner.py b/src/runner.py index 4dfb5b2..59e3126 100644 --- a/src/runner.py +++ b/src/runner.py @@ -1,3 +1,4 @@ +from typing import NoReturn from os import environ import json from sinks import TimescaleDB @@ -37,7 +38,7 @@ log = logging.getLogger('runner') -def worker(timescaleDB, binance, database): +def worker(timescaleDB: TimescaleDB, binance: Binance, database: str) -> None: for symbol in config['symbols']: log.info("Current Symbol: {}".format(symbol)) try: @@ -63,7 +64,7 @@ def worker(timescaleDB, binance, database): timescaleDB.write(database, symbol, klines_df) -def main(): +def main() -> NoReturn: log.debug(config) timescaleDB = TimescaleDB(**config['timescale']) binance = Binance(**config['binance']) diff --git a/src/sinks/timescaledb.py b/src/sinks/timescaledb.py index f415968..04cd406 100644 --- a/src/sinks/timescaledb.py +++ b/src/sinks/timescaledb.py @@ -1,10 +1,12 @@ +from typing import Union +import pandas as pd import psycopg2 from pgcopy import CopyManager from sources import binanceColumns class TimescaleDB(): - def __init__(self, host, port, username, password): + def __init__(self, host: str, port: Union[str, int], username: str, password: str): self.host = host self.port = port self.username = username @@ -12,7 +14,7 @@ def __init__(self, host, port, username, password): self.connStr = "postgres://{}:{}@{}:{}/".format( self.username, self.password, self.host, self.port) - def createDatabase(self, name, clear=False): + def createDatabase(self, name: str, clear: bool = False) -> None: try: connection = psycopg2.connect(self.connStr,) connection.autocommit = True @@ -24,7 +26,7 @@ def createDatabase(self, name, clear=False): if connection: connection.close() - def createTable(self, database, symbol, clear=False): + def createTable(self, database: str, symbol: str, clear: bool = False) -> None: with psycopg2.connect("{}{}".format(self.connStr, database.lower())) as conn: cursor = conn.cursor() if clear: @@ -43,14 +45,14 @@ def createTable(self, database, symbol, clear=False): conn.commit() cursor.close() - def write(self, database, symbol, df): + def write(self, database: str, symbol: str, df: pd.DataFrame) -> None: with psycopg2.connect("{}{}".format(self.connStr, database.lower())) as conn: mgr = CopyManager(conn, symbol.lower(), df.columns) self.deleteLastTimestamp(database, symbol) mgr.copy(df.to_numpy()) conn.commit() - def deleteLastTimestamp(self, database, symbol): + def deleteLastTimestamp(self, database: str, symbol: str) -> None: with psycopg2.connect("{}{}".format(self.connStr, database.lower())) as conn: ts = self.getLastTimestamp(database, symbol) if ts: @@ -60,7 +62,7 @@ def deleteLastTimestamp(self, database, symbol): conn.commit() cursor.close() - def getLastTimestamp(self, database, symbol): + def getLastTimestamp(self, database: str, symbol: str) -> Union[str, None]: with psycopg2.connect("{}{}".format(self.connStr, database.lower())) as conn: cursor = conn.cursor() cursor.execute( @@ -72,7 +74,7 @@ def getLastTimestamp(self, database, symbol): cursor.close() return ts - def getFirstTimestamp(self, database, symbol): + def getFirstTimestamp(self, database: str, symbol: str) -> Union[str, None]: with psycopg2.connect("{}{}".format(self.connStr, database.lower())) as conn: cursor = conn.cursor() cursor.execute( diff --git a/src/sources/binance.py b/src/sources/binance.py index 4737355..dd12aed 100644 --- a/src/sources/binance.py +++ b/src/sources/binance.py @@ -1,3 +1,4 @@ +from typing import Literal from binance import Client import numpy as np import pandas as pd @@ -18,12 +19,17 @@ class Binance(): - def __init__(self, api_key, api_secret): + def __init__(self, api_key: str, api_secret: str): self.api_key = api_key self.api_secret = api_secret self.client = Client(api_key, api_secret) - def get_klines(self, symbol, startTime, endTime='NOW', interval=Client.KLINE_INTERVAL_1MINUTE): + def get_klines(self, + symbol: str, + startTime: str, + endTime: str = 'NOW', + interval: Literal = Client.KLINE_INTERVAL_1MINUTE + ) -> pd.DataFrame: klines = self.client.get_historical_klines( symbol, interval, startTime, endTime) data = np.array(klines)