cog.yaml

# Configuration for Cog ⚙️
# Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md

build:
  # set to true if your model requires a GPU
  gpu: true
  cuda: "11.7"

  # a list of ubuntu apt packages to install
  # system_packages:
    # - "libgl1-mesa-glx"
    # - "libglib2.0-0"

  # python version in the form '3.8' or '3.8.12'
  python_version: "3.8"

  # a list of packages in the format <package-name>==<version>
  python_packages:
    - "torch==1.13.1"
    - "einops==0.5.0"
    - "packaging==23.1"
    - "transformers==4.28"
    - "tensorizer==1.1.0"

  # commands run after the environment is setup
  run:
    - pip install flash-attn==v1.0.3.post0
    - pip install triton==2.0.0.dev20221202
    - pip install xentropy-cuda-lib@git+https://github.com/HazyResearch/flash-attention.git@v0.2.8#subdirectory=csrc/xentropy
    - "echo 'deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main' | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list"
    - "curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -"
    - "apt-get update && apt-get install google-cloud-cli"

    # - "echo another command if needed"

# predict.py defines how predictions are run on your model
predict: "predict.py:Predictor"