diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml new file mode 100644 index 0000000..9d5dd12 --- /dev/null +++ b/.github/workflows/docker-ci.yml @@ -0,0 +1,47 @@ +name: docker build CI + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + docker: + strategy: + matrix: + dockerfile: ["server", "worker-cut", "worker-encode", "worker-merge"] + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push - Encode + if: matrix.dockerfile == 'worker-encode' + uses: docker/build-push-action@v4 + with: + context: . + file: ./deploy/Dockerfile-${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: lychee0/finalrip-${{ matrix.dockerfile }}:dev + + - name: Build and push + if: matrix.dockerfile != 'worker-encode' + uses: docker/build-push-action@v4 + with: + context: . + file: ./deploy/Dockerfile-${{ matrix.dockerfile }} + platforms: linux/amd64, linux/arm64 + push: true + tags: lychee0/finalrip-${{ matrix.dockerfile }}:dev diff --git a/.github/workflows/docker.yml b/.github/workflows/docker-release.yml similarity index 95% rename from .github/workflows/docker.yml rename to .github/workflows/docker-release.yml index 948230c..c868c49 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker-release.yml @@ -1,9 +1,6 @@ -name: docker build +name: docker build Release on: - push: - branches: - - main workflow_dispatch: jobs: diff --git a/Makefile b/Makefile index 1977592..5d7a9eb 100644 --- a/Makefile +++ b/Makefile @@ -25,3 +25,7 @@ lint: golangci-lint run pre-commit install # pip install pre-commit pre-commit run --all-files + +.PHONY: pytorch +pytorch: + docker buildx build -f .\deploy\Dockerfile-worker-encode-pytorch . \ No newline at end of file diff --git a/deploy/Dockerfile-worker-encode b/deploy/Dockerfile-worker-encode index e148f4b..81aac40 100644 --- a/deploy/Dockerfile-worker-encode +++ b/deploy/Dockerfile-worker-encode @@ -13,9 +13,28 @@ RUN make worker FROM continuumio/miniconda3:24.5.0-0 AS app +# prepare environment +RUN conda install python=3.10.* -y +RUN apt update && apt install -y libgl1-mesa-glx + +# install vapoursynth RUN conda install conda-forge::vapoursynth=69 -y + +# install vapoursynth C++ plugins +RUN conda install -n base libarchive -c main --force-reinstall -y RUN conda install tongyuantongyu::vapoursynth-bestsource=5 -y +RUN conda install tongyuantongyu::vapoursynth-fmtconv=30 -y + +# install python packages +RUN pip install numpy==1.26.4 +RUN pip install opencv-python==4.10.0.82 + +# install vapoursynth python plugins +RUN conda install tongyuantongyu::vapoursynth-mvsfunc=10.10 -y +RUN conda install tongyuantongyu::vapoursynth-vsutil=0.8.0 -y +RUN pip install git+https://github.com/HomeOfVapourSynthEvolution/havsfunc.git +# worker app WORKDIR /app COPY --from=builder /build/worker/worker /app/ diff --git a/deploy/Dockerfile-worker-encode-pytorch b/deploy/Dockerfile-worker-encode-pytorch new file mode 100644 index 0000000..d5af638 --- /dev/null +++ b/deploy/Dockerfile-worker-encode-pytorch @@ -0,0 +1,54 @@ +FROM golang:1.22-bookworm AS builder + +ENV GO111MODULE=on \ + CGO_ENABLED=0 + +WORKDIR /build + +COPY . . + +RUN go mod download + +RUN make worker + +FROM continuumio/miniconda3:24.5.0-0 AS app + +# prepare environment +RUN conda install python=3.10.* -y +RUN apt update && apt install -y libgl1-mesa-glx + +# install vapoursynth +RUN conda install conda-forge::vapoursynth=69 -y + +# install vapoursynth C++ plugins +RUN conda install -n base libarchive -c main --force-reinstall -y +RUN conda install tongyuantongyu::vapoursynth-bestsource=5 -y +RUN conda install tongyuantongyu::vapoursynth-fmtconv=30 -y + +# install vapoursynth python plugins +RUN conda install tongyuantongyu::vapoursynth-mvsfunc=10.10 -y +RUN conda install tongyuantongyu::vapoursynth-vsutil=0.8.0 -y +RUN pip install git+https://github.com/HomeOfVapourSynthEvolution/havsfunc.git + +# install python packages +RUN pip install numpy==1.26.4 +RUN pip install opencv-python==4.10.0.82 + +# install CUDA +RUN conda install conda-forge::cudatoolkit=11.8.0 -y +RUN conda install conda-forge::cudnn=8.9.7.* -y + +# install PyTorch +RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 + +# install AI packages +RUN pip install vsrealesrgan +RUN python -m vsrealesrgan + +# worker app +WORKDIR /app + +COPY --from=builder /build/worker/worker /app/ +COPY --from=builder /build/conf/finalrip.yml /app/conf/ + +CMD ["/app/worker", "encode"] diff --git a/deploy/docker-compose-run.yml b/deploy/docker-compose-run.yml index 9320e2a..753ba84 100644 --- a/deploy/docker-compose-run.yml +++ b/deploy/docker-compose-run.yml @@ -33,12 +33,21 @@ services: worker-encode: image: lychee0/finalrip-worker-encode:latest + # image: lychee0/finalrip-worker-encode-pytorch:latest container_name: finalrip-worker-encode restart: always environment: - FINALRIP_DB_HOST=mongodb - FINALRIP_REDIS_HOST=redis - FINALRIP_OSS_ENDPOINT=192.168.0.109:9000 + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # device_ids: + # - "0" + # capabilities: [gpu] networks: - backend diff --git a/module/ffmpeg/vapoursynth.go b/module/ffmpeg/vapoursynth.go index f5c6c4d..e9e0ad5 100644 --- a/module/ffmpeg/vapoursynth.go +++ b/module/ffmpeg/vapoursynth.go @@ -27,9 +27,9 @@ func EncodeVideo(encodeScript string, encodeParam string) error { condaInitScript = "@echo off\r\n" + "call \"%USERPROFILE%\\miniconda3\\condabin\\activate.bat\"\r\n" default: scriptPath = "temp_script.sh" - condaInitScript = "#!/bin/bash\n" + "conda activate\n" + condaInitScript = "#!/bin/bash\n" // linux 下默认激活 conda 环境 } - commandStr = condaInitScript + "vspipe -c y4m encode.py - | ffmpeg -i - " + encodeParam + " encoded.mkv" + commandStr = condaInitScript + "vspipe -c y4m encode.py - | " + encodeParam + " encoded.mkv" log.Logger.Info("commandStr: " + commandStr) // 清理临时文件