feat: encode docker pytorch (#2)

.github/workflows/docker-ci.yml

@@ -0,0 +1,47 @@
+name: docker build CI
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  docker:
+    strategy:
+      matrix:
+        dockerfile: ["server", "worker-cut", "worker-encode", "worker-merge"]
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push - Encode
+        if: matrix.dockerfile == 'worker-encode'
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./deploy/Dockerfile-${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: true
+          tags: lychee0/finalrip-${{ matrix.dockerfile }}:dev
+
+      - name: Build and push
+        if: matrix.dockerfile != 'worker-encode'
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./deploy/Dockerfile-${{ matrix.dockerfile }}
+          platforms: linux/amd64, linux/arm64
+          push: true
+          tags: lychee0/finalrip-${{ matrix.dockerfile }}:dev

.github/workflows/docker.yml → .github/workflows/docker-release.yml

@@ -1,9 +1,6 @@
-name: docker build
+name: docker build Release
 
 on:
-  push:
-    branches:
-      - main
   workflow_dispatch:
 
 jobs:

Makefile

@@ -25,3 +25,7 @@ lint:
 	golangci-lint run
 	pre-commit install # pip install pre-commit
 	pre-commit run --all-files
+
+.PHONY: pytorch
+pytorch:
+	docker buildx build -f .\deploy\Dockerfile-worker-encode-pytorch .

deploy/Dockerfile-worker-encode

@@ -13,9 +13,28 @@ RUN make worker
 
 FROM continuumio/miniconda3:24.5.0-0 AS app
 
+# prepare environment
+RUN conda install python=3.10.* -y
+RUN apt update && apt install -y libgl1-mesa-glx
+
+# install vapoursynth
 RUN conda install conda-forge::vapoursynth=69 -y
+
+# install vapoursynth C++ plugins
+RUN conda install -n base libarchive -c main --force-reinstall -y
 RUN conda install tongyuantongyu::vapoursynth-bestsource=5 -y
+RUN conda install tongyuantongyu::vapoursynth-fmtconv=30 -y
+
+# install python packages
+RUN pip install numpy==1.26.4
+RUN pip install opencv-python==4.10.0.82
+
+# install vapoursynth python plugins
+RUN conda install tongyuantongyu::vapoursynth-mvsfunc=10.10 -y
+RUN conda install tongyuantongyu::vapoursynth-vsutil=0.8.0 -y
+RUN pip install git+https://github.com/HomeOfVapourSynthEvolution/havsfunc.git
 
+# worker app
 WORKDIR /app
 
 COPY --from=builder /build/worker/worker /app/

deploy/Dockerfile-worker-encode-pytorch

@@ -0,0 +1,54 @@
+FROM golang:1.22-bookworm AS builder
+
+ENV GO111MODULE=on \
+    CGO_ENABLED=0
+
+WORKDIR /build
+
+COPY . .
+
+RUN go mod download
+
+RUN make worker
+
+FROM continuumio/miniconda3:24.5.0-0 AS app
+
+# prepare environment
+RUN conda install python=3.10.* -y
+RUN apt update && apt install -y libgl1-mesa-glx
+
+# install vapoursynth
+RUN conda install conda-forge::vapoursynth=69 -y
+
+# install vapoursynth C++ plugins
+RUN conda install -n base libarchive -c main --force-reinstall -y
+RUN conda install tongyuantongyu::vapoursynth-bestsource=5 -y
+RUN conda install tongyuantongyu::vapoursynth-fmtconv=30 -y
+
+# install vapoursynth python plugins
+RUN conda install tongyuantongyu::vapoursynth-mvsfunc=10.10 -y
+RUN conda install tongyuantongyu::vapoursynth-vsutil=0.8.0 -y
+RUN pip install git+https://github.com/HomeOfVapourSynthEvolution/havsfunc.git
+
+# install python packages
+RUN pip install numpy==1.26.4
+RUN pip install opencv-python==4.10.0.82
+
+# install CUDA
+RUN conda install conda-forge::cudatoolkit=11.8.0 -y
+RUN conda install conda-forge::cudnn=8.9.7.* -y
+
+# install PyTorch
+RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
+
+# install AI packages
+RUN pip install vsrealesrgan
+RUN python -m vsrealesrgan
+
+# worker app
+WORKDIR /app
+
+COPY --from=builder /build/worker/worker /app/
+COPY --from=builder /build/conf/finalrip.yml /app/conf/
+
+CMD ["/app/worker", "encode"]

deploy/docker-compose-run.yml

@@ -33,12 +33,21 @@ services:
 
   worker-encode:
     image: lychee0/finalrip-worker-encode:latest
+    #    image: lychee0/finalrip-worker-encode-pytorch:latest
     container_name: finalrip-worker-encode
     restart: always
     environment:
       - FINALRIP_DB_HOST=mongodb
       - FINALRIP_REDIS_HOST=redis
       - FINALRIP_OSS_ENDPOINT=192.168.0.109:9000
+    #    deploy:
+    #      resources:
+    #        reservations:
+    #          devices:
+    #            - driver: nvidia
+    #              device_ids:
+    #                - "0"
+    #              capabilities: [gpu]
     networks:
       - backend
 

module/ffmpeg/vapoursynth.go

@@ -27,9 +27,9 @@ func EncodeVideo(encodeScript string, encodeParam string) error {
 		condaInitScript = "@echo off\r\n" + "call \"%USERPROFILE%\\miniconda3\\condabin\\activate.bat\"\r\n"
 	default:
 		scriptPath = "temp_script.sh"
-		condaInitScript = "#!/bin/bash\n" + "conda activate\n"
+		condaInitScript = "#!/bin/bash\n" // linux 下默认激活 conda 环境
 	}
-	commandStr = condaInitScript + "vspipe -c y4m encode.py - | ffmpeg -i - " + encodeParam + " encoded.mkv"
+	commandStr = condaInitScript + "vspipe -c y4m encode.py - | " + encodeParam + " encoded.mkv"
 	log.Logger.Info("commandStr: " + commandStr)
 
 	// 清理临时文件