[Image generation] Image2image and inpainting pipelines (#1334)

- Added common infra for image to image and inpainting pipelines - Reused it in SD / LCM / SDXL pipelines - Generalized SD pipeline and inherit SDXL pipeline from it. SDXL overrides only SDXL specific blocks which work with second text encoder - Added Python API for Image2Image and Inpainting pipelines - Added Python and C++ samples
openvinotoolkit · Dec 11, 2024 · aef1591 · aef1591
1 parent c4857e3
commit aef1591
Show file tree

Hide file tree

Showing 65 changed files with 2,412 additions and 770 deletions.
diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml
@@ -591,7 +591,7 @@ jobs:
           PYTHONPATH: "./build"
 
   cpp-greedy_causal_lm-redpajama-3b-chat:
-    runs-on: ubuntu-20.04-4-cores
+    runs-on: ubuntu-20.04-8-cores
     defaults:
       run:
         shell: bash

diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml
@@ -47,11 +47,11 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
 
-      - name: Build app
+      - name: Build apps
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
 
       - name: Create virtual environment
         run: python3 -m venv openvino_lcm_cpp
@@ -66,20 +66,32 @@ jobs:
         run: |
           source openvino_lcm_cpp/bin/activate
           optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16
+          wget -O ./image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png
+          wget -O ./mask_image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png
 
-      - name: Run app
+      - name: Run heterogeneous_stable_diffusion
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
-          ${{ env.build_dir }}/samples/cpp/text2image/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York  with tall buildings at dusk golden hour cinematic lighting"
+          ${{ env.build_dir }}/samples/cpp/image_generation/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York  with tall buildings at dusk golden hour cinematic lighting"
 
-      - name: Run Python app
+      - name: Run heterogeneous_stable_diffusion.py
         run: |
           source openvino_lcm_cpp/bin/activate
           source ./ov/setupvars.sh
-          python ./samples/python/text2image/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+          python ./samples/python/image_generation/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
         env:
           PYTHONPATH: ${{ env.build_dir }}
 
+      - name: Run image2image
+        run: |
+          source ./ov/setupvars.sh
+          ${{ env.build_dir }}/samples/cpp/image_generation/image2image ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png
+
+      - name: Run inpainting
+        run: |
+          source ./ov/setupvars.sh
+          ${{ env.build_dir }}/samples/cpp/image_generation/inpainting ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png ./mask_image.png
+
   lcm_dreamshaper_v7_cpp-windows:
     runs-on: windows-2019
     defaults:
@@ -110,11 +122,11 @@ jobs:
       - name: Create virtual environment
         run: python -m venv openvino_lcm_cpp
 
-      - name: Build app
+      - name: Build apps
         run: |
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
 
       - name: Install python dependencies
         run: |
@@ -126,18 +138,38 @@ jobs:
         run: |
           . "./openvino_lcm_cpp/Scripts/Activate.ps1"
           optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16
+          Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png' -OutFile 'image.png'
+          Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png' -OutFile 'mask_image.png'
 
-      - name: Run app
+      - name: Run heterogeneous_stable_diffusion
         run: >
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
-          & "${{ env.build_dir }}/samples/cpp/text2image/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'"
+          & "${{ env.build_dir }}/samples/cpp/image_generation/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'"
+
+      - name: Run heterogeneous_stable_diffusion.py
+        run: |
+          . "./openvino_lcm_cpp/Scripts/Activate.ps1"
+          . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
+          $env:Path += "${{ env.build_dir }}\openvino_genai"
+          python .\samples\python\image_generation\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+        env:
+          PYTHONPATH: ${{ env.build_dir }}
+
+      - name: Run image2image.py
+        run: |
+          . "./openvino_lcm_cpp/Scripts/Activate.ps1"
+          . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
+          $env:Path += "${{ env.build_dir }}\openvino_genai"
+          python .\samples\python\image_generation\image2image.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png
+        env:
+          PYTHONPATH: ${{ env.build_dir }}
 
-      - name: Run Python app
+      - name: Run inpainting.py
         run: |
           . "./openvino_lcm_cpp/Scripts/Activate.ps1"
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           $env:Path += "${{ env.build_dir }}\openvino_genai"
-          python .\samples\python\text2image\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+          python .\samples\python\image_generation\inpainting.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png .\mask_image.png
         env:
           PYTHONPATH: ${{ env.build_dir }}
 

diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml
@@ -51,7 +51,7 @@ jobs:
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
 
       - name: Create virtual environment
         run: python3 -m venv openvino_sd_cpp
@@ -68,29 +68,29 @@ jobs:
           optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --weight-format fp16 --task stable-diffusion models/dreamlike-art-dreamlike-anime-1.0/FP16
           wget -O ./models/soulcard.safetensors https://civitai.com/api/download/models/72591
 
-      - name: Run main app
+      - name: Run text2image app
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
-          ${{ env.build_dir }}/samples/cpp/text2image/stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York  with tall buildings at dusk golden hour cinematic lighting"
+          ${{ env.build_dir }}/samples/cpp/image_generation/text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York  with tall buildings at dusk golden hour cinematic lighting"
 
-      - name: Run LoRA app
+      - name: Run lora_text2image app
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
-          ${{ env.build_dir }}/samples/cpp/text2image/lora_stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7
+          ${{ env.build_dir }}/samples/cpp/image_generation/lora_text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7
 
-      - name: Run Python main app
+      - name: Run text2image.py app
         run: |
           source openvino_sd_cpp/bin/activate
           source ./ov/setupvars.sh
-          python ./samples/python/text2image/main.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+          python ./samples/python/image_generation/text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
         env:
           PYTHONPATH: ${{ env.build_dir }}
 
-      - name: Run Python LoRA app
+      - name: Run lora_text2image.py app
         run: |
           source openvino_sd_cpp/bin/activate
           source ./ov/setupvars.sh
-          python ./samples/python/text2image/lora.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7
+          python ./samples/python/image_generation/lora_text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7
         env:
           PYTHONPATH: ${{ env.build_dir }}
 
@@ -125,7 +125,7 @@ jobs:
         run: |
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
 
       - name: Create virtual environment
         run: python -m venv openvino_sd_cpp
@@ -142,35 +142,35 @@ jobs:
           optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 models/dreamlike-art-dreamlike-anime-1.0/FP16
           Invoke-WebRequest -Uri 'https://civitai.com/api/download/models/72591' -OutFile 'models/soulcard.safetensors'
 
-      - name: Run main app
+      - name: Run text2image app
         run: |
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
-          "${{ env.build_dir }}/samples/cpp/text2image/Release/stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'"
+          "${{ env.build_dir }}/samples/cpp/image_generation/Release/text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'"
         env:
           PATH: ${{ env.build_dir }}\openvino_genai
 
-      - name: Run LoRA app
+      - name: Run lora_text2image app
         run: |
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
-          "${{ env.build_dir }}/samples/cpp/text2image/Release/lora_stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7"
+          "${{ env.build_dir }}/samples/cpp/image_generation/Release/lora_text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7"
         env:
           PATH: ${{ env.build_dir }}\openvino_genai
 
-      - name: Run Python main app
+      - name: Run text2image.py app
         run: |
           . "./openvino_sd_cpp/Scripts/Activate.ps1"
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           $env:Path += "${{ env.build_dir }}\openvino_genai"
-          python .\samples\python\text2image\main.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+          python .\samples\python\image_generation\text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
         env:
           PYTHONPATH: ${{ env.build_dir }}
 
-      - name: Run Python LoRA app
+      - name: Run lora_text2image.py app
         run: |
           . "./openvino_sd_cpp/Scripts/Activate.ps1"
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           $env:Path += "${{ env.build_dir }}\openvino_genai"
-          python .\samples\python\text2image\lora.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7
+          python .\samples\python\image_generation\lora_text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7
         env:
           PYTHONPATH: ${{ env.build_dir }}