From e300a2ae47c0f42685325a54be6bd006f5739d62 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Tue, 25 Apr 2023 17:23:37 +0400 Subject: [PATCH 001/134] Stable Diffusion quantization example (#294) * Intitial implementation * Fixed issus * Added custom scheduler definition. Changed README and demo * Added support of laion-aesthetic dataset * Fixed style * Applied some comments * Fixed Readme * Moved notebook * Applied comments. Do renamings * Fixes --- examples/openvino/stable-diffusion/README.md | 89 ++ .../stable-diffusion/requirements.txt | 4 + .../train_text_to_image_qat.py | 1043 +++++++++++++++++ notebooks/openvino/README.md | 1 + .../stable_diffusion_quantization.ipynb | 125 ++ 5 files changed, 1262 insertions(+) create mode 100644 examples/openvino/stable-diffusion/README.md create mode 100644 examples/openvino/stable-diffusion/requirements.txt create mode 100644 examples/openvino/stable-diffusion/train_text_to_image_qat.py create mode 100644 notebooks/openvino/stable_diffusion_quantization.ipynb diff --git a/examples/openvino/stable-diffusion/README.md b/examples/openvino/stable-diffusion/README.md new file mode 100644 index 0000000000..d7c2a8faff --- /dev/null +++ b/examples/openvino/stable-diffusion/README.md @@ -0,0 +1,89 @@ +# Stable Diffusion Quantization +This example demonstrates Quantization-aware Training (QAT) of Stable Diffusion using [NNCF](https://github.com/openvinotoolkit/nncf). Quantization is applyied to UNet model which is the most time-consuming element of the whole pipeline. The quantized model and the pipeline is exported to the OpenVINO format for inference with `OVStableDiffusionPipeline` helper. The original training code was taken from the Diffusers [repository](https://github.com/huggingface/diffusers/tree/main/examples/text_to_image) and modified to support QAT. + +Knowledge distillation and EMA techniques can be used to improve the model accuracy. + +This example supports model tuning on two datasets from the HuggingFace: +* [Pokemon BLIP captions](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) +* [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en) +* [laion2B-en-aesthetic](https://huggingface.co/datasets/laion/laion2B-en-aesthetic) + +But it can be easily extended to other datasets. +>**Note**: laion2B-en is being downloaded on-fly durint the fine-tuning process. No need to store it locally. + +## Prerequisites +* Install Optimum-Intel for OpenVINO: +```python +pip install optimum-intel[openvino] +``` +* Install example requirements: +```python +pip install -r requirements.txt +``` +>**Note**: The example requires `torch~=1.13` and does not work with PyTorch 2.0. + +## Running pre-optimized model +* General-purpose image generation model: +```python +from optimum.intel.openvino import OVStableDiffusionPipeline + +pipe = OVStableDiffusionPipeline.from_pretrained("OpenVINO/stable-diffusion-2-1-quantized", compile=False) +pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1) +pipe.compile() + +prompt = "sailing ship in storm by Rembrandt" +output = pipe(prompt, num_inference_steps=50, output_type="pil") +output.images[0].save("result.png") +``` +* Pokemon generation: +```python +from optimum.intel.openvino import OVStableDiffusionPipeline + +pipe = OVStableDiffusionPipeline.from_pretrained("OpenVINO/Stable-Diffusion-Pokemon-en-quantized", compile=False) +pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1) +pipe.compile() + +prompt = "cartoon bird" +output = pipe(prompt, num_inference_steps=50, output_type="pil") +output.images[0].save("result.png") +``` +* You can also run `pokemon_generation_demo.ipynb` notebook from the folder to compare FP32 pipeline with the optimized. + +## HW Requirements for QAT +The minimal HW setup for the run is GPU with 24GB of memory. + +>**NOTE**: Potentially you can set the number of training steps to 0 and it will lead to Post-Training Quantization. CPU should be enough in this case but you may need to modify the scipt. + +## Run QAT: +* QAT for pokemon generation model: +```python +python train_text_to_image_qat.py \ + --ema_device="cpu" \ + --use_kd \ + --model_id="svjack/Stable-Diffusion-Pokemon-en" \ + --center_crop \ + --random_flip \ + --gradient_checkpointing \ + --dataloader_num_workers=2 \ + --dataset_name="lambdalabs/pokemon-blip-captions" \ + --max_train_steps=4096 \ + --opt_init_steps=300 \ + --output_dir=sd-quantized-pokemon +``` + +* QAT on a laion-aesthetic dataset: +```python +python train_text_to_image_qat.py \ + --use_kd \ + --center_crop \ + --random_flip \ + --dataset_name="laion/laion2B-en-aesthetic" \ + --max_train_steps=2048 \ + --model_id="runwayml/stable-diffusion-v1-5" \ + --max_train_samples=15000 \ + --dataloader_num_workers=4 \ + --opt_init_steps=500 \ + --gradient_checkpointing \ + --tune_quantizers_only \ + --output_dir=sd-1-5-quantied-laion +``` \ No newline at end of file diff --git a/examples/openvino/stable-diffusion/requirements.txt b/examples/openvino/stable-diffusion/requirements.txt new file mode 100644 index 0000000000..ccd29e4d02 --- /dev/null +++ b/examples/openvino/stable-diffusion/requirements.txt @@ -0,0 +1,4 @@ +accelerate +diffusers +torch~=1.13 +nncf @ git+https://github.com/openvinotoolkit/nncf.git diff --git a/examples/openvino/stable-diffusion/train_text_to_image_qat.py b/examples/openvino/stable-diffusion/train_text_to_image_qat.py new file mode 100644 index 0000000000..a748d2c4e0 --- /dev/null +++ b/examples/openvino/stable-diffusion/train_text_to_image_qat.py @@ -0,0 +1,1043 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import itertools +import logging +import math +import os +import random +import tempfile +from functools import partial +from io import BytesIO +from pathlib import Path +from typing import Iterable, Optional + +import numpy as np +import requests +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed +from datasets import load_dataset +from diffusers import DDIMScheduler, DDPMScheduler, DiffusionPipeline, LMSDiscreteScheduler, StableDiffusionPipeline +from diffusers.optimization import get_scheduler +from huggingface_hub import HfFolder, Repository, whoami +from nncf import NNCFConfig +from nncf.common.logging import nncf_logger +from nncf.torch import create_compressed_model, register_default_init_args +from nncf.torch.initialization import PTInitializingDataLoader +from nncf.torch.layer_utils import CompressionParameter +from openvino._offline_transformations import apply_moc_transformations, compress_quantize_weights_transformation +from PIL import Image +from torchvision import transforms +from tqdm import tqdm + +from optimum.exporters.onnx import export_models, get_stable_diffusion_models_for_export +from optimum.intel import OVStableDiffusionPipeline +from optimum.utils import ( + DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, + DIFFUSION_MODEL_UNET_SUBFOLDER, + DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, + DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, +) + + +random.seed(42) +logger = get_logger(__name__) +nncf_logger.setLevel(logging.INFO) + + +def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None): + if token is None: + token = HfFolder.get_token() + if organization is None: + username = whoami(token)["name"] + return f"{username}/{model_id}" + else: + return f"{organization}/{model_id}" + + +def pokemon_preprocess_train(examples, train_transforms, tokenize_captions, image_column="image"): + image = examples[image_column] + examples["pixel_values"] = train_transforms(image.convert("RGB")) + examples["input_ids"] = tokenize_captions(examples) + return examples + + +def get_pil_from_url(url): + response = requests.get(url) + image = Image.open(BytesIO(response.content)) + return image.convert("RGB") + + +# Many of the images in laion2B dataset are unavailable +# This is a workaround to substitute such images with a backup or cached available examples +BACKUP_PAIR = ( + get_pil_from_url( + "https://thumbs.dreamstime.com/t/altai-mountains-mountain-lake-russia-siberia-chuya-ridge-49130812.jpg" + ), + "Altai mountains Stock Photography", +) +AVAILABLE_EXAMPLES = [] + + +def laion2B_preprocess_train(examples, train_transforms, tokenize_captions, image_column="URL"): + url = examples[image_column] + try: + image = get_pil_from_url(url) + AVAILABLE_EXAMPLES.append((url, examples["TEXT"])) + except Exception: + logger.info(f"Can't load image from url: {url}, using cache with size: {len(AVAILABLE_EXAMPLES)}") + if len(AVAILABLE_EXAMPLES) > 0: + backup_id = random.randint(0, len(AVAILABLE_EXAMPLES) - 1) + backup_example = AVAILABLE_EXAMPLES[backup_id] + try: + image = get_pil_from_url(backup_example[0]) + examples["TEXT"] = backup_example[1] + except Exception: + logger.info(f"Can't load image from cached url: {backup_example[0]}, using backup") + image = BACKUP_PAIR[0].copy() + examples["TEXT"] = BACKUP_PAIR[1] + else: + logger.info(f"Can't load image from url: {url}, using backup") + image = BACKUP_PAIR[0].copy() + examples["TEXT"] = BACKUP_PAIR[1] + + examples["pixel_values"] = train_transforms(image) + examples["input_ids"] = tokenize_captions(examples) + return examples + + +dataset_name_mapping = { + "lambdalabs/pokemon-blip-captions": { + "columns": ("image", "text"), + "preprocess_fn": pokemon_preprocess_train, + "streaming": False, + }, + "laion/laion2B-en": { + "columns": ("URL", "TEXT"), + "preprocess_fn": laion2B_preprocess_train, + "streaming": True, + }, + "laion/laion2B-en-aesthetic": { + "columns": ("URL", "TEXT"), + "preprocess_fn": laion2B_preprocess_train, + "streaming": True, + }, +} + + +# Adapted from torch-ema https://github.com/fadel/pytorch_ema/blob/master/torch_ema/ema.py#L14 +class EMAQUnet: + """ + Exponential Moving Average of unets weights + """ + + def __init__(self, parameters: Iterable[torch.nn.Parameter], decay=0.9999): + parameters = list(parameters) + self.shadow_params = [p.clone().detach() for p in parameters] + + self.decay = decay + self.optimization_step = 0 + + def get_decay(self, optimization_step): + """ + Compute the decay factor for the exponential moving average. + """ + value = (1 + optimization_step) / (10 + optimization_step) + return 1 - min(self.decay, value) + + @torch.no_grad() + def step(self, parameters): + parameters = list(parameters) + + self.optimization_step += 1 + self.decay = self.get_decay(self.optimization_step) + + for s_param, param in zip(self.shadow_params, parameters): + if param.requires_grad: + tmp = param.clone() + tmp = tmp.to(s_param.device) + # tmp = self.decay * (s_param - param.clone.to(s_param.device)) + tmp.sub_(s_param) + tmp.mul_(self.decay) + tmp.neg_() + s_param.sub_(tmp) + else: + s_param.copy_(param) + + torch.cuda.empty_cache() + + def copy_to(self, parameters: Iterable[torch.nn.Parameter]) -> None: + """ + Copy current averaged parameters into given collection of parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored moving averages. If `None`, the + parameters with which this `ExponentialMovingAverage` was + initialized will be used. + """ + parameters = list(parameters) + for s_param, param in zip(self.shadow_params, parameters): + param.data.copy_(s_param.data) + + def to(self, device=None, dtype=None) -> None: + r"""Move internal buffers of the ExponentialMovingAverage to `device`. + + Args: + device: like `device` argument to `torch.Tensor.to` + """ + # .to() on the tensors handles None correctly + self.shadow_params = [ + p.to(device=device, dtype=dtype) if p.is_floating_point() else p.to(device=device) + for p in self.shadow_params + ] + + +def parse_args(): + parser = argparse.ArgumentParser(description="Stable Diffusion 8-bit Quantization for OpenVINO") + parser.add_argument( + "--model_id", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--dataset_name", + type=str, + default="lambdalabs/pokemon-blip-captions", + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), + ) + parser.add_argument( + "--dataset_config_name", + type=str, + default=None, + help="The config of the Dataset, leave as None if there's only one config.", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-quantized", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=42, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--noise_scheduler", + type=str, + default=None, + choices=["DDIM", "DDPM", "LMSDiscrete"], + help="The noise scheduler for the Diffusion pipiline used for training.", + ) + parser.add_argument( + "--beta_start", + type=float, + default=0.00085, + help="Beta min value for noise scheduler.", + ) + parser.add_argument( + "--beta_end", + type=float, + default=0.012, + help="BetaMax value for noise scheduler.", + ) + parser.add_argument( + "--beta_schedule", + type=str, + default="scaled_linear", + help="Beta schedule type", + ) + parser.add_argument( + "--noise_schedule_steps", + type=int, + default=1000, + help=("The noise scheduler max train timestemps"), + ) + parser.add_argument( + "--center_crop", + default=False, + action="store_true", + help=( + "Whether to center crop the input images to the resolution. If not set, the images will be randomly" + " cropped. The images will be resized to the resolution first before cropping." + ), + ) + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=1) + parser.add_argument( + "--max_train_steps", + type=int, + default=15000, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=4, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-5, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=0, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--snr_gamma", + type=float, + default=None, + help="SNR weighting gamma to be used if rebalancing the loss. Recommended value is 5.0. " + "More details here: https://arxiv.org/abs/2303.09556.", + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--ema_device", + type=str, + default=None, + choices=["cpu", "cuda"], + help="Whether to use EMA model and where to store the EMA model.", + ) + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=15000, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=1, + help=( + "Max number of checkpoints to store. Passed as `total_limit` to the `Accelerator` `ProjectConfiguration`." + " See Accelerator::save_state https://huggingface.co/docs/accelerate/package_reference/accelerator#accelerate.Accelerator.save_state" + " for more docs" + ), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--opt_init_steps", + type=int, + default=300, + help=("Max number of initialization steps for quantization before the actual fine-tuning."), + ) + parser.add_argument( + "--opt_init_type", + type=str, + default="mean_min_max", + choices=["min_max", "mean_min_max", "threesigma"], + help="They way how to estimate activation quantization paramters at the initializatin step before QAT.", + ) + parser.add_argument( + "--tune_quantizers_only", + action="store_true", + default=False, + help="Whether to train quantization parameters only.", + ) + parser.add_argument("--use_kd", action="store_true", help="Use Knowledge Distillation to boost accuracy.") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + return args + + +def get_noise_scheduler(args): + scheduler_args = { + "beta_start": args.beta_start, + "beta_end": args.beta_end, + "beta_schedule": args.beta_schedule, + "num_train_timesteps": args.noise_schedule_steps, + } + if args.noise_scheduler == "DDIM": + noise_scheduler = DDIMScheduler(**scheduler_args) + elif args.noise_scheduler == "DDPM": + noise_scheduler = DDPMScheduler(**scheduler_args) + elif args.noise_scheduler == "LMSDiscrete": + noise_scheduler = LMSDiscreteScheduler(**scheduler_args) + else: + raise ValueError(f"Unknown noise schedule {args.noise_schedule}") + return noise_scheduler + + +def export_to_onnx(pipeline, save_dir): + unet = pipeline.unet + vae = pipeline.vae + text_encoder = pipeline.text_encoder + + unet.eval().cpu() + vae.eval().cpu() + text_encoder.eval().cpu() + + ONNX_WEIGHTS_NAME = "model.onnx" + + output_names = [ + os.path.join(DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, ONNX_WEIGHTS_NAME), + os.path.join(DIFFUSION_MODEL_UNET_SUBFOLDER, ONNX_WEIGHTS_NAME), + os.path.join(DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ONNX_WEIGHTS_NAME), + os.path.join(DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, ONNX_WEIGHTS_NAME), + ] + + with torch.no_grad(): + models_and_onnx_configs = get_stable_diffusion_models_for_export(pipeline) + pipeline.save_config(save_dir) + export_models( + models_and_onnx_configs=models_and_onnx_configs, output_dir=Path(save_dir), output_names=output_names + ) + + +def export_to_openvino(pipeline, onnx_dir, save_dir): + ov_pipe = OVStableDiffusionPipeline.from_pretrained( + model_id=onnx_dir, + from_onnx=True, + model_save_dir=save_dir, + tokenizer=pipeline.tokenizer, + scheduler=pipeline.scheduler, + feature_extractor=pipeline.feature_extractor, + compile=False, + ) + apply_moc_transformations(ov_pipe.unet.model, cf=False) + compress_quantize_weights_transformation(ov_pipe.unet.model) + ov_pipe.save_pretrained(save_dir) + + +class UnetInitDataset(torch.utils.data.Dataset): + def __init__(self, data): + super().__init__() + self.init_data = data + + def __len__(self): + return len(self.init_data) + + def __getitem__(self, index): + return self.init_data[index] + + +def prepare_nncf_init_data(pipeline, dataloader, args): + weight_dtype = torch.float32 + text_encoder = pipeline.text_encoder + vae = pipeline.vae + noise_scheduler = pipeline.scheduler + + nncf_init_data = [] + + logger.info(f"Fetching {args.opt_init_steps} for the initialization...") + for _, batch in tqdm(zip(range(args.opt_init_steps), itertools.islice(dataloader, 0, args.opt_init_steps))): + with torch.no_grad(): + # Convert images to latent space + latents = vae.encode(batch["pixel_values"].to(weight_dtype)).latent_dist.sample() + latents = latents * 0.18215 + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + encoder_hidden_states = text_encoder(batch["input_ids"])[0] + nncf_init_data.append( + ( + torch.squeeze(noisy_latents).to("cpu"), + torch.squeeze(timesteps).to("cpu"), + torch.squeeze(encoder_hidden_states).to("cpu"), + 0, + ) + ) + return nncf_init_data + + +# The config should work for Stable Diffusion v1.4-2.1 +def get_nncf_config(pipeline, dataloader, args): + text_encoder = pipeline.text_encoder + unet = pipeline.unet + nncf_config_dict = { + "input_info": [ + { # "keyword": "latent_model_input", + "sample_size": [1, unet.config["in_channels"], unet.config["sample_size"], unet.config["sample_size"]] + }, + {"sample_size": [1]}, # "keyword": "t", + { # "keyword": "encoder_hidden_states", + "sample_size": [1, text_encoder.config.max_position_embeddings, text_encoder.config.hidden_size] + }, + ], + "log_dir": args.output_dir, # The log directory for NNCF-specific logging outputs. + "compression": [ + { + "algorithm": "quantization", # Specify the algorithm here. + "preset": "mixed", + "initializer": { + "range": {"num_init_samples": args.opt_init_steps, "type": args.opt_init_type}, + "batchnorm_adaptation": {"num_bn_adaptation_samples": args.opt_init_steps}, + }, + "scope_overrides": {"activations": {"{re}.*baddbmm_0": {"mode": "symmetric"}}}, + "ignored_scopes": [ + "{re}.*__add___[0-2]", + "{re}.*layer_norm_0", + "{re}.*Attention.*/bmm_0", + "{re}.*__truediv__*", + "{re}.*group_norm_0", + "{re}.*mul___[0-2]", + "{re}.*silu_[0-2]", + ], + "export_to_onnx_standard_ops": True, + }, + ], + } + if args.use_kd: + nncf_config_dict["compression"].append({"algorithm": "knowledge_distillation", "type": "mse"}) # or ""softmax + + class UnetInitDataLoader(PTInitializingDataLoader): + def get_inputs(self, dataloader_output): + noisy_latents = dataloader_output[0].float().to(unet.device, non_blocking=True) + timesteps = dataloader_output[1].float().to(unet.device, non_blocking=True) + encoder_hidden_states = dataloader_output[2].float().to(unet.device, non_blocking=True) + return (noisy_latents, timesteps, encoder_hidden_states), {} + + def get_target(self, dataloader_output): + return dataloader_output[0] + + nncf_config = NNCFConfig.from_dict(nncf_config_dict) + nncf_config = register_default_init_args(nncf_config, UnetInitDataLoader(dataloader)) + return nncf_config + + +def main(): + args = parse_args() + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + logging_dir=logging_dir, + ) + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + + logger.info(accelerator.state, main_process_only=False) + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.push_to_hub: + if args.hub_model_id is None: + repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token) + else: + repo_name = args.hub_model_id + repo = Repository(args.output_dir, clone_from=repo_name) + + with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore: + if "step_*" not in gitignore: + gitignore.write("step_*\n") + if "epoch_*" not in gitignore: + gitignore.write("epoch_*\n") + elif args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + pipeline = DiffusionPipeline.from_pretrained(args.model_id) + + # Load models and create wrapper for stable diffusion + tokenizer = pipeline.tokenizer + text_encoder = pipeline.text_encoder + vae = pipeline.vae + unet = pipeline.unet + noise_scheduler = pipeline.scheduler + if args.noise_scheduler: + noise_scheduler = get_noise_scheduler(args) + + # Freeze vae and text_encoder + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + + if args.gradient_checkpointing: + unet.enable_gradient_checkpointing() + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + dataset_settings = dataset_name_mapping.get(args.dataset_name, None) + if dataset_settings is None: + raise ValueError( + f"Dataset {args.dataset_name} not supported. Please choose from {dataset_name_mapping.keys()}" + ) + + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, + streaming=dataset_settings["streaming"], + ) + else: + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, + ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + + # 6. Get the column names for input/target. + dataset_columns = dataset_settings["columns"] + caption_column = dataset_columns[1] + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images. + def tokenize_captions(examples, is_train=True): + captions = [] + caption = examples[caption_column] + if isinstance(caption, str): + captions.append(caption) + elif isinstance(caption, (list, np.ndarray)): + # take a random caption if there are multiple + captions.append(random.choice(caption) if is_train else caption[0]) + else: + raise ValueError(f"Caption column `{caption_column}` should contain either strings or lists of strings.") + inputs = tokenizer(captions[0], max_length=tokenizer.model_max_length, padding="do_not_pad", truncation=True) + input_ids = inputs.input_ids + return input_ids + + train_transforms = transforms.Compose( + [ + transforms.Resize((args.resolution, args.resolution), interpolation=transforms.InterpolationMode.BILINEAR), + transforms.CenterCrop(args.resolution) if args.center_crop else transforms.RandomCrop(args.resolution), + transforms.RandomHorizontalFlip() if args.random_flip else transforms.Lambda(lambda x: x), + transforms.ToTensor(), + transforms.Normalize([0.5], [0.5]), + ] + ) + + preprocess_fn = partial( + dataset_settings["preprocess_fn"], train_transforms=train_transforms, tokenize_captions=tokenize_captions + ) + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=42, buffer_size=args.max_train_samples) + # Set the training transforms + train_dataset = dataset["train"] + + def collate_fn(examples): + examples = [preprocess_fn(example) for example in examples] + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float() + input_ids = [example["input_ids"] for example in examples] + padded_tokens = tokenizer.pad({"input_ids": input_ids}, padding=True, return_tensors="pt") + return { + "pixel_values": pixel_values, + "input_ids": padded_tokens.input_ids, + "attention_mask": padded_tokens.attention_mask, + } + + train_dataloader = torch.utils.data.DataLoader( + train_dataset, collate_fn=collate_fn, batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers + ) + + unet = accelerator.prepare(unet) + vae.to(unet.device) + text_encoder.to(unet.device) + train_dataloader = accelerator.prepare_data_loader(train_dataloader) + orig_unet = unet # save link to original unet model for EMA + + ## Create initialization dataset for PTQ + nncf_init_data = prepare_nncf_init_data(pipeline, train_dataloader, args) + init_dataloader = torch.utils.data.DataLoader(UnetInitDataset(nncf_init_data), batch_size=1, num_workers=1) + nncf_config = get_nncf_config(pipeline, init_dataloader, args) + + # Quantize the model and initialize quantizer using init data + compression_controller, unet = create_compressed_model(unet, nncf_config) + + statistics_unet = compression_controller.statistics() + logger.info(statistics_unet.to_str()) + + del nncf_init_data, init_dataloader + torch.cuda.empty_cache() + + unet.train() + + if args.tune_quantizers_only: + for p in unet.parameters(): + if not isinstance(p, CompressionParameter): + p.requires_grad = False + + # Reinit + optimizer = optimizer_cls( + filter(lambda p: p.requires_grad, unet.parameters()), + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + dataset_len = args.max_train_samples if args.max_train_samples is not None else len(train_dataloader) + num_update_steps_per_epoch = math.ceil(dataset_len / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + unet, optimizer, train_dataloader, lr_scheduler + ) + + weight_dtype = torch.float32 + if args.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif args.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + + # Move text_encode and vae to gpu. + # For mixed precision training we cast the text_encoder and vae weights to half-precision + # as these models are only used for inference, keeping weights in full precision is not required. + text_encoder.to(accelerator.device, dtype=weight_dtype) + vae.to(accelerator.device, dtype=weight_dtype) + + # Create EMA for the unet. + if args.ema_device: + ema_unet = EMAQUnet(orig_unet.parameters()) + if args.ema_device == "cpu": + ema_unet.to("cpu") + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(dataset_len / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + accelerator.init_trackers("text2image-fine-tune", config=vars(args)) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {dataset_len}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + + # Only show the progress bar once on each machine. + progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("Steps") + global_step = 0 + + for epoch in range(args.num_train_epochs): + train_loss = 0.0 + compression_controller.scheduler.epoch_step() + + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(unet): + # Convert images to latent space + latents = vae.encode(batch["pixel_values"].to(weight_dtype)).latent_dist.sample() + latents = latents * 0.18215 + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + # Get the text embedding for conditioning + encoder_hidden_states = text_encoder(batch["input_ids"])[0] + + # Predict the noise residual and compute loss + noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample + loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="mean") + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + compression_loss_unet = compression_controller.loss() + loss = loss + compression_loss_unet + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(unet.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.ema_device: + ema_unet.step(orig_unet.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + unet = accelerator.unwrap_model(unet) + if args.ema_device: + ema_unet.copy_to(orig_unet.parameters()) + + if args.push_to_hub: + repo.push_to_hub(commit_message="End of training", blocking=False, auto_lfs_prune=True) + + accelerator.end_training() + + # Export optimized pipline to OpenVINO + export_unet = compression_controller.strip(do_copy=False) + export_pipeline = StableDiffusionPipeline( + text_encoder=text_encoder, + vae=vae, + unet=export_unet, + tokenizer=tokenizer, + scheduler=noise_scheduler, + safety_checker=pipeline.safety_checker, + feature_extractor=pipeline.feature_extractor, + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + export_to_onnx(export_pipeline, tmpdirname) + export_to_openvino(export_pipeline, tmpdirname, Path(args.output_dir) / "openvino") + + +if __name__ == "__main__": + main() diff --git a/notebooks/openvino/README.md b/notebooks/openvino/README.md index 058a0a2b38..611228dc35 100644 --- a/notebooks/openvino/README.md +++ b/notebooks/openvino/README.md @@ -12,4 +12,5 @@ The notebooks have been tested with Python 3.8 and 3.10 on Ubuntu Linux. |:----------|:-------------|:-------------|------:| | [How to run inference with the OpenVINO](https://github.com/huggingface/optimum-intel/blob/main/notebooks/openvino/optimum_openvino_inference.ipynb) | Explains how to export your model to OpenVINO and to run inference with OpenVINO Runtime on various tasks| [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/optimum-intel/blob/main/notebooks/openvino/optimum_openvino_inference.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/optimum-intel/blob/main/notebooks/openvino/optimum_openvino_inference.ipynb)| | [How to quantize a question answering model with OpenVINO NNCF](https://github.com/huggingface/optimum-intel/blob/main/notebooks/openvino/question_answering_quantization.ipynb) | Show how to apply post-training quantization on a question answering model using [NNCF](https://github.com/openvinotoolkit/nncf) and to accelerate inference with OpenVINO| [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/optimum-intel/blob/main/notebooks/openvino/question_answering_quantization.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/optimum-intel/blob/main/notebooks/openvino/question_answering_quantization.ipynb)| +| [Compare outputs of a quantized Stable Diffusion model with its full-precision counterpart](https://github.com/huggingface/optimum-intel/blob/main/notebooks/openvino/stable_diffusion_quantization.ipynb) | Show how to load and compare outputs from two Stable Diffusion models with different precision| [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/optimum-intel/blob/main/notebooks/openvino/stable_diffusion_quantization.ipynb)| [![Open in AWS Studio](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/huggingface/optimum-intel/blob/main/notebooks/openvino/stable_diffusion_quantization.ipynb)| diff --git a/notebooks/openvino/stable_diffusion_quantization.ipynb b/notebooks/openvino/stable_diffusion_quantization.ipynb new file mode 100644 index 0000000000..14b24996f6 --- /dev/null +++ b/notebooks/openvino/stable_diffusion_quantization.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Comparison of the results of the stable diffusion quantization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from optimum.intel.openvino import OVStableDiffusionPipeline\n", + "from diffusers.training_utils import set_seed\n", + "from IPython.display import display" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run the original pipeline\n", + "This pipeline was fine-tuned on the public [dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) with Pokemon images and the correspoinding captions. You can find the source model and the description [here](https://huggingface.co/svjack/Stable-Diffusion-Pokemon-en)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipe = OVStableDiffusionPipeline.from_pretrained(\"OpenVINO/stable-diffusion-pokemons-fp32\", compile=False)\n", + "pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1)\n", + "\n", + "pipe.compile()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's fix the seed for reproducibility.\n", + "set_seed(42)\n", + "\n", + "prompt = \"cartoon bird\"\n", + "output = pipe(prompt, num_inference_steps=50, output_type=\"pil\")\n", + "display(output.images[0])" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run the quantized pipeline\n", + "Now we run the quantized pipeline that was obtained with Quantization-Aware Training on the same dataset. The original model was used as a baseline for quantization. The resulted model can be found [here](https://huggingface.co/OpenVINO/Stable-Diffusion-Pokemon-en-quantized)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quantized_pipe = OVStableDiffusionPipeline.from_pretrained(\"OpenVINO/Stable-Diffusion-Pokemon-en-quantized\", compile=False)\n", + "quantized_pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1)\n", + "quantized_pipe.compile()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use the same seed to compare\n", + "set_seed(42)\n", + "\n", + "output = quantized_pipe(prompt, num_inference_steps=50, output_type=\"pil\")\n", + "display(output.images[0])" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now you can see the difference of the difference in the results and the time required to generate the image." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.10 ('stable_diffusion')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "7918409a64d3d4275e0103fc4443d9be5863d1df136c02ed032407c7ae821339" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 1423bbc9270e5f5faf70d896d5c1b58846881fbd Mon Sep 17 00:00:00 2001 From: jiqing-feng <107918818+jiqing-feng@users.noreply.github.com> Date: Tue, 25 Apr 2023 23:29:27 +0800 Subject: [PATCH 002/134] add use_cache in model_kwargs (#299) --- optimum/intel/generation/modeling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index 2abd34eec8..441efc037a 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -205,6 +205,7 @@ def _from_transformers( "subfolder": subfolder, "local_files_only": local_files_only, "force_download": force_download, + "use_cache": use_cache, "torch_dtype": torch_dtype, } From caae4be38104d9d7426960675755a8550f6f2eb2 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:30:05 +0200 Subject: [PATCH 003/134] Enable export of decoder of quantized models to be exported with pkv (#303) * Export decoder of LM only once for quantized models * fix default opset * add test * fix style --- optimum/intel/openvino/quantization.py | 74 +++++++++++++++----------- optimum/intel/openvino/trainer.py | 42 ++++----------- tests/openvino/test_quantization.py | 63 ++++++++++++++-------- 3 files changed, 94 insertions(+), 85 deletions(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 6d5f5dbda6..0c7ff2d891 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -32,6 +32,7 @@ from openvino._offline_transformations import compress_quantize_weights_transformation from openvino.runtime import Core from torch.onnx import export as onnx_export +from torch.utils._pytree import tree_map from torch.utils.data import DataLoader, RandomSampler from transformers import DataCollator, PreTrainedModel, default_data_collator @@ -171,14 +172,21 @@ def quantize( task=self.task, model_type=model_type, ) - onnx_config = onnx_config_class(self.model.config) + + if self.task == "text-generation": + onnx_config = onnx_config_class(self.model.config, use_past=self.model.config.use_cache) + else: + onnx_config = onnx_config_class(self.model.config) + compressed_model.eval() num_parameters = compressed_model.num_parameters() save_as_external_data = use_external_data_format(num_parameters) or quantization_config.save_onnx_model f = io.BytesIO() if not save_as_external_data else save_directory / ONNX_WEIGHTS_NAME # Export the compressed model to the ONNX format - self._onnx_export(compressed_model, onnx_config, model_inputs, quantization_config, f) + opset = min(onnx_config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) + opset = opset if not quantization_config.save_onnx_model else max(opset, MIN_ONNX_QDQ_OPSET) + _onnx_export_nncf_model(compressed_model, onnx_config, f, opset) # Load and save the compressed model model = core.read_model(f) if save_as_external_data else core.read_model(f.getvalue(), b"") @@ -190,35 +198,6 @@ def _save_pretrained(model: openvino.runtime.Model, output_path: str): compress_quantize_weights_transformation(model) openvino.runtime.serialize(model, output_path, output_path.replace(".xml", ".bin")) - def _onnx_export( - self, - model: NNCFNetwork, - config: OnnxConfig, - model_inputs: Dict, - ov_config: OVConfig, - f: Union[str, io.BytesIO], - ): - opset = min(config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) - opset = opset if not ov_config.save_onnx_model else max(opset, MIN_ONNX_QDQ_OPSET) - model_inputs = {k: v.to(model.device) for k, v in model_inputs.items()} - # Create ordered inputs for the ONNX export of NNCFNetwork as keyword arguments are currently not supported - inputs = tuple([model_inputs.pop(key, None) for key in self._export_input_names if len(model_inputs) != 0]) - - with torch.no_grad(): - # Disable node additions to be exported in the graph - model.disable_dynamic_graph_building() - onnx_export( - model, - inputs, - f=f, - input_names=list(config.inputs.keys()), - output_names=list(config.outputs.keys()), - dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), - do_constant_folding=True, - opset_version=opset, - ) - model.enable_dynamic_graph_building() - def _set_task(self): if self.task is None: self.task = HfApi().model_info(self.model.config._name_or_path).pipeline_tag @@ -299,3 +278,36 @@ def _get_calibration_dataloader( def _remove_unused_columns(self, dataset: Dataset): ignored_columns = list(set(dataset.column_names) - set(self._signature_columns)) return dataset.remove_columns(ignored_columns) + + +def _onnx_export_nncf_model(model: NNCFNetwork, config: OnnxConfig, output: Union[str, io.BytesIO], opset: int = None): + signature = inspect.signature(model.get_nncf_wrapped_model().forward) + signature = list(signature.parameters.keys()) + opset = opset or config.DEFAULT_ONNX_OPSET + model_inputs = config.generate_dummy_inputs(framework="pt") + # Create ordered inputs for the ONNX export of NNCFNetwork as keyword arguments are currently not supported + model_inputs = tuple(model_inputs.pop(key, None) for key in signature if len(model_inputs) != 0) + device = model.device + + def remap(value): + if isinstance(value, torch.Tensor): + value = value.to(device) + return value + + with config.patch_model_for_export(model.get_nncf_wrapped_model()): + model_inputs = tree_map(remap, model_inputs) + with torch.no_grad(): + model.eval() + # Disable node additions to be exported in the graph + model.disable_dynamic_graph_building() + onnx_export( + model, + model_inputs, + f=output, + input_names=list(config.inputs.keys()), + output_names=list(config.outputs.keys()), + dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), + do_constant_folding=True, + opset_version=opset, + ) + model.enable_dynamic_graph_building() diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index 51edb4e9bc..3e5b2b2ccc 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -19,7 +19,6 @@ import sys import time from collections import defaultdict -from itertools import chain from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Type, Union @@ -46,7 +45,6 @@ apply_moc_transformations, apply_user_transformations, ) -from torch.onnx import export as onnx_export from torch.utils.data import DataLoader, Dataset, RandomSampler from torch.utils.data.distributed import DistributedSampler from tqdm.auto import tqdm @@ -78,11 +76,10 @@ ) from optimum.exporters import TasksManager -from optimum.exporters.onnx import OnnxConfig from ..utils.constant import _TASK_ALIASES from .configuration import OVConfig -from .quantization import OVDataLoader +from .quantization import OVDataLoader, _onnx_export_nncf_model from .training_args import OVTrainingArguments from .utils import ( MAX_ONNX_OPSET, @@ -692,11 +689,18 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): model_type=model_type, ) - onnx_config = onnx_config_class(self.model.config) + if self.task == "text-generation": + onnx_config = onnx_config_class(self.model.config, use_past=self.model.config.use_cache) + else: + onnx_config = onnx_config_class(self.model.config) + num_parameters = self.model.num_parameters() save_as_external_data = use_external_data_format(num_parameters) or self.ov_config.save_onnx_model f = io.BytesIO() if not save_as_external_data else os.path.join(output_dir, ONNX_WEIGHTS_NAME) - self._onnx_export(self.model, onnx_config, self.ov_config, f) + + opset = min(onnx_config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) + opset = opset if not self.ov_config.save_onnx_model else max(opset, MIN_ONNX_QDQ_OPSET) + _onnx_export_nncf_model(self.model, onnx_config, f, opset) ov_model = core.read_model(f) if save_as_external_data else core.read_model(f.getvalue(), b"") # Prune IR if structured pruning is conducted on the model @@ -762,29 +766,3 @@ def _set_task(self): if self.task is None: raise ValueError("The model task defining the model topology needs to be specified for the ONNX export.") self.task = _TASK_ALIASES.get(self.task, self.task) - - def _onnx_export(self, model: NNCFNetwork, config: OnnxConfig, ov_config: OVConfig, f: Union[str, io.BytesIO]): - opset = min(config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) - opset = opset if not ov_config.save_onnx_model else max(opset, MIN_ONNX_QDQ_OPSET) - model_inputs = config.generate_dummy_inputs(framework="pt") - device = model.device - model_inputs = {k: v.to(device) for k, v in model_inputs.items()} - self._set_signature_columns_if_needed() # find model input names needed in ONNX export - # Create ordered inputs for the ONNX export of NNCFNetwork as keyword arguments are currently not supported - inputs = tuple([model_inputs.pop(key, None) for key in self._signature_columns if len(model_inputs) != 0]) - - with torch.no_grad(): - model.eval() - # Disable node additions to be exported in the graph - model.disable_dynamic_graph_building() - onnx_export( - model, - inputs, - f=f, - input_names=list(config.inputs.keys()), - output_names=list(config.outputs.keys()), - dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), - do_constant_folding=True, - opset_version=opset, - ) - model.enable_dynamic_graph_building() diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index fcfce98766..d1a0891f59 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# ruff: noqa + import tempfile import unittest from functools import partial @@ -23,6 +25,8 @@ from transformers import ( AutoModelForQuestionAnswering, AutoModelForSequenceClassification, + AutoModelForCausalLM, + AutoModelForTokenClassification, AutoTokenizer, TrainingArguments, default_data_collator, @@ -32,43 +36,64 @@ OVConfig, OVModelForQuestionAnswering, OVModelForSequenceClassification, + OVModelForCausalLM, + OVModelForTokenClassification, OVQuantizer, OVTrainer, ) +_TASK_TO_DATASET = { + "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), + "text-classification": ("glue", "sst2", "sentence"), +} + + +def get_num_quantized_nodes(ov_model): + num_fake_quantize = 0 + num_int8 = 0 + for elem in ov_model.model.get_ops(): + if "FakeQuantize" in elem.name: + num_fake_quantize += 1 + for i in range(elem.get_output_size()): + if "8" in elem.get_output_element_type(i).get_type_name(): + num_int8 += 1 + return num_fake_quantize, num_int8 + class OVQuantizerTest(unittest.TestCase): + # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( - ("distilbert-base-uncased-finetuned-sst-2-english", 50, 38), + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 43, 32), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 71, 1), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) - def test_static_quantization(self, model_name, expected_fake_quantize, expected_int8): + def test_static_quantization(self, model_cls, model_name, expected_fake_quantize, expected_int8): + task = model_cls.export_feature + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] + def preprocess_function(examples, tokenizer): - return tokenizer(examples["sentence"], padding="max_length", max_length=128, truncation=True) + return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) with tempfile.TemporaryDirectory() as tmp_dir: - transformers_model = AutoModelForSequenceClassification.from_pretrained(model_name) + transformers_model = model_cls.auto_model_class.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) - quantizer = OVQuantizer.from_pretrained(transformers_model) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + quantizer = OVQuantizer.from_pretrained(transformers_model, task=task) + calibration_dataset = quantizer.get_calibration_dataset( - "glue", - dataset_config_name="sst2", + dataset_name, + dataset_config_name=dataset_config_name, preprocess_function=partial(preprocess_function, tokenizer=tokenizer), num_samples=10, dataset_split="train", ) quantizer.quantize(save_directory=tmp_dir, calibration_dataset=calibration_dataset) - model = OVModelForSequenceClassification.from_pretrained(tmp_dir) + model = model_cls.from_pretrained(tmp_dir) - num_int8 = 0 - num_fake_quantize = 0 - for elem in model.model.get_ops(): - if "FakeQuantize" in elem.name: - num_fake_quantize += 1 - if "8" in elem.get_element_type().get_type_name(): - num_int8 += 1 + num_fake_quantize, num_int8 = get_num_quantized_nodes(model) self.assertEqual(expected_fake_quantize, num_fake_quantize) self.assertEqual(expected_int8, num_int8) @@ -156,13 +181,7 @@ def compute_metrics(p): trainer.save_model() model = OVModelForSequenceClassification.from_pretrained(tmp_dir) - num_int8 = 0 - num_fake_quantize = 0 - for elem in model.model.get_ops(): - if "FakeQuantize" in elem.name: - num_fake_quantize += 1 - if "8" in elem.get_element_type().get_type_name(): - num_int8 += 1 + num_fake_quantize, num_int8 = get_num_quantized_nodes(model) self.assertEqual(expected_fake_quantize, num_fake_quantize) self.assertEqual(expected_int8, num_int8) From c9512a35d3d084f33d6e9ad5db1f358f9e7945d4 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 2 May 2023 09:58:17 +0200 Subject: [PATCH 004/134] enable loading of INC quantized stable diffusion model (#305) * add inc stable diffusion pipeline * enable loading of quantized stable diffusion model * add inc loading stable diffusion * add model * update model * test * fix inc config loading --- .github/workflows/test_inc.yml | 2 +- .../run_diffusion_post_training.py | 19 +++--- .../textual-inversion/text2images.py | 21 +----- .../textual-inversion/textual_inversion.py | 20 +++--- optimum/intel/__init__.py | 17 ++++- optimum/intel/neural_compressor/__init__.py | 5 ++ .../neural_compressor/modeling_diffusion.py | 55 +++++++++++++++ .../intel/neural_compressor/quantization.py | 15 ++--- optimum/intel/neural_compressor/trainer.py | 3 +- optimum/intel/neural_compressor/utils.py | 8 +-- optimum/intel/utils/constant.py | 7 ++ ...neural_compressor_and_diffusers_objects.py | 26 +++++++ tests/neural_compressor/test_optimization.py | 67 +++++++++++++++---- 13 files changed, 198 insertions(+), 67 deletions(-) create mode 100644 optimum/intel/neural_compressor/modeling_diffusion.py create mode 100644 optimum/intel/utils/dummy_neural_compressor_and_diffusers_objects.py diff --git a/.github/workflows/test_inc.yml b/.github/workflows/test_inc.yml index b35747cff3..fd5fd16509 100644 --- a/.github/workflows/test_inc.yml +++ b/.github/workflows/test_inc.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[neural-compressor,ipex,tests] + pip install .[neural-compressor,ipex,diffusers,tests] - name: Test with Pytest run: | pytest tests/neural_compressor/ diff --git a/examples/neural_compressor/text-to-image/run_diffusion_post_training.py b/examples/neural_compressor/text-to-image/run_diffusion_post_training.py index 359afdd9ce..ba1ba7f0ab 100644 --- a/examples/neural_compressor/text-to-image/run_diffusion_post_training.py +++ b/examples/neural_compressor/text-to-image/run_diffusion_post_training.py @@ -32,8 +32,8 @@ from pytorch_fid import fid_score from torch.utils.data import Dataset -from optimum.intel.neural_compressor import INCQuantizer -from optimum.intel.neural_compressor.utils import load_quantized_model +from optimum.intel import INCQuantizer, INCStableDiffusionPipeline +from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME os.environ["CUDA_VISIBLE_DEVICES"] = "" @@ -254,31 +254,30 @@ def eval_func(model): ) quantization_config = PostTrainingQuantConfig(approach=args.quantization_approach) + pipeline.save_pretrained(args.output_dir) quantizer = INCQuantizer.from_pretrained(pipeline.unet, calibration_fn=calibration_func) quantizer.quantize( quantization_config=quantization_config, - save_directory=args.output_dir, + save_directory=os.path.join(args.output_dir, "unet"), calibration_dataset=CalibDataset() if args.quantization_approach == "static" else None, remove_unused_columns=False, + file_name=DIFFUSION_WEIGHTS_NAME, ) if args.apply_quantization and args.verify_loading: - loaded_model = load_quantized_model(args.output_dir, model=getattr(pipeline, "unet")) + int8_pipeline = INCStableDiffusionPipeline.from_pretrained(args.output_dir) result_optimized_model = eval_func(quantizer._quantized_model) - result_loaded_model = eval_func(loaded_model) + result_loaded_model = eval_func(int8_pipeline.unet) if result_loaded_model != result_optimized_model: logger.error("The quantized model was not successfully loaded.") else: logger.info("The quantized model was successfully loaded.") if args.benchmark and args.int8: - print("====int8 inference====") - loaded_model = load_quantized_model(args.output_dir, model=getattr(pipeline, "unet")) - loaded_model.eval() - setattr(pipeline, "unet", loaded_model) + int8_pipeline = INCStableDiffusionPipeline.from_pretrained(args.output_dir) generator = torch.Generator("cpu").manual_seed(args.seed) - benchmark(pipeline, generator) + benchmark(int8_pipeline, generator) def _mp_fn(index): diff --git a/examples/neural_compressor/textual-inversion/text2images.py b/examples/neural_compressor/textual-inversion/text2images.py index 64f515fdbb..dad01404da 100644 --- a/examples/neural_compressor/textual-inversion/text2images.py +++ b/examples/neural_compressor/textual-inversion/text2images.py @@ -3,10 +3,9 @@ import os import torch -from diffusers import AutoencoderKL, StableDiffusionPipeline, UNet2DConditionModel -from neural_compressor.utils.pytorch import load from PIL import Image -from transformers import CLIPTextModel, CLIPTokenizer + +from optimum.intel import INCStableDiffusionPipeline def parse_args(): @@ -86,23 +85,9 @@ def generate_images( args = parse_args() -# Load models and create wrapper for stable diffusion -tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer") -text_encoder = CLIPTextModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="text_encoder") -vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path, subfolder="vae") -unet = UNet2DConditionModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="unet") -pipeline = StableDiffusionPipeline.from_pretrained( - args.pretrained_model_name_or_path, text_encoder=text_encoder, vae=vae, unet=unet, tokenizer=tokenizer -) +pipeline = INCStableDiffusionPipeline.from_pretrained(args.output_dir).to(torch.device("cpu")) pipeline.safety_checker = lambda images, clip_input: (images, False) -if os.path.exists(os.path.join(args.pretrained_model_name_or_path, "best_model.pt")): - unet = load(args.pretrained_model_name_or_path, model=unet) - unet.eval() - setattr(pipeline, "unet", unet) -else: - unet = unet.to(torch.device("cuda", args.cuda_id)) -pipeline = pipeline.to(unet.device) grid, images = generate_images(pipeline, prompt=args.caption, num_images_per_prompt=args.images_num, seed=args.seed) grid.save(os.path.join(args.pretrained_model_name_or_path, "{}.png".format("_".join(args.caption.split())))) dirname = os.path.join(args.pretrained_model_name_or_path, "_".join(args.caption.split())) diff --git a/examples/neural_compressor/textual-inversion/textual_inversion.py b/examples/neural_compressor/textual-inversion/textual_inversion.py index 933d89fbf6..efe25080fd 100644 --- a/examples/neural_compressor/textual-inversion/textual_inversion.py +++ b/examples/neural_compressor/textual-inversion/textual_inversion.py @@ -21,7 +21,6 @@ from neural_compressor.config import DistillationConfig, IntermediateLayersKnowledgeDistillationLossConfig from neural_compressor.training import prepare_compression from neural_compressor.utils import logger -from neural_compressor.utils.pytorch import load from packaging import version from PIL import Image from torch.utils.data import Dataset @@ -29,6 +28,9 @@ from tqdm.auto import tqdm from transformers import CLIPTextModel, CLIPTokenizer +from optimum.intel import INCStableDiffusionPipeline +from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME + if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): PIL_INTERPOLATION = { @@ -949,7 +951,10 @@ def attention_fetcher(x): compression_manager.callbacks.on_train_end() # Save the resulting model and its corresponding configuration in the given directory - model.save(args.output_dir) + state_dict = model.state_dict() + if hasattr(model, "q_config"): + state_dict["best_configure"] = model.q_config + torch.save(state_dict, os.path.join(args.output_dir, "unet", DIFFUSION_WEIGHTS_NAME)) logger.info(f"Optimized model saved to: {args.output_dir}.") @@ -994,15 +999,8 @@ def attention_fetcher(x): accelerator.end_training() if args.do_quantization and args.verify_loading: - # Load the model obtained after Intel Neural Compressor quantization - loaded_model = load(args.output_dir, model=unet) - loaded_model.eval() - - setattr(pipeline, "unet", loaded_model) - if args.do_quantization: - pipeline = pipeline.to(torch.device("cpu")) - - loaded_model_images = generate_images(pipeline, prompt=prompt, seed=args.seed) + int8_pipeline = INCStableDiffusionPipeline.from_pretrained(args.output_dir).to(torch.device("cpu")) + loaded_model_images = generate_images(int8_pipeline, prompt=prompt, seed=args.seed) if loaded_model_images != optimized_model_images: logger.info("The quantized model was not successfully loaded.") else: diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index 4951d98173..e465240bb9 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -52,7 +52,6 @@ else: _import_structure["openvino"].extend(["OVConfig", "OVQuantizer", "OVTrainer", "OVTrainingArguments"]) - try: if not (is_openvino_available() and is_diffusers_available()): raise OptionalDependencyNotAvailable() @@ -109,6 +108,13 @@ "INCSeq2SeqTrainer", "INCTrainer", ] +try: + if not (is_neural_compressor_available() and is_diffusers_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + _import_structure["utils.dummy_neural_compressor_and_diffusers_objects"] = ["INCStableDiffusionPipeline"] +else: + _import_structure["neural_compressor"].append("INCStableDiffusionPipeline") if TYPE_CHECKING: @@ -175,6 +181,15 @@ INCSeq2SeqTrainer, INCTrainer, ) + + try: + if not (is_neural_compressor_available() and is_diffusers_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_neural_compressor_and_diffusers_objects import INCStableDiffusionPipeline + else: + from .neural_compressor import INCStableDiffusionPipeline + else: import sys diff --git a/optimum/intel/neural_compressor/__init__.py b/optimum/intel/neural_compressor/__init__.py index 58ebec4523..66db62ac1b 100644 --- a/optimum/intel/neural_compressor/__init__.py +++ b/optimum/intel/neural_compressor/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ..utils.import_utils import is_diffusers_available from .configuration import INCConfig from .quantization import ( INCModel, @@ -28,3 +29,7 @@ ) from .trainer import INCTrainer from .trainer_seq2seq import INCSeq2SeqTrainer + + +if is_diffusers_available(): + from .modeling_diffusion import INCStableDiffusionPipeline diff --git a/optimum/intel/neural_compressor/modeling_diffusion.py b/optimum/intel/neural_compressor/modeling_diffusion.py new file mode 100644 index 0000000000..c0031b0e55 --- /dev/null +++ b/optimum/intel/neural_compressor/modeling_diffusion.py @@ -0,0 +1,55 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from diffusers import StableDiffusionPipeline +from neural_compressor.utils.pytorch import load + +from ..utils.constant import DIFFUSION_WEIGHTS_NAME, WEIGHTS_NAME +from ..utils.import_utils import _torch_version, is_torch_version +from .configuration import INCConfig + + +class INCStableDiffusionPipeline(StableDiffusionPipeline): + @classmethod + def from_pretrained(cls, *args, **kwargs): + model = super(INCStableDiffusionPipeline, cls).from_pretrained(*args, low_cpu_mem_usage=False, **kwargs) + components = set(model.config.keys()).intersection({"vae", "text_encoder", "unet"}) + for name in components: + component = getattr(model, name, None) + name_or_path = "" + if hasattr(component, "_internal_dict"): + name_or_path = component._internal_dict["_name_or_path"] + elif hasattr(component, "name_or_path"): + name_or_path = component.name_or_path + if os.path.isdir(name_or_path): + folder_contents = os.listdir(name_or_path) + file_name = DIFFUSION_WEIGHTS_NAME if DIFFUSION_WEIGHTS_NAME in folder_contents else WEIGHTS_NAME + state_dict_path = os.path.join(name_or_path, file_name) + if os.path.exists(state_dict_path) and INCConfig.CONFIG_NAME in folder_contents: + msg = None + inc_config = INCConfig.from_pretrained(name_or_path) + if not is_torch_version("==", inc_config.torch_version): + msg = f"Quantized model was obtained with torch version {inc_config.torch_version} but {_torch_version} was found." + state_dict = torch.load(state_dict_path, map_location="cpu") + if "best_configure" in state_dict and state_dict["best_configure"] is not None: + try: + load(state_dict_path, component) + except Exception as e: + if msg is not None: + e.args += (msg,) + raise + return model diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index 44b7a618b2..16c9905920 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -20,12 +20,13 @@ from enum import Enum from itertools import chain from pathlib import Path -from typing import TYPE_CHECKING, Callable, ClassVar, Dict, Optional, Union +from typing import Callable, ClassVar, Dict, Optional, Union import torch from datasets import Dataset, load_dataset from huggingface_hub import HfApi, hf_hub_download from neural_compressor.adaptor.pytorch import PyTorch_FXAdaptor, _cfg_to_qconfig, _propagate_qconfig +from neural_compressor.config import PostTrainingQuantConfig from neural_compressor.experimental.export import torch_to_int8_onnx from neural_compressor.model.torch_model import IPEXModel, PyTorchModel from neural_compressor.quantization import fit @@ -57,7 +58,7 @@ from optimum.exporters.onnx import OnnxConfig from optimum.quantization_base import OptimumQuantizer -from ..utils.constant import _TASK_ALIASES +from ..utils.constant import _TASK_ALIASES, MIN_QDQ_ONNX_OPSET, ONNX_WEIGHTS_NAME, WEIGHTS_NAME from ..utils.import_utils import ( _neural_compressor_version, _torch_version, @@ -65,11 +66,7 @@ is_torch_version, ) from .configuration import INCConfig -from .utils import MIN_QDQ_ONNX_OPSET, ONNX_WEIGHTS_NAME, WEIGHTS_NAME, INCDataLoader, _cfgs_to_fx_cfgs - - -if TYPE_CHECKING: - from neural_compressor.config import PostTrainingQuantConfig +from .utils import INCDataLoader, _cfgs_to_fx_cfgs logger = logging.getLogger(__name__) @@ -141,6 +138,7 @@ def quantize( batch_size: int = 8, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + file_name: str = None, **kwargs, ): """ @@ -163,7 +161,7 @@ def quantize( save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) save_onnx_model = kwargs.pop("save_onnx_model", False) - output_path = save_directory.joinpath(WEIGHTS_NAME) + output_path = save_directory.joinpath(file_name or WEIGHTS_NAME) calibration_dataloader = None if INCQuantizationMode(quantization_config.approach) == INCQuantizationMode.STATIC: @@ -228,6 +226,7 @@ def _save_pretrained(model: Union[PyTorchModel, IPEXModel], output_path: str): logger.info(f"Model weights saved to {output_path}") return state_dict = model._model.state_dict() + if hasattr(model, "q_config"): state_dict["best_configure"] = model.q_config torch.save(state_dict, output_path) diff --git a/optimum/intel/neural_compressor/trainer.py b/optimum/intel/neural_compressor/trainer.py index ee29d2c351..eda52a5cea 100644 --- a/optimum/intel/neural_compressor/trainer.py +++ b/optimum/intel/neural_compressor/trainer.py @@ -63,10 +63,9 @@ from optimum.exporters import TasksManager -from ..utils.constant import _TASK_ALIASES +from ..utils.constant import _TASK_ALIASES, MIN_QDQ_ONNX_OPSET, ONNX_WEIGHTS_NAME, TRAINING_ARGS_NAME from ..utils.import_utils import is_neural_compressor_version from .configuration import INCConfig -from .utils import MIN_QDQ_ONNX_OPSET, ONNX_WEIGHTS_NAME, TRAINING_ARGS_NAME if is_apex_available(): diff --git a/optimum/intel/neural_compressor/utils.py b/optimum/intel/neural_compressor/utils.py index 573a970734..50e928292c 100644 --- a/optimum/intel/neural_compressor/utils.py +++ b/optimum/intel/neural_compressor/utils.py @@ -14,6 +14,7 @@ import logging import os +import warnings from collections import UserDict from typing import Dict @@ -22,15 +23,13 @@ from packaging import version from torch.utils.data import DataLoader +from ..utils.constant import WEIGHTS_NAME + logger = logging.getLogger(__name__) CONFIG_NAME = "best_configure.yaml" -WEIGHTS_NAME = "pytorch_model.bin" -TRAINING_ARGS_NAME = "training_args.bin" -ONNX_WEIGHTS_NAME = "model.onnx" -MIN_QDQ_ONNX_OPSET = 14 parsed_torch_version_base = version.parse(version.parse(torch.__version__).base_version) is_torch_less_than_1_13 = parsed_torch_version_base < version.parse("1.13.0") @@ -102,6 +101,7 @@ def load_quantized_model(checkpoint_dir_or_file: str, model: torch.nn.Module, ** model (`torch.nn.Module`): The original FP32 model. """ + warnings.warn("This function has been depreciated and will be removed in optimum-intel v1.9.") if os.path.isdir(checkpoint_dir_or_file): checkpoint_dir_or_file = os.path.join( os.path.abspath(os.path.expanduser(checkpoint_dir_or_file)), WEIGHTS_NAME diff --git a/optimum/intel/utils/constant.py b/optimum/intel/utils/constant.py index 7f1469de8a..413ccacca1 100644 --- a/optimum/intel/utils/constant.py +++ b/optimum/intel/utils/constant.py @@ -32,3 +32,10 @@ "summarization": "seq2seq-lm", "translation": "seq2seq-lm", } + + +WEIGHTS_NAME = "pytorch_model.bin" +DIFFUSION_WEIGHTS_NAME = "diffusion_pytorch_model.bin" +TRAINING_ARGS_NAME = "training_args.bin" +ONNX_WEIGHTS_NAME = "model.onnx" +MIN_QDQ_ONNX_OPSET = 14 diff --git a/optimum/intel/utils/dummy_neural_compressor_and_diffusers_objects.py b/optimum/intel/utils/dummy_neural_compressor_and_diffusers_objects.py new file mode 100644 index 0000000000..d98a09da37 --- /dev/null +++ b/optimum/intel/utils/dummy_neural_compressor_and_diffusers_objects.py @@ -0,0 +1,26 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .import_utils import DummyObject, requires_backends + + +class INCStableDiffusionPipeline(metaclass=DummyObject): + _backends = ["neural_compressor", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["neural_compressor", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["neural_compressor", "diffusers"]) diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index 2f848ed84a..66b26af1f8 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -21,6 +21,7 @@ import numpy as np import torch from datasets import load_dataset +from diffusers import StableDiffusionPipeline from neural_compressor.config import ( AccuracyCriterion, DistillationConfig, @@ -46,8 +47,10 @@ INCModelForQuestionAnswering, INCModelForSequenceClassification, INCQuantizer, + INCStableDiffusionPipeline, INCTrainer, ) +from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME from optimum.onnxruntime import ORTModelForSequenceClassification @@ -55,6 +58,14 @@ set_seed(1009) +def num_quantized_matmul_onnx_model(onnx_model): + num_quantized_matmul = 0 + for initializer in onnx_model.graph.initializer: + if "MatMul" in initializer.name and "quantized" in initializer.name: + num_quantized_matmul += 1 + return num_quantized_matmul + + class QuantizationTest(unittest.TestCase): def test_dynamic_quantization(self): model_name = "distilbert-base-uncased-finetuned-sst-2-english" @@ -78,10 +89,7 @@ def test_dynamic_quantization(self): self.assertTrue(inc_config.save_onnx_model) self.assertFalse(inc_config.quantization["is_static"]) - num_quantized_matmul = 0 - for initializer in onnx_model.graph.initializer: - if "MatMul" in initializer.name and "quantized" in initializer.name: - num_quantized_matmul += 1 + num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) ort_outputs = ort_model(**tokens) @@ -167,10 +175,7 @@ def preprocess_function(examples, tokenizer): self.assertTrue(inc_config.quantization["is_static"]) self.assertEqual(inc_config.quantization["dataset_num_samples"], num_samples) - num_quantized_matmul = 0 - for initializer in onnx_model.graph.initializer: - if "MatMul" in initializer.name and "quantized" in initializer.name: - num_quantized_matmul += 1 + num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) ort_outputs = ort_model(**tokens) @@ -257,10 +262,7 @@ def compute_metrics(p: EvalPrediction): self.assertTrue(inc_config.save_onnx_model) self.assertTrue(inc_config.quantization["is_static"]) - num_quantized_matmul = 0 - for initializer in onnx_model.graph.initializer: - if "MatMul" in initializer.name and "quantized" in initializer.name: - num_quantized_matmul += 1 + num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) ort_outputs = ort_model(**tokens) @@ -325,6 +327,47 @@ def compute_metrics(p: EvalPrediction): transformers_model(**tokens) # self.assertTrue(torch.allclose(ort_outputs.logits, transformers_outputs.logits, atol=1e-4)) + def test_dynamic_diffusion_model(self): + model_id = "hf-internal-testing/diffusers-stable-diffusion-tiny-all" + pipeline = StableDiffusionPipeline.from_pretrained(model_id) + pipeline.safety_checker = None + num_images_per_prompt, height, width, scale_factor = 1, 512, 512, 8 + latents_shape = ( + num_images_per_prompt, + pipeline.unet.in_channels, + height // scale_factor, + width // scale_factor, + ) + latents = np.random.randn(*latents_shape).astype(np.float32) + kwargs = { + "prompt": "sailing ship in storm by Leonardo da Vinci", + "num_inference_steps": 1, + "output_type": "np", + "num_images_per_prompt": num_images_per_prompt, + "height": height, + "width": width, + } + + pipeline.to("cpu") + quantization_config = PostTrainingQuantConfig(approach="dynamic") + quantizer = INCQuantizer.from_pretrained(pipeline.unet) + + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline.save_pretrained(tmp_dir) + quantizer.quantize( + quantization_config=quantization_config, + save_directory=os.path.join(tmp_dir, "unet"), + file_name=DIFFUSION_WEIGHTS_NAME, + ) + loaded_pipeline = INCStableDiffusionPipeline.from_pretrained(tmp_dir) + loaded_pipeline.to("cpu") + pipeline.unet = quantizer._quantized_model + with torch.no_grad(): + outputs = pipeline(latents=torch.from_numpy(latents), **kwargs).images + loaded_pipe_outputs = loaded_pipeline(latents=torch.from_numpy(latents), **kwargs).images + # Compare model outputs + self.assertTrue(np.allclose(loaded_pipe_outputs, outputs, atol=1e-4)) + class PruningTest(unittest.TestCase): def test_magnitude_pruning(self): From 78a41857a3e1cfd4859fcb12807abf082ee70767 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Wed, 3 May 2023 17:50:26 +0400 Subject: [PATCH 005/134] Added Token Merging optimization for Stable Diffusion on OpenVINO (#304) * Added Token Merging in combination with QAT for Stable Diffusion * Modifiedstable diffusion notebook * Chanded the SD readme * Style * Fixed requirements --- examples/openvino/stable-diffusion/README.md | 23 ++++++++++-- .../stable-diffusion/requirements.txt | 1 + .../train_text_to_image_qat.py | 29 +++++++++++++-- ...nb => stable_diffusion_optimization.ipynb} | 35 ++++++++++++++++++- 4 files changed, 82 insertions(+), 6 deletions(-) rename notebooks/openvino/{stable_diffusion_quantization.ipynb => stable_diffusion_optimization.ipynb} (75%) diff --git a/examples/openvino/stable-diffusion/README.md b/examples/openvino/stable-diffusion/README.md index d7c2a8faff..48cb99df29 100644 --- a/examples/openvino/stable-diffusion/README.md +++ b/examples/openvino/stable-diffusion/README.md @@ -1,9 +1,9 @@ # Stable Diffusion Quantization -This example demonstrates Quantization-aware Training (QAT) of Stable Diffusion using [NNCF](https://github.com/openvinotoolkit/nncf). Quantization is applyied to UNet model which is the most time-consuming element of the whole pipeline. The quantized model and the pipeline is exported to the OpenVINO format for inference with `OVStableDiffusionPipeline` helper. The original training code was taken from the Diffusers [repository](https://github.com/huggingface/diffusers/tree/main/examples/text_to_image) and modified to support QAT. +This example demonstrates how to apply Quantization-aware Training (QAT) from [NNCF](https://github.com/openvinotoolkit/nncf) and Token Merging method to optimize UNet model from Stable Diffusion pipeline. The optimized model and the pipeline are exported to the OpenVINO format for inference with `OVStableDiffusionPipeline` helper. The original training code was taken from the Diffusers [repository](https://github.com/huggingface/diffusers/tree/main/examples/text_to_image) and modified to support QAT. Knowledge distillation and EMA techniques can be used to improve the model accuracy. -This example supports model tuning on two datasets from the HuggingFace: +This example supports model tuning on three datasets from the HuggingFace: * [Pokemon BLIP captions](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) * [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en) * [laion2B-en-aesthetic](https://huggingface.co/datasets/laion/laion2B-en-aesthetic) @@ -23,6 +23,7 @@ pip install -r requirements.txt >**Note**: The example requires `torch~=1.13` and does not work with PyTorch 2.0. ## Running pre-optimized model +* You can also run the [notebook](../../../notebooks/openvino/stable_diffusion_optimization.ipynb) to compare FP32 pipeline with the optimized versions. * General-purpose image generation model: ```python from optimum.intel.openvino import OVStableDiffusionPipeline @@ -47,7 +48,6 @@ prompt = "cartoon bird" output = pipe(prompt, num_inference_steps=50, output_type="pil") output.images[0].save("result.png") ``` -* You can also run `pokemon_generation_demo.ipynb` notebook from the folder to compare FP32 pipeline with the optimized. ## HW Requirements for QAT The minimal HW setup for the run is GPU with 24GB of memory. @@ -71,6 +71,23 @@ python train_text_to_image_qat.py \ --output_dir=sd-quantized-pokemon ``` +* QAT + Token Merging (0.5 ratio) for pokemon generation model: +```python +python train_text_to_image_qat.py \ + --ema_device="cpu" \ + --use_kd \ + --model_id="svjack/Stable-Diffusion-Pokemon-en" \ + --center_crop \ + --random_flip \ + --gradient_checkpointing \ + --dataloader_num_workers=2 \ + --dataset_name="lambdalabs/pokemon-blip-captions" \ + --max_train_steps=8000 \ + --opt_init_steps=300 \ + --tome_ratio=0.5 \ + --output_dir=sd-quantized-pokemon +``` + * QAT on a laion-aesthetic dataset: ```python python train_text_to_image_qat.py \ diff --git a/examples/openvino/stable-diffusion/requirements.txt b/examples/openvino/stable-diffusion/requirements.txt index ccd29e4d02..8f8eb6a770 100644 --- a/examples/openvino/stable-diffusion/requirements.txt +++ b/examples/openvino/stable-diffusion/requirements.txt @@ -2,3 +2,4 @@ accelerate diffusers torch~=1.13 nncf @ git+https://github.com/openvinotoolkit/nncf.git +tomesd @ git+https://github.com/AlexKoff88/tomesd/tree/openvino diff --git a/examples/openvino/stable-diffusion/train_text_to_image_qat.py b/examples/openvino/stable-diffusion/train_text_to_image_qat.py index a748d2c4e0..1e6f6145c9 100644 --- a/examples/openvino/stable-diffusion/train_text_to_image_qat.py +++ b/examples/openvino/stable-diffusion/train_text_to_image_qat.py @@ -20,6 +20,7 @@ import os import random import tempfile +from copy import deepcopy from functools import partial from io import BytesIO from pathlib import Path @@ -27,6 +28,7 @@ import numpy as np import requests +import tomesd import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -486,6 +488,14 @@ def parse_args(): ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' ), ) + parser.add_argument( + "--tome_ratio", + type=float, + default=0, + help=( + "Token Merging ratio. If 0, no merging is applied" "More details here: https://arxiv.org/abs/2303.17604." + ), + ) parser.add_argument( "--opt_init_steps", type=int, @@ -667,8 +677,6 @@ def get_nncf_config(pipeline, dataloader, args): }, ], } - if args.use_kd: - nncf_config_dict["compression"].append({"algorithm": "knowledge_distillation", "type": "mse"}) # or ""softmax class UnetInitDataLoader(PTInitializingDataLoader): def get_inputs(self, dataloader_output): @@ -728,6 +736,15 @@ def main(): pipeline = DiffusionPipeline.from_pretrained(args.model_id) + if args.use_kd: + teacher_model = deepcopy(pipeline.unet) + + if args.tome_ratio > 0: + logger.info(f"Using Token Merging with ratio: {args.tome_ratio}") + tomesd.apply_patch( + pipeline, ratio=args.tome_ratio, use_rand=False + ) # Can also use pipe.unet in place of pipe here + # Load models and create wrapper for stable diffusion tokenizer = pipeline.tokenizer text_encoder = pipeline.text_encoder @@ -857,6 +874,8 @@ def collate_fn(examples): text_encoder.to(unet.device) train_dataloader = accelerator.prepare_data_loader(train_dataloader) orig_unet = unet # save link to original unet model for EMA + if args.use_kd: + teacher_model.to(unet.device) ## Create initialization dataset for PTQ nncf_init_data = prepare_nncf_init_data(pipeline, train_dataloader, args) @@ -978,8 +997,14 @@ def collate_fn(examples): # Predict the noise residual and compute loss noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample + loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="mean") + if args.use_kd: + with torch.no_grad(): + orig_output = teacher_model(noisy_latents, timesteps, encoder_hidden_states).sample + loss += F.mse_loss(noise_pred.float(), orig_output.float(), reduction="mean") + # Gather the losses across all processes for logging (if we use distributed training). avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() train_loss += avg_loss.item() / args.gradient_accumulation_steps diff --git a/notebooks/openvino/stable_diffusion_quantization.ipynb b/notebooks/openvino/stable_diffusion_optimization.ipynb similarity index 75% rename from notebooks/openvino/stable_diffusion_quantization.ipynb rename to notebooks/openvino/stable_diffusion_optimization.ipynb index 14b24996f6..6c79bc5df0 100644 --- a/notebooks/openvino/stable_diffusion_quantization.ipynb +++ b/notebooks/openvino/stable_diffusion_optimization.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Comparison of the results of the stable diffusion quantization" + "# Comparison of the results of the stable diffusion optimization" ] }, { @@ -87,6 +87,39 @@ "display(output.images[0])" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run the highly optimized pipeline\n", + "Here, we run the pipeline optimized with a combination of Token Merging Method and Quantization-aware training. The resulted model can be found [here](https://huggingface.co/OpenVINO/stable-diffusion-pokemons-tome-quantized)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "optimized_pipe = OVStableDiffusionPipeline.from_pretrained(\"OpenVINO/stable-diffusion-pokemons-tome-quantized\", compile=False)\n", + "optimized_pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1)\n", + "optimized_pipe.compile()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use the same seed to compare\n", + "set_seed(42)\n", + "\n", + "output = optimized_pipe(prompt, num_inference_steps=50, output_type=\"pil\")\n", + "display(output.images[0])" + ] + }, { "attachments": {}, "cell_type": "markdown", From 65d6bc46cd2c483117ee59c345835899e2fb1456 Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Wed, 3 May 2023 15:51:17 +0200 Subject: [PATCH 006/134] Use static shapes for SWIN model in training test (#306) Set static shape for swin model to workaround issue with dynamic batch size for tiny-swin in OpenVINO 2023.0 --- tests/openvino/test_training.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/openvino/test_training.py b/tests/openvino/test_training.py index 33a9649b8d..c9e529c55a 100644 --- a/tests/openvino/test_training.py +++ b/tests/openvino/test_training.py @@ -131,7 +131,9 @@ def run_ovtrainer_training_checks(self, desc: OVTrainerTestDescriptor): # check saved ovmodel IR and output ovmodel = self.get_ov_model() - self.check_if_ovmodel_is_dynamic(ovmodel, True) + # dynamic batch size for tiny-swin does not work in OpenVINO 2023.0 + is_swin = "swin" in desc.model_id.lower() + self.check_if_ovmodel_is_dynamic(ovmodel, expected_result=not is_swin) self.check_ovmodel_output_equals_torch_output(ovmodel, trainer.model) self.check_ovmodel_reshaping(ovmodel) @@ -621,19 +623,23 @@ def data_transform(examples, size=None): self.train_dataset = raw_dataset.select(range(8)) self.eval_dataset = raw_dataset.select(range(8, 12)) self.data_collator = default_data_collator + self.is_swin = "swin" in desc.model_id.lower() def get_ov_model(self, model_id=None) -> OVModel: # image models, e.g. swin, may require a determined image size model_id = model_id or self.output_dir size = (self.feature_extractor.size["height"], self.feature_extractor.size["width"]) ovmodel = self.ovmodel_cls.from_pretrained(model_id, compile=False) - ovmodel.reshape(-1, 3, *size) + # dynamic batch size for tiny-swin does not work in OpenVINO 2023.0 + batch_size = 1 if self.is_swin else -1 + ovmodel.reshape(batch_size, 3, *size) ovmodel.compile() return ovmodel def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model): torch_model = torch_model.eval() - for batch_size in [1, 4]: + batch_sizes = [1] if self.is_swin else [1, 4] + for batch_size in batch_sizes: self.trainer.args.per_device_eval_batch_size = batch_size for inputs in self.trainer.get_eval_dataloader(): self.assertEqual(inputs["pixel_values"].shape[0], batch_size) @@ -650,7 +656,8 @@ def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model): ) def check_ovmodel_reshaping(self, ovmodel: OVModel): - self.check_if_ovmodel_is_dynamic(ovmodel, True) + # dynamic batch size for tiny-swin does not work in OpenVINO 2023.0 + self.check_if_ovmodel_is_dynamic(ovmodel, not self.is_swin) size = (self.feature_extractor.size["height"], self.feature_extractor.size["width"]) dynamic_shape = [-1, 3, *size] for batch_size in [1, 4]: @@ -659,8 +666,9 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): self.check_if_ovmodel_is_dynamic(ovmodel, False) for input_ in ovmodel.model.inputs: self.assertSequenceEqual(list(input_.get_shape()), static_shape) - ovmodel.reshape(*dynamic_shape) - self.check_if_ovmodel_is_dynamic(ovmodel, True) + if not self.is_swin: + ovmodel.reshape(*dynamic_shape) + self.check_if_ovmodel_is_dynamic(ovmodel, True) QUANTIZATION_CONFIG_FOR_WAV2VEC2 = { From 109db5a66da37696f5f871d6433f24b702625fc5 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 5 May 2023 09:41:19 +0200 Subject: [PATCH 007/134] Set height and width by default for static models (#308) * Set height and width by defaulf for static models * fix warning --- optimum/intel/openvino/modeling_diffusion.py | 49 ++++++++++++++++++-- tests/openvino/test_modeling.py | 3 +- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 3405bfe9dd..d3179f2083 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -17,7 +17,7 @@ import os from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import openvino @@ -101,6 +101,7 @@ def __init__( self.feature_extractor = feature_extractor self.safety_checker = None self.preprocessors = [] + self._vae_scale_factor = 8 if self.is_dynamic: self.reshape(batch_size=-1, height=-1, width=-1, num_images_per_prompt=-1) @@ -414,14 +415,54 @@ def compile(self): self.vae_decoder._create_inference_request() self.unet._create_inference_request() - def __call__(self, *args, **kwargs): - guidance_scale = kwargs.get("guidance_scale", None) + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = 512, + width: Optional[int] = 512, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + **kwargs, + ): + _, _, _height, _width = self.unet.model.inputs[0].get_partial_shape() + + if _height.is_static: + _height = _height.get_length() * self._vae_scale_factor + if height != _height: + logger.warning( + f"`height` was set to {height} but the static model will output images of height {_height}." + "To fix the height, please reshape your model accordingly using the `.reshape()` method." + ) + height = _height + + if _width.is_static: + _width = _width.get_length() * self._vae_scale_factor + if width != _width: + logger.warning( + f"`width` was set to {width} but the static model will output images of width {_width}." + "To fix the width, please reshape your model accordingly using the `.reshape()` method." + ) + width = _width + if guidance_scale is not None and guidance_scale <= 1 and not self.is_dynamic: raise ValueError( f"`guidance_scale` was set to {guidance_scale}, static shapes are only supported for `guidance_scale` > 1, " "please set `dynamic_shapes` to `True` when loading the model." ) - return StableDiffusionPipelineMixin.__call__(self, *args, **kwargs) + + return StableDiffusionPipelineMixin.__call__( + self, + prompt=prompt, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) @classmethod def _load_config(cls, config_name_or_path: Union[str, os.PathLike], **kwargs): diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index caac969722..4aad4ea705 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -818,11 +818,12 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): ) self.assertFalse(pipeline.is_dynamic) pipeline.compile() + # Verify output shapes requirements not matching the static model don't impact the final outputs outputs = pipeline( [prompt] * batch_size, num_inference_steps=2, num_images_per_prompt=num_images_per_prompt, - height=height, + height=width, width=width, output_type="np", ).images From 229513bead35014d5247a7273b2ebf6f8c8695b6 Mon Sep 17 00:00:00 2001 From: "Cheng, Penghui" Date: Sat, 6 May 2023 17:01:43 +0800 Subject: [PATCH 008/134] Enhance quantization code for neural_compressor Signed-off-by: Cheng, Penghui --- .../intel/neural_compressor/configuration.py | 2 +- .../intel/neural_compressor/quantization.py | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/optimum/intel/neural_compressor/configuration.py b/optimum/intel/neural_compressor/configuration.py index e83add9f9b..164b68d664 100644 --- a/optimum/intel/neural_compressor/configuration.py +++ b/optimum/intel/neural_compressor/configuration.py @@ -14,7 +14,7 @@ from typing import Dict, Optional, Union -from neural_compressor.conf.pythonic_config import DistillationConfig, WeightPruningConfig, _BaseQuantizationConfig +from neural_compressor.config import DistillationConfig, WeightPruningConfig, _BaseQuantizationConfig from optimum.configuration_utils import BaseConfig diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index 16c9905920..400456f880 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -165,16 +165,21 @@ def quantize( calibration_dataloader = None if INCQuantizationMode(quantization_config.approach) == INCQuantizationMode.STATIC: + # Since PyTorch fx trace does not really require an example_inputs, only need calibration_dataset or calibration_fn here. + if calibration_dataset is None and self.calibration_fn is None: + raise ValueError( + "Post-training static quantization needs a calibration dataset or a calibration_function." + ) if calibration_dataset is None: - raise ValueError("Post-training static quantization needs a calibration dataset.") - - quantization_config.calibration_sampling_size = len(calibration_dataset) - calibration_dataloader = self._get_calibration_dataloader( - calibration_dataset=calibration_dataset, - batch_size=batch_size, - remove_unused_columns=remove_unused_columns, - data_collator=data_collator, - ) + calibration_dataloader = None + else: + quantization_config.calibration_sampling_size = len(calibration_dataset) + calibration_dataloader = self._get_calibration_dataloader( + calibration_dataset=calibration_dataset, + batch_size=batch_size, + remove_unused_columns=remove_unused_columns, + data_collator=data_collator, + ) if isinstance(self._original_model.config, PretrainedConfig): self._original_model.config.backend = quantization_config.backend @@ -193,7 +198,10 @@ def quantize( " accuracy tolerance has been found. Either the tolerance or the number of trials need to be increased." ) if isinstance(self._original_model.config, PretrainedConfig): + original_dtype = self._original_model.config.torch_dtype + self._original_model.config.torch_dtype = "int8" self._original_model.config.save_pretrained(save_directory) + self._original_model.config.torch_dtype = original_dtype self._quantized_model = compressed_model._model From 3714c3226da20a78d2fb1008da2e3592f294b7f1 Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Tue, 9 May 2023 00:29:53 +0800 Subject: [PATCH 009/134] fix regression of not using jit model (#310) also add UT to avoid such regression. UT will check if the optimized model is jit model. Signed-off-by: Wang, Yi A --- optimum/intel/ipex/inference.py | 4 ++-- setup.py | 2 +- tests/ipex/test_inference.py | 33 ++++++++++++++++++++++++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/optimum/intel/ipex/inference.py b/optimum/intel/ipex/inference.py index c7806fc755..127fccd3f9 100644 --- a/optimum/intel/ipex/inference.py +++ b/optimum/intel/ipex/inference.py @@ -46,8 +46,8 @@ def prepare_jit_inputs(model: PreTrainedModel, task: str): """ Prepare tuple inputs for jit trace model """ - task = _TASK_ALIASES[task] - if hasattr(model.config, "use_cache") and model.config.use_cache: + task = _TASK_ALIASES.get(task, task) + if "generation" in task and hasattr(model.config, "use_cache") and model.config.use_cache: task += "-with-past" onnx_config_class = TasksManager.get_exporter_config_constructor( exporter="onnx", diff --git a/setup.py b/setup.py index c3e0503527..b94283fb80 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ ], "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime"], "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217"], - "ipex": ["intel-extension-for-pytorch"], + "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, diff --git a/tests/ipex/test_inference.py b/tests/ipex/test_inference.py index ef47261f3e..4e538c385e 100644 --- a/tests/ipex/test_inference.py +++ b/tests/ipex/test_inference.py @@ -20,6 +20,7 @@ # TODO : add more tasks from transformers import ( AutoModelForCausalLM, + AutoModelForQuestionAnswering, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoTokenizer, @@ -54,27 +55,48 @@ class IPEXIntegrationTest(unittest.TestCase): ) TEXT_GENERATION_SUPPORTED_ARCHITECTURES = ( - "bloom", "gptj", "gpt2", "gpt_neo", - "gpt_neox", ) + QA_SUPPORTED_ARCHITECTURES = ( + "bert", + "distilbert", + "roberta", + ) + + @parameterized.expand(QA_SUPPORTED_ARCHITECTURES) + def test_question_answering_pipeline_inference(self, model_arch): + model_id = MODEL_NAMES[model_arch] + tokenizer = AutoTokenizer.from_pretrained(model_id) + model = AutoModelForQuestionAnswering.from_pretrained(model_id, torch_dtype=torch.float32) + pipe = pipeline("question-answering", model=model, tokenizer=tokenizer) + + with torch.inference_mode(): + outputs = pipe(question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris.") + with ipex_inference_mode(pipe, dtype=model.config.torch_dtype, verbose=False, jit=True) as ipex_pipe: + outputs_ipex = ipex_pipe( + question="Where was HuggingFace founded ?", context="HuggingFace was founded in Paris." + ) + self.assertTrue(isinstance(ipex_pipe.model._optimized, torch.jit.RecursiveScriptModule)) + self.assertEqual(outputs["start"], outputs_ipex["start"]) + self.assertEqual(outputs["end"], outputs_ipex["end"]) + @parameterized.expand(CLASSIFICATION_SUPPORTED_ARCHITECTURES) def test_classification_pipeline_inference(self, model_arch): model_id = MODEL_NAMES[model_arch] tokenizer = AutoTokenizer.from_pretrained(model_id) inputs = "This is a sample input" for task, auto_model_class in _CLASSIFICATION_TASK_TO_AUTOMODELS.items(): - model = auto_model_class.from_pretrained(model_id) + model = auto_model_class.from_pretrained(model_id, torch_dtype=torch.float32) pipe = pipeline(task, model=model, tokenizer=tokenizer) with torch.inference_mode(): outputs = pipe(inputs) - with ipex_inference_mode(pipe) as ipex_pipe: + with ipex_inference_mode(pipe, dtype=model.config.torch_dtype, verbose=False, jit=True) as ipex_pipe: outputs_ipex = ipex_pipe(inputs) - + self.assertTrue(isinstance(ipex_pipe.model._optimized, torch.jit.RecursiveScriptModule)) self.assertEqual(outputs[0]["score"], outputs_ipex[0]["score"]) @parameterized.expand(TEXT_GENERATION_SUPPORTED_ARCHITECTURES) @@ -91,4 +113,5 @@ def test_text_generation_pipeline_inference(self, model_arch): text_generator, dtype=model.config.torch_dtype, verbose=False, jit=True ) as ipex_text_generator: output_ipex = ipex_text_generator(inputs) + self.assertTrue(isinstance(ipex_text_generator.model._optimized, torch.jit.RecursiveScriptModule)) self.assertEqual(output[0]["generated_text"], output_ipex[0]["generated_text"]) From 68011830d48d82aa50bd4de71e6a05e6e676e4bf Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Tue, 16 May 2023 10:23:27 +0200 Subject: [PATCH 010/134] Limit transformers to <4.29 in [openvino] extra (#316) * Limit transformers version in [openvino] extra Prevent "RuntimeError: Default process group has not been initialized," with transformers 4.29 * Add accelerate requirement to neural-compressor and nncf extras PyTorch 4.29 raises this error: ImportError: Using the `Trainer` with `PyTorch` requires `accelerate`: Run `pip install --upgrade accelerate` --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b94283fb80..ada6ba129b 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,10 @@ "onnxruntime", "torch<2.0.0", # remove after neural-compressor next release "intel-extension-for-pytorch<2.0.0", + "accelerate" ], - "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217"], + "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime", "transformers<4.29"], + "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217", "accelerate"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From dacf94fb4273722082275ef8cf930cb1c91473f2 Mon Sep 17 00:00:00 2001 From: "Cheng, Penghui" Date: Wed, 17 May 2023 16:45:58 +0800 Subject: [PATCH 011/134] Introduce smooth quant method and enable smooth quant in language modeling examples (#302) * Introduce smooth quant method in optimization_inc.mdx and enable smooth quant in language modeling examples * Update doc * update description for smooth quant * Update examples/neural_compressor/language-modeling/README.md Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update examples/neural_compressor/language-modeling/README.md Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update document of smooth quant * Update Document for PyTorch quantization * Update docs/source/optimization_inc.mdx Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update docs/source/optimization_inc.mdx Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update docs/source/optimization_inc.mdx Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update test case code * Fixed code style error * fixed UT error * Fixed code style error * Update package version for neural-compressor and PyTorch * Fixed dependency issue for transformers 4.29 * fixed openvino UT error * Fixed UT error --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- docs/source/optimization_inc.mdx | 10 ++++++++++ .../neural_compressor/language-modeling/README.md | 15 +++++++++++++++ .../language-modeling/run_clm.py | 12 +++++++++++- setup.py | 4 ++-- tests/neural_compressor/test_optimization.py | 6 ++++-- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/source/optimization_inc.mdx b/docs/source/optimization_inc.mdx index 87a33fb0aa..1a46c161f0 100644 --- a/docs/source/optimization_inc.mdx +++ b/docs/source/optimization_inc.mdx @@ -108,6 +108,16 @@ quantizer.quantize( ) ``` +### Specify Quantization Recipes + +The [SmoothQuant](https://arxiv.org/abs/2211.10438) methodology is available for post-training quantization. This methodology usually improves the accuracy of the model in comparison to other post-training static quantization methodologies. This is done by migratesing the difficulty from activations to weights with a mathematically equivalent transformation. +```diff +- quantization_config = PostTrainingQuantConfig(approach="static") ++ recipes={"smooth_quant": True, "smooth_quant_args": {"alpha": 0.5, "folding": True}} ++ quantization_config = PostTrainingQuantConfig(approach="static", backend="ipex", recipes=recipes) +``` +Please refer to INC [documentation](https://github.com/intel/neural-compressor/blob/master/docs/source/smooth_quant.md) and the list of [models](https://github.com/intel/neural-compressor/blob/master/docs/source/smooth_quant.md#validated-models) quantized with the methodology for more details. + ## During training optimization The [`INCTrainer`] class provides an API to train your model while combining different compression techniques such as knowledge distillation, pruning and quantization. diff --git a/examples/neural_compressor/language-modeling/README.md b/examples/neural_compressor/language-modeling/README.md index b16cb7c5d9..1c8e98b9ee 100644 --- a/examples/neural_compressor/language-modeling/README.md +++ b/examples/neural_compressor/language-modeling/README.md @@ -21,6 +21,8 @@ and [`run_mlm.py`](https://github.com/huggingface/optimum-intel/blob/main/exampl allow us to apply different quantization approaches (such as dynamic, static and aware-training quantization) as well as pruning using the [Intel Neural Compressor ](https://github.com/intel/neural-compressor) library for language modeling tasks. +The SmoothQuant methodology is also available for post-training quantization. + For pruning, we support snip_momentum(default), snip_momentum_progressive, magnitude, magnitude_progressive, gradient, gradient_progressive, snip, snip_progressive and pattern_lock. You can refer to [the pruning details](https://github.com/intel/neural-compressor/tree/master/neural_compressor/pruner#pruning-types). > **_Note:_** At present, neural_compressor only support to prune linear and conv ops. So if we set a target sparsity is 0.9, it means that the pruning op's sparsity will be 0.9, not the whole model's sparsity is 0.9. For example: the embedding ops will not be pruned in the model. @@ -53,6 +55,19 @@ python run_clm.py \ --output_dir /tmp/clm_output ``` +The following example shows how to apply post-training static quantization using the SmoothQuant methodology on a GPT-Neo model : +```bash +python run_clm.py \ + --model_name_or_path EleutherAI/gpt-neo-125M \ + --dataset_name wikitext \ + --dataset_config_name wikitext-2-raw-v1 \ + --apply_quantization \ + --quantization_approach static \ + --smooth_quant \ + --do_eval \ + --verify_loading \ + --output_dir /tmp/clm_output + ### RoBERTa/BERT/DistilBERT and masked language modeling The following example fine-tunes RoBERTa on WikiText-2 while applying quantization aware training and snip_momentum pruning. We're using the raw diff --git a/examples/neural_compressor/language-modeling/run_clm.py b/examples/neural_compressor/language-modeling/run_clm.py index aca69f98ae..80913f0071 100644 --- a/examples/neural_compressor/language-modeling/run_clm.py +++ b/examples/neural_compressor/language-modeling/run_clm.py @@ -145,6 +145,14 @@ class OptimizationArguments: default="dynamic", metadata={"help": "Quantization approach. Supported approach are static, dynamic and aware_training."}, ) + smooth_quant: bool = field( + default=False, + metadata={"help": "Whether or not to quantize with smooth quant."}, + ) + smooth_quant_alpha: float = field( + default=0.5, + metadata={"help": "Set alpha of smooth quant argument."}, + ) num_calibration_samples: int = field( default=50, metadata={"help": "Number of examples to use for the calibration step resulting from static quantization."}, @@ -588,7 +596,9 @@ def compute_metrics(eval_preds): if optim_args.quantization_approach == "aware_training": quantization_config = QuantizationAwareTrainingConfig() else: - quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach) + if optim_args.smooth_quant: + recipes = {"smooth_quant": True, "smooth_quant_args": optim_args.smooth_quant_alpha} + quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, recipes=recipes) if optim_args.apply_pruning: if optim_args.end_step is None: diff --git a/setup.py b/setup.py index ada6ba129b..f14136a51a 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ "datasets>=1.4.0", "sentencepiece", "scipy", + "accelerate", # transformers 4.29 require accelerate for PyTorch ] TESTS_REQUIRE = ["pytest", "parameterized", "Pillow", "evaluate", "diffusers", "py-cpuinfo"] @@ -25,12 +26,11 @@ EXTRAS_REQUIRE = { "neural-compressor": [ - "neural-compressor>=2.1.0", + "neural-compressor>=2.1.1", "onnx", "onnxruntime", "torch<2.0.0", # remove after neural-compressor next release "intel-extension-for-pytorch<2.0.0", - "accelerate" ], "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime", "transformers<4.29"], "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217", "accelerate"], diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index 66b26af1f8..1206a9ac6f 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -186,9 +186,11 @@ def preprocess_function(examples, tokenizer): self.assertTrue(torch.equal(model_outputs.logits, loaded_model_outputs.logits)) # self.assertTrue(torch.allclose(ort_outputs.logits, loaded_model_outputs.logits, atol=1e-4)) - def test_ipex_static_quantization(self): + def test_ipex_static_quantization_with_smoothquant(self): model_name = "distilbert-base-uncased-finetuned-sst-2-english" - quantization_config = PostTrainingQuantConfig(approach="static", backend="ipex") + quantization_config = PostTrainingQuantConfig( + approach="static", backend="ipex", recipes={"smooth_quant": True, "smooth_quant_args": {"alpha": 0.5}} + ) model = AutoModelForSequenceClassification.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) tokens = tokenizer("This is a sample input", return_tensors="pt") From 4808b3401cdbc8328bbccad5a78510f0e5b16fc5 Mon Sep 17 00:00:00 2001 From: xinhe Date: Fri, 19 May 2023 14:24:52 +0800 Subject: [PATCH 012/134] Fix bug in example --- examples/neural_compressor/language-modeling/run_clm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/neural_compressor/language-modeling/run_clm.py b/examples/neural_compressor/language-modeling/run_clm.py index 80913f0071..fe28530183 100644 --- a/examples/neural_compressor/language-modeling/run_clm.py +++ b/examples/neural_compressor/language-modeling/run_clm.py @@ -597,7 +597,7 @@ def compute_metrics(eval_preds): quantization_config = QuantizationAwareTrainingConfig() else: if optim_args.smooth_quant: - recipes = {"smooth_quant": True, "smooth_quant_args": optim_args.smooth_quant_alpha} + recipes = {"smooth_quant": True, "smooth_quant_args": {"alpha": optim_args.smooth_quant_alpha}} quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, recipes=recipes) if optim_args.apply_pruning: From b8dc1929339e2d139fa2f1aee0593254e2fffd0a Mon Sep 17 00:00:00 2001 From: xinhe Date: Fri, 19 May 2023 14:38:40 +0800 Subject: [PATCH 013/134] fix compression_manager error --- optimum/intel/neural_compressor/trainer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/optimum/intel/neural_compressor/trainer.py b/optimum/intel/neural_compressor/trainer.py index eda52a5cea..65de84a93e 100644 --- a/optimum/intel/neural_compressor/trainer.py +++ b/optimum/intel/neural_compressor/trainer.py @@ -157,10 +157,10 @@ def __init__( self.model = self._compression_manager.model.model self.model_wrapped = self.model - for callback in self._compression_manager.callbacks.callbacks_list: - if isinstance(callback, DistillationCallbacks): - self.distillation_callback = callback - break + for callback in self._compression_manager.callbacks.callbacks_list: + if isinstance(callback, DistillationCallbacks): + self.distillation_callback = callback + break self.inc_config = INCConfig( quantization=self.quantization_config, From af974e398cdf6493932c3fb4367e92fd234ea3a0 Mon Sep 17 00:00:00 2001 From: jiqing-feng <107918818+jiqing-feng@users.noreply.github.com> Date: Mon, 22 May 2023 16:18:41 +0800 Subject: [PATCH 014/134] Integrate text-generation pipeline from inference.py to TSModelForCausalLM (#300) * Integrate ipex and TSModelForCausalLM * fix code style * add jit_trace file * fix code style * fix style * fix jit trace * fix style * remove tracing file * subclass _ModelFallbackWrapper to enable generation * . * rm * update with ipex test * fix format * outputs type * fix all task jit issue Signed-off-by: Wang, Yi A --------- Signed-off-by: Wang, Yi A Co-authored-by: Wang, Yi A --- optimum/intel/generation/modeling.py | 81 ++++++++--- optimum/intel/ipex/inference.py | 199 +++++++++------------------ tests/ipex/test_inference.py | 4 +- 3 files changed, 133 insertions(+), 151 deletions(-) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index 441efc037a..11fa90a0d6 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -21,7 +21,7 @@ import torch from huggingface_hub import hf_hub_download -from transformers import AutoConfig, AutoModelForCausalLM, PretrainedConfig +from transformers import AutoConfig, AutoModelForCausalLM, PretrainedConfig, PreTrainedModel from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.utils import WEIGHTS_NAME @@ -29,6 +29,7 @@ from optimum.modeling_base import OptimizedModel from optimum.utils import NormalizedConfigManager +from ..utils.constant import _TASK_ALIASES from ..utils.import_utils import is_torch_version, is_transformers_version @@ -41,6 +42,52 @@ logger = logging.getLogger(__name__) +def prepare_jit_inputs(model: PreTrainedModel, task: str, use_cache: bool = False): + task = _TASK_ALIASES.get(task, task) + signature = inspect.signature(model.forward) if hasattr(model, "forward") else inspect.signature(model.__call__) + onnx_config_class = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) + onnx_config = onnx_config_class(model.config) + if task == "text-generation" and use_cache: + onnx_config = onnx_config_class(model.config, use_past=True) + dummy_inputs = onnx_config.generate_dummy_inputs(framework="pt") + model_inputs = {key: dummy_inputs[key] for key in signature.parameters if dummy_inputs.get(key, None) is not None} + if task == "text-generation" and use_cache: + # WA jit.trace issue of model like llama in https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L464, or else, generation output will be incorrect + pkv = [] + for i in range(len(model_inputs["past_key_values"])): + pkv.append([]) + for j in range(len(model_inputs["past_key_values"][0])): + pkv[i].append(model_inputs["past_key_values"][i][j].to(model.dtype)) + pkv[i] = tuple(pkv[i]) + model_inputs["past_key_values"] = tuple(pkv) + i = model_inputs["input_ids"] + a = model_inputs["attention_mask"] + model_inputs["input_ids"] = torch.cat([torch.zeros(i.shape[0], 1), i], -1).to(i.dtype) + model_inputs["attention_mask"] = torch.cat([torch.zeros(a.shape[0], 1), a], -1).to(a.dtype) + return model_inputs + + +def jit_trace(model: PreTrainedModel, task: str, use_cache: bool = False): + model_inputs = prepare_jit_inputs(model, task, use_cache) + torch._C._jit_set_texpr_fuser_enabled(False) + if "past_key_values" in model_inputs.keys(): + model.config.return_dict = False + if is_torch_version(">", "2.0.1"): + traced_model = torch.jit.trace(model, example_kwarg_inputs=model_inputs, strict=False) + else: + traced_model = torch.jit.trace(model, example_inputs=tuple(model_inputs.values()), strict=False) + else: + if is_torch_version(">=", "2.0.0"): + traced_model = torch.jit.trace(model, example_kwarg_inputs=model_inputs, strict=False) + else: + traced_model = torch.jit.trace(model, example_inputs=tuple(model_inputs.values()), strict=False) + traced_model = torch.jit.freeze(traced_model.eval()) + traced_model(**model_inputs) + traced_model(**model_inputs) + + return traced_model + + class TSModelForCausalLM(OptimizedModel, GenerationMixin): auto_model_class = AutoModelForCausalLM export_feature = "text-generation" @@ -63,6 +110,7 @@ def __init__( self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model.to(self._device) self.normalized_config = NormalizedConfigManager.get_normalized_config_class(config.model_type)(config) + self.model_dtype = kwargs.get("model_dtype", None) if is_transformers_version("<=", "4.25.1"): self.generation_config = None @@ -107,6 +155,8 @@ def forward( if self.config.model_type != "bloom": new_shape = [input_ids.shape[0], num_attention_heads, 0, d_k] empty_tensor = torch.empty(size=new_shape) + if self.model_dtype is not None: + empty_tensor = empty_tensor.to(self.model_dtype) past_key_values = tuple(tuple(empty_tensor for _ in range(nb_pkv)) for _ in range(num_layers)) pkv = tuple(empty_tensor for _ in range(nb_pkv)) else: @@ -116,13 +166,23 @@ def forward( new_shape = [input_ids.shape[0] * num_attention_heads, d_k, 0] else: new_shape = [input_ids.shape[0] * num_attention_heads, 0, d_k] - pkv = pkv + (torch.empty(size=new_shape),) + empty_tensor = torch.empty(size=new_shape) + if self.model_dtype is not None: + empty_tensor = empty_tensor.to(self.model_dtype) + pkv = pkv + (empty_tensor,) past_key_values = tuple(tuple(pkv) for _ in range(num_layers)) inputs["past_key_values"] = past_key_values outputs = self.model(**inputs) - return CausalLMOutputWithPast(logits=outputs[0], past_key_values=outputs[1] if self.use_cache else None) + if isinstance(outputs, tuple): + outputs = CausalLMOutputWithPast(logits=outputs[0], past_key_values=outputs[1] if self.use_cache else None) + else: + outputs = CausalLMOutputWithPast( + logits=outputs["logits"], past_key_values=outputs["past_key_values"] if self.use_cache else None + ) + + return outputs @classmethod def _from_pretrained( @@ -210,20 +270,7 @@ def _from_transformers( } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - model.config.return_dict = False - signature = inspect.signature(model.forward) if hasattr(model, "forward") else inspect.signature(model.call) - onnx_config_class = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) - onnx_config = onnx_config_class(model.config, use_past=use_cache) - dummy_inputs = onnx_config.generate_dummy_inputs(framework="pt") - model_inputs = { - key: dummy_inputs[key] for key in signature.parameters if dummy_inputs.get(key, None) is not None - } - - if use_cache: - traced_model = torch.jit.trace(model, example_inputs=tuple(model_inputs.values())) - else: - traced_model = torch.jit.trace(model, example_kwarg_inputs=model_inputs) - traced_model = torch.jit.freeze(traced_model.eval()) + traced_model = jit_trace(model, task, use_cache) save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) torch.jit.save(traced_model, save_dir_path / WEIGHTS_NAME) diff --git a/optimum/intel/ipex/inference.py b/optimum/intel/ipex/inference.py index 127fccd3f9..961c48dce7 100644 --- a/optimum/intel/ipex/inference.py +++ b/optimum/intel/ipex/inference.py @@ -1,17 +1,13 @@ -import inspect import logging -from typing import Tuple, Union +from typing import Union import torch from torch import nn -from transformers import GenerationMixin, PreTrainedModel, add_start_docstrings -from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers import add_start_docstrings from transformers.pipelines import Pipeline from transformers.utils import is_ipex_available -from optimum.exporters.tasks import TasksManager - -from ..utils.constant import _TASK_ALIASES +from ..generation.modeling import TSModelForCausalLM, jit_trace logger = logging.getLogger(__name__) @@ -26,41 +22,6 @@ import intel_extension_for_pytorch as ipex -def ordered_inputs(inputs: str, model: PreTrainedModel): - """ - Order input dict and convert input dict to tuple since jit traced model only support tuple input. - """ - if hasattr(model, "forward"): - sig = inspect.signature(model.forward) - else: - sig = inspect.signature(model.call) - - return tuple( - inputs[key] - for key in sig.parameters - if inputs.get(key, None) is not None and not isinstance(inputs.get(key, None), bool) - ) - - -def prepare_jit_inputs(model: PreTrainedModel, task: str): - """ - Prepare tuple inputs for jit trace model - """ - task = _TASK_ALIASES.get(task, task) - if "generation" in task and hasattr(model.config, "use_cache") and model.config.use_cache: - task += "-with-past" - onnx_config_class = TasksManager.get_exporter_config_constructor( - exporter="onnx", - model=model, - task=task, - ) - onnx_config = onnx_config_class(model.config) - dummy_inputs = onnx_config.generate_dummy_inputs(framework="pt") - inputs = ordered_inputs(dummy_inputs, model) - - return inputs - - class _ModelFallbackWrapper: __slots__ = ("_optimized", "_default") @@ -81,52 +42,15 @@ def __getattr__(self, item): return self.item -class _ModelGenerationWrapper(GenerationMixin): - __slots__ = ("_optimized", "_default") - - def __init__(self, optimized, default): - self._optimized = optimized - self._default = default - - def __call__(self, *args, **kwargs): - try: - trace_graph_inputs = ordered_inputs(kwargs, self._default) - if args: - trace_graph_inputs = args + trace_graph_inputs - trace_graph_inputs = tuple(trace_graph_inputs) - outputs = self._optimized(*trace_graph_inputs) - lm_logits = outputs[0] - past_key_values = outputs[1] - fixed_output = CausalLMOutputWithPast( - loss=None, - logits=lm_logits, - past_key_values=past_key_values, - hidden_states=None, - attentions=None, - ) - return fixed_output - except Exception: - return self._default(*args, **kwargs) - +class _ModelGenerationWrapper(_ModelFallbackWrapper): def __getattr__(self, item): - return getattr(self._default, item) - - def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs - ): - return self._default.prepare_inputs_for_generation( - input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, **kwargs - ) - - def _reorder_cache( - self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor - ) -> Tuple[Tuple[torch.Tensor]]: - """ - This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or - [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct - beam_idx at every generation step. - """ - return self._default._reorder_cache(past_key_values, beam_idx) + if not item.startswith("__"): + try: + return getattr(self._optimized, item) + except Exception: + return getattr(self._default, item) + else: + return self.item @add_start_docstrings( @@ -142,8 +66,8 @@ def __init__( self, model: Union[nn.Module, Pipeline], dtype: torch.dtype = torch.float32, - verbose: bool = False, jit: bool = False, + **kwargs, ): """ Args: @@ -154,14 +78,13 @@ def __init__( Acceptable type are `torch.float32` (default) and `torch.bfloat16`. Please note `torch.bfloat16` requires `avx512_bf16` instructions set as present on 4th Generation of Intel Xeon Scalable CPUs (Sapphire Rapids). - verbose (`boolean = False`, *optional*): - Enable IPEx verbose output to see the kernels and optimizations applied. + jit (`boolean = False`, *optional*): + Enable jit to accelerate inference speed """ if not is_ipex_available(): raise ImportError(IPEX_NOT_AVAILABLE_ERROR_MSG) self._model = model - self._verbose = ipex.utils.verbose.VERBOSE_ON if verbose else ipex.utils.verbose.VERBOSE_OFF self._dtype = dtype self._graph_mode = False # Let's keep for future use when it doesn't hang anymore self._original = None @@ -171,49 +94,59 @@ def __enter__(self): if self._model.framework == "pt": with torch.inference_mode(): try: - with ipex.verbose(self._verbose): - ipex.enable_onednn_fusion(True) - if isinstance(self._model, Pipeline): - self._original = self._model.model - - model = ipex.optimize( - self._model.model, - dtype=self._dtype, - graph_mode=self._graph_mode, - level="O1", - auto_kernel_selection=True, - ) - - # Enable automatic mixed precision (AMP) if we are going to target `bfloat16` - with torch.cpu.amp.autocast(enabled=(self._dtype == torch.bfloat16)), torch.no_grad(): - if self._model.tokenizer is not None and self._jit: - try: - jit_inputs = prepare_jit_inputs(self._model.model, self._model.task) - model = torch.jit.trace(model, jit_inputs, strict=False) - model = torch.jit.freeze(model) - model(*jit_inputs) - model(*jit_inputs) - except Exception as e: - logger.warning(f"failed to use PyTorch jit mode due to: {e}.") + ipex.enable_onednn_fusion(True) + if isinstance(self._model, Pipeline): + self._original = self._model.model + + model = ipex.optimize( + self._model.model, + dtype=self._dtype, + graph_mode=self._graph_mode, + level="O1", + auto_kernel_selection=True, + ) + + # Enable automatic mixed precision (AMP) if we are going to target `bfloat16` + with torch.cpu.amp.autocast( + enabled=(self._dtype == torch.bfloat16 and self._original.dtype != torch.bfloat16) + ), torch.no_grad(): + if self._jit: + try: + use_cache = False + if hasattr(self._original.config, "use_cache") and self._original.config.use_cache: + use_cache = True + model = jit_trace( + model=model, + task=self._model.task, + use_cache=use_cache, + ) + if self._model.task == "text-generation": + model = TSModelForCausalLM( + model=model, + config=self._original.config, + use_cache=use_cache, + model_dtype=self._original.dtype, + ) + except Exception as e: + logger.warning(f"failed to use PyTorch jit mode due to: {e}.") # Patching model with the new one - if self._model.task == "text-generation": - self._model.model = _ModelGenerationWrapper(model, self._original) - else: - self._model.model = _ModelFallbackWrapper(model, self._original) - return self._model - else: - self._original = self._model - model = ipex.optimize( - self._model, - dtype=self._dtype, - graph_mode=self._graph_mode, - level="O1", - auto_kernel_selection=True, - ) - - # Enable automatic mixed precision (AMP) if we are going to target `bfloat16` - with torch.cpu.amp.autocast(enabled=(self._dtype == torch.bfloat16)): - return model + self._model.model = _ModelGenerationWrapper(model, self._original) + return self._model + else: + self._original = self._model + model = ipex.optimize( + self._model, + dtype=self._dtype, + graph_mode=self._graph_mode, + level="O1", + auto_kernel_selection=True, + ) + + # Enable automatic mixed precision (AMP) if we are going to target `bfloat16` + with torch.cpu.amp.autocast( + enabled=(self._dtype == torch.bfloat16 and self._original.dtype != torch.bfloat16) + ): + return model except RuntimeError: return self._model else: diff --git a/tests/ipex/test_inference.py b/tests/ipex/test_inference.py index 4e538c385e..d5cf571150 100644 --- a/tests/ipex/test_inference.py +++ b/tests/ipex/test_inference.py @@ -28,6 +28,7 @@ ) from optimum.intel import inference_mode as ipex_inference_mode +from optimum.intel.generation.modeling import TSModelForCausalLM MODEL_NAMES = { @@ -113,5 +114,6 @@ def test_text_generation_pipeline_inference(self, model_arch): text_generator, dtype=model.config.torch_dtype, verbose=False, jit=True ) as ipex_text_generator: output_ipex = ipex_text_generator(inputs) - self.assertTrue(isinstance(ipex_text_generator.model._optimized, torch.jit.RecursiveScriptModule)) + self.assertTrue(isinstance(ipex_text_generator.model._optimized, TSModelForCausalLM)) + self.assertTrue(isinstance(ipex_text_generator.model._optimized.model, torch.jit.RecursiveScriptModule)) self.assertEqual(output[0]["generated_text"], output_ipex[0]["generated_text"]) From 8020eb07a315e3e89a1666ae6968bb8a1018da72 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Thu, 25 May 2023 18:11:08 +0200 Subject: [PATCH 015/134] Suppress torch version mismatch warning with NNCF (#311) --- optimum/intel/openvino/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index c7389b3a4c..3f8737e1b9 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -11,14 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from ..utils.import_utils import is_diffusers_available, is_nncf_available from .utils import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME if is_nncf_available(): + import nncf + + # Suppress version mismatch logging + nncf.set_log_level(logging.ERROR) from nncf.torch import patch_torch_operators + nncf.set_log_level(logging.INFO) + patch_torch_operators() from .configuration import OVConfig From b0cc0847a24e0afbf80a123e9f98fff665512753 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 26 May 2023 14:16:45 +0200 Subject: [PATCH 016/134] add SD blog post link to documentation (#324) --- docs/source/optimization_ov.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/optimization_ov.mdx b/docs/source/optimization_ov.mdx index feb6e30c46..7ee942d2f3 100644 --- a/docs/source/optimization_ov.mdx +++ b/docs/source/optimization_ov.mdx @@ -203,6 +203,7 @@ More on available algorithms in NNCF, see documentation [here](https://github.co For complete JPQD scripts, please refer to examples provided [here](https://github.com/huggingface/optimum-intel/tree/main/examples/openvino). +Quantization-Aware Training (QAT) and knowledge distillation can also be combined in order to optimize Stable Diffusion models while maintaining accuracy. For more details, take a look at this [blog post](https://huggingface.co/blog/train-optimize-sd-intel). ## Inference with Transformers pipeline From 8ce2c6be521ca25daff3f97f769f12af2649f2f2 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 26 May 2023 14:27:39 +0200 Subject: [PATCH 017/134] Add argument to execute code present on the Hub (#325) --- optimum/intel/openvino/modeling_base.py | 2 ++ optimum/intel/openvino/modeling_base_seq2seq.py | 2 ++ optimum/intel/openvino/modeling_decoder.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index ad0ffa3524..2db8867613 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -237,6 +237,7 @@ def _from_transformers( subfolder: str = "", local_files_only: bool = False, task: Optional[str] = None, + trust_remote_code: bool = False, **kwargs, ): """ @@ -267,6 +268,7 @@ def _from_transformers( "subfolder": subfolder, "local_files_only": local_files_only, "force_download": force_download, + "trust_remote_code": trust_remote_code, } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index 3e98929c33..6cd8c5b177 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -272,6 +272,7 @@ def _from_transformers( local_files_only: bool = False, task: Optional[str] = None, use_cache: bool = True, + trust_remote_code: bool = False, **kwargs, ): """ @@ -310,6 +311,7 @@ def _from_transformers( "subfolder": subfolder, "local_files_only": local_files_only, "force_download": force_download, + "trust_remote_code": trust_remote_code, } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 0db1b95422..50d39938ba 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -136,6 +136,7 @@ def _from_transformers( local_files_only: bool = False, task: Optional[str] = None, use_cache: bool = True, + trust_remote_code: bool = False, **kwargs, ): model_file_name = ONNX_WEIGHTS_NAME @@ -152,6 +153,7 @@ def _from_transformers( "subfolder": subfolder, "local_files_only": local_files_only, "force_download": force_download, + "trust_remote_code": trust_remote_code, } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) From fe624226727ebc4f7bf6da8ccf01d60913815aa0 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Tue, 30 May 2023 21:38:31 +0400 Subject: [PATCH 018/134] Improved Stable Diffusion optimization example (#326) * Added Token Merging in combination with QAT for Stable Diffusion * Modifiedstable diffusion notebook * Chanded the SD readme * Style * Fixed requirements * Added an option for agressive quantization * Enhanced data loading * SD optimization improvements * Update examples/openvino/stable-diffusion/README.md Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update examples/openvino/stable-diffusion/train_text_to_image_qat.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Apply comments --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- examples/openvino/stable-diffusion/README.md | 11 ++- .../train_text_to_image_qat.py | 89 +++++++++++++++++-- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/examples/openvino/stable-diffusion/README.md b/examples/openvino/stable-diffusion/README.md index 48cb99df29..2f2fb1c82e 100644 --- a/examples/openvino/stable-diffusion/README.md +++ b/examples/openvino/stable-diffusion/README.md @@ -3,13 +3,15 @@ This example demonstrates how to apply Quantization-aware Training (QAT) from [N Knowledge distillation and EMA techniques can be used to improve the model accuracy. -This example supports model tuning on three datasets from the HuggingFace: +This example supports model tuning on the following datasets from the HuggingFace: * [Pokemon BLIP captions](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) * [laion2B-en](https://huggingface.co/datasets/laion/laion2B-en) * [laion2B-en-aesthetic](https://huggingface.co/datasets/laion/laion2B-en-aesthetic) +* [laion-art](https://huggingface.co/datasets/laion/laion-art) +* [laion400m](https://huggingface.co/datasets/laion/laion400m) But it can be easily extended to other datasets. ->**Note**: laion2B-en is being downloaded on-fly durint the fine-tuning process. No need to store it locally. +>**Note**: laion2B* datasets are being downloaded on-fly during the fine-tuning process. No need to store them locally. ## Prerequisites * Install Optimum-Intel for OpenVINO: @@ -103,4 +105,7 @@ python train_text_to_image_qat.py \ --gradient_checkpointing \ --tune_quantizers_only \ --output_dir=sd-1-5-quantied-laion -``` \ No newline at end of file +``` + +## References +* [Optimizing Stable Diffusion for Intel CPUs with NNCF and 🤗 Optimum](https://huggingface.co/blog/train-optimize-sd-intel) \ No newline at end of file diff --git a/examples/openvino/stable-diffusion/train_text_to_image_qat.py b/examples/openvino/stable-diffusion/train_text_to_image_qat.py index 1e6f6145c9..147490c586 100644 --- a/examples/openvino/stable-diffusion/train_text_to_image_qat.py +++ b/examples/openvino/stable-diffusion/train_text_to_image_qat.py @@ -46,6 +46,7 @@ from nncf.torch.layer_utils import CompressionParameter from openvino._offline_transformations import apply_moc_transformations, compress_quantize_weights_transformation from PIL import Image +from requests.packages.urllib3.exceptions import InsecureRequestWarning from torchvision import transforms from tqdm import tqdm @@ -59,6 +60,8 @@ ) +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + random.seed(42) logger = get_logger(__name__) nncf_logger.setLevel(logging.INFO) @@ -82,7 +85,7 @@ def pokemon_preprocess_train(examples, train_transforms, tokenize_captions, imag def get_pil_from_url(url): - response = requests.get(url) + response = requests.get(url, verify=False, timeout=20) image = Image.open(BytesIO(response.content)) return image.convert("RGB") @@ -98,11 +101,21 @@ def get_pil_from_url(url): AVAILABLE_EXAMPLES = [] -def laion2B_preprocess_train(examples, train_transforms, tokenize_captions, image_column="URL"): +def check_text_data(data): + if isinstance(data, str): + return True + if isinstance(data, list): + return all(isinstance(x, str) for x in data) + return False + + +def laion2B_preprocess_train(examples, train_transforms, tokenize_captions, image_column="URL", text_column="TEXT"): url = examples[image_column] try: image = get_pil_from_url(url) - AVAILABLE_EXAMPLES.append((url, examples["TEXT"])) + if not check_text_data(examples[text_column]): + raise ValueError("Text data is not valid") + AVAILABLE_EXAMPLES.append((url, examples[text_column])) except Exception: logger.info(f"Can't load image from url: {url}, using cache with size: {len(AVAILABLE_EXAMPLES)}") if len(AVAILABLE_EXAMPLES) > 0: @@ -110,15 +123,15 @@ def laion2B_preprocess_train(examples, train_transforms, tokenize_captions, imag backup_example = AVAILABLE_EXAMPLES[backup_id] try: image = get_pil_from_url(backup_example[0]) - examples["TEXT"] = backup_example[1] + examples[text_column] = backup_example[1] except Exception: logger.info(f"Can't load image from cached url: {backup_example[0]}, using backup") image = BACKUP_PAIR[0].copy() - examples["TEXT"] = BACKUP_PAIR[1] + examples[text_column] = BACKUP_PAIR[1] else: logger.info(f"Can't load image from url: {url}, using backup") image = BACKUP_PAIR[0].copy() - examples["TEXT"] = BACKUP_PAIR[1] + examples[text_column] = BACKUP_PAIR[1] examples["pixel_values"] = train_transforms(image) examples["input_ids"] = tokenize_captions(examples) @@ -141,6 +154,16 @@ def laion2B_preprocess_train(examples, train_transforms, tokenize_captions, imag "preprocess_fn": laion2B_preprocess_train, "streaming": True, }, + "laion/laion-art": { + "columns": ("URL", "TEXT"), + "preprocess_fn": laion2B_preprocess_train, + "streaming": True, + }, + "laion/laion400m": { + "columns": ("url", "caption"), + "preprocess_fn": partial(laion2B_preprocess_train, image_column="url", text_column="caption"), + "streaming": True, + }, } @@ -399,6 +422,16 @@ def parse_args(): choices=["cpu", "cuda"], help="Whether to use EMA model and where to store the EMA model.", ) + parser.add_argument( + "--quantization_mode", + type=str, + default="moderate", + choices=["moderate", "aggressive"], + help=( + "'aggressive' mode quantizes all MatMul operations while 'moderate' keeps MatMul that applies attention mask non-quantized." + " The later allows preserving a better accuracy while keeping the similar inference performance after optimization." + ), + ) parser.add_argument( "--non_ema_revision", type=str, @@ -644,7 +677,7 @@ def prepare_nncf_init_data(pipeline, dataloader, args): def get_nncf_config(pipeline, dataloader, args): text_encoder = pipeline.text_encoder unet = pipeline.unet - nncf_config_dict = { + moderate_quantization_config = { "input_info": [ { # "keyword": "latent_model_input", "sample_size": [1, unet.config["in_channels"], unet.config["sample_size"], unet.config["sample_size"]] @@ -673,6 +706,42 @@ def get_nncf_config(pipeline, dataloader, args): "{re}.*mul___[0-2]", "{re}.*silu_[0-2]", ], + "overflow_fix": "disable", + "export_to_onnx_standard_ops": True, + }, + ], + } + + aggressive_quantization_config = { + "input_info": [ + { # "keyword": "latent_model_input", + "sample_size": [1, unet.config["in_channels"], unet.config["sample_size"], unet.config["sample_size"]] + }, + {"sample_size": [1]}, # "keyword": "t", + { # "keyword": "encoder_hidden_states", + "sample_size": [1, text_encoder.config.max_position_embeddings, text_encoder.config.hidden_size] + }, + ], + "log_dir": args.output_dir, # The log directory for NNCF-specific logging outputs. + "compression": [ + { + "algorithm": "quantization", # Specify the algorithm here. + "preset": "mixed", + "initializer": { + "range": {"num_init_samples": args.opt_init_steps, "type": args.opt_init_type}, + "batchnorm_adaptation": {"num_bn_adaptation_samples": args.opt_init_steps}, + }, + "scope_overrides": { + "activations": {"{re}.*baddbmm_0": {"mode": "symmetric"}, "{re}.*bmm_0": {"mode": "symmetric"}} + }, + "ignored_scopes": [ + "{re}.*layer_norm_0", + "{re}.*__truediv__*", + "{re}.*group_norm_0", + "{re}.*mul___[0-2]", + "{re}.*silu_[0-2]", + ], + "overflow_fix": "disable", "export_to_onnx_standard_ops": True, }, ], @@ -688,7 +757,11 @@ def get_inputs(self, dataloader_output): def get_target(self, dataloader_output): return dataloader_output[0] - nncf_config = NNCFConfig.from_dict(nncf_config_dict) + quantization_config = ( + aggressive_quantization_config if args.quantization_mode == "aggressive" else moderate_quantization_config + ) + + nncf_config = NNCFConfig.from_dict(quantization_config) nncf_config = register_default_init_args(nncf_config, UnetInitDataLoader(dataloader)) return nncf_config From 62238aca7f8e464cd7a469aaf2434f129b972283 Mon Sep 17 00:00:00 2001 From: Chang Wang Date: Wed, 31 May 2023 16:12:36 +0800 Subject: [PATCH 019/134] improve inference for TSModelForCausalLM model without attention_mask (#321) * improve inference for TSModelForCausalLM model without attention_mask * fix style Signed-off-by: changwangss * Update optimum/intel/generation/modeling.py agree Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --------- Signed-off-by: changwangss Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- optimum/intel/generation/modeling.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index 11fa90a0d6..c66df36e11 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -139,6 +139,9 @@ def forward( past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, **kwargs, ) -> CausalLMOutputWithPast: + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + inputs = { "input_ids": input_ids, "attention_mask": attention_mask, @@ -220,17 +223,6 @@ def _from_pretrained( model_save_dir = Path(model_cache_path).parent model = cls.load_model(model_cache_path) - # IPEX jit model need 2 iterations to convert model to int8 model - onnx_config_class = TasksManager.get_exporter_config_constructor( - model_type=config.model_type.replace("_", "-"), - exporter="onnx", - task=cls.export_feature, - ) - onnx_config = onnx_config_class(config, use_past=use_cache) - model_inputs = onnx_config.generate_dummy_inputs(framework="pt") - for i in range(2): - model(**model_inputs) - return cls( model, config=config, From 9286c99993d423ae0710f99124f3c4a64b7c5b8f Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 31 May 2023 18:30:27 +0200 Subject: [PATCH 020/134] Increase tolerance for speedup enabled by cache (#331) --- tests/generation/test_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generation/test_modeling.py b/tests/generation/test_modeling.py index a6c3f7d811..6093164358 100644 --- a/tests/generation/test_modeling.py +++ b/tests/generation/test_modeling.py @@ -50,7 +50,7 @@ class ModelingIntegrationTest(unittest.TestCase): "gpt_neo", ) GENERATION_LENGTH = 100 - SPEEDUP_CACHE = 1.2 + SPEEDUP_CACHE = 1.1 @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): From cb361d5f45542eb0807aa7a4685894a0f5386862 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 31 May 2023 18:41:49 +0200 Subject: [PATCH 021/134] Fix INC Trainer for transformers > v4.29.2 (#330) --- optimum/intel/neural_compressor/trainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimum/intel/neural_compressor/trainer.py b/optimum/intel/neural_compressor/trainer.py index 65de84a93e..ac235e5061 100644 --- a/optimum/intel/neural_compressor/trainer.py +++ b/optimum/intel/neural_compressor/trainer.py @@ -137,6 +137,8 @@ def __init__( self._compression_manager = None self.distillation_callback = None self.save_onnx_model = save_onnx_model + # TODO : To deprecate once support transformers > 4.30.0 + self.deepspeed = None # Attach dtype and architecture to the config self.dtype = "int8" if quantization_config is not None else str(get_parameter_dtype(self.model)).split(".")[1] From a0ca358603314b82decf6bc2933a71547ffc1eef Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 31 May 2023 18:42:16 +0200 Subject: [PATCH 022/134] Fix OVTrainer for transformers >= v4.29.0 (#328) --- optimum/intel/openvino/trainer.py | 14 +++++++++++--- setup.py | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index 3e5b2b2ccc..af5b55ddce 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -78,6 +78,7 @@ from optimum.exporters import TasksManager from ..utils.constant import _TASK_ALIASES +from ..utils.import_utils import is_transformers_version from .configuration import OVConfig from .quantization import OVDataLoader, _onnx_export_nncf_model from .training_args import OVTrainingArguments @@ -282,9 +283,16 @@ def _inner_training_loop( if args.gradient_checkpointing: self.model.gradient_checkpointing_enable() - if self.args.local_rank != -1: - if self.compression_controller is not None: - self.compression_controller.distributed() + if is_transformers_version("<", "4.29.0"): + is_distributed = self.args.local_rank != -1 + else: + from accelerate.utils import DistributedType + + is_distributed = self.args.distributed_state.distributed_type != DistributedType.NO + + if self.compression_controller is not None and is_distributed: + self.compression_controller.distributed() + model = self._wrap_model(self.model_wrapped) if is_sagemaker_mp_enabled() and resume_from_checkpoint is not None: diff --git a/setup.py b/setup.py index f14136a51a..280da592f0 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,8 @@ "torch<2.0.0", # remove after neural-compressor next release "intel-extension-for-pytorch<2.0.0", ], - "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime", "transformers<4.29"], - "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217", "accelerate"], + "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime"], + "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From ad12e62377ad7816239d1eed7803c82ca4212165 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 31 May 2023 18:46:48 +0200 Subject: [PATCH 023/134] Fix openvino model inference for transformers >= v4.29.3 (#329) --- optimum/intel/openvino/modeling_base.py | 7 ++++++- optimum/intel/openvino/trainer.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 2db8867613..dfabafb5aa 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -55,12 +55,17 @@ } +# workaround to enable compatibility between openvino models and transformers pipelines +class PreTrainedModel(OptimizedModel): + pass + + @add_start_docstrings( """ Base OVModel class. """, ) -class OVBaseModel(OptimizedModel): +class OVBaseModel(PreTrainedModel): _AUTOMODELS_TO_TASKS = {cls_name: task for task, cls_name in TasksManager._TASKS_TO_AUTOMODELS.items()} auto_model_class = None export_feature = None diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index af5b55ddce..2d70593188 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -191,6 +191,8 @@ def __init__( self.compression_controller, self.model = create_compressed_model(self.model, nncf_config) self.model_wrapped = self.model + # TODO : To deprecate once support transformers > 4.30.0 + self.deepspeed = None def _set_signature_columns_if_needed(self): if self._signature_columns is None: From 3ccbb1bf3826382013485bc68628d6a184596750 Mon Sep 17 00:00:00 2001 From: Jan Iwaszkiewicz Date: Thu, 1 Jun 2023 14:04:23 +0200 Subject: [PATCH 024/134] Apply new Python API features from OpenVINO 2023.0 release (#265) * Apply shared memory and CompiledModel * Minor fixes * Re-run jobs * For some reason it needs to be reformmated * ruff fixes * Apply changes to CasualLM * Bump OV version * Remove unused helper * Adjust diffiusion modeling * Update optimum/intel/openvino/modeling_diffusion.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- optimum/intel/openvino/modeling.py | 31 +++---- optimum/intel/openvino/modeling_base.py | 5 +- optimum/intel/openvino/modeling_decoder.py | 19 +---- optimum/intel/openvino/modeling_diffusion.py | 29 ++++--- optimum/intel/openvino/modeling_seq2seq.py | 86 +++++++------------- setup.py | 4 +- 6 files changed, 61 insertions(+), 113 deletions(-) diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index 5a3a323ad1..b999a4116e 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -178,8 +178,7 @@ def forward( inputs["token_type_ids"] = token_type_ids # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return SequenceClassifierOutput(logits=logits) @@ -245,8 +244,7 @@ def forward( inputs["token_type_ids"] = token_type_ids # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) start_logits = ( torch.from_numpy(outputs["start_logits"]).to(self.device) if not np_inputs else outputs["start_logits"] ) @@ -316,8 +314,7 @@ def forward( inputs["token_type_ids"] = token_type_ids # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return TokenClassifierOutput(logits=logits) @@ -382,13 +379,12 @@ def forward( inputs["token_type_ids"] = token_type_ids # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} - - last_hidden_state = outputs["last_hidden_state"] - if not np_inputs: - last_hidden_state = torch.from_numpy(last_hidden_state).to(self.device) - + outputs = self.request(inputs) + last_hidden_state = ( + torch.from_numpy(outputs["last_hidden_state"]).to(self.device) + if not np_inputs + else outputs["last_hidden_state"] + ) return BaseModelOutput(last_hidden_state=last_hidden_state) @@ -453,8 +449,7 @@ def forward( inputs["token_type_ids"] = token_type_ids # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return MaskedLMOutput(logits=logits) @@ -512,8 +507,7 @@ def forward( } # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return ImageClassifierOutput(logits=logits) @@ -578,7 +572,6 @@ def forward( inputs["attention_mask"] = attention_mask # Run inference - outputs = self.request.infer(inputs) - outputs = {key.get_any_name(): value for key, value in outputs.items()} + outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return SequenceClassifierOutput(logits=logits) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index dfabafb5aa..a7b594794c 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -313,11 +313,10 @@ def _from_transformers( def compile(self): if self.request is None: - logger.info("Compiling the model and creating the inference request ...") + logger.info("Compiling the model...") cache_dir = Path(self.model_save_dir).joinpath("model_cache") ov_config = {**self.ov_config, "CACHE_DIR": str(cache_dir)} - compiled_model = core.compile_model(self.model, self._device, ov_config) - self.request = compiled_model.create_infer_request() + self.request = core.compile_model(self.model, self._device, ov_config) def _reshape( self, diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 50d39938ba..fc93603921 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -71,10 +71,6 @@ """ -def _contiguous_helper(tensor: np.ndarray) -> np.ndarray: - return tensor if tensor.flags["C_CONTIGUOUS"] else np.ascontiguousarray(tensor) - - @add_start_docstrings( """ Base OVBaseDecoderModel class. @@ -245,15 +241,10 @@ def forward( if past_key_values is not None: # Flatten the past_key_values past_key_values = tuple( - _contiguous_helper(np.array(past_key_value)) - for pkv_per_layer in past_key_values - for past_key_value in pkv_per_layer + np.array(past_key_value) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer ) # Add the past_key_values to the decoder inputs - inputs = { - input_name: Tensor(past_key_value, shared_memory=True) - for input_name, past_key_value in zip(self.key_value_input_names, past_key_values) - } + inputs = dict(zip(self.key_value_input_names, past_key_values)) # Create empty past_key_values for decoder_with_past first generation step elif self.use_cache: @@ -278,12 +269,8 @@ def forward( inputs["attention_mask"] = np.array(attention_mask) # Run inference - self.request.start_async(inputs) - self.request.wait() + outputs = self.request(inputs, shared_memory=True) - outputs = { - key.get_any_name(): value.data for key, value in zip(self.request.model_outputs, self.request.outputs) - } logits = torch.from_numpy(outputs["logits"]).to(self.device) if self.use_cache: diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index d3179f2083..f83f12fdde 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -411,9 +411,9 @@ def clear_requests(self): self.unet.request = None def compile(self): - self.text_encoder._create_inference_request() - self.vae_decoder._create_inference_request() - self.unet._create_inference_request() + self.text_encoder._compile() + self.vae_decoder._compile() + self.unet._compile() def __call__( self, @@ -486,11 +486,10 @@ def __init__( self.ov_config = ov_config or self.parent_model.ov_config self.request = None - def _create_inference_request(self): + def _compile(self): if self.request is None: - logger.info("Compiling the encoder and creating the inference request ...") - compiled_model = core.compile_model(self.model, self.device, self.ov_config) - self.request = compiled_model.create_infer_request() + logger.info("Compiling the encoder...") + self.request = core.compile_model(self.model, self.device, self.ov_config) @property def device(self): @@ -499,18 +498,18 @@ def device(self): class OVModelTextEncoder(OVModelPart): def __call__(self, input_ids: np.ndarray): - self._create_inference_request() + self._compile() inputs = { "input_ids": input_ids, } - outputs = self.request.infer(inputs) + outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) class OVModelUnet(OVModelPart): def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_states: np.ndarray): - self._create_inference_request() + self._compile() inputs = { "sample": sample, @@ -518,27 +517,27 @@ def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_stat "encoder_hidden_states": encoder_hidden_states, } - outputs = self.request.infer(inputs) + outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) class OVModelVaeDecoder(OVModelPart): def __call__(self, latent_sample: np.ndarray): - self._create_inference_request() + self._compile() inputs = { "latent_sample": latent_sample, } - outputs = self.request.infer(inputs) + outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) class OVModelVaeEncoder(OVModelPart): def __call__(self, sample: np.ndarray): - self._create_inference_request() + self._compile() inputs = { "sample": sample, } - outputs = self.request.infer(inputs) + outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index c00cc8ef16..a9eba68ecf 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -16,11 +16,10 @@ from pathlib import Path from typing import Dict, Optional, Tuple -import numpy as np import openvino import torch import transformers -from openvino.runtime import Core, Tensor +from openvino.runtime import Core from transformers import AutoConfig, AutoModelForSeq2SeqLM from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput @@ -126,10 +125,6 @@ """ -def _contiguous_helper(tensor: np.ndarray) -> np.ndarray: - return tensor if tensor.flags["C_CONTIGUOUS"] else np.ascontiguousarray(tensor) - - @add_start_docstrings( """ Sequence-to-sequence model with a language modeling head for OpenVINO inference. @@ -287,10 +282,10 @@ def clear_requests(self): self.decoder_with_past.request = None def compile(self): - self.encoder._create_inference_request() - self.decoder._create_inference_request() + self.encoder._compile() + self.decoder._compile() if self.use_cache: - self.decoder_with_past._create_inference_request() + self.decoder_with_past._compile() class OVEncoder: @@ -318,36 +313,29 @@ def forward( attention_mask: torch.LongTensor = None, **kwargs, ) -> BaseModelOutput: - self._create_inference_request() + self._compile() - # Check if inputs are c-like, if not - convert them - input_ids = _contiguous_helper(np.array(input_ids)) - - inputs = { - "input_ids": Tensor(input_ids, shared_memory=True), - } + # Model inputs + inputs = {"input_ids": input_ids} # Add the attention_mask inputs when needed if "attention_mask" in self.input_names: - attention_mask = _contiguous_helper(np.array(attention_mask)) - inputs["attention_mask"] = Tensor(attention_mask, shared_memory=True) + inputs["attention_mask"] = attention_mask # Run inference - self.request.start_async(inputs) - self.request.wait() - - last_hidden_state = torch.from_numpy(self.request.get_tensor("last_hidden_state").data).to(self.device) + last_hidden_state = torch.from_numpy(self.request(inputs, shared_memory=True)["last_hidden_state"]).to( + self.device + ) return BaseModelOutput(last_hidden_state=last_hidden_state) def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) - def _create_inference_request(self): + def _compile(self): if self.request is None: - logger.info("Compiling the encoder and creating the inference request ...") - compiled_model = core.compile_model(self.model, self._device, self.ov_config) - self.request = compiled_model.create_infer_request() + logger.info("Compiling the encoder...") + self.request = core.compile_model(self.model, self._device, self.ov_config) class OVDecoder: @@ -387,56 +375,39 @@ def forward( encoder_attention_mask: Optional[torch.LongTensor] = None, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, ) -> Seq2SeqLMOutput: - self._create_inference_request() - + self._compile() + # Model inputs inputs = {} if past_key_values is not None: # Flatten the past_key_values past_key_values = tuple( - _contiguous_helper(np.array(past_key_value)) - for pkv_per_layer in past_key_values - for past_key_value in pkv_per_layer + past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer ) # Add the past_key_values to the decoder inputs - inputs = { - input_name: Tensor(past_key_value, shared_memory=True) - for input_name, past_key_value in zip(self.key_value_input_names, past_key_values) - } + inputs = dict(zip(self.key_value_input_names, past_key_values)) - # Check if inputs are c-like, if not - convert them - input_ids = _contiguous_helper(np.array(input_ids)) - inputs["input_ids"] = Tensor(input_ids, shared_memory=True) + inputs["input_ids"] = input_ids # Add the encoder_attention_mask inputs when needed if "encoder_attention_mask" in self.input_names and encoder_attention_mask is not None: - encoder_attention_mask = _contiguous_helper(np.array(encoder_attention_mask)) - inputs["encoder_attention_mask"] = Tensor(encoder_attention_mask, shared_memory=True) + inputs["encoder_attention_mask"] = encoder_attention_mask # Add the encoder_hidden_states inputs when needed if "encoder_hidden_states" in self.input_names and encoder_hidden_states is not None: - encoder_hidden_states = _contiguous_helper(np.array(encoder_hidden_states)) - inputs["encoder_hidden_states"] = Tensor(encoder_hidden_states, shared_memory=True) + inputs["encoder_hidden_states"] = encoder_hidden_states # Run inference - self.request.start_async(inputs) - self.request.wait() - - outputs = {} - for key, value in zip(self.request.model_outputs, self.request.outputs): - output_names = key.get_names() - output_name = "logits" if "logits" in output_names else next(iter(output_names)) - outputs[output_name] = value.data - + outputs = self.request(inputs, shared_memory=True) logits = torch.from_numpy(outputs["logits"]).to(self.device) # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the # self-attention layer and 2 to the cross-attention layer) out_past_key_values = tuple( - torch.from_numpy(outputs[key]).to(self.device) - for key in outputs - if ("key_values" in key or "present" in key) + torch.from_numpy(outputs[next(iter(key))]).to(self.device) + for key in outputs.names() + if ("key_values" in next(iter(key)) or "present" in next(iter(key))) ) # Tuple of tuple of length `n_layers`, with each tuple of length equal to: @@ -458,8 +429,7 @@ def forward( def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) - def _create_inference_request(self): + def _compile(self): if self.request is None: - logger.info("Compiling the decoder and creating the inference request ...") - compiled_model = core.compile_model(self.model, self._device, self.ov_config) - self.request = compiled_model.create_infer_request() + logger.info("Compiling the decoder...") + self.request = core.compile_model(self.model, self._device, self.ov_config) diff --git a/setup.py b/setup.py index 280da592f0..cd8f40ef85 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,8 @@ "torch<2.0.0", # remove after neural-compressor next release "intel-extension-for-pytorch<2.0.0", ], - "openvino": ["openvino>=2023.0.0.dev20230217", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0.dev20230217"], + "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], + "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From f98fbb4a69501369bb03d43629d0ed4246b4a356 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:06:37 +0200 Subject: [PATCH 025/134] Remove torch version constraint (#332) --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index cd8f40ef85..385edca0d4 100644 --- a/setup.py +++ b/setup.py @@ -25,13 +25,7 @@ QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] EXTRAS_REQUIRE = { - "neural-compressor": [ - "neural-compressor>=2.1.1", - "onnx", - "onnxruntime", - "torch<2.0.0", # remove after neural-compressor next release - "intel-extension-for-pytorch<2.0.0", - ], + "neural-compressor": ["neural-compressor>=2.1.1", "onnx", "onnxruntime"], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], From c48f06ad3303df8f1c530db6f953c23f6e71553c Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:44:12 +0200 Subject: [PATCH 026/134] Update documentation to highlight the INC quantization CLI (#333) * Update documentation to highlight the INC quantization CLI * fix typo --- docs/source/optimization_inc.mdx | 45 ++++++++------------------------ 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/docs/source/optimization_inc.mdx b/docs/source/optimization_inc.mdx index 1a46c161f0..be65fb98b4 100644 --- a/docs/source/optimization_inc.mdx +++ b/docs/source/optimization_inc.mdx @@ -21,33 +21,23 @@ Note that quantization is currently only supported for CPUs (only CPU backends a ### Dynamic quantization -To apply dynamic quantization on a fine-tuned DistilBERT, we first need to create the corresponding configuration describing the quantization details as well as the quantizer object used to later apply quantization: - -```python -from transformers import AutoModelForQuestionAnswering -from neural_compressor.config import PostTrainingQuantConfig -from optimum.intel import INCQuantizer - -model_name = "distilbert-base-cased-distilled-squad" -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -# The directory where the quantized model will be saved -save_dir = "dynamic_quantization" +You can easily add dynamic quantization on your model by using the following command line: -# Load the quantization configuration detailing the quantization we wish to apply -quantization_config = PostTrainingQuantConfig(approach="dynamic") -quantizer = INCQuantizer.from_pretrained(model) -# Apply dynamic quantization and save the resulting model -quantizer.quantize(quantization_config=quantization_config, save_directory=save_dir) +```bash +optimum-cli inc quantize --model distilbert-base-cased-distilled-squad --output quantized_distilbert ``` -The accuracy tolerance along with an adapted evaluation function can also be specified in order to find a quantized model meeting the specified accuracy tolerance. +When applying post-training quantization, an accuracy tolerance along with an adapted evaluation function can also be specified in order to find a quantized model meeting the specified constraints. This can be done for both dynamic and static quantization. ```python import evaluate +from optimum.intel import INCQuantizer from datasets import load_dataset -from transformers import AutoTokenizer, pipeline -from neural_compressor.config import AccuracyCriterion, TuningCriterion +from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline +from neural_compressor.config import AccuracyCriterion, TuningCriterion, PostTrainingQuantConfig +model_name = "distilbert-base-cased-distilled-squad" +model = AutoModelForQuestionAnswering.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) eval_dataset = load_dataset("squad", split="validation").select(range(64)) task_evaluator = evaluate.evaluator("question-answering") @@ -66,7 +56,7 @@ quantization_config = PostTrainingQuantConfig( approach="dynamic", accuracy_criterion=accuracy_criterion, tuning_criterion=tuning_criterion ) quantizer = INCQuantizer.from_pretrained(model, eval_fn=eval_fn) -quantizer.quantize(quantization_config=quantization_config, save_directory=save_dir) +quantizer.quantize(quantization_config=quantization_config, save_directory="dynamic_quantization") ``` ### Static quantization @@ -110,7 +100,7 @@ quantizer.quantize( ### Specify Quantization Recipes -The [SmoothQuant](https://arxiv.org/abs/2211.10438) methodology is available for post-training quantization. This methodology usually improves the accuracy of the model in comparison to other post-training static quantization methodologies. This is done by migratesing the difficulty from activations to weights with a mathematically equivalent transformation. +The [SmoothQuant](https://arxiv.org/abs/2211.10438) methodology is available for post-training quantization. This methodology usually improves the accuracy of the model in comparison to other post-training static quantization methodologies. This is done by migrating the difficulty from activations to weights with a mathematically equivalent transformation. ```diff - quantization_config = PostTrainingQuantConfig(approach="static") + recipes={"smooth_quant": True, "smooth_quant_args": {"alpha": 0.5, "folding": True}} @@ -270,16 +260,3 @@ outputs = pipe_cls(text) ``` Check out the [`examples`](https://github.com/huggingface/optimum-intel/tree/main/examples) directory for more sophisticated usage. - - - -## Apply quantization using the CLI - -Intel Neural Compressor dynamic quantization can be applied on your model through the Optimum command-line. - -You can easily add dynamic quantization on your model by using the following command line: - -```bash -optimum-cli inc quantize --model distilbert-base-cased-distilled-squad --output ./quantized_distilbert -``` - From f10eef0ba22e062d8d0a606da39a57f9751f8ba7 Mon Sep 17 00:00:00 2001 From: "Abhishek C.V. Salian" <43337429+AbhishekSalian@users.noreply.github.com> Date: Mon, 5 Jun 2023 13:48:41 +0530 Subject: [PATCH 027/134] Fix INC trainer generation evaluation (#335) --- optimum/intel/neural_compressor/trainer_seq2seq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/intel/neural_compressor/trainer_seq2seq.py b/optimum/intel/neural_compressor/trainer_seq2seq.py index 29ef987fa4..619e23eaff 100644 --- a/optimum/intel/neural_compressor/trainer_seq2seq.py +++ b/optimum/intel/neural_compressor/trainer_seq2seq.py @@ -157,8 +157,8 @@ def prediction_step( # XXX: adapt synced_gpus for fairscale as well gen_kwargs = { - "max_length": self._max_length if self._max_length is not None else self.config.max_length, - "num_beams": self._num_beams if self._num_beams is not None else self.config.num_beams, + "max_length": self._max_length if self._max_length is not None else self.model.config.max_length, + "num_beams": self._num_beams if self._num_beams is not None else self.model.config.num_beams, "synced_gpus": True if is_deepspeed_zero3_enabled() else False, } From a61837c1746e6e0592102384bb852e5ea1ad1033 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 6 Jun 2023 19:37:16 +0200 Subject: [PATCH 028/134] Fix inc trainer for seq2seq models (#336) * fix inc trainer * test refactorization * fix style * add fix * add test * fix onnx version * requirement onnxruntime version for inc * fix styme * add test * add seq2seq QAT test * fix style --- .../neural_compressor/trainer_seq2seq.py | 4 +- optimum/intel/neural_compressor/utils.py | 12 + setup.py | 2 +- tests/neural_compressor/test_optimization.py | 686 ++++++++++-------- 4 files changed, 413 insertions(+), 291 deletions(-) diff --git a/optimum/intel/neural_compressor/trainer_seq2seq.py b/optimum/intel/neural_compressor/trainer_seq2seq.py index 619e23eaff..123ced6030 100644 --- a/optimum/intel/neural_compressor/trainer_seq2seq.py +++ b/optimum/intel/neural_compressor/trainer_seq2seq.py @@ -210,8 +210,8 @@ def _pad_tensors_to_max_len(self, tensor, max_length): self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id ) else: - if self.config.pad_token_id is not None: - pad_token_id = self.config.pad_token_id + if self.model.config.pad_token_id is not None: + pad_token_id = self.model.config.pad_token_id else: raise ValueError("Pad_token_id must be set in the configuration of the model, in order to pad tensors") diff --git a/optimum/intel/neural_compressor/utils.py b/optimum/intel/neural_compressor/utils.py index 50e928292c..aff8035a00 100644 --- a/optimum/intel/neural_compressor/utils.py +++ b/optimum/intel/neural_compressor/utils.py @@ -31,6 +31,18 @@ CONFIG_NAME = "best_configure.yaml" +_HEAD_TO_AUTOMODELS = { + "fill-mask": "INCModelForMaskedLM", + "text-generation": "INCModelForCausalLM", + "text2text-generation": "INCModelForSeq2SeqLM", + "text-classification": "INCModelForSequenceClassification", + "token-classification": "INCModelForTokenClassification", + "question-answering": "INCModelForQuestionAnswering", + "multiple-choice": "INCModelForMultipleChoice", + "stable-diffusion": "INCStableDiffusionPipeline", +} + + parsed_torch_version_base = version.parse(version.parse(torch.__version__).base_version) is_torch_less_than_1_13 = parsed_torch_version_base < version.parse("1.13.0") diff --git a/setup.py b/setup.py index 385edca0d4..4e69e8c6d7 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] EXTRAS_REQUIRE = { - "neural-compressor": ["neural-compressor>=2.1.1", "onnx", "onnxruntime"], + "neural-compressor": ["neural-compressor>=2.1.1", "onnx", "onnxruntime<1.15.0"], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index 1206a9ac6f..67be425f88 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# ruff: noqa + import os import tempfile import unittest @@ -31,32 +33,49 @@ WeightPruningConfig, ) from onnx import load as onnx_load +from parameterized import parameterized from transformers import ( AutoModelForQuestionAnswering, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction, TrainingArguments, + Seq2SeqTrainingArguments, default_data_collator, pipeline, + BertTokenizer, + EncoderDecoderModel, set_seed, ) from optimum.intel import ( INCConfig, + INCModelForCausalLM, + INCModelForSeq2SeqLM, INCModelForQuestionAnswering, INCModelForSequenceClassification, + INCModelForMaskedLM, + INCModelForTokenClassification, INCQuantizer, INCStableDiffusionPipeline, INCTrainer, + INCSeq2SeqTrainer, ) -from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME -from optimum.onnxruntime import ORTModelForSequenceClassification +from optimum.intel.neural_compressor.utils import _HEAD_TO_AUTOMODELS +from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME, ONNX_WEIGHTS_NAME +from optimum.onnxruntime import ORTModelForCausalLM, ORTModelForSequenceClassification +from optimum.pipelines import ORT_SUPPORTED_TASKS os.environ["CUDA_VISIBLE_DEVICES"] = "" set_seed(1009) +_TASK_TO_DATASET = { + "text-classification": ("glue", "sst2", "sentence"), + "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), + "text2text-generation": ("cnn_dailymail", "3.0.0", ("article", "highlights")), +} + def num_quantized_matmul_onnx_model(onnx_model): num_quantized_matmul = 0 @@ -66,268 +85,157 @@ def num_quantized_matmul_onnx_model(onnx_model): return num_quantized_matmul -class QuantizationTest(unittest.TestCase): - def test_dynamic_quantization(self): - model_name = "distilbert-base-uncased-finetuned-sst-2-english" - expected_quantized_matmuls = 36 - quantization_config = PostTrainingQuantConfig(approach="dynamic") - model = AutoModelForSequenceClassification.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - quantizer = INCQuantizer.from_pretrained(model) +def _preprocess_function(examples, tokenizer, column_name): + return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) - with tempfile.TemporaryDirectory() as tmp_dir: - quantizer.quantize( - quantization_config=quantization_config, - save_directory=tmp_dir, - save_onnx_model=True, - ) - loaded_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) - onnx_model = onnx_load(os.path.join(tmp_dir, "model.onnx")) - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) - self.assertFalse(inc_config.quantization["is_static"]) - num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) - self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) +def _compute_metrics(outputs, metric): + return metric.compute(predictions=np.argmax(outputs.predictions, axis=1), references=outputs.label_ids) - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - with torch.no_grad(): - model_outputs = quantizer._quantized_model(**tokens) - loaded_model_outputs = loaded_model(**tokens) - self.assertTrue(torch.equal(model_outputs.logits, loaded_model_outputs.logits)) - # self.assertTrue(torch.allclose(ort_outputs.logits, loaded_model_outputs.logits, atol=1e-4)) - def test_dynamic_accuracy_strategy_quantization(self): - model_name = "distilbert-base-cased-distilled-squad" - model = AutoModelForQuestionAnswering.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - eval_dataset = load_dataset("squad", split="validation").select(range(64)) - task_evaluator = evaluate.evaluator("question-answering") - qa_pipeline = pipeline("question-answering", model=model, tokenizer=tokenizer) - tolerance_criterion = 0.1 +def _generate_dataset(quantizer, tokenizer, num_samples=10): + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[quantizer.task] + dataset = quantizer.get_calibration_dataset( + dataset_name, + dataset_config_name=dataset_config_name, + preprocess_function=partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), + num_samples=num_samples, + dataset_split="train", + ) + return dataset - def eval_fn(model): - qa_pipeline.model = model - metrics = task_evaluator.compute(model_or_pipeline=qa_pipeline, data=eval_dataset, metric="squad") - return metrics["f1"] - original_model_metric = eval_fn(model) - tuning_criterion = TuningCriterion(max_trials=10) - accuracy_criterion = AccuracyCriterion(tolerable_loss=tolerance_criterion) - quantization_config = PostTrainingQuantConfig( - approach="dynamic", accuracy_criterion=accuracy_criterion, tuning_criterion=tuning_criterion - ) - tokenizer("This is a sample input", return_tensors="pt") +class OptimizationTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( + ("text-classification", "hf-internal-testing/tiny-random-bert", 30), + # ("text-generation", "hf-internal-testing/tiny-random-BloomForCausalLM", 1), # TODO : enable causal lm task once INC ONNX export fixed + ) + + SUPPORTED_ARCHITECTURES_DYNAMIC = SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS + ( + ("fill-mask", "hf-internal-testing/tiny-random-DistilBertForMaskedLM", 30), + ("token-classification", "hf-internal-testing/tiny-random-AlbertForTokenClassification", 30), + ) + @parameterized.expand(SUPPORTED_ARCHITECTURES_DYNAMIC) + def test_dynamic_quantization(self, task, model_name, expected_quantized_matmuls): + quantization_config = PostTrainingQuantConfig(approach="dynamic") + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + quantizer = INCQuantizer.from_pretrained(model, task=task) + save_onnx_model = "generation" in task with tempfile.TemporaryDirectory() as tmp_dir: - quantizer = INCQuantizer.from_pretrained(model, eval_fn=eval_fn) quantizer.quantize( quantization_config=quantization_config, save_directory=tmp_dir, - save_onnx_model=True, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=quantizer._quantized_model, + task=task, + tokenizer=tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=False, + load_onnx_model=save_onnx_model, ) - loaded_model = INCModelForQuestionAnswering.from_pretrained(tmp_dir) - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) - self.assertFalse(inc_config.quantization["is_static"]) - - quantized_model_metric = eval_fn(loaded_model) - # Verification accuracy loss is under 5% - self.assertGreaterEqual(quantized_model_metric, original_model_metric * (1 - tolerance_criterion)) - def test_static_quantization(self): - model_name = "distilbert-base-uncased-finetuned-sst-2-english" - expected_quantized_matmuls = 36 + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_static_quantization(self, task, model_name, expected_quantized_matmuls): num_samples = 10 quantization_config = PostTrainingQuantConfig(approach="static") - model = AutoModelForSequenceClassification.from_pretrained(model_name) + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token - def preprocess_function(examples, tokenizer): - return tokenizer(examples["sentence"], padding="max_length", max_length=128, truncation=True) - - quantizer = INCQuantizer.from_pretrained(model) - calibration_dataset = quantizer.get_calibration_dataset( - "glue", - dataset_config_name="sst2", - preprocess_function=partial(preprocess_function, tokenizer=tokenizer), - num_samples=num_samples, - dataset_split="train", - ) - quantizer = INCQuantizer.from_pretrained(model) + quantizer = INCQuantizer.from_pretrained(model, task=task) + calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=num_samples) + save_onnx_model = task != "text-generation" with tempfile.TemporaryDirectory() as tmp_dir: quantizer.quantize( quantization_config=quantization_config, calibration_dataset=calibration_dataset, save_directory=tmp_dir, - save_onnx_model=True, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=quantizer._quantized_model, + task=task, + tokenizer=tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=True, + num_samples=num_samples, + load_onnx_model=save_onnx_model, ) - loaded_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) - onnx_model = onnx_load(os.path.join(tmp_dir, "model.onnx")) - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) - self.assertTrue(inc_config.quantization["is_static"]) - self.assertEqual(inc_config.quantization["dataset_num_samples"], num_samples) - - num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) - self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) - - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - with torch.no_grad(): - model_outputs = quantizer._quantized_model(**tokens) - loaded_model_outputs = loaded_model(**tokens) - self.assertTrue(torch.equal(model_outputs.logits, loaded_model_outputs.logits)) - # self.assertTrue(torch.allclose(ort_outputs.logits, loaded_model_outputs.logits, atol=1e-4)) - def test_ipex_static_quantization_with_smoothquant(self): - model_name = "distilbert-base-uncased-finetuned-sst-2-english" - quantization_config = PostTrainingQuantConfig( - approach="static", backend="ipex", recipes={"smooth_quant": True, "smooth_quant_args": {"alpha": 0.5}} - ) - model = AutoModelForSequenceClassification.from_pretrained(model_name) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_ipex_static_quantization_with_smoothquant(self, task, model_name, expected_quantized_matmuls): + recipes = {"smooth_quant": True, "smooth_quant_args": {"alpha": 0.5}} + num_samples = 10 + quantization_config = PostTrainingQuantConfig(approach="static", backend="ipex", recipes=recipes) + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - - def preprocess_function(examples, tokenizer): - return tokenizer(examples["sentence"], padding="max_length", max_length=128, truncation=True) - - quantizer = INCQuantizer.from_pretrained(model) - calibration_dataset = quantizer.get_calibration_dataset( - "glue", - dataset_config_name="sst2", - preprocess_function=partial(preprocess_function, tokenizer=tokenizer), - num_samples=10, - dataset_split="train", - ) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + quantizer = INCQuantizer.from_pretrained(model, task=task) + calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=num_samples) with tempfile.TemporaryDirectory() as tmp_dir: - quantizer = INCQuantizer.from_pretrained(model) quantizer.quantize( quantization_config=quantization_config, calibration_dataset=calibration_dataset, save_directory=tmp_dir, save_onnx_model=False, ) - transformers_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertFalse(inc_config.save_onnx_model) - self.assertTrue(inc_config.quantization["is_static"]) - - with torch.no_grad(): - transformers_outputs = transformers_model(**tokens) - model_outputs = quantizer._quantized_model(**tokens) - - self.assertTrue(torch.equal(model_outputs["logits"], transformers_outputs["logits"])) - - def test_aware_training_quantization(self): - model_name = "distilbert-base-uncased" - expected_quantized_matmuls = 36 - quantization_config = QuantizationAwareTrainingConfig() - model = AutoModelForSequenceClassification.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - metric = evaluate.load("accuracy") - dataset = load_dataset("glue", "sst2") - dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding="max_length", max_length=128), batched=True - ) - - def compute_metrics(p: EvalPrediction): - return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) - - with tempfile.TemporaryDirectory() as tmp_dir: - trainer = INCTrainer( - model=model, - quantization_config=quantization_config, - task="text-classification", - args=TrainingArguments(tmp_dir, num_train_epochs=1.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(8)), - eval_dataset=dataset["validation"].select(range(8)), - compute_metrics=compute_metrics, + self.check_model_outputs( + q_model=quantizer._quantized_model, + task=task, tokenizer=tokenizer, - data_collator=default_data_collator, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=True, + load_onnx_model=False, + num_samples=num_samples, ) - trainer.train() - trainer.evaluate() - trainer.save_model(save_onnx_model=True) - loaded_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) - onnx_model = onnx_load(os.path.join(tmp_dir, "model.onnx")) - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) - self.assertTrue(inc_config.quantization["is_static"]) - num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) - self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) + def test_dynamic_accuracy_strategy_quantization(self): + model_name = "distilbert-base-cased-distilled-squad" + model = AutoModelForQuestionAnswering.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + eval_dataset = load_dataset("squad", split="validation").select(range(64)) + task_evaluator = evaluate.evaluator("question-answering") + qa_pipeline = pipeline("question-answering", model=model, tokenizer=tokenizer) + tolerance_criterion = 0.1 - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - trainer.model.eval() - loaded_model.eval() - with torch.no_grad(): - model_outputs = trainer.model(**tokens) - loaded_model_outputs = loaded_model(**tokens) - # self.assertTrue(torch.allclose(ort_outputs.logits, loaded_model_outputs.logits, atol=1e-4)) - self.assertTrue(torch.equal(model_outputs.logits, loaded_model_outputs.logits)) + def eval_fn(model): + qa_pipeline.model = model + metrics = task_evaluator.compute(model_or_pipeline=qa_pipeline, data=eval_dataset, metric="squad") + return metrics["f1"] - def test_aware_training_quantization_pruning(self): - model_name = "distilbert-base-uncased" - target_sparsity = 0.9 - pruning_config = WeightPruningConfig( - pruning_type="magnitude", - start_step=0, - end_step=15, - target_sparsity=target_sparsity, - pruning_scope="local", - ) - quantization_config = QuantizationAwareTrainingConfig() - model = AutoModelForSequenceClassification.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - metric = evaluate.load("accuracy") - dataset = load_dataset("glue", "sst2") - dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding="max_length", max_length=128), batched=True + original_model_metric = eval_fn(model) + tuning_criterion = TuningCriterion(max_trials=10) + accuracy_criterion = AccuracyCriterion(tolerable_loss=tolerance_criterion) + quantization_config = PostTrainingQuantConfig( + approach="dynamic", accuracy_criterion=accuracy_criterion, tuning_criterion=tuning_criterion ) - - def compute_metrics(p: EvalPrediction): - return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) + quantizer = INCQuantizer.from_pretrained(model, eval_fn=eval_fn) with tempfile.TemporaryDirectory() as tmp_dir: - trainer = INCTrainer( - model=model, + quantizer.quantize( quantization_config=quantization_config, - pruning_config=pruning_config, - task="sequence-classification", - args=TrainingArguments(tmp_dir, num_train_epochs=1.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(8)), - eval_dataset=dataset["validation"].select(range(8)), - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, + save_directory=tmp_dir, + save_onnx_model=True, ) - trainer.train() - trainer.evaluate() - trainer.save_model(save_onnx_model=True) - + loaded_model = INCModelForQuestionAnswering.from_pretrained(tmp_dir) inc_config = INCConfig.from_pretrained(tmp_dir) self.assertTrue(inc_config.save_onnx_model) - self.assertTrue(inc_config.quantization["is_static"]) + self.assertFalse(inc_config.quantization["is_static"]) - transformers_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - with torch.no_grad(): - transformers_model(**tokens) - # self.assertTrue(torch.allclose(ort_outputs.logits, transformers_outputs.logits, atol=1e-4)) + quantized_model_metric = eval_fn(loaded_model) + # Verification accuracy loss is under 5% + self.assertGreaterEqual(quantized_model_metric, original_model_metric * (1 - tolerance_criterion)) def test_dynamic_diffusion_model(self): model_id = "hf-internal-testing/diffusers-stable-diffusion-tiny-all" @@ -370,10 +278,63 @@ def test_dynamic_diffusion_model(self): # Compare model outputs self.assertTrue(np.allclose(loaded_pipe_outputs, outputs, atol=1e-4)) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_aware_training_quantization(self, task, model_name, expected_quantized_matmuls): + quantization_config = QuantizationAwareTrainingConfig() + save_onnx_model = True + + with tempfile.TemporaryDirectory() as tmp_dir: + trainer = self.get_trainer( + model_name=model_name, + task=task, + save_directory=tmp_dir, + q_config=quantization_config, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=trainer.model, + task=task, + tokenizer=trainer.tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=True, + load_onnx_model=save_onnx_model, + ) + + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_aware_training_quantization_pruning(self, task, model_name, expected_quantized_matmuls): + quantization_config = QuantizationAwareTrainingConfig() + target_sparsity = 0.9 + pruning_config = WeightPruningConfig( + pruning_type="magnitude", + start_step=0, + end_step=15, + target_sparsity=target_sparsity, + pruning_scope="local", + ) + save_onnx_model = True + + with tempfile.TemporaryDirectory() as tmp_dir: + trainer = self.get_trainer( + model_name=model_name, + task=task, + save_directory=tmp_dir, + q_config=quantization_config, + p_config=pruning_config, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=trainer.model, + task=task, + tokenizer=trainer.tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=True, + load_onnx_model=save_onnx_model, + ) -class PruningTest(unittest.TestCase): - def test_magnitude_pruning(self): - model_name = "distilbert-base-uncased" + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_magnitude_pruning(self, task, model_name, expected_quantized_matmuls): target_sparsity = 0.9 # end_step should be training_args.num_train_epochs * (len(train_dataset) // training_args.per_device_train_batch_size) pruning_config = WeightPruningConfig( @@ -383,93 +344,242 @@ def test_magnitude_pruning(self): target_sparsity=target_sparsity, pruning_scope="local", ) - model = AutoModelForSequenceClassification.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - metric = evaluate.load("accuracy") - dataset = load_dataset("glue", "sst2") - dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding="max_length", max_length=128), batched=True - ) - - def compute_metrics(p: EvalPrediction): - return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) + save_onnx_model = True with tempfile.TemporaryDirectory() as tmp_dir: - trainer = INCTrainer( - model=model, - pruning_config=pruning_config, - task="text-classification", - args=TrainingArguments(tmp_dir, num_train_epochs=2.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(64)), - eval_dataset=dataset["validation"].select(range(4)), - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, + trainer = self.get_trainer( + model_name=model_name, + task=task, + save_directory=tmp_dir, + p_config=pruning_config, + save_onnx_model=save_onnx_model, + num_train_samples=64, + ) + self.check_model_outputs( + q_model=trainer.model, + task=task, + tokenizer=trainer.tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=0, + is_static=True, + load_onnx_model=save_onnx_model, ) - trainer.train() - trainer.evaluate() - trainer.save_model(save_onnx_model=True) - - inc_config = INCConfig.from_pretrained(tmp_dir) - transformers_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - with torch.no_grad(): - transformers_outputs = transformers_model(**tokens) - self.assertTrue(torch.allclose(ort_outputs.logits, transformers_outputs.logits, atol=1e-4)) sparsity = trainer.get_model_sparsity() - self.assertGreaterEqual(sparsity, target_sparsity * 100 / 2) - self.assertTrue(inc_config.save_onnx_model) + inc_config = INCConfig.from_pretrained(tmp_dir) + # Factor modified from 2 to 4 for tiny random model compatibility + self.assertGreaterEqual(sparsity, target_sparsity * 100 / 4) self.assertEqual(inc_config.pruning["sparsity"], round(sparsity, 2)) self.assertEqual(inc_config.pruning["approach"], "magnitude") self.assertEqual(inc_config.pruning["pattern"], "4x1") + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_distillation(self, task, model_name, expected_quantized_matmuls): + teacher_model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) + distillation_config = DistillationConfig(teacher_model=teacher_model) + save_onnx_model = True -class DistillationTest(unittest.TestCase): - def test_knowledge_distillation(self): - model_name = "distilbert-base-uncased" - model = AutoModelForSequenceClassification.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - tokens = tokenizer("This is a sample input", return_tensors="pt") - metric = evaluate.load("accuracy") - dataset = load_dataset("glue", "sst2") - dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding="max_length", max_length=128), batched=True + with tempfile.TemporaryDirectory() as tmp_dir: + trainer = self.get_trainer( + model_name=model_name, + task=task, + save_directory=tmp_dir, + d_config=distillation_config, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=trainer.model, + task=task, + tokenizer=trainer.tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=0, + is_static=True, + load_onnx_model=save_onnx_model, + ) + inc_config = INCConfig.from_pretrained(tmp_dir) + self.assertEqual(inc_config.distillation["teacher_model_name_or_path"], model_name) + self.assertEqual(inc_config.distillation["temperature"], 1.0) + + def test_seq2seq_aware_training_quantization(self): + quantization_config = QuantizationAwareTrainingConfig() + save_onnx_model = True + batch_size = 2 + train_dataset = load_dataset("cnn_dailymail", "3.0.0", split="train[:1%]") + val_dataset = load_dataset("cnn_dailymail", "3.0.0", split="validation[:1%]") + train_dataset = train_dataset.select(range(4)) + val_dataset = val_dataset.select(range(4)) + + model = EncoderDecoderModel.from_encoder_decoder_pretrained("prajjwal1/bert-tiny", "prajjwal1/bert-tiny") + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + model.config.vocab_size = model.config.encoder.vocab_size + model.config.eos_token_id = tokenizer.sep_token_id + model.config.decoder_start_token_id = tokenizer.cls_token_id + model.config.max_length = 128 + columns = ["input_ids", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"] + + def _map_to_encoder_decoder_inputs(batch): + inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512) + outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=128) + batch["input_ids"] = inputs.input_ids + batch["attention_mask"] = inputs.attention_mask + + batch["decoder_input_ids"] = outputs.input_ids + batch["labels"] = outputs.input_ids.copy() + batch["labels"] = [ + [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"] + ] + batch["decoder_attention_mask"] = outputs.attention_mask + return batch + + def _compute_metrics(pred): + labels_ids = pred.label_ids + pred_ids = pred.predictions + pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True) + label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True) + accuracy = sum([int(pred_str[i] == label_str[i]) for i in range(len(pred_str))]) / len(pred_str) + return {"accuracy": accuracy} + + train_dataset = train_dataset.map( + _map_to_encoder_decoder_inputs, + batched=True, + batch_size=batch_size, + remove_columns=["article", "highlights"], ) - distillation_config = DistillationConfig(teacher_model=model) + train_dataset.set_format(type="torch", columns=columns) - def compute_metrics(p: EvalPrediction): - return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) + val_dataset = val_dataset.map( + _map_to_encoder_decoder_inputs, + batched=True, + batch_size=batch_size, + remove_columns=["article", "highlights"], + ) + val_dataset.set_format(type="torch", columns=columns) with tempfile.TemporaryDirectory() as tmp_dir: - trainer = INCTrainer( + training_args = Seq2SeqTrainingArguments( + output_dir=tmp_dir, + per_device_train_batch_size=batch_size, + per_device_eval_batch_size=batch_size, + predict_with_generate=True, + evaluation_strategy="steps", + do_train=True, + do_eval=True, + warmup_steps=0, + eval_steps=1, + logging_steps=1, + num_train_epochs=1.0, + ) + + trainer = INCSeq2SeqTrainer( model=model, - distillation_config=distillation_config, - task="sequence-classification", - args=TrainingArguments(tmp_dir, num_train_epochs=2.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(8)), - eval_dataset=dataset["validation"].select(range(8)), - compute_metrics=compute_metrics, + quantization_config=quantization_config, + args=training_args, + compute_metrics=_compute_metrics, + train_dataset=train_dataset, + eval_dataset=val_dataset, tokenizer=tokenizer, - data_collator=default_data_collator, ) - trainer._set_task() - self.assertEqual(trainer.task, "text-classification") + trainer.train() trainer.evaluate() - trainer.save_model(save_onnx_model=True) + trainer.save_model() + trainer.model.eval() + loaded_model = INCModelForSeq2SeqLM.from_pretrained(tmp_dir) + tokens = tokenizer("This is a sample input", return_tensors="pt") + decoder_inputs = { + "decoder_input_ids": torch.ones((1, 1), dtype=torch.long) * model.config.decoder_start_token_id + } - inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) - self.assertEqual(inc_config.distillation["teacher_model_name_or_path"], model_name) - self.assertEqual(inc_config.distillation["temperature"], 1.0) + with torch.no_grad(): + model_outputs = trainer.model(**tokens, **decoder_inputs) + loaded_model_outputs = loaded_model(**tokens, **decoder_inputs) + + self.assertTrue("logits" in loaded_model_outputs) + self.assertIsInstance(loaded_model_outputs.logits, torch.Tensor) + # Compare tensor outputs + self.assertTrue(torch.allclose(loaded_model_outputs.logits, model_outputs.logits, atol=1e-4)) + + def check_model_outputs( + self, + q_model, + task, + tokenizer, + save_directory, + expected_quantized_matmuls, + is_static=True, + load_onnx_model=True, + num_samples=None, + file_name=ONNX_WEIGHTS_NAME, + ): + tokens = tokenizer("This is a sample input", return_tensors="pt") + inc_model = eval(_HEAD_TO_AUTOMODELS[task]).from_pretrained(save_directory) + model_kwargs = ( + {"decoder_file_name": file_name, "use_cache": False} + if task == "text-generation" + else {"file_name": file_name} + ) + inc_config = INCConfig.from_pretrained(save_directory) + self.assertEqual(inc_config.save_onnx_model, load_onnx_model) + + if num_samples is not None: + self.assertEqual(inc_config.quantization["dataset_num_samples"], num_samples) - transformers_model = INCModelForSequenceClassification.from_pretrained(tmp_dir) - ort_model = ORTModelForSequenceClassification.from_pretrained(tmp_dir) + if load_onnx_model: + onnx_model = onnx_load(os.path.join(save_directory, file_name)) + num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) + + if num_quantized_matmul > 0: + self.assertEqual(inc_config.quantization["is_static"], is_static) + + self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) + ort_model = ORT_SUPPORTED_TASKS[task]["class"][0].from_pretrained(save_directory, **model_kwargs) ort_outputs = ort_model(**tokens) self.assertTrue("logits" in ort_outputs) - with torch.no_grad(): - transformers_outputs = transformers_model(**tokens) - self.assertTrue(torch.allclose(ort_outputs.logits, transformers_outputs.logits, atol=1e-4)) + + with torch.no_grad(): + model_outputs = q_model(**tokens) + inc_model_outputs = inc_model(**tokens) + self.assertTrue(torch.equal(model_outputs["logits"], inc_model_outputs["logits"])) + # self.assertTrue(torch.allclose(ort_outputs.logits, inc_model_outputs.logits, atol=1e-4)) + + @staticmethod + def get_trainer( + model_name, + task, + save_directory, + q_config=None, + p_config=None, + d_config=None, + save_onnx_model=True, + num_train_samples=8, + num_eval_samples=8, + ): + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + metric = evaluate.load("accuracy") + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] + dataset = load_dataset(dataset_name, dataset_config_name) + dataset = dataset.map( + partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), batched=True + ) + + trainer = INCTrainer( + model=model, + quantization_config=q_config, + pruning_config=p_config, + distillation_config=d_config, + task=task, + args=TrainingArguments(save_directory, num_train_epochs=2.0, do_train=True, do_eval=True), + train_dataset=dataset["train"].select(range(num_train_samples)), + eval_dataset=dataset["validation"].select(range(num_eval_samples)), + compute_metrics=partial(_compute_metrics, metric=metric), + tokenizer=tokenizer, + data_collator=default_data_collator, + ) + trainer.train() + trainer.evaluate() + trainer.save_model(save_onnx_model=save_onnx_model) + trainer.model.eval() + return trainer From a55a05928983c45e842db88850028785cfe0534a Mon Sep 17 00:00:00 2001 From: Lyalyushkin Nikolay Date: Tue, 6 Jun 2023 20:11:55 +0200 Subject: [PATCH 029/134] Prepare to NNCF v2.5 release (#314) * prepare to NNCF v2.5 release * use nncf 2.5.0 in setup.py * remove NNCFNetwork import from trainer.py --------- Co-authored-by: Helena Kloosterman --- optimum/intel/openvino/quantization.py | 4 ++-- optimum/intel/openvino/trainer.py | 8 +------- setup.py | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 0c7ff2d891..7dcde5ba73 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -281,7 +281,7 @@ def _remove_unused_columns(self, dataset: Dataset): def _onnx_export_nncf_model(model: NNCFNetwork, config: OnnxConfig, output: Union[str, io.BytesIO], opset: int = None): - signature = inspect.signature(model.get_nncf_wrapped_model().forward) + signature = inspect.signature(model.forward) signature = list(signature.parameters.keys()) opset = opset or config.DEFAULT_ONNX_OPSET model_inputs = config.generate_dummy_inputs(framework="pt") @@ -294,7 +294,7 @@ def remap(value): value = value.to(device) return value - with config.patch_model_for_export(model.get_nncf_wrapped_model()): + with config.patch_model_for_export(model): model_inputs = tree_map(remap, model_inputs) with torch.no_grad(): model.eval() diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index 2d70593188..20b9f1f550 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -36,7 +36,6 @@ from nncf.torch import create_compressed_model from nncf.torch.composite_compression import PTCompositeCompressionAlgorithmController from nncf.torch.compression_method_api import PTCompressionAlgorithmController -from nncf.torch.nncf_network import NNCFNetwork from nncf.torch.quantization.algo import QuantizationController from openvino._offline_transformations import compress_quantize_weights_transformation from openvino.runtime import Core, PartialShape, serialize @@ -197,10 +196,7 @@ def __init__( def _set_signature_columns_if_needed(self): if self._signature_columns is None: # Inspect model forward signature to keep only the arguments it accepts. - if isinstance(self.model, NNCFNetwork): - signature = inspect.signature(self.model.get_nncf_wrapped_model().forward) - else: - signature = inspect.signature(self.model.forward) + signature = inspect.signature(self.model.forward) self._signature_columns = list(signature.parameters.keys()) # Labels may be named label or label_ids, the default data collator handles that. self._signature_columns += list(set(["label", "label_ids"] + self.label_names)) @@ -665,8 +661,6 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): if not isinstance(self.model, PreTrainedModel): unwrapped_model = unwrap_model(self.model) - if isinstance(unwrapped_model, NNCFNetwork): - unwrapped_model = unwrapped_model.get_nncf_wrapped_model() is_pretrained_model = isinstance(unwrapped_model, PreTrainedModel) if state_dict is None: state_dict = self.model.state_dict() diff --git a/setup.py b/setup.py index 4e69e8c6d7..5602860f0a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ EXTRAS_REQUIRE = { "neural-compressor": ["neural-compressor>=2.1.1", "onnx", "onnxruntime<1.15.0"], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.4.0", "openvino-dev>=2023.0.0"], + "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From 1b0f4a58841ed3f87b775971236839b7381bbe88 Mon Sep 17 00:00:00 2001 From: Lyalyushkin Nikolay Date: Wed, 7 Jun 2023 10:15:14 +0200 Subject: [PATCH 030/134] Test training of examples with torchrun call (#319) * Test training of examples with torchrun call * fixed style --- tests/openvino/test_training_examples.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/openvino/test_training_examples.py b/tests/openvino/test_training_examples.py index 4528e526ad..8a33ba42e0 100644 --- a/tests/openvino/test_training_examples.py +++ b/tests/openvino/test_training_examples.py @@ -149,7 +149,7 @@ def test_single_card_training(self, _, desc: TrainingExampleDescriptor): self.env[CUDA_VISIBLE_DEVICES] = str(self.available_cuda_device_ids[0]) with tempfile.TemporaryDirectory() as output_dir: - args = [sys.executable, desc.filename, *desc.get_args_with_output_dir(output_dir)] + args = ["torchrun", "--nproc_per_node=1", desc.filename, *desc.get_args_with_output_dir(output_dir)] proc = subprocess.Popen( args=args, cwd=desc.cwd, @@ -184,9 +184,7 @@ def test_distributed_data_parallel_training(self, _, desc: TrainingExampleDescri self.env[CUDA_VISIBLE_DEVICES] = ",".join(map(str, self.available_cuda_device_ids[:2])) with tempfile.TemporaryDirectory() as output_dir: args = [ - sys.executable, - "-m", - "torch.distributed.run", + "torchrun", "--rdzv_backend=c10d", "--rdzv_endpoint=localhost:0", "--nnodes=1", From 8109a15ba0af0e5a7e79dceb5ea9efdff59522cf Mon Sep 17 00:00:00 2001 From: Lyalyushkin Nikolay Date: Wed, 7 Jun 2023 10:16:15 +0200 Subject: [PATCH 031/134] Corrected docs to run JPQD in DDP mode (#315) * cmd to run JPQD in DDP mode * suggestion from Alexander and note about hyperparameters tuning * removed not needed instructions --- examples/openvino/audio-classification/README.md | 6 +++--- examples/openvino/image-classification/README.md | 4 ++-- examples/openvino/question-answering/README.md | 9 ++------- examples/openvino/text-classification/README.md | 3 ++- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/examples/openvino/audio-classification/README.md b/examples/openvino/audio-classification/README.md index 39896a6acf..1a6f1ddce8 100644 --- a/examples/openvino/audio-classification/README.md +++ b/examples/openvino/audio-classification/README.md @@ -18,7 +18,7 @@ limitations under the License. This folder contains [`run_audio_classification.py`](https://github.com/huggingface/optimum/blob/main/examples/openvino/audio-classification/run_audio_classification.py), a script to fine-tune a 🤗 Transformers model on the 🗣️ [Keyword Spotting subset](https://huggingface.co/datasets/superb#ks) of the SUPERB dataset while applying Quantization-Aware Training (QAT). QAT can be easily applied by replacing the Transformers [`Trainer`](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#trainer) with the Optimum [`OVTrainer`]. Any model from our [hub](https://huggingface.co/models) can be fine-tuned and quantized, as long as the model is supported by the [`AutoModelForAudioClassification`](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoModelForAudioClassification) API. -### Fintuning Wav2Vec2 on Keyword Spotting with QAT +### Fine-tuning Wav2Vec2 on Keyword Spotting with QAT The following command shows how to fine-tune [Wav2Vec2-base](https://huggingface.co/facebook/wav2vec2-base) on the 🗣️ [Keyword Spotting subset](https://huggingface.co/datasets/superb#ks) of the SUPERB dataset with Quantization-Aware Training (QAT). The `OVTrainer` uses a default quantization configuration which should work in many cases, but we can also customize the algorithm details. Here, we quantize the Wav2Vec2-base model with a custom configuration file specified by `--nncf_compression_config`. For more details on the quantization configuration, see NNCF documentation [here](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Quantization.md). @@ -60,7 +60,7 @@ On a single V100 GPU, this script should run in ~45 minutes and yield a quantize `OVTrainer` also provides advanced optimization workflow via NNCF to structurally prune, quantize and distill. Following is an example of joint pruning, quantization and distillation on Wav2Vec2-base model for keyword spotting task. To enable JPQD optimization, use an alternative configuration specified with `--nncf_compression_config`. For more details on how to configure the pruning algorithm, see NNCF documentation [here](https://github.com/openvinotoolkit/nncf/blob/develop/nncf/experimental/torch/sparsity/movement/MovementSparsity.md). ```bash -python run_audio_classification.py \ +torchrun --nproc-per-node=1 run_audio_classification.py \ --model_name_or_path facebook/wav2vec2-base \ --teacher_model_name_or_path anton-l/wav2vec2-base-ft-keyword-spotting \ --nncf_compression_config configs/wav2vec2-base-jpqd.json \ @@ -92,4 +92,4 @@ python run_audio_classification.py \ --seed 0 ``` -This script should take about 3 hours on a single V100 GPU and produce a quantized Wav2Vec2-base model with ~80% structured sparsity in its linear layers. The model accuracy should converge to about 97.5%. +This script should take about 3 hours on a single V100 GPU and produce a quantized Wav2Vec2-base model with ~80% structured sparsity in its linear layers. The model accuracy should converge to about 97.5%. For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters might be required to achieve the same results as on a single GPU. diff --git a/examples/openvino/image-classification/README.md b/examples/openvino/image-classification/README.md index 6b42fdbe42..25d7cbc541 100644 --- a/examples/openvino/image-classification/README.md +++ b/examples/openvino/image-classification/README.md @@ -48,7 +48,7 @@ On a single V100 GPU, this example takes about 1 minute and yields a quantized m `OVTrainer` also provides advanced optimization workflow via NNCF to structurally prune, quantize and distill. Following is an example of joint pruning, quantization and distillation on Swin-base model for food101 dataset. To enable JPQD optimization, use an alternative configuration specified with `--nncf_compression_config`. For more details on how to configure the pruning algorithm, see NNCF documentation [here](https://github.com/openvinotoolkit/nncf/blob/develop/nncf/experimental/torch/sparsity/movement/MovementSparsity.md). ```bash -python run_image_classification.py \ +torchrun --nproc-per-node=1 run_image_classification.py \ --model_name_or_path microsoft/swin-base-patch4-window7-224 \ --teacher_model_name_or_path skylord/swin-finetuned-food101 \ --distillation_weight 0.9 \ @@ -75,4 +75,4 @@ python run_image_classification.py \ --nncf_compression_config configs/swin-base-jpqd.json ``` -This example results in a quantized swin-base model with ~40% sparsity in its linear layers of the transformer blocks, giving 90.7% accuracy on food101 and taking about 12.5 hours on a single V100 GPU. +This example results in a quantized swin-base model with ~40% sparsity in its linear layers of the transformer blocks, giving 90.7% accuracy on food101 and taking about 12.5 hours on a single V100 GPU. For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters might be required to achieve the same results as on a single GPU. diff --git a/examples/openvino/question-answering/README.md b/examples/openvino/question-answering/README.md index 24ac373c6d..c57d332e63 100644 --- a/examples/openvino/question-answering/README.md +++ b/examples/openvino/question-answering/README.md @@ -47,17 +47,12 @@ python run_qa.py \ ``` ### Joint Pruning, Quantization and Distillation (JPQD) for BERT on SQuAD1.0 -`OVTrainer` also provides an advanced optimization workflow through the NNCF when Transformer model can be structurally pruned along with 8-bit quantization and distillation. Below is an example which demonstrates how to jointly prune, quantize BERT-base for SQuAD 1.0 using NNCF config `--nncf_compression_config` and distill from BERT-large teacher. This example closely resembles the movement sparsification work of [Lagunas et al., 2021, Block Pruning For Faster Transformers](https://arxiv.org/pdf/2109.04838.pdf). This example takes about 12 hours with a single V100 GPU and ~40% of the weights of the Transformer blocks were pruned. +`OVTrainer` also provides an advanced optimization workflow through the NNCF when Transformer model can be structurally pruned along with 8-bit quantization and distillation. Below is an example which demonstrates how to jointly prune, quantize BERT-base for SQuAD 1.0 using NNCF config `--nncf_compression_config` and distill from BERT-large teacher. This example closely resembles the movement sparsification work of [Lagunas et al., 2021, Block Pruning For Faster Transformers](https://arxiv.org/pdf/2109.04838.pdf). This example takes about 12 hours with a single V100 GPU and ~40% of the weights of the Transformer blocks were pruned. For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters qmight be required to achieve the same results as on a single GPU. More on how to configure movement sparsity, see NNCF documentation [here](https://github.com/openvinotoolkit/nncf/blob/develop/nncf/experimental/torch/sparsity/movement/MovementSparsity.md). -To run the JPQD example, please install optimum-intel from source. This command will install or upgrade optimum-intel and all necessary dependencies: - -```python -m pip install --upgrade "git+https://github.com/huggingface/optimum-intel.git#egg=optimum-intel[openvino, nncf]" -``` - ```bash -python run_qa.py \ +torchrun --nproc-per-node=1 run_qa.py \ --model_name_or_path bert-base-uncased \ --dataset_name squad \ --teacher_model_name_or_path bert-large-uncased-whole-word-masking-finetuned-squad \ diff --git a/examples/openvino/text-classification/README.md b/examples/openvino/text-classification/README.md index d10d8f7430..0128220c89 100644 --- a/examples/openvino/text-classification/README.md +++ b/examples/openvino/text-classification/README.md @@ -58,7 +58,7 @@ To run the JPQD example, please install optimum-intel from source. This command ```bash TASK_NAME=sst2 -python run_glue.py \ +torchrun --nproc-per-node=1 run_glue.py \ --model_name_or_path bert-base-uncased \ --task_name $TASK_NAME \ --teacher_model_name_or_path yoshitomo-matsubara/bert-large-uncased-sst2 \ @@ -83,3 +83,4 @@ python run_glue.py \ ``` On a single V100 GPU, this script should run in ~1.8 hours, and yield accuracy of **92.2%** with ~40% of the weights of the Transformer blocks pruned. +For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters might be required to achieve the same results as on a single GPU. From 2d2af36fb652ba9b3c79a9e9c6066746ebf0e499 Mon Sep 17 00:00:00 2001 From: Lyalyushkin Nikolay Date: Wed, 7 Jun 2023 12:10:49 +0200 Subject: [PATCH 032/134] Explicitly set batch=1 for NNCF in order to avoid issue with Wave2Vec (#312) * explicitly set batch=1 for NNCF in order to avoid issue with Wave2Vec * More safe changes. affects only pruning scenario * renamed variable * Corrections * correction * fixed style --- optimum/intel/openvino/configuration.py | 4 ++-- optimum/intel/openvino/trainer.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/optimum/intel/openvino/configuration.py b/optimum/intel/openvino/configuration.py index e98a833251..31aad18426 100644 --- a/optimum/intel/openvino/configuration.py +++ b/optimum/intel/openvino/configuration.py @@ -56,10 +56,10 @@ def __init__( self._enable_standard_onnx_export_option() self.optimum_version = kwargs.pop("optimum_version", None) - def add_input_info(self, model_inputs: Dict): + def add_input_info(self, model_inputs: Dict, force_batch_one: bool = False): self.input_info = [ { - "sample_size": list(value.shape), + "sample_size": [1] + list(value.shape[1:]) if force_batch_one else list(value.shape), "type": "long" if value.dtype is torch.int64 else "float", "keyword": name, } diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index 20b9f1f550..f5b724b950 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -168,7 +168,8 @@ def __init__( model_inputs = next(iter(train_dataloader)) for label_name in self.label_names: model_inputs.pop(label_name) - self.ov_config.add_input_info(model_inputs) + force_batch_one = self._is_pruning_enabled() + self.ov_config.add_input_info(model_inputs, force_batch_one) nncf_config = NNCFConfig.from_dict(self.ov_config.__dict__) nncf_config.register_extra_structs( [ @@ -770,3 +771,12 @@ def _set_task(self): if self.task is None: raise ValueError("The model task defining the model topology needs to be specified for the ONNX export.") self.task = _TASK_ALIASES.get(self.task, self.task) + + def _is_pruning_enabled(compression: Union[Dict, List, None]): + if isinstance(compression, dict) and compression["algorithm"] == "movement_pruning": + return True + if isinstance(compression, list): + for algo_config in compression: + if algo_config["algorithm"] == "movement_pruning": + return True + return False From 15b3902316b04a2bb51cec5829b759110d1f1cba Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:41:52 +0200 Subject: [PATCH 033/134] fix ts generation modeling (#341) --- optimum/intel/generation/modeling.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index c66df36e11..fdc6f4c1e6 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -88,7 +88,11 @@ def jit_trace(model: PreTrainedModel, task: str, use_cache: bool = False): return traced_model -class TSModelForCausalLM(OptimizedModel, GenerationMixin): +class PreTrainedModel(OptimizedModel): + pass + + +class TSModelForCausalLM(PreTrainedModel, GenerationMixin): auto_model_class = AutoModelForCausalLM export_feature = "text-generation" main_input_name = "input_ids" From 86cdea9893493c106d3be516c7a47c9acc4c8a9e Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 9 Jun 2023 15:55:28 +0200 Subject: [PATCH 034/134] Fix doc generation (#343) * Fix doc generation * trigger workflow --- .github/workflows/build_pr_documentation.yml | 13 +++++-------- .github/workflows/delete_doc_comment.yml | 17 +++++++++++------ .../workflows/delete_doc_comment_trigger.yml | 12 ++++++++++++ 3 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/delete_doc_comment_trigger.yml diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 2cf1dc6537..f567cf0b7e 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -28,10 +28,6 @@ jobs: repository: 'huggingface/optimum-intel' path: optimum-intel - - name: Set environment variables - run: | - echo "write_token=$(echo 'ghp_'$(wget -qO- lysand.re/doc-build-dev)'bm')" >> $GITHUB_ENV - - name: Setup environment run: | pip uninstall -y doc-builder @@ -53,7 +49,7 @@ jobs: sudo chmod -R ugo+rwx intel-doc-build cd intel-doc-build sudo mv optimum.intel optimum-intel - doc-builder push optimum-intel --doc_build_repo_id "hf-doc-build/doc-build-dev" --token "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" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-intel/commit/$COMMIT_SHA" --n_retries 5 + doc-builder push optimum-intel --doc_build_repo_id "hf-doc-build/doc-build-dev" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-intel/commit/$COMMIT_SHA" --n_retries 5 shell: bash - name: Find doc comment @@ -64,19 +60,20 @@ jobs: body-includes: docs for this PR - name: Add doc comment if not present - uses: thollander/actions-comment-pull-request@v1 + uses: thollander/actions-comment-pull-request@v2 if: steps.find_comment.outputs.comment-id == '' with: message: 'The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum-intel/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint.' - GITHUB_TOKEN: ${{ env.write_token }} + pr_number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ secrets.COMMENT_BOT_TOKEN }} - name: Update doc comment if necessary if: github.event.action == 'reopened' && steps.find_comment.outputs.comment-id != '' uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ steps.find_comment.outputs.comment-id }} - token: ${{ env.write_token }} + token: ${{ secrets.COMMENT_BOT_TOKEN }} edit-mode: replace body: | The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum-intel/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint. diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml index d47bdfc654..768c348c7a 100644 --- a/.github/workflows/delete_doc_comment.yml +++ b/.github/workflows/delete_doc_comment.yml @@ -1,13 +1,18 @@ name: Delete PR documentation on: - pull_request: - types: [ closed ] - + workflow_run: + workflows: ["Delete doc comment trigger"] + types: + - completed + paths: + - "optimum/**.py" + - "docs/**" + - ".github/workflows/build_pr_documentation.yml" + - ".github/workflows/delete_doc_comment.yml" jobs: delete: uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main - with: - pr_number: ${{ github.event.number }} - package: optimum-intel + secrets: + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/delete_doc_comment_trigger.yml b/.github/workflows/delete_doc_comment_trigger.yml new file mode 100644 index 0000000000..f87d9bd4dc --- /dev/null +++ b/.github/workflows/delete_doc_comment_trigger.yml @@ -0,0 +1,12 @@ +name: Delete doc comment trigger + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment_trigger.yml@main + with: + pr_number: ${{ github.event.number }} From cbb66c91fbc5d664b8e3e99e5af47c0864fabc94 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Fri, 9 Jun 2023 19:41:02 +0400 Subject: [PATCH 035/134] small improvements in diffusion pipeline for openvino (#342) * small improvements in diffusion pipeline * fix typo, add test, add workaround for torch 2.0.1 onnx export * fix type detection and formatting * use is_torch_version from utils --- optimum/intel/openvino/modeling_diffusion.py | 202 +++++++++++++++++-- tests/openvino/test_modeling.py | 19 ++ 2 files changed, 203 insertions(+), 18 deletions(-) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index f83f12fdde..074830baea 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -39,6 +39,7 @@ DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ) +from ..utils import is_torch_version from .modeling_base import OVBaseModel from .utils import ONNX_WEIGHTS_NAME, OV_TO_NP_TYPE, OV_XML_FILE_NAME @@ -285,6 +286,8 @@ def _from_transformers( feature_extractor: Optional["CLIPFeatureExtractor"] = None, **kwargs, ): + if is_torch_version(">", "1.13.1") and is_torch_version("<=", "2.0.1"): + register_custom_scaled_dot_product_attention_export() if task is None: task = cls._auto_model_to_task(cls.auto_model_class) save_dir = TemporaryDirectory() @@ -339,6 +342,20 @@ def to(self, device: str): def device(self) -> str: return self._device.lower() + @property + def height(self) -> int: + height = self.unet.model.inputs[0].get_partial_shape()[2] + if height.is_dynamic: + return -1 + return height.get_length() * self._vae_scale_factor + + @property + def width(self) -> int: + width = self.unet.model.inputs[0].get_partial_shape()[3] + if width.is_dynamic: + return -1 + return width.get_length() * self._vae_scale_factor + def _reshape_unet( self, model: openvino.runtime.Model, @@ -426,24 +443,21 @@ def __call__( num_images_per_prompt: Optional[int] = 1, **kwargs, ): - _, _, _height, _width = self.unet.model.inputs[0].get_partial_shape() - - if _height.is_static: - _height = _height.get_length() * self._vae_scale_factor - if height != _height: - logger.warning( - f"`height` was set to {height} but the static model will output images of height {_height}." - "To fix the height, please reshape your model accordingly using the `.reshape()` method." - ) + _height = self.height + _width = self.width + + if _height != -1 and height != _height: + logger.warning( + f"`height` was set to {height} but the static model will output images of height {_height}." + "To fix the height, please reshape your model accordingly using the `.reshape()` method." + ) height = _height - if _width.is_static: - _width = _width.get_length() * self._vae_scale_factor - if width != _width: - logger.warning( - f"`width` was set to {width} but the static model will output images of width {_width}." - "To fix the width, please reshape your model accordingly using the `.reshape()` method." - ) + if _width != -1 and width != _width: + logger.warning( + f"`width` was set to {width} but the static model will output images of width {_width}." + "To fix the width, please reshape your model accordingly using the `.reshape()` method." + ) width = _width if guidance_scale is not None and guidance_scale <= 1 and not self.is_dynamic: @@ -474,7 +488,11 @@ def _save_config(self, save_directory): class OVModelPart: def __init__( - self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + self, + model: openvino.runtime.Model, + parent_model: OVBaseModel, + ov_config: Optional[Dict[str, str]] = None, + model_name: str = "encoder", ): self.model = model self.parent_model = parent_model @@ -485,10 +503,11 @@ def __init__( } self.ov_config = ov_config or self.parent_model.ov_config self.request = None + self._model_name = model_name def _compile(self): if self.request is None: - logger.info("Compiling the encoder...") + logger.info(f"Compiling the {self._model_name}...") self.request = core.compile_model(self.model, self.device, self.ov_config) @property @@ -497,6 +516,11 @@ def device(self): class OVModelTextEncoder(OVModelPart): + def __init__( + self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + ): + super().__init__(model, parent_model, ov_config, "text_encoder") + def __call__(self, input_ids: np.ndarray): self._compile() @@ -508,6 +532,11 @@ def __call__(self, input_ids: np.ndarray): class OVModelUnet(OVModelPart): + def __init__( + self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + ): + super().__init__(model, parent_model, ov_config, "unet") + def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_states: np.ndarray): self._compile() @@ -522,6 +551,11 @@ def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_stat class OVModelVaeDecoder(OVModelPart): + def __init__( + self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + ): + super().__init__(model, parent_model, ov_config, "vae_decoder") + def __call__(self, latent_sample: np.ndarray): self._compile() @@ -533,6 +567,11 @@ def __call__(self, latent_sample: np.ndarray): class OVModelVaeEncoder(OVModelPart): + def __init__( + self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + ): + super().__init__(model, parent_model, ov_config, "vae_encoder") + def __call__(self, sample: np.ndarray): self._compile() @@ -541,3 +580,130 @@ def __call__(self, sample: np.ndarray): } outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) + + +def register_custom_scaled_dot_product_attention_export(): + import torch + + @torch.onnx.symbolic_helper.parse_args("v", "v", "v", "v", "f", "b", "v") + def scaled_dot_product_attention( + g: torch.onnx._internal.jit_utils.GraphContext, + query: torch._C.Value, + key: torch._C.Value, + value: torch._C.Value, + attn_mask: Optional[torch._C.Value] = None, + dropout_p: float = 0.0, + is_causal: bool = False, + scale: Optional[torch._C.Value] = None, + ): + assert (not is_causal) or ( + is_causal and torch.onnx.symbolic_helper._is_none(attn_mask) + ), "is_causal and attn_mask cannot be set at the same time" + + scale = torch.onnx.symbolic_helper._maybe_get_const(scale, "f") + if scale is None: + scale = _attention_scale(g, query) + + if is_causal: + attn_mask = _causal_attention_mask(g, query, key) + key_shape_builtin = torch.onnx.symbolic_helper._get_tensor_rank(key) + key_transposed_axes = list(range(key_shape_builtin)) + key_transposed_axes[-1], key_transposed_axes[-2] = ( + key_transposed_axes[-2], + key_transposed_axes[-1], + ) + key_transposed = g.op("Transpose", key, perm_i=key_transposed_axes) + query_scaled = g.op("Mul", query, g.op("Sqrt", scale)) + key_transposed_scaled = g.op("Mul", key_transposed, g.op("Sqrt", scale)) + mul_qk = g.op("MatMul", query_scaled, key_transposed_scaled) + if attn_mask is None or torch.onnx.symbolic_helper._is_none(attn_mask): + mul_qk_add = mul_qk + elif torch.onnx._type_utils.JitScalarType.from_value(attn_mask) == torch.onnx._type_utils.JitScalarType.BOOL: + # Turn the Boolean mask to float: attn_mask.masked_fill(not attn_mask, -float('inf')) + const_zero = g.op("Constant", value_t=torch.tensor([0.0])) + const_neg_inf = g.op("Constant", value_t=torch.tensor([-float("inf")])) + attn_mask = g.op("Where", attn_mask, const_zero, const_neg_inf) + mul_qk_add = g.op("Add", mul_qk, attn_mask) + elif torch.onnx._type_utils.JitScalarType.from_value(attn_mask) == torch.onnx._type_utils.JitScalarType.FLOAT: + mul_qk_add = g.op("Add", mul_qk, attn_mask) + else: + raise ValueError( + f"Unsupported type for attn_mask: {torch.onnx._type_utils.JitScalarType.from_value(attn_mask)}" + ) + + attn_weight = g.op("Softmax", mul_qk_add, axis_i=-1) + + if dropout_p != 0: + attn_weight = g.op( + "Dropout", + attn_weight, + g.op("Constant", value_t=torch.tensor(dropout_p, dtype=torch.float)), + ) + + return g.op("MatMul", attn_weight, value) + + def _attention_scale(g: torch.onnx._internal.jit_utils.GraphContext, query: torch._C.Value) -> torch._C.Value: + """Calculate the scale factor for the attention result. + + Args: + query: Tensor of shape [..., L, E] + + Returns: + Scalar scale factor := 1 / math.sqrt(query.size(-1)) + """ + query_shape = g.op("Shape", query) + query_shape_last = g.op( + "Slice", + query_shape, + g.op("Constant", value_t=torch.tensor([-1], dtype=torch.int64)), + g.op("Constant", value_t=torch.tensor([torch.onnx._constants.INT64_MAX], dtype=torch.int64)), + ) + embedding_size = g.op( + "Cast", + query_shape_last, + to_i=torch.onnx._type_utils.JitScalarType.from_value(query).onnx_type(), + ) + const_one = g.op("Constant", value_t=torch.tensor([1.0], dtype=torch.float)) + scale = g.op("Div", const_one, g.op("Sqrt", embedding_size)) + return scale + + def _causal_attention_mask( + g: torch.onnx._internal.jit_utils.GraphContext, query: torch._C.Value, key: torch._C.Value + ) -> torch._C.Value: + """Create a causal mask for the given query and key tensors. + + Equivalent to:: + mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0) + attn_mask = torch.zeros(L, S, dtype=torch.float) + attn_mask = attn_mask.masked_fill(not mask, -float('inf')) + + Args: + query: Tensor of shape [..., L, E] + key: Tensor of shape [..., S, E] + + Returns: + Tensor of shape [L, S] + """ + + query_shape = g.op("Shape", query) + key_shape = g.op("Shape", key) + + last_idx = g.op("Constant", value_t=torch.tensor([-1], dtype=torch.int64)) + second_last_idx = g.op("Constant", value_t=torch.tensor([-2], dtype=torch.int64)) + target_length = g.op("Slice", query_shape, second_last_idx, last_idx) + source_length = g.op("Slice", key_shape, second_last_idx, last_idx) + # attn_mask = torch.ones(L, S) := { + size = g.op("Concat", target_length, source_length, axis_i=0) + const_one = g.op("Constant", value_t=torch.tensor([1.0])) + attn_mask = g.op("Expand", const_one, size) + # } + attn_mask = g.op("Trilu", attn_mask, upper_i=0) + # The causal mask has 0s in the lower triangle and -inf in the upper triangle. + const_zero = g.op("Constant", value_t=torch.tensor([0.0])) + const_neg_inf = g.op("Constant", value_t=torch.tensor([-float("inf")])) + attn_mask = g.op("Where", g.op("Equal", attn_mask, const_zero), const_neg_inf, const_zero) + return attn_mask + + torch.onnx.register_custom_op_symbolic( + "aten::scaled_dot_product_attention", scaled_dot_product_attention, opset_version=14 + ) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 4aad4ea705..7d555464d3 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -852,3 +852,22 @@ def test_image_reproducibility(self, model_arch: str): # Compare model outputs self.assertTrue(np.array_equal(outputs_1.images[0], outputs_2.images[0])) self.assertFalse(np.array_equal(outputs_1.images[0], outputs_3.images[0])) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + @require_diffusers + def test_height_width_properties(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + batch_size = 1 + num_images_per_prompt = 4 + height = 128 + width = 64 + pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True, compile=False, dynamic=True) + self.assertTrue(pipeline.is_dynamic) + self.assertEqual(pipeline.height, -1) + self.assertEqual(pipeline.width, -1) + pipeline.reshape( + batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images_per_prompt + ) + self.assertFalse(pipeline.is_dynamic) + self.assertEqual(pipeline.height, height) + self.assertEqual(pipeline.width, width) From 571f6c33823d950e57851bcac10dbf0d84eb802c Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 9 Jun 2023 18:59:28 +0200 Subject: [PATCH 036/134] Fix OV model for BLOOM architecture (#340) * Fix OpenVINO model BLOOM * Fix llama architecture * fix style --- optimum/intel/openvino/modeling_decoder.py | 8 ++ optimum/intel/openvino/modeling_utils.py | 91 ++++++++++++++++++++++ tests/openvino/test_modeling.py | 10 ++- 3 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 optimum/intel/openvino/modeling_utils.py diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index fc93603921..975a593905 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -31,6 +31,7 @@ from ..utils.import_utils import is_transformers_version from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel +from .modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .utils import ONNX_WEIGHTS_NAME @@ -155,6 +156,13 @@ def _from_transformers( onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config, use_past=use_cache) + # TODO : create ModelPatcher to patch each architecture + if model.config.model_type == "bloom": + model.transformer._prepare_attn_mask = _prepare_attn_mask + + if model.config.model_type == "llama": + model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + # Export the model to the ONNX format export(model=model, config=onnx_config, output=save_dir_path / model_file_name) diff --git a/optimum/intel/openvino/modeling_utils.py b/optimum/intel/openvino/modeling_utils.py new file mode 100644 index 0000000000..c7be049990 --- /dev/null +++ b/optimum/intel/openvino/modeling_utils.py @@ -0,0 +1,91 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple + +import torch + + +# Modified from transformers.models.bloom.modeling_bloom._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, + device: torch.device, + past_key_values_length: int, + dtype: torch.dtype = torch.bool, +) -> torch.BoolTensor: + """ + Make causal mask used for bi-directional self-attention. + """ + batch_size, target_length = input_ids_shape + mask = torch.zeros((target_length, target_length + past_key_values_length), dtype=dtype, device=device) + seq_ids = torch.arange(target_length, device=device) + + mask[:, past_key_values_length:] = ( + (seq_ids[:, None] < seq_ids[None, :]) * torch.finfo(dtype).min + if torch.is_floating_point(mask) + else seq_ids[:, None] < seq_ids[None, :] + ) + + return mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length) + + +# Modified from transformers.models..bloom.modeling_bloom._prepare_attn_mask +def _prepare_attn_mask( + attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int +) -> torch.BoolTensor: + from transformers.models.bloom.modeling_bloom import _expand_mask + + # create causal mask + # [batch_size, seq_length] -> [batch_size, 1, tgt_length, src_length] + combined_attention_mask = None + device = attention_mask.device + _, src_length = input_shape + + combined_attention_mask = _make_causal_mask( + input_shape, device=device, past_key_values_length=past_key_values_length + ) + # [batch_size, seq_length] -> [batch_size, 1, tgt_length, src_length]_prepare_decoder_attention_mask + expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask + ) + + return combined_attention_mask + + +# Modified from transformers.models.llama.modeling_llama._prepare_decoder_attention_mask +def _prepare_decoder_attention_mask(attention_mask, input_shape, inputs_embeds, past_key_values_length): + from transformers.models.llama.modeling_llama import _expand_mask + + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + + combined_attention_mask = _make_causal_mask( + input_shape, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + dtype=inputs_embeds.dtype, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 7d555464d3..8cd671f606 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -81,10 +81,12 @@ "bloom": "hf-internal-testing/tiny-random-BloomModel", "bigbird_pegasus": "hf-internal-testing/tiny-random-bigbird_pegasus", "distilbert": "hf-internal-testing/tiny-random-distilbert", + # "gpt_bigcode": "bigcode/tiny_starcoder_py", "gptj": "hf-internal-testing/tiny-random-GPTJModel", "gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel", "gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", "gpt2": "hf-internal-testing/tiny-random-gpt2", + "llama": "trl-internal-testing/tiny-random-LlamaForCausalLM", "marian": "sshleifer/tiny-marian-en-de", "mbart": "hf-internal-testing/tiny-random-mbart", "m2m_100": "valhalla/m2m100_tiny_random", @@ -417,6 +419,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "gpt2", "gpt_neo", "gpt_neox", + # "llama", ) GENERATION_LENGTH = 100 SPEEDUP_CACHE = 1.2 @@ -429,15 +432,16 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForCausalLM.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) - tokens = tokenizer("This is a sample", return_tensors="pt") + tokens = tokenizer( + "This is a sample", return_tensors="pt", return_token_type_ids=False if model_arch == "llama" else None + ) ov_outputs = ov_model(**tokens) self.assertTrue("logits" in ov_outputs) self.assertIsInstance(ov_outputs.logits, torch.Tensor) with torch.no_grad(): transformers_outputs = transformers_model(**tokens) # Compare tensor outputs - atol = 1e-1 if model_arch == "bloom" else 1e-4 - self.assertTrue(torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=atol)) + self.assertTrue(torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=1e-4)) gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) From 44500eb47e7801182e36e5ae9ed31b163d3fb5c8 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 9 Jun 2023 19:38:01 +0200 Subject: [PATCH 037/134] Fix TS model for BLOOM architecture (#344) --- optimum/intel/generation/modeling.py | 8 ++++++++ optimum/intel/openvino/modeling_decoder.py | 2 +- optimum/intel/{openvino => utils}/modeling_utils.py | 0 tests/generation/test_modeling.py | 3 +-- 4 files changed, 10 insertions(+), 3 deletions(-) rename optimum/intel/{openvino => utils}/modeling_utils.py (100%) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index fdc6f4c1e6..d524deb09c 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -31,6 +31,7 @@ from ..utils.constant import _TASK_ALIASES from ..utils.import_utils import is_torch_version, is_transformers_version +from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask if is_transformers_version("<", "4.25.0"): @@ -266,6 +267,13 @@ def _from_transformers( } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) + + if model.config.model_type == "bloom": + model.transformer._prepare_attn_mask = _prepare_attn_mask + + if model.config.model_type == "llama": + model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + traced_model = jit_trace(model, task, use_cache) save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 975a593905..26a3e0424d 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -30,8 +30,8 @@ from optimum.utils import NormalizedConfigManager from ..utils.import_utils import is_transformers_version +from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .utils import ONNX_WEIGHTS_NAME diff --git a/optimum/intel/openvino/modeling_utils.py b/optimum/intel/utils/modeling_utils.py similarity index 100% rename from optimum/intel/openvino/modeling_utils.py rename to optimum/intel/utils/modeling_utils.py diff --git a/tests/generation/test_modeling.py b/tests/generation/test_modeling.py index 6093164358..0fd668ad8f 100644 --- a/tests/generation/test_modeling.py +++ b/tests/generation/test_modeling.py @@ -66,8 +66,7 @@ def test_compare_to_transformers(self, model_arch): with torch.no_grad(): trfs_outputs = trfs_model(**tokens) # Compare outputs with original transformers model - atol = 1e-1 if model_arch == "bloom" else 1e-4 - self.assertTrue(torch.allclose(outputs.logits, trfs_outputs.logits, atol=atol)) + self.assertTrue(torch.allclose(outputs.logits, trfs_outputs.logits, atol=1e-4)) # Compare outputs with loaded model with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) From 7c05310510016b4bb8be562aa366b33239f775bf Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Mon, 12 Jun 2023 14:57:31 +0200 Subject: [PATCH 038/134] Dev version --- optimum/intel/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/version.py b/optimum/intel/version.py index 2325839058..65db31ad88 100644 --- a/optimum/intel/version.py +++ b/optimum/intel/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.8.1.dev0" +__version__ = "1.9.1.dev0" From d95534d374da203ee75111e0eb51ae1bcd431553 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:30:30 +0200 Subject: [PATCH 039/134] Add INC model loading test (#348) --- tests/neural_compressor/test_modeling.py | 68 ++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/neural_compressor/test_modeling.py diff --git a/tests/neural_compressor/test_modeling.py b/tests/neural_compressor/test_modeling.py new file mode 100644 index 0000000000..f6aca1c671 --- /dev/null +++ b/tests/neural_compressor/test_modeling.py @@ -0,0 +1,68 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import unittest + +from parameterized import parameterized +from transformers import set_seed + +from optimum.exporters import TasksManager +from optimum.intel import ( # noqa + INCConfig, + INCModelForCausalLM, + INCModelForMaskedLM, + INCModelForQuestionAnswering, + INCModelForSeq2SeqLM, + INCModelForSequenceClassification, + INCModelForTokenClassification, + INCQuantizer, + INCSeq2SeqTrainer, + INCStableDiffusionPipeline, + INCTrainer, +) +from optimum.intel.neural_compressor.utils import _HEAD_TO_AUTOMODELS + + +os.environ["CUDA_VISIBLE_DEVICES"] = "" +set_seed(1009) + + +MODEL_NAMES_TO_TASK = ( + ("echarlaix/distilbert-base-uncased-finetuned-sst-2-english-int8-dynamic", "text-classification"), + ("echarlaix/distilbert-sst2-inc-dynamic-quantization-magnitude-pruning-0.1", "text-classification"), + ("hf-internal-testing/tiny-random-bert", "fill-mask"), + ("Intel/bert-base-uncased-squad-int8-static", "question-answering"), + ("hf-internal-testing/tiny-random-gpt2", "text-generation"), + ("Intel/t5-small-xsum-int8-dynamic", "text2text-generation"), + # ("echarlaix/stable-diffusion-v1-5-inc-int8-dynamic", "stable-diffusion") +) + + +class INCModelingTest(unittest.TestCase): + @parameterized.expand(MODEL_NAMES_TO_TASK) + def test_modeling(self, model_id, task): + inc_model = eval(_HEAD_TO_AUTOMODELS[task]).from_pretrained(model_id) # TRANSFORMERS_AUTO_CLASS + model_type = inc_model.config.model_type.replace("_", "-") + config_class = TasksManager.get_exporter_config_constructor( + exporter="onnx", + model=inc_model, + task=task, + model_name=model_id, + model_type=model_type, + ) + config = config_class(inc_model.config) + model_inputs = config.generate_dummy_inputs(framework="pt") + inc_model(**model_inputs) From 28b4b3ad08c766ef79e3b3de0dcfa5311ddff6e3 Mon Sep 17 00:00:00 2001 From: regisss <15324346+regisss@users.noreply.github.com> Date: Thu, 15 Jun 2023 01:09:23 +0200 Subject: [PATCH 040/134] Fix PR doc build from forks (#350) --- .github/workflows/build_pr_documentation.yml | 33 ++++--------------- .github/workflows/upload_pr_documentation.yml | 16 +++++++++ 2 files changed, 22 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/upload_pr_documentation.yml diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index f567cf0b7e..9825356234 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -43,37 +43,16 @@ jobs: make doc BUILD_DIR=intel-doc-build VERSION=pr_$PR_NUMBER COMMIT_SHA_SUBPACKAGE=$COMMIT_SHA CLONE_URL=$PR_CLONE_URL cd .. - - name: Push to repositories + - name: Save commit_sha & pr_number run: | cd optimum-intel sudo chmod -R ugo+rwx intel-doc-build cd intel-doc-build sudo mv optimum.intel optimum-intel - doc-builder push optimum-intel --doc_build_repo_id "hf-doc-build/doc-build-dev" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-intel/commit/$COMMIT_SHA" --n_retries 5 - shell: bash + echo ${{ env.COMMIT_SHA }} > ./commit_sha + echo ${{ env.PR_NUMBER }} > ./pr_number - - name: Find doc comment - uses: peter-evans/find-comment@v2 - id: find_comment + - uses: actions/upload-artifact@v3 with: - issue-number: ${{ env.PR_NUMBER }} - body-includes: docs for this PR - - - name: Add doc comment if not present - uses: thollander/actions-comment-pull-request@v2 - if: steps.find_comment.outputs.comment-id == '' - - with: - message: 'The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum-intel/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint.' - pr_number: ${{ env.PR_NUMBER }} - GITHUB_TOKEN: ${{ secrets.COMMENT_BOT_TOKEN }} - - - name: Update doc comment if necessary - if: github.event.action == 'reopened' && steps.find_comment.outputs.comment-id != '' - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ steps.find_comment.outputs.comment-id }} - token: ${{ secrets.COMMENT_BOT_TOKEN }} - edit-mode: replace - body: | - The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum-intel/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint. + name: doc-build-artifact + path: optimum-intel/intel-doc-build/ diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml new file mode 100644 index 0000000000..1012ac8f09 --- /dev/null +++ b/.github/workflows/upload_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Upload PR Documentation + +on: + workflow_run: + workflows: ["Build PR Documentation"] + types: + - completed + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main + with: + package_name: optimum-intel + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} From f3f7afaea863d5e42c692dd412f003378d761a9f Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 15 Jun 2023 03:15:46 +0200 Subject: [PATCH 041/134] Fix inference for causal lm models (#351) * fix inference * add tests * raise error when architecture not validated * fix style --- optimum/intel/openvino/modeling_decoder.py | 29 +++- setup.py | 12 +- tests/openvino/test_modeling.py | 151 +++++++++++++++++++-- 3 files changed, 174 insertions(+), 18 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 26a3e0424d..f4457983cd 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -71,6 +71,21 @@ ``` """ +_SUPPORTED_ARCHITECTURES = ( + "bart", + "blenderbot", + "blenderbot-small", + "bloom", + # "codegen", + "gpt2", + "gpt_neo", + "gpt_neox", + "llama", + "marian", + "opt", + "pegasus", +) + @add_start_docstrings( """ @@ -153,15 +168,23 @@ def _from_transformers( "trust_remote_code": trust_remote_code, } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) + config.is_decoder = True + config.is_encoder_decoder = False + if config.model_type not in _SUPPORTED_ARCHITECTURES: + raise ValueError( + f"Unrecognized architecture : {config.model_type}, only :{', '.join(_SUPPORTED_ARCHITECTURES)} architectures are supported." + ) + onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config, use_past=use_cache) # TODO : create ModelPatcher to patch each architecture - if model.config.model_type == "bloom": + if config.model_type == "bloom": model.transformer._prepare_attn_mask = _prepare_attn_mask - - if model.config.model_type == "llama": + elif config.model_type == "llama": model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + elif config.model_type in {"blenderbot-small", "blenderbot", "opt", "pegasus", "bart"}: + model.model.decoder._prepare_decoder_attention_mask = _prepare_decoder_attention_mask # Export the model to the ONNX format export(model=model, config=onnx_config, output=save_dir_path / model_file_name) diff --git a/setup.py b/setup.py index 5602860f0a..55f1d42b1e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,17 @@ "accelerate", # transformers 4.29 require accelerate for PyTorch ] -TESTS_REQUIRE = ["pytest", "parameterized", "Pillow", "evaluate", "diffusers", "py-cpuinfo"] +TESTS_REQUIRE = [ + "pytest", + "parameterized", + "Pillow", + "evaluate", + "diffusers", + "py-cpuinfo", + "sacremoses", + "torchaudio", + "rjieba", +] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 8cd671f606..bdddaf66ea 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -38,6 +38,7 @@ AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoTokenizer, + GenerationConfig, PretrainedConfig, pipeline, set_seed, @@ -76,27 +77,71 @@ MODEL_NAMES = { - "bart": "hf-internal-testing/tiny-random-bart", + "albert": "hf-internal-testing/tiny-random-albert", + "audio_spectrogram_transformer": "Ericwang/tiny-random-ast", + "beit": "hf-internal-testing/tiny-random-BeitForImageClassification", "bert": "hf-internal-testing/tiny-random-bert", - "bloom": "hf-internal-testing/tiny-random-BloomModel", + "bart": "hf-internal-testing/tiny-random-bart", "bigbird_pegasus": "hf-internal-testing/tiny-random-bigbird_pegasus", + "blenderbot-small": "hf-internal-testing/tiny-random-BlenderbotModel", + "blenderbot": "hf-internal-testing/tiny-random-blenderbot", + "bloom": "hf-internal-testing/tiny-random-BloomModel", + "camembert": "hf-internal-testing/tiny-random-camembert", + "convbert": "hf-internal-testing/tiny-random-ConvBertForSequenceClassification", + "codegen": "hf-internal-testing/tiny-random-CodeGenModel", + "data2vec_text": "hf-internal-testing/tiny-random-Data2VecTextModel", + "data2vec_vision": "hf-internal-testing/tiny-random-Data2VecVisionModel", + "data2vec_audio": "hf-internal-testing/tiny-random-Data2VecAudioModel", + "deberta": "hf-internal-testing/tiny-random-deberta", + "deberta_v2": "hf-internal-testing/tiny-random-DebertaV2Model", + "deit": "hf-internal-testing/tiny-random-deit", + "convnext": "hf-internal-testing/tiny-random-convnext", "distilbert": "hf-internal-testing/tiny-random-distilbert", + "electra": "hf-internal-testing/tiny-random-electra", + "flaubert": "hf-internal-testing/tiny-random-flaubert", # "gpt_bigcode": "bigcode/tiny_starcoder_py", - "gptj": "hf-internal-testing/tiny-random-GPTJModel", + "gpt2": "hf-internal-testing/tiny-random-gpt2", "gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel", "gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", - "gpt2": "hf-internal-testing/tiny-random-gpt2", - "llama": "trl-internal-testing/tiny-random-LlamaForCausalLM", - "marian": "sshleifer/tiny-marian-en-de", + "gptj": "hf-internal-testing/tiny-random-GPTJModel", + "hubert": "hf-internal-testing/tiny-random-HubertModel", + "ibert": "hf-internal-testing/tiny-random-ibert", + "levit": "hf-internal-testing/tiny-random-LevitModel", + "longt5": "hf-internal-testing/tiny-random-LongT5Model", + "llama": "fxmarty/tiny-llama-fast-tokenizer", + "m2m_100": "hf-internal-testing/tiny-random-m2m_100", + "opt": "hf-internal-testing/tiny-random-OPTModel", + "marian": "sshleifer/tiny-marian-en-de", # hf-internal-testing ones are broken "mbart": "hf-internal-testing/tiny-random-mbart", - "m2m_100": "valhalla/m2m100_tiny_random", + "mobilebert": "hf-internal-testing/tiny-random-MobileBertModel", + "mobilenet_v1": "google/mobilenet_v1_0.75_192", + "mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model", + "mobilevit": "hf-internal-testing/tiny-random-mobilevit", + "mt5": "hf-internal-testing/tiny-random-mt5", + "nystromformer": "hf-internal-testing/tiny-random-NystromformerModel", + "pegasus": "hf-internal-testing/tiny-random-pegasus", + "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel", + "resnet": "hf-internal-testing/tiny-random-resnet", "roberta": "hf-internal-testing/tiny-random-roberta", + "roformer": "hf-internal-testing/tiny-random-roformer", + "segformer": "hf-internal-testing/tiny-random-SegformerModel", + "squeezebert": "hf-internal-testing/tiny-random-squeezebert", "stable-diffusion": "hf-internal-testing/tiny-stable-diffusion-torch", + "sew": "hf-internal-testing/tiny-random-SEWModel", + "sew_d": "hf-internal-testing/tiny-random-SEWDModel", + "swin": "hf-internal-testing/tiny-random-SwinModel", "t5": "hf-internal-testing/tiny-random-t5", + "unispeech": "hf-internal-testing/tiny-random-unispeech", + "unispeech_sat": "hf-internal-testing/tiny-random-UnispeechSatModel", "vit": "hf-internal-testing/tiny-random-vit", + "wavlm": "hf-internal-testing/tiny-random-WavlmModel", "wav2vec2": "anton-l/wav2vec2-random-tiny-classifier", + "wav2vec2-conformer": "hf-internal-testing/tiny-random-wav2vec2-conformer", + "xlm": "hf-internal-testing/tiny-random-xlm", + "xlm_roberta": "hf-internal-testing/tiny-xlm-roberta", } + TENSOR_ALIAS_TO_TYPE = { "pt": torch.Tensor, "np": np.ndarray, @@ -207,9 +252,23 @@ def test_load_from_hub_and_save_stable_diffusion_model(self): class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( + "albert", "bert", + # "camembert", + "convbert", + # "data2vec_text", + # "deberta_v2", "distilbert", + "electra", + "flaubert", + "ibert", + # "mobilebert", + # "nystromformer", "roberta", + "roformer", + "squeezebert", + "xlm", + # "xlm_roberta", ) @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -415,11 +474,19 @@ def test_pipeline(self, model_arch): class OVModelForCausalLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( + "bart", + "blenderbot", + "blenderbot-small", "bloom", + # "codegen", + # "data2vec-text", # TODO : enable when enabled in exporters "gpt2", "gpt_neo", "gpt_neox", # "llama", + "marian", + "opt", + "pegasus", ) GENERATION_LENGTH = 100 SPEEDUP_CACHE = 1.2 @@ -449,6 +516,7 @@ def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] tokenizer = AutoTokenizer.from_pretrained(model_id) model = OVModelForCausalLM.from_pretrained(model_id, from_transformers=True, use_cache=False, compile=False) + model.config.encoder_no_repeat_ngram_size = 0 model.to("cpu") model.half() model.compile() @@ -467,7 +535,8 @@ def test_multiple_inputs(self, model_arch): tokenizer.pad_token = tokenizer.eos_token texts = ["this is a simple input", "this is a second simple input", "this is a third simple input"] tokens = tokenizer(texts, padding=True, return_tensors="pt") - outputs = model.generate(**tokens, max_new_tokens=20, num_beams=2) + generation_config = GenerationConfig(encoder_no_repeat_ngram_size=0, max_new_tokens=20, num_beams=2) + outputs = model.generate(**tokens, generation_config=generation_config) self.assertIsInstance(outputs, torch.Tensor) self.assertEqual(outputs.shape[0], 3) @@ -511,9 +580,23 @@ def test_compare_with_and_without_past_key_values(self): class OVModelForMaskedLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( + # "albert", "bert", + # "camembert", + # "convbert", + # "data2vec_text", + # "deberta", + # "deberta_v2", "distilbert", + # "electra", + # "flaubert", + # "ibert", + # "mobilebert", "roberta", + # "roformer", + # "squeezebert", + # "xlm", + # "xlm_roberta", ) @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -550,7 +633,21 @@ def test_pipeline(self, model_arch): class OVModelForImageClassificationIntegrationTest(unittest.TestCase): - SUPPORTED_ARCHITECTURES = ("vit",) + SUPPORTED_ARCHITECTURES = ( + "beit", + "convnext", + # "data2vec_vision", + # "deit", + "levit", + "mobilenet_v1", + "mobilenet_v2", + "mobilevit", + # "poolformer", + "resnet", + # "segformer", + # "swin", + "vit", + ) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): @@ -590,9 +687,15 @@ def test_pipeline(self, model_arch): class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( "bart", + # "bigbird_pegasus", + "blenderbot", + "blenderbot-small", + # "longt5", + "m2m_100", "marian", "mbart", - "m2m_100", + # "mt5", + "pegasus", "t5", ) @@ -710,7 +813,26 @@ def test_compare_with_and_without_past_key_values(self): class OVModelForAudioClassificationIntegrationTest(unittest.TestCase): - SUPPORTED_ARCHITECTURES = ("wav2vec2",) + SUPPORTED_ARCHITECTURES = ( + # "audio_spectrogram_transformer", + # "data2vec_audio", + # "hubert", + # "sew", + # "sew_d", + # "wav2vec2-conformer", + "unispeech", + # "unispeech_sat", + # "wavlm", + "wav2vec2", + # "wav2vec2-conformer", + ) + + def _generate_random_audio_data(self): + np.random.seed(10) + t = np.linspace(0, 5.0, int(5.0 * 22050), endpoint=False) + # generate pure sine wave at 220 Hz + audio_data = 0.5 * np.sin(2 * np.pi * 220 * t) + return audio_data @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): @@ -720,12 +842,13 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForAudioClassification.from_pretrained(model_id) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) - wavs = [np.random.random(16000)] - inputs = preprocessor(wavs, sampling_rate=preprocessor.sampling_rate, return_tensors="pt") + inputs = preprocessor(self._generate_random_audio_data(), return_tensors="pt") + with torch.no_grad(): transformers_outputs = transformers_model(**inputs) + for input_type in ["pt", "np"]: - inputs = preprocessor(wavs, sampling_rate=preprocessor.sampling_rate, return_tensors=input_type) + inputs = preprocessor(self._generate_random_audio_data(), return_tensors=input_type) ov_outputs = ov_model(**inputs) self.assertIn("logits", ov_outputs) self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) From 8c71217abb1d6c03533dad2532585551b7d43ff8 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:37:19 +0200 Subject: [PATCH 042/134] Add openvino tests (#353) --- optimum/intel/openvino/modeling_decoder.py | 17 +++++++------- tests/openvino/test_modeling.py | 26 +++++++++++----------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index f4457983cd..910de676cb 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -71,12 +71,12 @@ ``` """ -_SUPPORTED_ARCHITECTURES = ( +_SUPPORTED_ARCHITECTURES = { "bart", "blenderbot", "blenderbot-small", "bloom", - # "codegen", + "codegen", "gpt2", "gpt_neo", "gpt_neox", @@ -84,7 +84,7 @@ "marian", "opt", "pegasus", -) +} @add_start_docstrings( @@ -151,6 +151,12 @@ def _from_transformers( trust_remote_code: bool = False, **kwargs, ): + if config.model_type not in _SUPPORTED_ARCHITECTURES: + logger.warning( + f"This architecture : {config.model_type} was not validated, only :{', '.join(_SUPPORTED_ARCHITECTURES)} architectures were " + "validated, use at your own risk." + ) + model_file_name = ONNX_WEIGHTS_NAME if task is None: @@ -170,11 +176,6 @@ def _from_transformers( model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) config.is_decoder = True config.is_encoder_decoder = False - if config.model_type not in _SUPPORTED_ARCHITECTURES: - raise ValueError( - f"Unrecognized architecture : {config.model_type}, only :{', '.join(_SUPPORTED_ARCHITECTURES)} architectures are supported." - ) - onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config, use_past=use_cache) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index bdddaf66ea..1a8e35d55d 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -88,7 +88,7 @@ "bloom": "hf-internal-testing/tiny-random-BloomModel", "camembert": "hf-internal-testing/tiny-random-camembert", "convbert": "hf-internal-testing/tiny-random-ConvBertForSequenceClassification", - "codegen": "hf-internal-testing/tiny-random-CodeGenModel", + "codegen": "hf-internal-testing/tiny-random-CodeGenForCausalLM", "data2vec_text": "hf-internal-testing/tiny-random-Data2VecTextModel", "data2vec_vision": "hf-internal-testing/tiny-random-Data2VecVisionModel", "data2vec_audio": "hf-internal-testing/tiny-random-Data2VecAudioModel", @@ -107,7 +107,7 @@ "hubert": "hf-internal-testing/tiny-random-HubertModel", "ibert": "hf-internal-testing/tiny-random-ibert", "levit": "hf-internal-testing/tiny-random-LevitModel", - "longt5": "hf-internal-testing/tiny-random-LongT5Model", + "longt5": "hf-internal-testing/tiny-random-longt5", "llama": "fxmarty/tiny-llama-fast-tokenizer", "m2m_100": "hf-internal-testing/tiny-random-m2m_100", "opt": "hf-internal-testing/tiny-random-OPTModel", @@ -117,7 +117,7 @@ "mobilenet_v1": "google/mobilenet_v1_0.75_192", "mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model", "mobilevit": "hf-internal-testing/tiny-random-mobilevit", - "mt5": "hf-internal-testing/tiny-random-mt5", + "mt5": "stas/mt5-tiny-random", "nystromformer": "hf-internal-testing/tiny-random-NystromformerModel", "pegasus": "hf-internal-testing/tiny-random-pegasus", "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel", @@ -478,7 +478,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "blenderbot", "blenderbot-small", "bloom", - # "codegen", + "codegen", # "data2vec-text", # TODO : enable when enabled in exporters "gpt2", "gpt_neo", @@ -580,22 +580,22 @@ def test_compare_with_and_without_past_key_values(self): class OVModelForMaskedLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( - # "albert", + "albert", "bert", # "camembert", # "convbert", # "data2vec_text", - # "deberta", + "deberta", # "deberta_v2", "distilbert", - # "electra", - # "flaubert", - # "ibert", + "electra", + "flaubert", + "ibert", # "mobilebert", "roberta", - # "roformer", - # "squeezebert", - # "xlm", + "roformer", + "squeezebert", + "xlm", # "xlm_roberta", ) @@ -694,7 +694,7 @@ class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase): "m2m_100", "marian", "mbart", - # "mt5", + "mt5", "pegasus", "t5", ) From a03bb812caf1b15b015d9249dfb9360db2c3468a Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 16 Jun 2023 17:54:01 +0200 Subject: [PATCH 043/134] fix link documentation (#355) --- docs/source/inference.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index c2cc752085..792916bf7c 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. # Optimum Inference with OpenVINO -Optimum Intel can be used to load optimized models from the [Hugging Face Hub](hf.co/models) and create pipelines to run inference with OpenVINO Runtime without rewriting your APIs. +Optimum Intel can be used to load optimized models from the [Hugging Face Hub](https://huggingface.co/models?library=openvino&sort=downloads) and create pipelines to run inference with OpenVINO Runtime without rewriting your APIs. ## Switching from Transformers to Optimum Inference From cbb85ed7453238d67a54f88fffdcb55cc07ef468 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Fri, 16 Jun 2023 21:34:39 +0400 Subject: [PATCH 044/134] Add support of OVModel in quantizer (#346) * Added quantization of OVModel * Style * Try to handle pkv with patching forward (does not work) * Make CausalLMs working * Style * Update optimum/intel/openvino/quantization.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update optimum/intel/openvino/quantization.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Added quantization tests for OVModel * Style * Reverted to read_model * Updated references --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- optimum/intel/openvino/configuration.py | 4 +- optimum/intel/openvino/modeling_base.py | 24 +++- optimum/intel/openvino/quantization.py | 142 +++++++++++++++++++++++- tests/openvino/test_quantization.py | 82 ++++++++++++-- 4 files changed, 238 insertions(+), 14 deletions(-) diff --git a/optimum/intel/openvino/configuration.py b/optimum/intel/openvino/configuration.py index 31aad18426..30778856f3 100644 --- a/optimum/intel/openvino/configuration.py +++ b/optimum/intel/openvino/configuration.py @@ -30,8 +30,8 @@ "scope_overrides": {"activations": {"{re}.*matmul_0": {"mode": "symmetric"}}}, "ignored_scopes": [ "{re}.*Embedding*", - "{re}.*__add___[0-1]", - "{re}.*layer_norm_0", + "{re}.*__add___*", + "{re}.*layer_norm_*", "{re}.*matmul_1", "{re}.*__truediv__*", ], diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index a7b594794c..eb8476fefa 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -114,11 +114,28 @@ def load_model(file_name: Union[str, Path]): file_name (`str` or `Path`): The path of the model ONNX or XML file. """ + + def fix_op_names_duplicates(model: openvino.runtime.Model): + names = set() + for op in model.get_ops(): + friendly_name = op.get_friendly_name() + while True: + if friendly_name not in names: + break + friendly_name += "_" + names.add(friendly_name) + op.set_friendly_name(friendly_name) + return model + if isinstance(file_name, str): file_name = Path(file_name) bin_file_name = file_name.with_suffix(".bin") if file_name.suffix == ".xml" else None - return core.read_model(file_name, bin_file_name) + model = core.read_model(file_name, bin_file_name) + if file_name.suffix == ".onnx": + model = fix_op_names_duplicates(model) # should be called during model conversion to IR + + return model def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): """ @@ -187,7 +204,6 @@ def _from_pretrained( "The file names `ov_model.xml` and `ov_model.bin` will be soon deprecated." "Make sure to rename your file to respectively `openvino_model.xml` and `openvino_model.bin`" ) - model = cls.load_model(file_name) model_save_dir = model_id # Download the model from the hub else: @@ -227,7 +243,9 @@ def _from_pretrained( "Make sure to rename your file to respectively `openvino_model.xml` and `openvino_model.bin`" ) model_save_dir = Path(model_cache_path).parent - model = cls.load_model(file_names[0]) + file_name = file_names[0] + + model = cls.load_model(file_name) return cls(model, config=config, model_save_dir=model_save_dir, **kwargs) @classmethod diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 7dcde5ba73..c68d6d94ef 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Callable, Dict, Optional, Tuple, Union +import nncf import openvino import torch import transformers @@ -42,6 +43,8 @@ from ..utils.constant import _TASK_ALIASES from .configuration import OVConfig +from .modeling_base import OVBaseModel +from .modeling_decoder import OVBaseDecoderModel from .utils import ( MAX_ONNX_OPSET, MIN_ONNX_QDQ_OPSET, @@ -86,12 +89,12 @@ def __init__(self, model: transformers.PreTrainedModel, task: Optional[str] = No ) self.task = task or feature self.seed = seed + self.input_names = None signature = inspect.signature(self.model.forward) self._signature_columns = list(signature.parameters.keys()) self._export_input_names = [ column for column in self._signature_columns if column not in {"label", "labels", "label_ids"} ] - self.input_names = None @classmethod def from_pretrained(cls, model: PreTrainedModel, **kwargs): @@ -104,9 +107,10 @@ def quantize( save_directory: Union[str, Path], quantization_config: OVConfig = None, file_name: Optional[str] = None, - batch_size: int = 8, + batch_size: int = 1, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + **kwargs, ): """ Quantize a model given the optimization specifications defined in `quantization_config`. @@ -131,17 +135,151 @@ def quantize( ```python >>> from optimum.intel.openvino import OVQuantizer, OVModelForSequenceClassification >>> from transformers import AutoModelForSequenceClassification + >>> model = OVModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english", export=True) + >>> # or >>> model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english") >>> OVQuantizer.from_pretrained(model, task="text-classification") >>> quantizer.quantize(calibration_dataset=calibration_dataset, save_directory="./quantized_model") >>> optimized_model = OVModelForSequenceClassification.from_pretrained("./quantized_model") ``` """ + if isinstance(self.model, OVBaseDecoderModel) and self.model.use_cache: + self._quantize_ovcausallm( + calibration_dataset, + save_directory, + batch_size, + data_collator, + remove_unused_columns, + **kwargs, + ) + elif isinstance(self.model, OVBaseModel): + self._quantize_ovbasemodel( + calibration_dataset, + save_directory, + batch_size, + data_collator, + remove_unused_columns, + **kwargs, + ) + elif isinstance(self.model, torch.nn.Module): + self._quantize_torchmodel( + calibration_dataset, + save_directory, + quantization_config, + file_name, + batch_size, + data_collator, + remove_unused_columns, + ) + else: + raise TypeError(f"Unsupported model type: {type(self.model)}") + + def _quantize_ovbasemodel( + self, + calibration_dataset: Dataset, + save_directory: Union[str, Path], + batch_size: int = 1, + data_collator: Optional[DataCollator] = None, + remove_unused_columns: bool = True, + **kwargs, + ): + save_directory = Path(save_directory) + save_directory.mkdir(parents=True, exist_ok=True) + + calibration_dataloader = self._get_calibration_dataloader( + calibration_dataset=calibration_dataset, + batch_size=batch_size, + remove_unused_columns=remove_unused_columns, + data_collator=data_collator, + ) + + quantization_dataset = nncf.Dataset(calibration_dataloader, lambda x: x) + quantized_model = nncf.quantize( + self.model.model, + quantization_dataset, + model_type=nncf.ModelType.TRANSFORMER if not kwargs.get("model_type") else kwargs.get("model_type"), + fast_bias_correction=kwargs.get("fast_bias_correction", True), + subset_size=300 if not kwargs.get("subset_size") else kwargs.get("subset_size"), + **kwargs, + ) + self.model.model = quantized_model + self.model.save_pretrained(save_directory) + + def _quantize_ovcausallm( + self, + calibration_dataset: Dataset, + save_directory: Union[str, Path], + batch_size: int = 1, + data_collator: Optional[DataCollator] = None, + remove_unused_columns: bool = True, + **kwargs, + ): + save_directory = Path(save_directory) + save_directory.mkdir(parents=True, exist_ok=True) + + calibration_dataloader = self._get_calibration_dataloader( + calibration_dataset=calibration_dataset, + batch_size=batch_size, + remove_unused_columns=remove_unused_columns, + data_collator=data_collator, + ) + + # Prefeth past_key_values + self.model.compile() + subset_size = kwargs.get("subset_size", 300) + data_cache = [] + + class InferRequestWrapper: + def __init__(self, request): + self.request = request + + def __call__(self, *args, **kwargs): + data_cache.append(*args) + return self.request(*args, *kwargs) + + def __getattr__(self, attr): + if attr in self.__dict__: + return getattr(self, attr) + return getattr(self.request, attr) + + self.model.request = InferRequestWrapper(self.model.request) + for i, data in enumerate(calibration_dataloader): + self.model.generate(**data, max_new_tokens=10) + if len(data_cache) >= subset_size: + break + self.model.request = self.model.request.request + + # Actual model quantization + quantization_dataset = nncf.Dataset(data_cache, lambda x: x) + quantized_model = nncf.quantize( + self.model.model, + quantization_dataset, + model_type=nncf.ModelType.TRANSFORMER if not kwargs.get("model_type") else kwargs.get("model_type"), + fast_bias_correction=True + if not kwargs.get("fast_bias_correction") + else kwargs.get("fast_bias_correction"), + subset_size=subset_size, + **kwargs, + ) + self.model.model = quantized_model + self.model.save_pretrained(save_directory) + + def _quantize_torchmodel( + self, + calibration_dataset: Dataset, + save_directory: Union[str, Path], + quantization_config: OVConfig = None, + file_name: Optional[str] = None, + batch_size: int = 1, + data_collator: Optional[DataCollator] = None, + remove_unused_columns: bool = True, + ): save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) file_name = file_name if file_name is not None else OV_XML_FILE_NAME output_path = save_directory.joinpath(file_name) output_path = output_path.with_suffix(".xml").as_posix() + calibration_dataloader = self._get_calibration_dataloader( calibration_dataset=calibration_dataset, batch_size=batch_size, diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index d1a0891f59..5115b4894e 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -63,12 +63,12 @@ def get_num_quantized_nodes(ov_model): class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( - (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 43, 32), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 71, 1), + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 42, 32), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 41, 21), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) - def test_static_quantization(self, model_cls, model_name, expected_fake_quantize, expected_int8): + def test_automodel_static_quantization(self, model_cls, model_name, expected_fake_quantize, expected_int8): task = model_cls.export_feature dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] @@ -93,9 +93,10 @@ def preprocess_function(examples, tokenizer): model = model_cls.from_pretrained(tmp_dir) - num_fake_quantize, num_int8 = get_num_quantized_nodes(model) - self.assertEqual(expected_fake_quantize, num_fake_quantize) - self.assertEqual(expected_int8, num_int8) + # TODO: uncomment once move to a newer version of NNCF which has some fixes + # num_fake_quantize, num_int8 = get_num_quantized_nodes(model) + # self.assertEqual(expected_fake_quantize, num_fake_quantize) + # self.assertEqual(expected_int8, num_int8) tokens = tokenizer("This is a sample input", return_tensors="pt") outputs = model(**tokens) @@ -106,12 +107,46 @@ def preprocess_function(examples, tokenizer): loaded_config = OVConfig.from_pretrained(tmp_dir) self.assertEqual(expected_config.to_dict()["compression"], loaded_config.to_dict()["compression"]) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_ovmodel_static_quantization(self, model_cls, model_name, expected_fake_quantize, expected_int8): + task = model_cls.export_feature + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] + + def preprocess_function(examples, tokenizer): + return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) + + with tempfile.TemporaryDirectory() as tmp_dir: + transformers_model = model_cls.from_pretrained(model_name, export=True) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + quantizer = OVQuantizer.from_pretrained(transformers_model, task=task) + + calibration_dataset = quantizer.get_calibration_dataset( + dataset_name, + dataset_config_name=dataset_config_name, + preprocess_function=partial(preprocess_function, tokenizer=tokenizer), + num_samples=10, + dataset_split="train", + ) + quantizer.quantize(save_directory=tmp_dir, calibration_dataset=calibration_dataset) + + model = model_cls.from_pretrained(tmp_dir) + + num_fake_quantize, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_fake_quantize, num_fake_quantize) + self.assertEqual(expected_int8, num_int8) + + tokens = tokenizer("This is a sample input", return_tensors="pt") + outputs = model(**tokens) + self.assertTrue("logits" in outputs) + class OVQuantizerQATest(unittest.TestCase): SUPPORTED_ARCHITECTURES = (("hf-internal-testing/tiny-random-BertForQuestionAnswering",),) @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_static_quantization(self, model_name): + def test_automodel_static_quantization(self, model_name): def preprocess_function(examples, tokenizer): return tokenizer( examples["question"], examples["context"], padding="max_length", max_length=64, truncation=True @@ -143,6 +178,39 @@ def preprocess_function(examples, tokenizer): except RuntimeError: self.fail("Loading BERT QA model a second time failed") + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_ovmodel_static_quantization(self, model_name): + def preprocess_function(examples, tokenizer): + return tokenizer( + examples["question"], examples["context"], padding="max_length", max_length=64, truncation=True + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + transformers_model = OVModelForQuestionAnswering.from_pretrained(model_name, export=True) + tokenizer = AutoTokenizer.from_pretrained(model_name) + quantizer = OVQuantizer.from_pretrained(transformers_model) + calibration_dataset = quantizer.get_calibration_dataset( + "squadshifts", + dataset_config_name="new_wiki", + preprocess_function=partial(preprocess_function, tokenizer=tokenizer), + num_samples=10, + dataset_split="test", + ) + quantizer.quantize(save_directory=tmp_dir, calibration_dataset=calibration_dataset) + + # Test that inference on quantized model works + model = OVModelForQuestionAnswering.from_pretrained(tmp_dir) + tokens = tokenizer.encode_plus( + "This is a sample question", "This is a sample context", add_special_tokens=True, return_tensors="pt" + ) + model(**tokens, return_dict=True) + + # Test loading model a second time to catch issues with caching + try: + model = OVModelForQuestionAnswering.from_pretrained(tmp_dir) + except RuntimeError: + self.fail("Loading BERT QA model a second time failed") + class OVTrainerTest(unittest.TestCase): SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = (("distilbert-base-uncased", 50, 38),) From 7d35330a7db8e5625f5d5824872309d61c930d67 Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Mon, 19 Jun 2023 18:08:11 +0300 Subject: [PATCH 045/134] Add cache_dir argument for get_calibration_dataset (#356) * Add cache_dir argument for get_calibration_dataset * fix comments --- optimum/intel/openvino/quantization.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index c68d6d94ef..5f7e6824b4 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -358,6 +358,7 @@ def get_calibration_dataset( preprocess_function: Optional[Callable] = None, preprocess_batch: bool = True, use_auth_token: bool = False, + cache_dir: Optional[str] = None, ) -> Dataset: """ Create the calibration `datasets.Dataset` to use for the post-training static quantization calibration step. @@ -378,11 +379,17 @@ def get_calibration_dataset( Whether the `preprocess_function` should be batched. use_auth_token (`bool`, defaults to `False`): Whether to use the token generated when running `transformers-cli login`. + cache_dir (`str`, *optional*): + Caching directory for a calibration dataset. Returns: The calibration `datasets.Dataset` to use for the post-training static quantization calibration step. """ calibration_dataset = load_dataset( - dataset_name, name=dataset_config_name, split=dataset_split, use_auth_token=use_auth_token + dataset_name, + name=dataset_config_name, + split=dataset_split, + use_auth_token=use_auth_token, + cache_dir=cache_dir, ) if num_samples is not None: From c56d3b42618d746915d9ce663d23c0b5a88b0e9e Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Wed, 21 Jun 2023 12:11:26 +0400 Subject: [PATCH 046/134] Improved performance of decoders (#354) * Improved performance of decoders * Improved performance of Seq2seq models * Style * Adjusted quantization logic * Style * Temporal changes * Temporal * Make it working * Some improvements * Style * Update optimum/intel/openvino/modeling_decoder.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- optimum/intel/openvino/modeling_decoder.py | 38 ++++++++++------------ optimum/intel/openvino/modeling_seq2seq.py | 18 +++++----- optimum/intel/openvino/quantization.py | 27 ++++++++++++--- 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 910de676cb..91da03ad28 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -237,6 +237,11 @@ def reshape(self, batch_size: int, sequence_length: int): logger.warning("Static shapes are not supported for causal language model.") return self + def compile(self): + if self.request is None: + super().compile() + self.request = self.request.create_infer_request() + @add_start_docstrings( """ @@ -273,7 +278,7 @@ def forward( if past_key_values is not None: # Flatten the past_key_values past_key_values = tuple( - np.array(past_key_value) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer ) # Add the past_key_values to the decoder inputs inputs = dict(zip(self.key_value_input_names, past_key_values)) @@ -301,15 +306,14 @@ def forward( inputs["attention_mask"] = np.array(attention_mask) # Run inference - outputs = self.request(inputs, shared_memory=True) + self.request.start_async(inputs, shared_memory=True) + self.request.wait() - logits = torch.from_numpy(outputs["logits"]).to(self.device) + logits = torch.from_numpy(self.request.get_tensor("logits").data).to(self.device) if self.use_cache: # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the self-attention layer) - past_key_values = tuple( - torch.from_numpy(outputs[key]).to(self.device) for key in self.key_value_output_names - ) + past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names) # Tuple of tuple of length `n_layers`, with each tuple of length equal to 2 (k/v of self-attention) past_key_values = tuple( past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) @@ -345,13 +349,13 @@ def _reorder_cache( [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. """ + if self.config.model_type == "bloom": return self._reorder_cache_bloom(past_key_values, beam_idx) # from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel._reorder_cache return tuple( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) - for layer_past in past_key_values + tuple(np.take(past_state, beam_idx, 0) for past_state in layer_past) for layer_past in past_key_values ) # Copied from transformers.models.bloom.modeling_bloom.BloomForCausalLM._reorder_cache @@ -365,16 +369,10 @@ def _reorder_cache_bloom( """ standardized_past = self._convert_to_standard_cache(past_key_values, batch_size=len(beam_idx)) - # Get a copy of `beam_idx` on all the devices where we need those indices. - device_to_beam_idx = { - past_state.device: beam_idx.to(past_state.device) - for layer_past in past_key_values - for past_state in layer_past - } reordered_past = tuple( ( - layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]), - layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]), + np.take(layer_past[0], beam_idx, 0), + np.take(layer_past[1], beam_idx, 0), ) for layer_past in standardized_past ) @@ -392,8 +390,8 @@ def _convert_to_bloom_cache(past_key_value: Tuple[Tuple[torch.Tensor]]) -> Tuple # value: [batch_size, num_heads, seq_length, head_dim] -> [batch_size * num_heads, seq_length, head_dim] return tuple( ( - layer_past[0].view(batch_size_times_num_heads, head_dim, seq_length), - layer_past[1].view(batch_size_times_num_heads, seq_length, head_dim), + layer_past[0].reshape((batch_size_times_num_heads, head_dim, seq_length)), + layer_past[1].reshape((batch_size_times_num_heads, seq_length, head_dim)), ) for layer_past in past_key_value ) @@ -414,8 +412,8 @@ def _convert_to_standard_cache( # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim] return tuple( ( - layer_past[0].view(batch_size, num_heads, head_dim, seq_length), - layer_past[1].view(batch_size, num_heads, seq_length, head_dim), + layer_past[0].reshape((batch_size, num_heads, head_dim, seq_length)), + layer_past[1].reshape((batch_size, num_heads, seq_length, head_dim)), ) for layer_past in past_key_value ) diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index a9eba68ecf..c56f608daf 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Dict, Optional, Tuple +import numpy as np import openvino import torch import transformers @@ -249,7 +250,7 @@ def _reorder_cache(past, beam_idx) -> Tuple[Tuple[torch.FloatTensor]]: for layer_past in past: # Cached cross_attention states don't have to be reordered -> they are always the same reordered_past += ( - tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:], + tuple(np.take(past_state, beam_idx, 0) for past_state in layer_past[:2]) + layer_past[2:], ) return reordered_past @@ -355,6 +356,8 @@ def __init__(self, model: openvino.runtime.Model, device: str, ov_config: Dict): self.device = torch.device("cpu") self.input_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.inputs)} self.key_value_input_names = [key for key in self.input_names if "key_values" in key] + self.output_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.outputs)} + self.key_value_output_names = [key for key in self.output_names if "key_values" in key or "present" in key] is_legacy = any("past_key_values" in key.get_any_name() for key in self.model.outputs) if len(self.key_value_input_names) > 0 and not is_legacy: @@ -399,16 +402,13 @@ def forward( inputs["encoder_hidden_states"] = encoder_hidden_states # Run inference - outputs = self.request(inputs, shared_memory=True) - logits = torch.from_numpy(outputs["logits"]).to(self.device) + self.request.start_async(inputs, shared_memory=True) + self.request.wait() + logits = torch.from_numpy(self.request.get_tensor("logits").data).to(self.device) # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 corresponds to the # self-attention layer and 2 to the cross-attention layer) - out_past_key_values = tuple( - torch.from_numpy(outputs[next(iter(key))]).to(self.device) - for key in outputs.names() - if ("key_values" in next(iter(key)) or "present" in next(iter(key))) - ) + out_past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names) # Tuple of tuple of length `n_layers`, with each tuple of length equal to: # * 4 for the decoder without cache (k/v of self-attention + k/v of cross-attention) @@ -432,4 +432,4 @@ def __call__(self, *args, **kwargs): def _compile(self): if self.request is None: logger.info("Compiling the decoder...") - self.request = core.compile_model(self.model, self._device, self.ov_config) + self.request = core.compile_model(self.model, self._device, self.ov_config).create_infer_request() diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 5f7e6824b4..9d4c18b157 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -17,7 +17,7 @@ import logging from itertools import chain from pathlib import Path -from typing import Callable, Dict, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional, Tuple, Union import nncf import openvino @@ -31,7 +31,7 @@ from nncf.torch.initialization import PTInitializingDataLoader from nncf.torch.nncf_network import NNCFNetwork from openvino._offline_transformations import compress_quantize_weights_transformation -from openvino.runtime import Core +from openvino.runtime import Core, Tensor from torch.onnx import export as onnx_export from torch.utils._pytree import tree_map from torch.utils.data import DataLoader, RandomSampler @@ -237,14 +237,33 @@ def __call__(self, *args, **kwargs): data_cache.append(*args) return self.request(*args, *kwargs) + def infer(self, inputs: Any = None, shared_memory: bool = False): + data_cache.append(inputs) + return self.request.infer(inputs, shared_memory) + + def start_async( + self, + inputs: Any = None, + userdata: Any = None, + shared_memory: bool = False, + ): + data_cache.append(inputs) + self.request.infer(inputs, shared_memory) + + def wait(self): + pass + + def get_tensor(self, name: str): + return Tensor(self.request.results[name]) + def __getattr__(self, attr): if attr in self.__dict__: return getattr(self, attr) return getattr(self.request, attr) self.model.request = InferRequestWrapper(self.model.request) - for i, data in enumerate(calibration_dataloader): - self.model.generate(**data, max_new_tokens=10) + for _, data in enumerate(calibration_dataloader): + self.model.generate(**data, max_new_tokens=100) if len(data_cache) >= subset_size: break self.model.request = self.model.request.request From 47428484598986443686b7c8ba825b4a3d7b4f73 Mon Sep 17 00:00:00 2001 From: qgao007 <108324932+qgao007@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:09:00 +0800 Subject: [PATCH 047/134] Fix causal LM INC example for post-training quantization (#358) Signed-off-by: Gao, Qun --- examples/neural_compressor/language-modeling/run_clm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/neural_compressor/language-modeling/run_clm.py b/examples/neural_compressor/language-modeling/run_clm.py index fe28530183..54f1e7b617 100644 --- a/examples/neural_compressor/language-modeling/run_clm.py +++ b/examples/neural_compressor/language-modeling/run_clm.py @@ -598,6 +598,8 @@ def compute_metrics(eval_preds): else: if optim_args.smooth_quant: recipes = {"smooth_quant": True, "smooth_quant_args": {"alpha": optim_args.smooth_quant_alpha}} + else: + recipes = {} quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, recipes=recipes) if optim_args.apply_pruning: From e21b38576a419e741511d0d73af4003ea9ded4f1 Mon Sep 17 00:00:00 2001 From: Yujie Pan Date: Mon, 26 Jun 2023 22:01:54 +0800 Subject: [PATCH 048/134] Update JPQD config in audio-classification example (#320) * update wav2vec2 config * update accuracy --- examples/openvino/audio-classification/README.md | 13 +++++++------ .../configs/wav2vec2-base-jpqd.json | 15 +++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/openvino/audio-classification/README.md b/examples/openvino/audio-classification/README.md index 1a6f1ddce8..8b3366c960 100644 --- a/examples/openvino/audio-classification/README.md +++ b/examples/openvino/audio-classification/README.md @@ -61,9 +61,10 @@ On a single V100 GPU, this script should run in ~45 minutes and yield a quantize ```bash torchrun --nproc-per-node=1 run_audio_classification.py \ - --model_name_or_path facebook/wav2vec2-base \ + --model_name_or_path anton-l/wav2vec2-base-ft-keyword-spotting \ --teacher_model_name_or_path anton-l/wav2vec2-base-ft-keyword-spotting \ --nncf_compression_config configs/wav2vec2-base-jpqd.json \ + --freeze_feature_encoder False \ --distillation_weight 0.9 \ --dataset_name superb \ --dataset_config_name ks \ @@ -74,11 +75,11 @@ torchrun --nproc-per-node=1 run_audio_classification.py \ --do_train \ --fp16 \ --optim adamw_torch \ - --learning_rate 2e-4 \ + --learning_rate 7e-5 \ --max_length_seconds 1 \ --attention_mask False \ - --warmup_ratio 0.1 \ - --num_train_epochs 15 \ + --warmup_ratio 0.5 \ + --num_train_epochs 12 \ --per_device_train_batch_size 32 \ --gradient_accumulation_steps 4 \ --per_device_eval_batch_size 64 \ @@ -89,7 +90,7 @@ torchrun --nproc-per-node=1 run_audio_classification.py \ --save_strategy epoch \ --load_best_model_at_end False \ --save_total_limit 3 \ - --seed 0 + --seed 42 ``` -This script should take about 3 hours on a single V100 GPU and produce a quantized Wav2Vec2-base model with ~80% structured sparsity in its linear layers. The model accuracy should converge to about 97.5%. For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters might be required to achieve the same results as on a single GPU. +This script should take about 3 hours on a single V100 GPU and produce a quantized Wav2Vec2-base model with ~60% structured sparsity in its linear layers. The model accuracy should converge to about 97.5%. For launching the script on multiple GPUs specify `--nproc-per-node=`. Note, that different batch size and other hyperparameters might be required to achieve the same results as on a single GPU. diff --git a/examples/openvino/audio-classification/configs/wav2vec2-base-jpqd.json b/examples/openvino/audio-classification/configs/wav2vec2-base-jpqd.json index 90e31fc457..c58903da17 100644 --- a/examples/openvino/audio-classification/configs/wav2vec2-base-jpqd.json +++ b/examples/openvino/audio-classification/configs/wav2vec2-base-jpqd.json @@ -2,9 +2,9 @@ { "algorithm": "movement_sparsity", "params": { - "warmup_start_epoch": 3, - "warmup_end_epoch": 8, - "importance_regularization_factor": 0.1, + "warmup_start_epoch": 1, + "warmup_end_epoch": 6, + "importance_regularization_factor": 0.045, "enable_structured_masking": true }, "sparse_structure_by_scopes": [ @@ -39,8 +39,12 @@ "overflow_fix": "enable", "initializer": { "range": { - "num_init_samples": 300, - "type": "mean_min_max" + "num_init_samples": 512, + "type": "percentile", + "params": { + "min_percentile": 0.01, + "max_percentile": 99.99 + } }, "batchnorm_adaptation": { "num_bn_adaptation_samples": 0 @@ -54,7 +58,6 @@ } }, "ignored_scopes": [ - "{re}.*feature_extractor.*", "{re}.*__add___[0-1]", "{re}.*layer_norm_0" ] From 86f0aa4d6afa0c3f8efce5223e7b37cfaf7dd209 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 27 Jun 2023 00:23:30 +0200 Subject: [PATCH 049/134] Fix INC integration compatibility issues for INC > v2.1.1 (#338) * update transformers version * check ipex version compatible with static quantization * add missing import * Fix compatibility issues for INC > v2.1.1 * remove installs from source * fix compatibility with INC * deprecate onnx export for dynamic quantized model as deprecated in INC>=2.2.0 * deprecate ONNX export deprecated in INC * add IPEX version check for generation * fix IPEX quantizatino * take opset max * fix style * set task * fix style --- .github/workflows/test_ipex.yml | 1 - .../intel/neural_compressor/configuration.py | 3 +- .../intel/neural_compressor/quantization.py | 42 +++++++++++++++---- optimum/intel/neural_compressor/trainer.py | 10 +++-- optimum/intel/utils/import_utils.py | 9 ++++ setup.py | 2 +- tests/neural_compressor/test_optimization.py | 13 +++--- 7 files changed, 58 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test_ipex.yml b/.github/workflows/test_ipex.yml index 64d8e7f6af..82c9e8c7f7 100644 --- a/.github/workflows/test_ipex.yml +++ b/.github/workflows/test_ipex.yml @@ -31,7 +31,6 @@ jobs: run: | python -m pip install --upgrade pip pip install .[ipex,tests] - pip install torch==1.13.0 intel-extension-for-pytorch==1.13.0 - name: Test with Pytest run: | pytest tests/ipex/ diff --git a/optimum/intel/neural_compressor/configuration.py b/optimum/intel/neural_compressor/configuration.py index 164b68d664..32d5e95375 100644 --- a/optimum/intel/neural_compressor/configuration.py +++ b/optimum/intel/neural_compressor/configuration.py @@ -79,7 +79,8 @@ def _create_pruning_config(config: Union[Dict, WeightPruningConfig]): @staticmethod def _create_distillation_config(config: Union[Dict, DistillationConfig]): if isinstance(config, DistillationConfig): - criterion = next(iter(config.criterion.values())) + criterion = getattr(config.criterion, "config", config.criterion) + criterion = next(iter(criterion.values())) config = { "teacher_model_name_or_path": config.teacher_model.config._name_or_path, "temperature": criterion.temperature, diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index 400456f880..07223a7b06 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -24,7 +24,7 @@ import torch from datasets import Dataset, load_dataset -from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub import hf_hub_download from neural_compressor.adaptor.pytorch import PyTorch_FXAdaptor, _cfg_to_qconfig, _propagate_qconfig from neural_compressor.config import PostTrainingQuantConfig from neural_compressor.experimental.export import torch_to_int8_onnx @@ -60,8 +60,10 @@ from ..utils.constant import _TASK_ALIASES, MIN_QDQ_ONNX_OPSET, ONNX_WEIGHTS_NAME, WEIGHTS_NAME from ..utils.import_utils import ( + _ipex_version, _neural_compressor_version, _torch_version, + is_ipex_version, is_neural_compressor_version, is_torch_version, ) @@ -72,6 +74,7 @@ logger = logging.getLogger(__name__) NEURAL_COMPRESSOR_MINIMUM_VERSION = "2.1.0" +IPEX_MINIMUM_VERSION = "2.1.0" if is_neural_compressor_version("<", NEURAL_COMPRESSOR_MINIMUM_VERSION): raise ImportError( @@ -163,6 +166,7 @@ def quantize( save_onnx_model = kwargs.pop("save_onnx_model", False) output_path = save_directory.joinpath(file_name or WEIGHTS_NAME) calibration_dataloader = None + self._set_task() if INCQuantizationMode(quantization_config.approach) == INCQuantizationMode.STATIC: # Since PyTorch fx trace does not really require an example_inputs, only need calibration_dataset or calibration_fn here. @@ -181,6 +185,24 @@ def quantize( data_collator=data_collator, ) + # Disable ONNX export for post-training quantized model as deprecated in neural-compressor>=2.2.0 + if save_onnx_model: + logger.warning( + "ONNX export for post-training quantized model is no longer supported by neural-compressor>=2.2.0. " + "To apply quantization on an ONNX model, check out optimum.onnxruntime.ORTQuantizer" + ) + save_onnx_model = False + + if ( + quantization_config.backend == "ipex" + and is_ipex_version("<", IPEX_MINIMUM_VERSION) + and "generation" in self.task + ): + raise ImportError( + f"Found an incompatible version of intel-extension-for-pytorch. Found version {_ipex_version}, " + f"but only version {IPEX_MINIMUM_VERSION} or higher is supported." + ) + if isinstance(self._original_model.config, PretrainedConfig): self._original_model.config.backend = quantization_config.backend @@ -206,7 +228,6 @@ def quantize( self._quantized_model = compressed_model._model if save_onnx_model: - self._set_task() model_type = self._original_model.config.model_type.replace("_", "-") model_name = getattr(self._original_model, "name", None) onnx_config_class = TasksManager.get_exporter_config_constructor( @@ -246,14 +267,14 @@ def _onnx_export( config: OnnxConfig, output_path: Union[str, Path], ): - opset = min(config.DEFAULT_ONNX_OPSET, MIN_QDQ_ONNX_OPSET) + opset = max(config.DEFAULT_ONNX_OPSET, MIN_QDQ_ONNX_OPSET) dynamic_axes = dict(chain(config.inputs.items(), config.outputs.items())) inputs = config.generate_dummy_inputs(framework="pt") device = model.model.device inputs = {k: v.to(device) for k, v in inputs.items()} + torch_to_int8_onnx( - fp32_model=self._original_model.to(device), - int8_model=model.model, + model.model, q_config=model.q_config, save_path=str(output_path), example_inputs=inputs, @@ -265,10 +286,13 @@ def _onnx_export( def _set_task(self): if self.task is None: - self.task = HfApi().model_info(self._original_model.config._name_or_path).pipeline_tag - if self.task is None: - raise ValueError( - "The task defining the model topology could not be extracted and needs to be specified for the ONNX export." + try: + self.task = TasksManager.infer_task_from_model(self._original_model.config._name_or_path) + except Exception as e: + self.task = "default" + logger.warning( + f"The task could not be automatically inferred and will be set to {self.task}. " + f"Please provide the task argument with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}" ) self.task = _TASK_ALIASES.get(self.task, self.task) diff --git a/optimum/intel/neural_compressor/trainer.py b/optimum/intel/neural_compressor/trainer.py index ac235e5061..705a83c900 100644 --- a/optimum/intel/neural_compressor/trainer.py +++ b/optimum/intel/neural_compressor/trainer.py @@ -596,6 +596,11 @@ def _save( state_dict["best_configure"] = self._compression_manager.model.q_config torch.save(state_dict, output_model_file) + # Disable ONNX export for quantized model as deprecated in neural-compressor>=2.2.0 + if save_onnx_model and self.dtype == "int8": + logger.warning("ONNX export for quantized model is no longer supported by neural-compressor>=2.2.0. ") + save_onnx_model = False + # Export the compressed model to the ONNX format if save_onnx_model: self._set_task() @@ -637,8 +642,7 @@ def _onnx_export(self, model: nn.Module, config: "OnnxConfig", output_path: str) if self.dtype == "int8": torch_to_int8_onnx( - fp32_model=self._compression_manager.model.fp32_model.to(device), - int8_model=model, + model, q_config=self._compression_manager.model.q_config, save_path=output_path, example_inputs=inputs, @@ -649,7 +653,7 @@ def _onnx_export(self, model: nn.Module, config: "OnnxConfig", output_path: str) ) else: torch_to_fp32_onnx( - fp32_model=model, + model, save_path=output_path, example_inputs=inputs, opset_version=opset, diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index 38bce51192..36e2229174 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -182,6 +182,15 @@ def is_torch_version(operation: str, version: str): return compare_versions(parse(_torch_version), operation, version) +def is_ipex_version(operation: str, version: str): + """ + Compare the current ipex version to a given reference with an operation. + """ + if not _ipex_available: + return False + return compare_versions(parse(_ipex_version), operation, version) + + DIFFUSERS_IMPORT_ERROR = """ {0} requires the diffusers library but it was not found in your environment. You can install it with pip: `pip install diffusers`. Please note that you may need to restart your runtime after installation. diff --git a/setup.py b/setup.py index 55f1d42b1e..54f0b66b4d 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] EXTRAS_REQUIRE = { - "neural-compressor": ["neural-compressor>=2.1.1", "onnx", "onnxruntime<1.15.0"], + "neural-compressor": ["neural-compressor>=2.2.0", "onnx", "onnxruntime<1.15.0"], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index 67be425f88..a885590e93 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -122,7 +122,7 @@ def test_dynamic_quantization(self, task, model_name, expected_quantized_matmuls model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) quantizer = INCQuantizer.from_pretrained(model, task=task) - save_onnx_model = "generation" in task + save_onnx_model = False with tempfile.TemporaryDirectory() as tmp_dir: quantizer.quantize( quantization_config=quantization_config, @@ -150,7 +150,7 @@ def test_static_quantization(self, task, model_name, expected_quantized_matmuls) quantizer = INCQuantizer.from_pretrained(model, task=task) calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=num_samples) - save_onnx_model = task != "text-generation" + save_onnx_model = False with tempfile.TemporaryDirectory() as tmp_dir: quantizer.quantize( @@ -226,11 +226,11 @@ def eval_fn(model): quantizer.quantize( quantization_config=quantization_config, save_directory=tmp_dir, - save_onnx_model=True, + save_onnx_model=False, ) loaded_model = INCModelForQuestionAnswering.from_pretrained(tmp_dir) inc_config = INCConfig.from_pretrained(tmp_dir) - self.assertTrue(inc_config.save_onnx_model) + self.assertFalse(inc_config.save_onnx_model) self.assertFalse(inc_config.quantization["is_static"]) quantized_model_metric = eval_fn(loaded_model) @@ -281,7 +281,7 @@ def test_dynamic_diffusion_model(self): @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) def test_aware_training_quantization(self, task, model_name, expected_quantized_matmuls): quantization_config = QuantizationAwareTrainingConfig() - save_onnx_model = True + save_onnx_model = False with tempfile.TemporaryDirectory() as tmp_dir: trainer = self.get_trainer( @@ -312,7 +312,7 @@ def test_aware_training_quantization_pruning(self, task, model_name, expected_qu target_sparsity=target_sparsity, pruning_scope="local", ) - save_onnx_model = True + save_onnx_model = False with tempfile.TemporaryDirectory() as tmp_dir: trainer = self.get_trainer( @@ -401,7 +401,6 @@ def test_distillation(self, task, model_name, expected_quantized_matmuls): def test_seq2seq_aware_training_quantization(self): quantization_config = QuantizationAwareTrainingConfig() - save_onnx_model = True batch_size = 2 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split="train[:1%]") val_dataset = load_dataset("cnn_dailymail", "3.0.0", split="validation[:1%]") From 14cd284f9e691531b0234c7a8913750080056a33 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Thu, 29 Jun 2023 16:32:19 +0300 Subject: [PATCH 050/134] 8-bit weight compression (#357) * Added Token Merging in combination with QAT for Stable Diffusion * Modifiedstable diffusion notebook * Chanded the SD readme * Style * Fixed requirements * Added an option for agressive quantization * Enhanced data loading * SD optimization improvements * Update examples/openvino/stable-diffusion/README.md Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Update examples/openvino/stable-diffusion/train_text_to_image_qat.py Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * Apply comments * Compress weights * Added weights compression config * Make it working but config.json is absent * Make it working * Come up with a unified implementation * Style * Clean up * Style * Fixed Trainer * Fixed export, test, changed to symmetric quantizaiton of weights * Fixed set_task issue * Style * Update optimum/intel/openvino/configuration.py Co-authored-by: Helena Kloosterman * Update optimum/intel/openvino/quantization.py Co-authored-by: Helena Kloosterman * Update optimum/intel/openvino/quantization.py Co-authored-by: Helena Kloosterman * Update tests/openvino/test_quantization.py Co-authored-by: Helena Kloosterman * Update tests/openvino/test_quantization.py Co-authored-by: Helena Kloosterman * Applied comments * Style * Apply comments * move compress_weights to quantize (#359) * move compress_weights to quantize * raise an exception when calibration dataset is not provided --------- Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Co-authored-by: Helena Kloosterman Co-authored-by: Ella Charlaix --- optimum/intel/openvino/configuration.py | 44 +++++- optimum/intel/openvino/quantization.py | 171 +++++++++++++----------- optimum/intel/openvino/trainer.py | 41 +++++- optimum/intel/openvino/utils.py | 2 +- tests/openvino/test_quantization.py | 38 +++++- 5 files changed, 211 insertions(+), 85 deletions(-) diff --git a/optimum/intel/openvino/configuration.py b/optimum/intel/openvino/configuration.py index 30778856f3..a45ee281f6 100644 --- a/optimum/intel/openvino/configuration.py +++ b/optimum/intel/openvino/configuration.py @@ -29,14 +29,50 @@ }, "scope_overrides": {"activations": {"{re}.*matmul_0": {"mode": "symmetric"}}}, "ignored_scopes": [ - "{re}.*Embedding*", - "{re}.*__add___*", - "{re}.*layer_norm_*", + "{re}.*Embedding.*", + "{re}.*add___.*", + "{re}.*layer_norm_.*", "{re}.*matmul_1", - "{re}.*__truediv__*", + "{re}.*__truediv__.*", ], } +INT8_WEIGHT_COMPRESSION_CONFIG = { + "algorithm": "quantization", + "weights": { + "mode": "symmetric", + "bits": 8, + "target_scopes": [ + "{re}.*Embedding.*", + "{re}.*matmul_.*", + "{re}.*addmm_.*", + "{re}.*baddmm_.*", + "{re}.*linear_.*", + ], + "ignored_scopes": [ + "{re}.*conv_*", + ], + }, + "activations": { + "ignored_scopes": [ + "{re}.*add___.*", + "{re}.*__radd___.*", + "{re}.*layer_norm_.*", + "{re}.*__truediv__.*", + "{re}.*__mul___.*", + "{re}.*__rmul___.*", + "{re}.*tanh_.*", + "{re}.*pow_.*", + "{re}.*matmul_.*", + "{re}.*addmm_.*", + "{re}.*baddmm_.*", + "{re}.*linear_.*", + "{re}.*conv_.*", + ], + }, + "overflow_fix": "disable", +} + class OVConfig(BaseConfig): CONFIG_NAME = "openvino_config.json" diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 9d4c18b157..462e8b9593 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -13,9 +13,8 @@ # limitations under the License. import inspect -import io import logging -from itertools import chain +import os from pathlib import Path from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -24,25 +23,21 @@ import torch import transformers from datasets import Dataset, load_dataset -from huggingface_hub import HfApi from nncf import NNCFConfig from nncf.torch import create_compressed_model, register_default_init_args from nncf.torch.dynamic_graph.io_handling import wrap_nncf_model_inputs_with_objwalk from nncf.torch.initialization import PTInitializingDataLoader -from nncf.torch.nncf_network import NNCFNetwork from openvino._offline_transformations import compress_quantize_weights_transformation from openvino.runtime import Core, Tensor -from torch.onnx import export as onnx_export -from torch.utils._pytree import tree_map -from torch.utils.data import DataLoader, RandomSampler +from torch.utils.data import DataLoader, RandomSampler, TensorDataset from transformers import DataCollator, PreTrainedModel, default_data_collator from optimum.exporters import TasksManager -from optimum.exporters.onnx import OnnxConfig +from optimum.exporters.onnx import export from optimum.quantization_base import OptimumQuantizer from ..utils.constant import _TASK_ALIASES -from .configuration import OVConfig +from .configuration import INT8_WEIGHT_COMPRESSION_CONFIG, OVConfig from .modeling_base import OVBaseModel from .modeling_decoder import OVBaseDecoderModel from .utils import ( @@ -50,7 +45,6 @@ MIN_ONNX_QDQ_OPSET, ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME, - use_external_data_format, ) @@ -83,10 +77,10 @@ def __init__(self, model: transformers.PreTrainedModel, task: Optional[str] = No feature = kwargs.pop("feature", None) if feature is not None: logger.warning("`feature` is deprecated and will be removed in a future version. Use `task` instead.") - if task is not None and task != feature: - logger.warning( - f"Both `feature` and `task` were specified. {task} will be used to define the model topology for the model ONNX export." - ) + if task is not None and task != feature: + logger.warning( + f"Both `feature` and `task` were specified. {task} will be used to define the model topology for the model ONNX export." + ) self.task = task or feature self.seed = seed self.input_names = None @@ -103,13 +97,14 @@ def from_pretrained(cls, model: PreTrainedModel, **kwargs): def quantize( self, - calibration_dataset: Dataset, - save_directory: Union[str, Path], + calibration_dataset: Dataset = None, + save_directory: Union[str, Path] = None, quantization_config: OVConfig = None, file_name: Optional[str] = None, batch_size: int = 1, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + weights_only: bool = False, **kwargs, ): """ @@ -130,19 +125,51 @@ def quantize( The function to use to form a batch from a list of elements of the calibration dataset. remove_unused_columns (`bool`, defaults to `True`): Whether or not to remove the columns unused by the model forward method. + weights_only (`bool`, defaults to `False`): + Compress weights to integer precision (8-bit by default) while keeping activations + floating-point. Fits best for LLM footprint reduction and performance acceleration. - Example: + Examples: ```python >>> from optimum.intel.openvino import OVQuantizer, OVModelForSequenceClassification >>> from transformers import AutoModelForSequenceClassification >>> model = OVModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english", export=True) >>> # or >>> model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english") - >>> OVQuantizer.from_pretrained(model, task="text-classification") + >>> quantizer = OVQuantizer.from_pretrained(model, task="text-classification") >>> quantizer.quantize(calibration_dataset=calibration_dataset, save_directory="./quantized_model") >>> optimized_model = OVModelForSequenceClassification.from_pretrained("./quantized_model") ``` + + ```python + >>> from optimum.intel.openvino import OVQuantizer, OVModelForCausalLM + >>> from transformers import AutoModelForCausalLM + >>> model = AutoModelForCausalLM.from_pretrained("databricks/dolly-v2-3b") + >>> quantizer = OVQuantizer.from_pretrained(model, task="text-generation") + >>> quantizer.quantize(save_directory="./quantized_model", weights_only=True) + >>> optimized_model = OVModelForCausalLM.from_pretrained("./quantized_model") + ``` """ + if save_directory is None: + # TODO : can be set to self.model.config.name_or_path for OVModels when not provided + raise ValueError("`save_directory` needs to be specified") + + if weights_only: + if isinstance(self.model, OVBaseModel): + raise ValueError( + "`weights_only` currently not supported for `OVModels`, only available for torch.nn.Module." + ) + if calibration_dataset is not None: + logger.warning( + "`calibration_dataset` was provided but will not be used as `weights_only` is set to `True`." + ) + else: + if calibration_dataset is None: + raise ValueError( + "`calibration_dataset` is needed to compute the activations range during the calibration step and was not provided. " + "In case you only want to apply quantization on the weights, please set `weights_only=True`." + ) + if isinstance(self.model, OVBaseDecoderModel) and self.model.use_cache: self._quantize_ovcausallm( calibration_dataset, @@ -170,6 +197,7 @@ def quantize( batch_size, data_collator, remove_unused_columns, + weights_only, ) else: raise TypeError(f"Unsupported model type: {type(self.model)}") @@ -199,7 +227,6 @@ def _quantize_ovbasemodel( quantization_dataset, model_type=nncf.ModelType.TRANSFORMER if not kwargs.get("model_type") else kwargs.get("model_type"), fast_bias_correction=kwargs.get("fast_bias_correction", True), - subset_size=300 if not kwargs.get("subset_size") else kwargs.get("subset_size"), **kwargs, ) self.model.model = quantized_model @@ -263,7 +290,7 @@ def __getattr__(self, attr): self.model.request = InferRequestWrapper(self.model.request) for _, data in enumerate(calibration_dataloader): - self.model.generate(**data, max_new_tokens=100) + self.model.generate(**data, max_new_tokens=10) if len(data_cache) >= subset_size: break self.model.request = self.model.request.request @@ -277,7 +304,6 @@ def __getattr__(self, attr): fast_bias_correction=True if not kwargs.get("fast_bias_correction") else kwargs.get("fast_bias_correction"), - subset_size=subset_size, **kwargs, ) self.model.model = quantized_model @@ -292,63 +318,85 @@ def _quantize_torchmodel( batch_size: int = 1, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + weights_only: bool = False, ): + self._set_task() save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) file_name = file_name if file_name is not None else OV_XML_FILE_NAME output_path = save_directory.joinpath(file_name) output_path = output_path.with_suffix(".xml").as_posix() + model_type = self.model.config.model_type.replace("_", "-") + onnx_config_class = TasksManager.get_exporter_config_constructor( + exporter="onnx", + model=self.model, + task=self.task, + model_type=model_type, + ) + + if weights_only: + calibration_dataset = TensorDataset(torch.tensor([0.0, 1.0])) + calibration_dataset.column_names = [] + remove_unused_columns = False + onnx_config = onnx_config_class(self.model.config) + + def data_collator(batch): + return onnx_config.generate_dummy_inputs(framework="pt") + calibration_dataloader = self._get_calibration_dataloader( calibration_dataset=calibration_dataset, batch_size=batch_size, remove_unused_columns=remove_unused_columns, data_collator=data_collator, ) - model_inputs = next(iter(calibration_dataloader)) + if quantization_config is None: logger.info( "No configuration describing the quantization process was provided, a default OVConfig will be generated." ) - quantization_config = OVConfig() + quantization_config = OVConfig(compression=INT8_WEIGHT_COMPRESSION_CONFIG) if weights_only else OVConfig() + + model_inputs = next(iter(calibration_dataloader)) quantization_config.add_input_info(model_inputs) nncf_config = NNCFConfig.from_dict(quantization_config.__dict__) nncf_config = register_default_init_args(nncf_config, calibration_dataloader) controller, compressed_model = create_compressed_model( self.model, nncf_config, wrap_inputs_fn=wrap_nncf_model_inputs_with_objwalk ) - controller.prepare_for_export() - - self._set_task() + compressed_model = controller.strip(do_copy=False) + task = self.task + model = self.model self.model.config.save_pretrained(save_directory) - model_type = self.model.config.model_type.replace("_", "-") - onnx_config_class = TasksManager.get_exporter_config_constructor( - exporter="onnx", - model=self.model, - task=self.task, - model_type=model_type, - ) - if self.task == "text-generation": - onnx_config = onnx_config_class(self.model.config, use_past=self.model.config.use_cache) + if task == "text-generation": + onnx_config = onnx_config_class(model.config, use_past=model.config.use_cache) else: - onnx_config = onnx_config_class(self.model.config) + onnx_config = onnx_config_class(model.config) - compressed_model.eval() - num_parameters = compressed_model.num_parameters() - save_as_external_data = use_external_data_format(num_parameters) or quantization_config.save_onnx_model - f = io.BytesIO() if not save_as_external_data else save_directory / ONNX_WEIGHTS_NAME + onnx_path = save_directory / ONNX_WEIGHTS_NAME - # Export the compressed model to the ONNX format + # Export the model to the ONNX format opset = min(onnx_config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) - opset = opset if not quantization_config.save_onnx_model else max(opset, MIN_ONNX_QDQ_OPSET) - _onnx_export_nncf_model(compressed_model, onnx_config, f, opset) + opset = max(opset, MIN_ONNX_QDQ_OPSET) + export( + model=compressed_model, + config=onnx_config, + opset=opset, + output=onnx_path, + ) # Load and save the compressed model - model = core.read_model(f) if save_as_external_data else core.read_model(f.getvalue(), b"") + model = core.read_model(onnx_path) self._save_pretrained(model, output_path) quantization_config.save_pretrained(save_directory) + if not quantization_config.save_onnx_model: + os.remove(onnx_path) + try: + os.remove(f"{onnx_path}_data") + except FileNotFoundError: + pass @staticmethod def _save_pretrained(model: openvino.runtime.Model, output_path: str): @@ -357,7 +405,7 @@ def _save_pretrained(model: openvino.runtime.Model, output_path: str): def _set_task(self): if self.task is None: - self.task = HfApi().model_info(self.model.config._name_or_path).pipeline_tag + self.task = TasksManager.infer_task_from_model(self.model.config._name_or_path) if self.task is None: raise ValueError( "The task defining the model topology could not be extracted and needs to be specified for the ONNX export." @@ -442,36 +490,3 @@ def _get_calibration_dataloader( def _remove_unused_columns(self, dataset: Dataset): ignored_columns = list(set(dataset.column_names) - set(self._signature_columns)) return dataset.remove_columns(ignored_columns) - - -def _onnx_export_nncf_model(model: NNCFNetwork, config: OnnxConfig, output: Union[str, io.BytesIO], opset: int = None): - signature = inspect.signature(model.forward) - signature = list(signature.parameters.keys()) - opset = opset or config.DEFAULT_ONNX_OPSET - model_inputs = config.generate_dummy_inputs(framework="pt") - # Create ordered inputs for the ONNX export of NNCFNetwork as keyword arguments are currently not supported - model_inputs = tuple(model_inputs.pop(key, None) for key in signature if len(model_inputs) != 0) - device = model.device - - def remap(value): - if isinstance(value, torch.Tensor): - value = value.to(device) - return value - - with config.patch_model_for_export(model): - model_inputs = tree_map(remap, model_inputs) - with torch.no_grad(): - model.eval() - # Disable node additions to be exported in the graph - model.disable_dynamic_graph_building() - onnx_export( - model, - model_inputs, - f=output, - input_names=list(config.inputs.keys()), - output_names=list(config.outputs.keys()), - dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), - do_constant_folding=True, - opset_version=opset, - ) - model.enable_dynamic_graph_building() diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index f5b724b950..ab4c2c217a 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -19,6 +19,7 @@ import sys import time from collections import defaultdict +from itertools import chain from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Type, Union @@ -36,6 +37,7 @@ from nncf.torch import create_compressed_model from nncf.torch.composite_compression import PTCompositeCompressionAlgorithmController from nncf.torch.compression_method_api import PTCompressionAlgorithmController +from nncf.torch.nncf_network import NNCFNetwork from nncf.torch.quantization.algo import QuantizationController from openvino._offline_transformations import compress_quantize_weights_transformation from openvino.runtime import Core, PartialShape, serialize @@ -44,6 +46,8 @@ apply_moc_transformations, apply_user_transformations, ) +from torch.onnx import export as onnx_export +from torch.utils._pytree import tree_map from torch.utils.data import DataLoader, Dataset, RandomSampler from torch.utils.data.distributed import DistributedSampler from tqdm.auto import tqdm @@ -75,11 +79,12 @@ ) from optimum.exporters import TasksManager +from optimum.exporters.onnx import OnnxConfig from ..utils.constant import _TASK_ALIASES from ..utils.import_utils import is_transformers_version from .configuration import OVConfig -from .quantization import OVDataLoader, _onnx_export_nncf_model +from .quantization import OVDataLoader from .training_args import OVTrainingArguments from .utils import ( MAX_ONNX_OPSET, @@ -109,6 +114,40 @@ NNCF_LOG_FILE_NAME = "nncf_output.log" +def _onnx_export_nncf_model(model: NNCFNetwork, config: OnnxConfig, output: Union[str, io.BytesIO], opset: int = None): + # TODO: remove it when fix controller.strip(copy=True) behavior + signature = inspect.signature(model.forward) + signature = list(signature.parameters.keys()) + opset = opset or config.DEFAULT_ONNX_OPSET + model_inputs = config.generate_dummy_inputs(framework="pt") + # Create ordered inputs for the ONNX export of NNCFNetwork as keyword arguments are currently not supported + model_inputs = tuple(model_inputs.pop(key, None) for key in signature if len(model_inputs) != 0) + device = model.device + + def remap(value): + if isinstance(value, torch.Tensor): + value = value.to(device) + return value + + with config.patch_model_for_export(model): + model_inputs = tree_map(remap, model_inputs) + with torch.no_grad(): + model.eval() + # Disable node additions to be exported in the graph + model.disable_dynamic_graph_building() + onnx_export( + model, + model_inputs, + f=output, + input_names=list(config.inputs.keys()), + output_names=list(config.outputs.keys()), + dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), + do_constant_folding=True, + opset_version=opset, + ) + model.enable_dynamic_graph_building() + + class OVTrainer(Trainer): """ OVTrainer enables NNCF quantization aware training. diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index d61ec85ae5..c5d003ba9e 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -28,7 +28,7 @@ ONNX_DECODER_WITH_PAST_NAME = "decoder_with_past_model.onnx" MAX_ONNX_OPSET_2022_2_0 = 10 -MAX_ONNX_OPSET = 13 +MAX_ONNX_OPSET = 16 MIN_ONNX_QDQ_OPSET = 13 EXTERNAL_DATA_FORMAT_SIZE_LIMIT = 2 * 1024 * 1024 * 1024 diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 5115b4894e..811ee59824 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -41,6 +41,7 @@ OVQuantizer, OVTrainer, ) +from optimum.intel.openvino.configuration import INT8_WEIGHT_COMPRESSION_CONFIG _TASK_TO_DATASET = { "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), @@ -93,7 +94,7 @@ def preprocess_function(examples, tokenizer): model = model_cls.from_pretrained(tmp_dir) - # TODO: uncomment once move to a newer version of NNCF which has some fixes + # TODO: uncomment once move to a newer version of NNCF which has some fixes (addmm, baddmm) # num_fake_quantize, num_int8 = get_num_quantized_nodes(model) # self.assertEqual(expected_fake_quantize, num_fake_quantize) # self.assertEqual(expected_int8, num_int8) @@ -142,6 +143,41 @@ def preprocess_function(examples, tokenizer): self.assertTrue("logits" in outputs) +class OVWeightCompressionTest(unittest.TestCase): + # TODO : add models + SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 39), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 5), + ) + + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) + def test_automodel_weight_compression(self, model_cls, model_name, expected_int8): + task = model_cls.export_feature + + with tempfile.TemporaryDirectory() as tmp_dir: + transformers_model = model_cls.auto_model_class.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + quantizer = OVQuantizer.from_pretrained(transformers_model, task=task) + quantizer.quantize(save_directory=tmp_dir, weights_only=True) + model = model_cls.from_pretrained(tmp_dir) + + # TODO: uncomment once move to a newer version of NNCF which has some fixes + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_int8, num_int8) + + tokens = tokenizer("This is a sample input", return_tensors="pt") + outputs = model(**tokens) + self.assertTrue("logits" in outputs) + + # Verify that that the configuration is correctly saved and loaded + expected_config = OVConfig(compression=INT8_WEIGHT_COMPRESSION_CONFIG) + loaded_config = OVConfig.from_pretrained(tmp_dir) + self.assertEqual(expected_config.to_dict()["compression"], loaded_config.to_dict()["compression"]) + + class OVQuantizerQATest(unittest.TestCase): SUPPORTED_ARCHITECTURES = (("hf-internal-testing/tiny-random-BertForQuestionAnswering",),) From 69434ed149102fc6cf2cc8c898fbe8858b484378 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 30 Jun 2023 17:16:13 +0200 Subject: [PATCH 051/134] Fix openvino model integration compatibility for optimum > v1.9.0 (#365) * fix openvino * fix style --- optimum/intel/openvino/modeling_base.py | 5 ++--- optimum/intel/openvino/modeling_base_seq2seq.py | 4 +--- optimum/intel/openvino/modeling_decoder.py | 11 +++-------- optimum/intel/openvino/modeling_diffusion.py | 4 ++-- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index eb8476fefa..74e5a5b2c9 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -281,8 +281,7 @@ def _from_transformers( kwargs (`Dict`, *optional*): kwargs will be passed to the model during initialization """ - if task is None: - task = cls._auto_model_to_task(cls.auto_model_class) + task = task or cls.export_feature model_kwargs = { "revision": revision, @@ -295,8 +294,8 @@ def _from_transformers( } model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - model_type = model.config.model_type.replace("_", "-") + onnx_config_class = TasksManager.get_exporter_config_constructor( exporter="onnx", model=model, diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index 6cd8c5b177..2120435d35 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -297,9 +297,7 @@ def _from_transformers( encoder_file_name = os.path.join("encoder", ONNX_ENCODER_NAME) decoder_file_name = os.path.join("decoder", ONNX_DECODER_NAME) decoder_with_past_file_name = os.path.join("decoder_with_past", ONNX_DECODER_WITH_PAST_NAME) - - if task is None: - task = cls._auto_model_to_task(cls.auto_model_class) + task = task or cls.export_feature save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 91da03ad28..2c5520b6f3 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -156,12 +156,7 @@ def _from_transformers( f"This architecture : {config.model_type} was not validated, only :{', '.join(_SUPPORTED_ARCHITECTURES)} architectures were " "validated, use at your own risk." ) - - model_file_name = ONNX_WEIGHTS_NAME - - if task is None: - task = cls._auto_model_to_task(cls.auto_model_class) - + task = task or cls.export_feature save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) model_kwargs = { @@ -188,7 +183,7 @@ def _from_transformers( model.model.decoder._prepare_decoder_attention_mask = _prepare_decoder_attention_mask # Export the model to the ONNX format - export(model=model, config=onnx_config, output=save_dir_path / model_file_name) + export(model=model, config=onnx_config, output=save_dir_path / ONNX_WEIGHTS_NAME) return cls._from_pretrained( model_id=save_dir_path, @@ -198,7 +193,7 @@ def _from_transformers( revision=revision, force_download=force_download, cache_dir=cache_dir, - file_name=model_file_name, + file_name=ONNX_WEIGHTS_NAME, local_files_only=local_files_only, use_cache=use_cache, **kwargs, diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 074830baea..318e0c1fd1 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -288,8 +288,8 @@ def _from_transformers( ): if is_torch_version(">", "1.13.1") and is_torch_version("<=", "2.0.1"): register_custom_scaled_dot_product_attention_export() - if task is None: - task = cls._auto_model_to_task(cls.auto_model_class) + task = task or cls.export_feature + save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) From 6a9b778115771c0972b7e66a3d0df88bc55d8ef0 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:43:43 +0200 Subject: [PATCH 052/134] Remove deprecated default openvino file name (#369) --- optimum/intel/openvino/modeling_base.py | 55 ++---- .../intel/openvino/modeling_base_seq2seq.py | 79 ++------- optimum/intel/openvino/modeling_diffusion.py | 166 +----------------- optimum/intel/openvino/quantization.py | 2 +- optimum/intel/openvino/trainer.py | 2 +- setup.py | 2 +- tests/openvino/test_quantization.py | 6 +- 7 files changed, 40 insertions(+), 272 deletions(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 74e5a5b2c9..8955a03870 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -20,7 +20,6 @@ import openvino from huggingface_hub import hf_hub_download -from huggingface_hub.utils import EntryNotFoundError from openvino._offline_transformations import apply_moc_transformations, compress_model_transformation from openvino.runtime import Core from transformers import PretrainedConfig @@ -137,7 +136,7 @@ def fix_op_names_duplicates(model: openvino.runtime.Model): return model - def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): + def _save_pretrained(self, save_directory: Union[str, Path]): """ Saves the model to the OpenVINO IR format so that it can be re-loaded using the [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. @@ -145,12 +144,9 @@ def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional Arguments: save_directory (`str` or `Path`): The directory where to save the model files. - file_name(`str`, *optional*): - The model file name to use when saving the model. Overwrites the default file names. """ - file_name = file_name if file_name is not None else OV_XML_FILE_NAME - dst_path = os.path.join(save_directory, file_name) - openvino.runtime.serialize(self.model, dst_path, dst_path.replace(".xml", ".bin")) + dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) + openvino.runtime.serialize(self.model, dst_path) @classmethod def _from_pretrained( @@ -198,12 +194,6 @@ def _from_pretrained( # Load the model from local directory if os.path.isdir(model_id): file_name = os.path.join(model_id, file_name) - if os.path.isfile(os.path.join(model_id, "ov_model.xml")): - file_name = os.path.join(model_id, "ov_model.xml") - logger.warning( - "The file names `ov_model.xml` and `ov_model.bin` will be soon deprecated." - "Make sure to rename your file to respectively `openvino_model.xml` and `openvino_model.bin`" - ) model_save_dir = model_id # Download the model from the hub else: @@ -212,36 +202,17 @@ def _from_pretrained( if not from_onnx: model_file_names.append(file_name.replace(".xml", ".bin")) file_names = [] - try: - for file_name in model_file_names: - model_cache_path = hf_hub_download( - repo_id=model_id, - filename=file_name, - use_auth_token=use_auth_token, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - local_files_only=local_files_only, - ) - file_names.append(model_cache_path) - except EntryNotFoundError: - file_names = [] - model_file_names = ["ov_model.xml", "ov_model.bin"] - for file_name in model_file_names: - model_cache_path = hf_hub_download( - repo_id=model_id, - filename=file_name, - use_auth_token=use_auth_token, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - local_files_only=local_files_only, - ) - file_names.append(model_cache_path) - logger.warning( - "The file names `ov_model.xml` and `ov_model.bin` will be soon deprecated." - "Make sure to rename your file to respectively `openvino_model.xml` and `openvino_model.bin`" + for file_name in model_file_names: + model_cache_path = hf_hub_download( + repo_id=model_id, + filename=file_name, + use_auth_token=use_auth_token, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + local_files_only=local_files_only, ) + file_names.append(model_cache_path) model_save_dir = Path(model_cache_path).parent file_name = file_names[0] diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index 2120435d35..a839d5c3a8 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -20,7 +20,6 @@ import openvino from huggingface_hub import hf_hub_download -from huggingface_hub.utils import EntryNotFoundError from openvino._offline_transformations import apply_moc_transformations, compress_model_transformation from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings @@ -88,14 +87,7 @@ def __init__( self.generation_config = GenerationConfig.from_model_config(config) if self.can_generate() else None - def _save_pretrained( - self, - save_directory: Union[str, Path], - encoder_file_name: Optional[str] = None, - decoder_file_name: Optional[str] = None, - decoder_with_past_file_name: Optional[str] = None, - **kwargs, - ): + def _save_pretrained(self, save_directory: Union[str, Path]): """ Saves the model to the OpenVINO IR format so that it can be re-loaded using the [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. @@ -103,25 +95,16 @@ def _save_pretrained( Arguments: save_directory (`str` or `Path`): The directory where to save the model files. - encoder_file_name(`str`, *optional*): - The encoder model file name. Overwrites the default file name and allows one to save the encoder model - with a different name. - decoder_file_name(`str`, *optional*): - The decoder model file name. Overwrites the default file name and allows one to save the decoder model - with a different name. - decoder_with_past_file_name(`str`, *optional*): - The decoder with past key values model file name overwriting the default file name, allowing to save - the decoder model with a different name. """ src_files = [self.encoder_model, self.decoder_model] - dst_file_names = [encoder_file_name or OV_ENCODER_NAME, decoder_file_name or OV_DECODER_NAME] + dst_file_names = [OV_ENCODER_NAME, OV_DECODER_NAME] if self.use_cache: src_files.append(self.decoder_with_past_model) - dst_file_names.append(decoder_with_past_file_name or OV_DECODER_WITH_PAST_NAME) + dst_file_names.append(OV_DECODER_WITH_PAST_NAME) for src_file, dst_file_name in zip(src_files, dst_file_names): dst_path = os.path.join(save_directory, dst_file_name) - openvino.runtime.serialize(src_file, dst_path, dst_path.replace(".xml", ".bin")) + openvino.runtime.serialize(src_file, dst_path) @classmethod def _from_pretrained( @@ -181,16 +164,6 @@ def _from_pretrained( # Load model from a local directory if os.path.isdir(model_id): - if os.path.isfile(os.path.join(model_id, "ov_encoder_model.xml")): - encoder_file_name = "ov_encoder_model.xml" - encoder_file_name = "ov_decoder_model.xml" - encoder_file_name = "ov_decoder_with_past_model.xml" - logger.warning( - "The file names `ov_encoder_model.xml`, `ov_decoder_model.xml` and `ov_decoder_with_past_model.xml` " - "will be soon deprecated. Make sure to rename your file to respectively `openvino_encoder_model.xml`, " - "`openvino_decoder_model.xml` and `openvino_decoder_with_past_model.xml`" - ) - encoder = cls.load_model(os.path.join(model_id, encoder_file_name)) decoder = cls.load_model(os.path.join(model_id, decoder_file_name)) decoder_with_past = ( @@ -209,41 +182,17 @@ def _from_pretrained( for key in list(model_file_names.keys()): model_file_names[key + "_bin"] = model_file_names[key].replace(".xml", ".bin") file_names = model_file_names.copy() - try: - for name, file_name in model_file_names.items(): - model_cache_path = hf_hub_download( - repo_id=model_id, - filename=file_name, - use_auth_token=use_auth_token, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - local_files_only=local_files_only, - ) - file_names[name] = model_cache_path - except EntryNotFoundError: - model_file_names = {"encoder": "ov_encoder_model.xml", "decoder": "ov_decoder_model.xml"} - if use_cache: - model_file_names["decoder_with_past"] = "ov_decoder_with_past_model.xml" - for key in list(model_file_names.keys()): - model_file_names[key + "_bin"] = model_file_names[key].replace(".xml", ".bin") - file_names = model_file_names.copy() - for name, file_name in model_file_names.items(): - model_cache_path = hf_hub_download( - repo_id=model_id, - filename=file_name, - use_auth_token=use_auth_token, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - local_files_only=local_files_only, - ) - file_names[name] = model_cache_path - logger.warning( - "The file names `ov_encoder_model.xml`, `ov_decoder_model.xml` and `ov_decoder_with_past_model.xml` " - "will be soon deprecated. Make sure to rename your file to respectively `openvino_encoder_model.xml`, " - "`openvino_decoder_model.xml` and `openvino_decoder_with_past_model.xml`" + for name, file_name in model_file_names.items(): + model_cache_path = hf_hub_download( + repo_id=model_id, + filename=file_name, + use_auth_token=use_auth_token, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + local_files_only=local_files_only, ) + file_names[name] = model_cache_path model_save_dir = Path(model_cache_path).parent encoder = cls.load_model(file_names["encoder"]) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 318e0c1fd1..4ac701d443 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -39,7 +39,6 @@ DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ) -from ..utils import is_torch_version from .modeling_base import OVBaseModel from .utils import ONNX_WEIGHTS_NAME, OV_TO_NP_TYPE, OV_XML_FILE_NAME @@ -123,49 +122,28 @@ def __init__( self._internal_dict.pop("vae", None) - def _save_pretrained( - self, - save_directory: Union[str, Path], - vae_decoder_file_name: str = OV_XML_FILE_NAME, - text_encoder_file_name: str = OV_XML_FILE_NAME, - unet_file_name: str = OV_XML_FILE_NAME, - vae_encoder_file_name: str = OV_XML_FILE_NAME, - **kwargs, - ): + def _save_pretrained(self, save_directory: Union[str, Path]): """ Saves the model to the OpenVINO IR format so that it can be re-loaded using the [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. Arguments: save_directory (`str` or `Path`): - The directory where to save the model files. - vae_decoder_file_name (`str`, defaults to `optimum.intel.openvino.utils.OV_XML_FILE_NAME`): - The VAE decoder model file name. Overwrites the default file name and allows one to save the VAE decoder model - with a different name. - text_encoder_file_name (`str`, defaults to `optimum.intel.openvino.utils.OV_XML_FILE_NAME`): - The text encoder model file name. Overwrites the default file name and allows one to save the text encoder model - with a different name. - unet_file_name (`str`, defaults to `optimum.intel.openvino.utils.OV_XML_FILE_NAME`): - The U-NET model file name. Overwrites the default file name and allows one to save the U-NET model - with a different name. - vae_encoder_file_name (`str`, defaults to `optimum.intel.openvino.utils.OV_XML_FILE_NAME`): - The VAE encoder model file name. Overwrites the default file name and allows one to save the VAE decoder model - with a different name. + The directory where to save the model files """ save_directory = Path(save_directory) src_to_dst_file = { - self.vae_decoder.model: save_directory / DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER / vae_decoder_file_name, - self.text_encoder.model: save_directory / DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER / text_encoder_file_name, - self.unet.model: save_directory / DIFFUSION_MODEL_UNET_SUBFOLDER / unet_file_name, + self.vae_decoder.model: save_directory / DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER / OV_XML_FILE_NAME, + self.text_encoder.model: save_directory / DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER / OV_XML_FILE_NAME, + self.unet.model: save_directory / DIFFUSION_MODEL_UNET_SUBFOLDER / OV_XML_FILE_NAME, } if self.vae_encoder is not None: src_to_dst_file[self.vae_encoder.model] = ( - save_directory / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / vae_encoder_file_name + save_directory / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / OV_XML_FILE_NAME ) - for src_file, dst_path in src_to_dst_file.items(): dst_path.parent.mkdir(parents=True, exist_ok=True) - openvino.runtime.serialize(src_file, str(dst_path), str(dst_path.with_suffix(".bin"))) + openvino.runtime.serialize(src_file, str(dst_path)) self.tokenizer.save_pretrained(save_directory.joinpath("tokenizer")) self.scheduler.save_pretrained(save_directory.joinpath("scheduler")) @@ -286,10 +264,7 @@ def _from_transformers( feature_extractor: Optional["CLIPFeatureExtractor"] = None, **kwargs, ): - if is_torch_version(">", "1.13.1") and is_torch_version("<=", "2.0.1"): - register_custom_scaled_dot_product_attention_export() task = task or cls.export_feature - save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) @@ -580,130 +555,3 @@ def __call__(self, sample: np.ndarray): } outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) - - -def register_custom_scaled_dot_product_attention_export(): - import torch - - @torch.onnx.symbolic_helper.parse_args("v", "v", "v", "v", "f", "b", "v") - def scaled_dot_product_attention( - g: torch.onnx._internal.jit_utils.GraphContext, - query: torch._C.Value, - key: torch._C.Value, - value: torch._C.Value, - attn_mask: Optional[torch._C.Value] = None, - dropout_p: float = 0.0, - is_causal: bool = False, - scale: Optional[torch._C.Value] = None, - ): - assert (not is_causal) or ( - is_causal and torch.onnx.symbolic_helper._is_none(attn_mask) - ), "is_causal and attn_mask cannot be set at the same time" - - scale = torch.onnx.symbolic_helper._maybe_get_const(scale, "f") - if scale is None: - scale = _attention_scale(g, query) - - if is_causal: - attn_mask = _causal_attention_mask(g, query, key) - key_shape_builtin = torch.onnx.symbolic_helper._get_tensor_rank(key) - key_transposed_axes = list(range(key_shape_builtin)) - key_transposed_axes[-1], key_transposed_axes[-2] = ( - key_transposed_axes[-2], - key_transposed_axes[-1], - ) - key_transposed = g.op("Transpose", key, perm_i=key_transposed_axes) - query_scaled = g.op("Mul", query, g.op("Sqrt", scale)) - key_transposed_scaled = g.op("Mul", key_transposed, g.op("Sqrt", scale)) - mul_qk = g.op("MatMul", query_scaled, key_transposed_scaled) - if attn_mask is None or torch.onnx.symbolic_helper._is_none(attn_mask): - mul_qk_add = mul_qk - elif torch.onnx._type_utils.JitScalarType.from_value(attn_mask) == torch.onnx._type_utils.JitScalarType.BOOL: - # Turn the Boolean mask to float: attn_mask.masked_fill(not attn_mask, -float('inf')) - const_zero = g.op("Constant", value_t=torch.tensor([0.0])) - const_neg_inf = g.op("Constant", value_t=torch.tensor([-float("inf")])) - attn_mask = g.op("Where", attn_mask, const_zero, const_neg_inf) - mul_qk_add = g.op("Add", mul_qk, attn_mask) - elif torch.onnx._type_utils.JitScalarType.from_value(attn_mask) == torch.onnx._type_utils.JitScalarType.FLOAT: - mul_qk_add = g.op("Add", mul_qk, attn_mask) - else: - raise ValueError( - f"Unsupported type for attn_mask: {torch.onnx._type_utils.JitScalarType.from_value(attn_mask)}" - ) - - attn_weight = g.op("Softmax", mul_qk_add, axis_i=-1) - - if dropout_p != 0: - attn_weight = g.op( - "Dropout", - attn_weight, - g.op("Constant", value_t=torch.tensor(dropout_p, dtype=torch.float)), - ) - - return g.op("MatMul", attn_weight, value) - - def _attention_scale(g: torch.onnx._internal.jit_utils.GraphContext, query: torch._C.Value) -> torch._C.Value: - """Calculate the scale factor for the attention result. - - Args: - query: Tensor of shape [..., L, E] - - Returns: - Scalar scale factor := 1 / math.sqrt(query.size(-1)) - """ - query_shape = g.op("Shape", query) - query_shape_last = g.op( - "Slice", - query_shape, - g.op("Constant", value_t=torch.tensor([-1], dtype=torch.int64)), - g.op("Constant", value_t=torch.tensor([torch.onnx._constants.INT64_MAX], dtype=torch.int64)), - ) - embedding_size = g.op( - "Cast", - query_shape_last, - to_i=torch.onnx._type_utils.JitScalarType.from_value(query).onnx_type(), - ) - const_one = g.op("Constant", value_t=torch.tensor([1.0], dtype=torch.float)) - scale = g.op("Div", const_one, g.op("Sqrt", embedding_size)) - return scale - - def _causal_attention_mask( - g: torch.onnx._internal.jit_utils.GraphContext, query: torch._C.Value, key: torch._C.Value - ) -> torch._C.Value: - """Create a causal mask for the given query and key tensors. - - Equivalent to:: - mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0) - attn_mask = torch.zeros(L, S, dtype=torch.float) - attn_mask = attn_mask.masked_fill(not mask, -float('inf')) - - Args: - query: Tensor of shape [..., L, E] - key: Tensor of shape [..., S, E] - - Returns: - Tensor of shape [L, S] - """ - - query_shape = g.op("Shape", query) - key_shape = g.op("Shape", key) - - last_idx = g.op("Constant", value_t=torch.tensor([-1], dtype=torch.int64)) - second_last_idx = g.op("Constant", value_t=torch.tensor([-2], dtype=torch.int64)) - target_length = g.op("Slice", query_shape, second_last_idx, last_idx) - source_length = g.op("Slice", key_shape, second_last_idx, last_idx) - # attn_mask = torch.ones(L, S) := { - size = g.op("Concat", target_length, source_length, axis_i=0) - const_one = g.op("Constant", value_t=torch.tensor([1.0])) - attn_mask = g.op("Expand", const_one, size) - # } - attn_mask = g.op("Trilu", attn_mask, upper_i=0) - # The causal mask has 0s in the lower triangle and -inf in the upper triangle. - const_zero = g.op("Constant", value_t=torch.tensor([0.0])) - const_neg_inf = g.op("Constant", value_t=torch.tensor([-float("inf")])) - attn_mask = g.op("Where", g.op("Equal", attn_mask, const_zero), const_neg_inf, const_zero) - return attn_mask - - torch.onnx.register_custom_op_symbolic( - "aten::scaled_dot_product_attention", scaled_dot_product_attention, opset_version=14 - ) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 462e8b9593..bda901e5d2 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -401,7 +401,7 @@ def data_collator(batch): @staticmethod def _save_pretrained(model: openvino.runtime.Model, output_path: str): compress_quantize_weights_transformation(model) - openvino.runtime.serialize(model, output_path, output_path.replace(".xml", ".bin")) + openvino.runtime.serialize(model, output_path) def _set_task(self): if self.task is None: diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index ab4c2c217a..b9e89fcc85 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -772,7 +772,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): compress_quantize_weights_transformation(ov_model) # Serialize IR xml and bin - serialize(ov_model, output_path, output_path.replace(".xml", ".bin")) + serialize(ov_model, output_path) def _get_compression_controller_by_cls( self, controller_cls: Type[PTCompressionAlgorithmController] diff --git a/setup.py b/setup.py index 54f0b66b4d..c8cdce500f 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ assert False, "Error: Could not open '%s' due %s\n" % (filepath, error) INSTALL_REQUIRE = [ - "optimum>=1.8.0", + "optimum>=1.8.8", "transformers>=4.20.0", "datasets>=1.4.0", "sentencepiece", diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 811ee59824..18fbd2107e 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -72,6 +72,7 @@ class OVQuantizerTest(unittest.TestCase): def test_automodel_static_quantization(self, model_cls, model_name, expected_fake_quantize, expected_int8): task = model_cls.export_feature dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] + file_name = "openvino_quantized_model.xml" def preprocess_function(examples, tokenizer): return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) @@ -90,9 +91,8 @@ def preprocess_function(examples, tokenizer): num_samples=10, dataset_split="train", ) - quantizer.quantize(save_directory=tmp_dir, calibration_dataset=calibration_dataset) - - model = model_cls.from_pretrained(tmp_dir) + quantizer.quantize(save_directory=tmp_dir, calibration_dataset=calibration_dataset, file_name=file_name) + model = model_cls.from_pretrained(tmp_dir, file_name=file_name) # TODO: uncomment once move to a newer version of NNCF which has some fixes (addmm, baddmm) # num_fake_quantize, num_int8 = get_num_quantized_nodes(model) From 26d82a8022eb7753ef58e26947f6b709c5cf8888 Mon Sep 17 00:00:00 2001 From: Yi30 <106061964+yiliu30@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:27:31 +0800 Subject: [PATCH 053/134] Accelerate Accuracy-aware Tuning by Leveraging Multi-nodes (#360) * disable torch distribution initialization Signed-off-by: yiliu30 * add docs Signed-off-by: yiliu30 * corrected typo Signed-off-by: yiliu30 * updated the example link Signed-off-by: yiliu30 * correct typo Signed-off-by: yiliu30 * update docs Signed-off-by: yiliu30 * update example doc Signed-off-by: yiliu30 * Update doc Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * corrected typo Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * corrected typo Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> * reformatted code style Signed-off-by: yiliu30 --------- Signed-off-by: yiliu30 Co-authored-by: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> --- docs/source/optimization_inc.mdx | 19 ++++++++++ .../text-classification/README.md | 37 +++++++++++++++++++ .../run_glue_post_training.py | 8 +++- .../run_task_in_distributed_mode.sh | 18 +++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 examples/neural_compressor/text-classification/run_task_in_distributed_mode.sh diff --git a/docs/source/optimization_inc.mdx b/docs/source/optimization_inc.mdx index be65fb98b4..27e7f703ad 100644 --- a/docs/source/optimization_inc.mdx +++ b/docs/source/optimization_inc.mdx @@ -108,6 +108,25 @@ The [SmoothQuant](https://arxiv.org/abs/2211.10438) methodology is available for ``` Please refer to INC [documentation](https://github.com/intel/neural-compressor/blob/master/docs/source/smooth_quant.md) and the list of [models](https://github.com/intel/neural-compressor/blob/master/docs/source/smooth_quant.md#validated-models) quantized with the methodology for more details. + +### Distributed Acuracy-aware Tuning +One challenge in model quantization is identifying the optimal configuration that balances accuracy and performance. Distributed tuning speeds up this time-consuming process by parallelizing it across multiple nodes, which accelerates the tuning process in linear scaling. + +To utilize distributed tuning, please set the `quant_level` to `1` and run it with `mpirun`. + + +```diff +- quantization_config = PostTrainingQuantConfig(approach="static") ++ quantization_config = PostTrainingQuantConfig(approach="static", quant_level=1) +``` + +```shell +mpirun -np +``` + +Please refer to INC [documentation](https://github.com/intel/neural-compressor/blob/master/docs/source/tuning_strategies.md#distributed-tuning) and [text-classification](https://github.com/huggingface/optimum-intel/tree/main/examples/neural_compressor/text-classification) example for more details. + + ## During training optimization The [`INCTrainer`] class provides an API to train your model while combining different compression techniques such as knowledge distillation, pruning and quantization. diff --git a/examples/neural_compressor/text-classification/README.md b/examples/neural_compressor/text-classification/README.md index 73aa92e3ed..0511335478 100644 --- a/examples/neural_compressor/text-classification/README.md +++ b/examples/neural_compressor/text-classification/README.md @@ -27,6 +27,8 @@ For pruning, we support snip_momentum(default), snip_momentum_progressive, magni > **_Note:_** At present, neural_compressor only support to prune linear and conv ops. So if we set a target sparsity is 0.9, it means that the pruning op's sparsity will be 0.9, not the whole model's sparsity is 0.9. For example: the embedding ops will not be pruned in the model. +### Post-training Quantization + The following example applies post-training static quantization on a DistilBERT fine-tuned on the sst-2 task. ```bash @@ -42,6 +44,41 @@ python run_glue_post_training.py \ ``` In order to apply dynamic or static, `quantization_approach` must be set to respectively `dynamic` or `static`. + +### Distributed Tuning with Multi-node +To speedup the accuracy-aware tuning process, you can use the [distributed tuning](https://github.com/intel/neural-compressor/blob/master/docs/source/tuning_strategies.md#distributed-tuning). + +- Prerequisites: + - [Open MPI](https://www.open-mpi.org/faq/?category=building#easy-build) + - [mpi4py](https://mpi4py.readthedocs.io/en/stable/install.html#using-pip) + + +> **_Note:_** User can also install Open MPI with [Conda](https://anaconda.org/conda-forge/openmpi). + +In `run_glue_post_training.py`, set `quant_level` to 1 as the following statement: + +```python +from neural_compressor import PostTrainingQuantConfig +quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, quant_level=1) +``` + +> **_Note:_** Please also add the below code snippet at the beginning of the example to disable the initialization of torch distribution. +> +>`os.environ.setdefault('OMPI_COMM_WORLD_SIZE', '-1') if os.environ.get('OMPI_COMM_WORLD_SIZE', -1) != -1 else None` + +And then, modify the `run_task_in_distributed_mode.sh` according to the cluster information. Below is the explanation for each parameter. + +- `` is the number of processes, recommend to set it with the number of hosts plus one. + +- `` is the number of threads, recommend to set it with the number of physical cores on one node. + +- `` is the host name, and argument --host ``,``... can be replaced with `--hostfile `, when each line in `` is a host name. + +- `-mca btl_tcp_if_include ` is used to set the network communication interface between hosts. For example, `` can be set to `192.168.20.0/24` to allow MPI communication between all hosts under `192.168.20.*` network segment. + + +### Knowledge Distillation with Quantization Aware Training + The following example fine-tunes DistilBERT on the sst-2 task while applying knowledge distillation with quantization aware training. ```bash diff --git a/examples/neural_compressor/text-classification/run_glue_post_training.py b/examples/neural_compressor/text-classification/run_glue_post_training.py index 4f7a83d9a8..fafc941550 100644 --- a/examples/neural_compressor/text-classification/run_glue_post_training.py +++ b/examples/neural_compressor/text-classification/run_glue_post_training.py @@ -19,6 +19,12 @@ import logging import os + + +# Disable the initialization of the torch distribution +if os.environ.get("OMPI_COMM_WORLD_SIZE", -1) != -1: + os.environ["OMPI_COMM_WORLD_SIZE"] = "-1" + import sys from dataclasses import dataclass, field from typing import Optional @@ -443,7 +449,7 @@ def preprocess_function(examples): f"Unknown quantization approach. Supported approach are {supported_approach}." f"{optim_args.quantization_approach} was given." ) - quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach) + quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, quant_level=1) # Apply post-training quantization quantizer = INCQuantizer.from_pretrained(model) diff --git a/examples/neural_compressor/text-classification/run_task_in_distributed_mode.sh b/examples/neural_compressor/text-classification/run_task_in_distributed_mode.sh new file mode 100644 index 0000000000..6a9b46ad19 --- /dev/null +++ b/examples/neural_compressor/text-classification/run_task_in_distributed_mode.sh @@ -0,0 +1,18 @@ +mpirun -np \ + -mca btl_tcp_if_include \ + -x OMP_NUM_THREADS= \ + --host ,,... \ + bash -c """ + source ~/miniconda3/etc/profile.d/conda.sh + conda activate conda_env + cd path/to/optimum-intel/examples/neural_compressor/text-classification + python -u ./run_glue_post_training.py \ + --model_name_or_path distilbert-base-uncased-finetuned-sst-2-english \ + --task_name sst2 \ + --apply_quantization \ + --quantization_approach static \ + --num_calibration_samples 50 \ + --do_eval \ + --verify_loading \ + --output_dir /tmp/sst2_output + """ \ No newline at end of file From 3dc83edd990c97f01cab88e883f90813624b7590 Mon Sep 17 00:00:00 2001 From: "Cheng, Penghui" Date: Wed, 5 Jul 2023 20:15:06 +0800 Subject: [PATCH 054/134] Rewrite INCModelForCahsalLM class to support load quantized model and JIT model. (#318) * Support sparsity kernel for inference with neural engine Signed-off-by: Cheng, Penghui * Fixed typo issue * Fixed issue of loading neural engine ir * Update code * Refactorize the INCModelForCausalLM and TSModelForCausalLM * Update code style * Add UT for INCModelForCausalLM * Fixed code style issue * Fixed UT error * Fixed UT error * Fixed IPEX version limitation * Fixed UT error * Limited IPEX version * Update code * Update code * add bloom model to UT test * remove invalid package * Fixed UT error * Update code * Remove neural_engine part code * Fixed code style issue * remove unused package * Clean the unused code * Fixed UT error * Add the smooth quant in text-generation example * Remove unused code * Fixed typo * Fixed ruff check error * Updated code * Updated code * Update code * Update code * Removed unused code * Fixed UT error * Update code * Fixed UT error --------- Signed-off-by: Cheng, Penghui --- docs/source/reference_inc.mdx | 2 +- .../text-generation/README.md | 21 +- .../text-generation/run_generation.py | 138 ++++++++- optimum/intel/generation/__init__.py | 2 +- optimum/intel/generation/modeling.py | 264 ++++++++--------- optimum/intel/neural_compressor/__init__.py | 2 +- .../intel/neural_compressor/modeling_base.py | 267 ++++++++++++++++++ .../neural_compressor/modeling_decoder.py | 49 ++++ .../intel/neural_compressor/quantization.py | 22 +- optimum/intel/neural_compressor/trainer.py | 8 +- optimum/intel/neural_compressor/utils.py | 1 + setup.py | 6 +- tests/neural_compressor/test_optimization.py | 34 +++ 13 files changed, 670 insertions(+), 146 deletions(-) create mode 100644 optimum/intel/neural_compressor/modeling_base.py create mode 100644 optimum/intel/neural_compressor/modeling_decoder.py diff --git a/docs/source/reference_inc.mdx b/docs/source/reference_inc.mdx index ec16a31001..54fd6c00fc 100644 --- a/docs/source/reference_inc.mdx +++ b/docs/source/reference_inc.mdx @@ -43,7 +43,7 @@ specific language governing permissions and limitations under the License. ## INCModelForCausalLM -[[autodoc]] neural_compressor.quantization.INCModelForCausalLM +[[autodoc]] neural_compressor.modeling_decoder.INCModelForCausalLM ## INCModelForSeq2SeqLM diff --git a/examples/neural_compressor/text-generation/README.md b/examples/neural_compressor/text-generation/README.md index 1e238dc83c..9ed232fff3 100644 --- a/examples/neural_compressor/text-generation/README.md +++ b/examples/neural_compressor/text-generation/README.md @@ -20,11 +20,28 @@ Based on the script [`run_generation.py`](https://github.com/huggingface/transfo The original generation task only supported the PyTorch eager model. By calling the `TSModelForCausalLM` class, we can now support a TorchScript model for generation tasks. +This example also allows us to apply different quantization approaches (such as dynamic, static, The example applies post-training static quantization on a gptj model). + Example usage: +### apply_quantization with post-training static +```bash +python run_generation.py \ + --model_type=gptj \ + --model_name_or_path=EleutherAI/gpt-j-6b \ + --apply_quantization \ + --quantization_approach static\ + --smooth_quant \ + --smooth_quant_alpha 0.7 +``` +### Use JIT model and apply_quantization with post-training static ```bash python run_generation.py \ - --model_type=gpt2 \ - --model_name_or_path=gpt2 \ + --model_type=gptj \ + --model_name_or_path=EleutherAI/gpt-j-6b \ + --apply_quantization \ + --quantization_approach static\ + --smooth_quant \ + --smooth_quant_alpha 0.7 \ --jit ``` diff --git a/examples/neural_compressor/text-generation/run_generation.py b/examples/neural_compressor/text-generation/run_generation.py index 054187de82..e06bba4102 100755 --- a/examples/neural_compressor/text-generation/run_generation.py +++ b/examples/neural_compressor/text-generation/run_generation.py @@ -20,14 +20,20 @@ import argparse import logging +import tempfile import numpy as np import torch +from datasets import load_dataset +from neural_compressor import PostTrainingQuantConfig +from torch.nn.functional import pad +from torch.utils.data import DataLoader from transformers import ( CTRLLMHeadModel, CTRLTokenizer, GPT2LMHeadModel, GPT2Tokenizer, + GPTJForCausalLM, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, TransfoXLLMHeadModel, @@ -38,7 +44,7 @@ XLNetTokenizer, ) -from optimum.intel.generation.modeling import TSModelForCausalLM +from optimum.intel.neural_compressor import INCModelForCausalLM, INCQuantizer logging.basicConfig( @@ -51,6 +57,7 @@ MAX_LENGTH = int(10000) # Hardcoded max length to avoid infinite loop MODEL_CLASSES = { + "gptj": (GPTJForCausalLM, GPT2Tokenizer), "gpt2": (GPT2LMHeadModel, GPT2Tokenizer), "ctrl": (CTRLLMHeadModel, CTRLTokenizer), "openai-gpt": (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), @@ -152,6 +159,51 @@ def adjust_length_to_model(length, max_sequence_length): return length +class datasets_processor: + def __init__( + self, + dataset, + tokenizer, + batch_size=8, + pad_val=1, + pad_max=512, + is_calib=False, + ): + self.dataset = dataset + self.tokenizer = tokenizer + self.batch_size = batch_size + self.pad_val = pad_val + self.pad_max = pad_max + self.is_calib = is_calib + + # tokenize the dataset + self.dataset = self.dataset.map(self.tokenize_function, batched=True) + self.dataset.set_format(type="torch", columns=["input_ids"]) + + @torch.no_grad() + def tokenize_function(self, examples): + example = self.tokenizer(examples["text"]) + return example + + @torch.no_grad() + def collate_batch(self, batch): + input_ids_padded = [] + last_ind = [] + for text in batch: + input_ids = text["input_ids"] + pad_len = self.pad_max - input_ids.shape[0] + last_ind.append(input_ids.shape[0] - 1) + if self.is_calib: + input_ids = input_ids[: self.pad_max] if len(input_ids) > self.pad_max else input_ids + else: + input_ids = pad(input_ids, (0, pad_len), value=self.pad_val) + input_ids_padded.append(input_ids) + return ( + torch.vstack(input_ids_padded), + torch.tensor(last_ind), + ) + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -205,6 +257,30 @@ def main(): type=str, help="Output directory where to save the resulting model", ) + parser.add_argument( + "--apply_quantization", + action="store_true", + help="Whether or not to apply quantization.", + ) + parser.add_argument( + "--quantization_approach", + default="static", + type=str, + help="Quantization approach. Supported approach are static, dynamic.", + ) + parser.add_argument( + "--smooth_quant", + action="store_true", + help="Whether or not to quantize with smooth quant.", + ) + parser.add_argument( + "--smooth_quant_alpha", + default=0.6, + type=float, + help="Set alpha of smooth quant argument.", + ) + parser.add_argument("--dataset_name", nargs="?", default="NeelNanda/pile-10k", const="NeelNanda/pile-10k") + parser.add_argument("--calib_iters", default=100, type=int, help="calibration iters.") args = parser.parse_args() args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") @@ -223,8 +299,10 @@ def main(): tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path) - if args.jit: - model = TSModelForCausalLM.from_pretrained(args.model_name_or_path, export=True) + if args.apply_quantization: + model = model_class.from_pretrained(args.model_name_or_path) + elif args.jit: + model = INCModelForCausalLM.from_pretrained(args.model_name_or_path, export=True) else: model = model_class.from_pretrained(args.model_name_or_path) @@ -234,6 +312,60 @@ def main(): model.to(args.device) + if args.apply_quantization: + # This is just an example for calibration_fn. If you want to achieve good accuracy, + # you must perform a calibration on your real dataset. + calib_dataset = load_dataset(args.dataset_name, split="train") + calib_dataset = calib_dataset.shuffle(seed=42) + calib_dataset = datasets_processor( + calib_dataset, + tokenizer, + 1, + is_calib=True, + ) + calib_size = 1 + calib_dataloader = DataLoader( + calib_dataset.dataset, + batch_size=calib_size, + shuffle=False, + collate_fn=calib_dataset.collate_batch, + ) + + def calibration_fn(p_model): + tmp_model = INCModelForCausalLM(p_model, model.config, use_cache=False) + for i, (input_ids, last_ind) in enumerate(calib_dataloader): + input_bs, input_len = input_ids.shape + attention_mask = torch.ones(input_bs, input_len) + if i >= args.calib_iters: + break + tmp_model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + max_new_tokens=32, + temperature=0.9, + num_beams=4, + do_sample=False, + ) + + example_inputs = {"input_ids": torch.randint(100, (1, 32)), "attention_mask": torch.ones(1, 32)} + quantization_config = PostTrainingQuantConfig( + approach=args.quantization_approach, + recipes={ + "smooth_quant": args.smooth_quant, + "smooth_quant_args": {"alpha": args.smooth_quant_alpha, "folding": True}, + }, + example_inputs=example_inputs, + ) + model.config.return_dict = False + quantizer = INCQuantizer.from_pretrained(model, calibration_fn=calibration_fn) + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + save_directory=tmp_dir, + save_onnx_model=False, + ) + model = INCModelForCausalLM.from_pretrained(tmp_dir, export=args.jit) + args.length = adjust_length_to_model( args.length, max_sequence_length=model.config.max_position_embeddings diff --git a/optimum/intel/generation/__init__.py b/optimum/intel/generation/__init__.py index 867f26bdec..588146f7ad 100644 --- a/optimum/intel/generation/__init__.py +++ b/optimum/intel/generation/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. -from .modeling import TSModelForCausalLM +from .modeling import BaseModelForCausalLM, TSModelForCausalLM diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index d524deb09c..1a93375bbe 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -93,7 +93,7 @@ class PreTrainedModel(OptimizedModel): pass -class TSModelForCausalLM(PreTrainedModel, GenerationMixin): +class BaseModelForCausalLM(PreTrainedModel, GenerationMixin): auto_model_class = AutoModelForCausalLM export_feature = "text-generation" main_input_name = "input_ids" @@ -107,13 +107,11 @@ def __init__( use_cache: bool = True, **kwargs, ): - self.model = model - self.config = config + super(BaseModelForCausalLM, self).__init__(model=model, config=config) self.model_save_dir = model_save_dir self.preprocessors = kwargs.get("preprocessors", []) self.use_cache = use_cache self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.model.to(self._device) self.normalized_config = NormalizedConfigManager.get_normalized_config_class(config.model_type)(config) self.model_dtype = kwargs.get("model_dtype", None) @@ -128,6 +126,13 @@ def __init__( AutoConfig.register(self.base_model_prefix, AutoConfig) self.auto_model_class.register(AutoConfig, self.__class__) + def can_generate(self) -> bool: + return True + + @property + def device(self) -> torch.device: + return self._device + @staticmethod def load_model(file_name: Union[str, Path]): model = torch.jit.load(file_name) @@ -137,6 +142,115 @@ def load_model(file_name: Union[str, Path]): def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): torch.jit.save(self.model, os.path.join(save_directory, WEIGHTS_NAME)) + # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): + past_key_values = past_key_values or kwargs.get("past", None) + + if self.use_cache and past_key_values is not None: + input_ids = input_ids[:, -1:] + + # `past_key_values` may be in the stardard format (e.g. in contrastive search), converts to bloom's format if needed + if past_key_values is not None and self.config.model_type == "bloom": + if past_key_values[0][0].shape[0] == input_ids.shape[0]: + past_key_values = self._convert_to_bloom_cache(past_key_values) + + return { + "input_ids": input_ids, + "past_key_values": past_key_values, + "use_cache": self.use_cache, + "position_ids": None, + "attention_mask": kwargs.get("attention_mask", None), + "token_type_ids": None, + } + + def _reorder_cache( + self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor + ) -> Tuple[Tuple[torch.Tensor]]: + """ + This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + [`~PreTrainedModel.beam_sample`] is called. + This is required to match `past_key_values` with the correct beam_idx at every generation step. + """ + if self.config.model_type == "bloom": + return self._reorder_cache_bloom(past_key_values, beam_idx) + + # from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel._reorder_cache + return tuple( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) + for layer_past in past_key_values + ) + + # Copied from transformers.models.bloom.modeling_bloom.BloomForCausalLM._reorder_cache + def _reorder_cache_bloom( + self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor + ) -> Tuple[Tuple[torch.Tensor]]: + """ + This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or + [`~PreTrainedModel.beam_sample`] is called for bloom architecture. + This is required to match `past_key_values` with the correct beam_idx at every generation step. + """ + standardized_past = self._convert_to_standard_cache(past_key_values, batch_size=len(beam_idx)) + + # Get a copy of `beam_idx` on all the devices where we need those indices. + device_to_beam_idx = { + past_state.device: beam_idx.to(past_state.device) + for layer_past in past_key_values + for past_state in layer_past + } + reordered_past = tuple( + ( + layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]), + layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]), + ) + for layer_past in standardized_past + ) + return self._convert_to_bloom_cache(reordered_past) + + # Copied from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._convert_to_bloom_cache + @staticmethod + def _convert_to_bloom_cache(past_key_value: Tuple[Tuple[torch.Tensor]]) -> Tuple[Tuple[torch.Tensor]]: + """ + Converts the cache to the format expected by Bloom, i.e. to tuple(tuple([batch_size * num_heads, ...])) + """ + batch_size, num_heads, head_dim, seq_length = past_key_value[0][0].shape + batch_size_times_num_heads = batch_size * num_heads + # key: [batch_size, num_heads, head_dim, seq_length] -> [batch_size * num_heads, head_dim, seq_length] + # value: [batch_size, num_heads, seq_length, head_dim] -> [batch_size * num_heads, seq_length, head_dim] + return tuple( + ( + layer_past[0].view(batch_size_times_num_heads, head_dim, seq_length), + layer_past[1].view(batch_size_times_num_heads, seq_length, head_dim), + ) + for layer_past in past_key_value + ) + + # Adapted from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._convert_to_standard_cache + def _convert_to_standard_cache( + self, past_key_value: Tuple[Tuple[torch.Tensor]], batch_size: int + ) -> Tuple[Tuple[torch.Tensor]]: + """ + Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size, num_heads, ...])) + """ + if self.config.model_type != "bloom": + return past_key_value + + batch_size_times_num_heads, head_dim, seq_length = past_key_value[0][0].shape + num_heads = batch_size_times_num_heads // batch_size + # key: [batch_size * num_heads, head_dim, seq_length] -> [batch_size, num_heads, head_dim, seq_length] + # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim] + return tuple( + ( + layer_past[0].view(batch_size, num_heads, head_dim, seq_length), + layer_past[1].view(batch_size, num_heads, seq_length, head_dim), + ) + for layer_past in past_key_value + ) + + def to(self, device: Union[torch.device, str]): + self._device = device if isinstance(device, torch.device) else torch.device(device) + self.model.to(self._device) + return self + def forward( self, input_ids: torch.LongTensor = None, @@ -183,14 +297,28 @@ def forward( inputs["past_key_values"] = past_key_values outputs = self.model(**inputs) - if isinstance(outputs, tuple): - outputs = CausalLMOutputWithPast(logits=outputs[0], past_key_values=outputs[1] if self.use_cache else None) + if isinstance(outputs, (list, tuple)): + logits = outputs[0] + past_key_values = outputs[1] if self.use_cache else None else: - outputs = CausalLMOutputWithPast( - logits=outputs["logits"], past_key_values=outputs["past_key_values"] if self.use_cache else None - ) + logits = outputs["logits"] + past_key_values = outputs["past_key_values"] if self.use_cache else None + return CausalLMOutputWithPast(logits=logits, past_key_values=past_key_values) - return outputs + +class TSModelForCausalLM(BaseModelForCausalLM): + def __init__( + self, + model, + config: PretrainedConfig = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + use_cache: bool = True, + **kwargs, + ): + super(TSModelForCausalLM, self).__init__( + model=model, config=config, model_save_dir=model_save_dir, use_cache=use_cache, **kwargs + ) + self.model.to(self._device) @classmethod def _from_pretrained( @@ -291,119 +419,3 @@ def _from_transformers( local_files_only=local_files_only, **kwargs, ) - - def can_generate(self) -> bool: - return True - - @property - def device(self) -> torch.device: - return self._device - - def to(self, device: Union[torch.device, str]): - self._device = device if isinstance(device, torch.device) else torch.device(device) - self.model.to(self._device) - return self - - # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation - def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): - past_key_values = past_key_values or kwargs.get("past", None) - - if self.use_cache and past_key_values is not None: - input_ids = input_ids[:, -1:] - - # `past_key_values` may be in the stardard format (e.g. in contrastive search), converts to bloom's format if needed - if past_key_values is not None and self.config.model_type == "bloom": - if past_key_values[0][0].shape[0] == input_ids.shape[0]: - past_key_values = self._convert_to_bloom_cache(past_key_values) - - return { - "input_ids": input_ids, - "past_key_values": past_key_values, - "use_cache": self.use_cache, - "position_ids": None, - "attention_mask": kwargs.get("attention_mask", None), - "token_type_ids": None, - } - - def _reorder_cache( - self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor - ) -> Tuple[Tuple[torch.Tensor]]: - """ - This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or - [`~PreTrainedModel.beam_sample`] is called. - This is required to match `past_key_values` with the correct beam_idx at every generation step. - """ - if self.config.model_type == "bloom": - return self._reorder_cache_bloom(past_key_values, beam_idx) - - # from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel._reorder_cache - return tuple( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) - for layer_past in past_key_values - ) - - # Copied from transformers.models.bloom.modeling_bloom.BloomForCausalLM._reorder_cache - def _reorder_cache_bloom( - self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor - ) -> Tuple[Tuple[torch.Tensor]]: - """ - This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or - [`~PreTrainedModel.beam_sample`] is called for bloom architecture. - This is required to match `past_key_values` with the correct beam_idx at every generation step. - """ - standardized_past = self._convert_to_standard_cache(past_key_values, batch_size=len(beam_idx)) - - # Get a copy of `beam_idx` on all the devices where we need those indices. - device_to_beam_idx = { - past_state.device: beam_idx.to(past_state.device) - for layer_past in past_key_values - for past_state in layer_past - } - reordered_past = tuple( - ( - layer_past[0].index_select(0, device_to_beam_idx[layer_past[0].device]), - layer_past[1].index_select(0, device_to_beam_idx[layer_past[0].device]), - ) - for layer_past in standardized_past - ) - return self._convert_to_bloom_cache(reordered_past) - - # Copied from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._convert_to_bloom_cache - @staticmethod - def _convert_to_bloom_cache(past_key_value: Tuple[Tuple[torch.Tensor]]) -> Tuple[Tuple[torch.Tensor]]: - """ - Converts the cache to the format expected by Bloom, i.e. to tuple(tuple([batch_size * num_heads, ...])) - """ - batch_size, num_heads, head_dim, seq_length = past_key_value[0][0].shape - batch_size_times_num_heads = batch_size * num_heads - # key: [batch_size, num_heads, head_dim, seq_length] -> [batch_size * num_heads, head_dim, seq_length] - # value: [batch_size, num_heads, seq_length, head_dim] -> [batch_size * num_heads, seq_length, head_dim] - return tuple( - ( - layer_past[0].view(batch_size_times_num_heads, head_dim, seq_length), - layer_past[1].view(batch_size_times_num_heads, seq_length, head_dim), - ) - for layer_past in past_key_value - ) - - # Adapted from transformers.models.bloom.modeling_bloom.BloomPreTrainedModel._convert_to_standard_cache - def _convert_to_standard_cache( - self, past_key_value: Tuple[Tuple[torch.Tensor]], batch_size: int - ) -> Tuple[Tuple[torch.Tensor]]: - """ - Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size, num_heads, ...])) - """ - if self.config.model_type != "bloom": - return past_key_value - - batch_size_times_num_heads, head_dim, seq_length = past_key_value[0][0].shape - num_heads = batch_size_times_num_heads // batch_size - # key: [batch_size * num_heads, head_dim, seq_length] -> [batch_size, num_heads, head_dim, seq_length] - # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim] - return tuple( - ( - layer_past[0].view(batch_size, num_heads, head_dim, seq_length), - layer_past[1].view(batch_size, num_heads, seq_length, head_dim), - ) - for layer_past in past_key_value - ) diff --git a/optimum/intel/neural_compressor/__init__.py b/optimum/intel/neural_compressor/__init__.py index 66db62ac1b..2e4250f2a8 100644 --- a/optimum/intel/neural_compressor/__init__.py +++ b/optimum/intel/neural_compressor/__init__.py @@ -14,9 +14,9 @@ from ..utils.import_utils import is_diffusers_available from .configuration import INCConfig +from .modeling_decoder import INCModelForCausalLM from .quantization import ( INCModel, - INCModelForCausalLM, INCModelForMaskedLM, INCModelForMultipleChoice, INCModelForQuestionAnswering, diff --git a/optimum/intel/neural_compressor/modeling_base.py b/optimum/intel/neural_compressor/modeling_base.py new file mode 100644 index 0000000000..0eaed698e8 --- /dev/null +++ b/optimum/intel/neural_compressor/modeling_base.py @@ -0,0 +1,267 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Optional, Union + +import torch +from huggingface_hub import hf_hub_download +from transformers import PretrainedConfig +from transformers.file_utils import add_start_docstrings +from transformers.utils import is_ipex_available + +from optimum.exporters import TasksManager + +from ..generation.modeling import jit_trace +from ..utils.import_utils import is_torch_version +from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask +from .quantization import INCModel +from .utils import WEIGHTS_NAME + + +logger = logging.getLogger(__name__) + + +MODEL_START_DOCSTRING = r""" + This model check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving) + Parameters: + model (`PyTorch model`): is the main class used to run inference. + config (`transformers.PretrainedConfig`): [PretrainedConfig](https://huggingface.co/docs/transformers/main_classes/configuration#transformers.PretrainedConfig) + is the Model configuration class with all the parameters of the model. + device (`str`, defaults to `"cpu"`): + The device type for which the model will be optimized for. The resulting compiled model will contains nodes specific to this device. +""" + + +@add_start_docstrings( + """ + Base INCBaseModel class. + """, +) +class INCBaseModel: + _AUTOMODELS_TO_TASKS = {cls_name: task for task, cls_name in TasksManager._TASKS_TO_AUTOMODELS.items()} + base_model_prefix = "inc_model" + + def __init__( + self, + model, + config: PretrainedConfig = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + use_cache: bool = True, + **kwargs, + ): + super(INCBaseModel, self).__init__( + model=model, config=config, model_save_dir=model_save_dir, use_cache=use_cache, **kwargs + ) + if getattr(self.config, "backend", None) == "ipex": + if not is_ipex_available(): + raise ImportError( + "Intel PyTorch Extensions was not found." + "please make sure you've installed the package or run " + "pip install intel_extension_for_pytorch" + ) + else: + # Need import intel_extension_for_pytorch for ipex model + import intel_extension_for_pytorch as ipex + + # Just to avoid to change by ruff. + logger.info("intel_extension_for_pytorch version is " + ipex.__version__) + + def _save_pretrained(self, save_directory: Union[str, Path], **kwargs): + if getattr(self.config, "torchscript", False): + torch.jit.save(self.model, os.path.join(save_directory, WEIGHTS_NAME)) + else: + state_dict = self.model.state_dict() + torch.save(state_dict, os.path.join(save_directory, WEIGHTS_NAME)) + logger.info(f"Model weights saved to {save_directory}") + + @classmethod + def _from_pretrained( + cls, + model_id: Union[str, Path], + config: PretrainedConfig, + use_auth_token: Optional[Union[bool, str, None]] = None, + revision: Optional[Union[str, None]] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + file_name: Optional[str] = None, + local_files_only: bool = False, + use_cache: bool = True, + torch_dtype: Optional[Union[str, "torch.dtype"]] = None, + **kwargs, + ): + """ + Loads a model and its configuration file from a directory or the HF Hub. + + Arguments: + model_id (`str` or `Path`): + The directory from which to load the model. + Can be either: + - The model id of a pretrained model hosted inside a model repo on huggingface.co. + - The path to a directory containing the model weights. + use_auth_token (`str` or `bool`): + The token to use as HTTP bearer authorization for remote files. Needed to load models from a private + repository. + revision (`str`, *optional*): + The specific model version to use. It can be a branch name, a tag name, or a commit id. + cache_dir (`Union[str, Path]`, *optional*): + The path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + file_name(`str`, *optional*): + The file name of the model to load. Overwrites the default file name and allows one to load the model + with a different name. This argument will be deprecated in next release. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + """ + if file_name is not None: + logger.warning("The argument of `file_name` will be deprecated in next release.") + else: + file_name = WEIGHTS_NAME + model_kwargs = { + "revision": revision, + "use_auth_token": use_auth_token, + "cache_dir": cache_dir, + "local_files_only": local_files_only, + "force_download": force_download, + } + if getattr(config, "torchscript", None): + # Load the model from local directory + if os.path.isdir(model_id): + file_name = os.path.join(model_id, file_name) + model_save_dir = model_id + # Download the model from the hub + else: + model_cache_path = hf_hub_download( + repo_id=model_id, + filename=file_name, + **model_kwargs, + ) + model_save_dir = Path(model_cache_path).parent + model = cls.load_model(file_name) + else: + model_save_dir = None + task = cls.export_feature + if config.torch_dtype != "int8" and config.torch_dtype != torch.int8: + model = TasksManager.get_model_from_task(task, model_id, torch_dtype=torch_dtype, **model_kwargs) + else: + INCModel.TRANSFORMERS_AUTO_CLASS = cls.auto_model_class + model = INCModel.from_pretrained(model_id, q_model_name=file_name, **model_kwargs) + + model.eval() + + return cls( + model, + config=config, + model_save_dir=model_save_dir, + use_cache=use_cache, + **kwargs, + ) + + @classmethod + def _from_transformers( + cls, + model_id: str, + config: PretrainedConfig, + use_auth_token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + subfolder: str = "", + local_files_only: bool = False, + use_cache: bool = True, + torch_dtype: Optional[Union[str, "torch.dtype"]] = None, + **kwargs, + ): + """ + Export a vanilla Transformers model into a TorchScript model using `torch.jit.trace`. + + Arguments: + model_id (`str` or `Path`): + The directory from which to load the model. + Can be either: + - The model id of a pretrained model hosted inside a model repo on huggingface.co. + - The path to a directory containing the model weights. save_dir (`str` or `Path`): + The directory where the exported ONNX model should be saved, default to + `transformers.file_utils.default_cache_path`, which is the cache directory for transformers. + config (`PretrainedConfig`) : + an object of PretrainedConfig. + use_auth_token (`str` or `bool`): + Is needed to load models from a private repository + revision (`str`): + Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id + kwargs (`Dict`, *optional*): + kwargs will be passed to the model during initialization + """ + if is_torch_version("<", "2.0.0"): + raise ImportError("`torch>=2.0.0` is needed to trace your model") + + task = cls.export_feature + model_kwargs = { + "revision": revision, + "use_auth_token": use_auth_token, + "cache_dir": cache_dir, + "subfolder": subfolder, + "local_files_only": local_files_only, + "force_download": force_download, + "torch_dtype": torch_dtype, + } + + if config.torch_dtype != "int8" and config.torch_dtype != torch.int8: + model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) + else: + file_name = kwargs.get("file_name", None) + if file_name is not None: + logger.warning("The argument of `file_name` will be deprecated in next release.") + INCModel.TRANSFORMERS_AUTO_CLASS = cls.auto_model_class + model = INCModel.from_pretrained(model_id, q_model_name=file_name, **model_kwargs) + + if model.config.model_type == "bloom": + model.transformer._prepare_attn_mask = _prepare_attn_mask + + if model.config.model_type == "llama": + model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + traced_model = jit_trace(model, task, use_cache) + save_dir = TemporaryDirectory() + save_dir_path = Path(save_dir.name) + torch.jit.save(traced_model, save_dir_path / WEIGHTS_NAME) + config.torchscript = True + + return cls._from_pretrained( + model_id=save_dir_path, + config=config, + use_cache=use_cache, + use_auth_token=use_auth_token, + revision=revision, + force_download=force_download, + cache_dir=cache_dir, + local_files_only=local_files_only, + **kwargs, + ) + + @classmethod + def _auto_model_to_task(cls, auto_model_class): + """ + Get the task corresponding to a class (for example AutoModelForXXX in transformers). + """ + return cls._AUTOMODELS_TO_TASKS[auto_model_class.__name__] + + def eval(self): + self.model.eval() diff --git a/optimum/intel/neural_compressor/modeling_decoder.py b/optimum/intel/neural_compressor/modeling_decoder.py new file mode 100644 index 0000000000..8e5618122f --- /dev/null +++ b/optimum/intel/neural_compressor/modeling_decoder.py @@ -0,0 +1,49 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Optional, Union + +from transformers import PretrainedConfig +from transformers.file_utils import add_start_docstrings + +from optimum.intel.generation import BaseModelForCausalLM + +from .modeling_base import MODEL_START_DOCSTRING, INCBaseModel + + +logger = logging.getLogger(__name__) + + +@add_start_docstrings( + """ + Neural-compressor Model with a causal language modeling head on top (linear layer with weights tied to the input + embeddings). + """, + MODEL_START_DOCSTRING, +) +class INCModelForCausalLM(INCBaseModel, BaseModelForCausalLM): + def __init__( + self, + model, + config: PretrainedConfig = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + use_cache: bool = True, + **kwargs, + ): + super(INCModelForCausalLM, self).__init__( + model=model, config=config, model_save_dir=model_save_dir, use_cache=use_cache, **kwargs + ) diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index 07223a7b06..ee1dbc978e 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -220,10 +220,16 @@ def quantize( " accuracy tolerance has been found. Either the tolerance or the number of trials need to be increased." ) if isinstance(self._original_model.config, PretrainedConfig): - original_dtype = self._original_model.config.torch_dtype - self._original_model.config.torch_dtype = "int8" - self._original_model.config.save_pretrained(save_directory) - self._original_model.config.torch_dtype = original_dtype + # If backend is IPEX, then the quantized model is JIT model which will drop the config attribute, + # so need set config from original_model. + model_config = copy.deepcopy(self._original_model.config) + model_config.torch_dtype = "int8" + if isinstance(compressed_model, IPEXModel): + model_config.torchscript = True + model_config.backend = "ipex" + else: + compressed_model._model.config = model_config + model_config.save_pretrained(save_directory) self._quantized_model = compressed_model._model @@ -481,6 +487,8 @@ def from_pretrained(cls, model_name_or_path: str, q_model_name: Optional[str] = Returns: q_model: Quantized model. """ + if q_model_name is not None: + logger.warning("The argument of `q_model_name` will be deprecated in next release.") download_kwarg_default = [ ("cache_dir", None), ("force_download", False), @@ -580,7 +588,7 @@ def from_pretrained(cls, model_name_or_path: str, q_model_name: Optional[str] = except Exception: logger.info("Couldn't verify torch version.") - if getattr(config, "backend", None) == "ipex": + if getattr(config, "backend", None) == "ipex" or getattr(config, "torchscript", False): # NOTE: Will improve to use load function when Intel Neural Compressor next 2.1 release. # return load(state_dict_path) load_model = torch.jit.load(state_dict_path) @@ -621,10 +629,6 @@ class INCModelForSeq2SeqLM(INCModel): TRANSFORMERS_AUTO_CLASS = AutoModelForSeq2SeqLM -class INCModelForCausalLM(INCModel): - TRANSFORMERS_AUTO_CLASS = AutoModelForCausalLM - - class INCModelForMaskedLM(INCModel): TRANSFORMERS_AUTO_CLASS = AutoModelForMaskedLM diff --git a/optimum/intel/neural_compressor/trainer.py b/optimum/intel/neural_compressor/trainer.py index 705a83c900..8e8fec1758 100644 --- a/optimum/intel/neural_compressor/trainer.py +++ b/optimum/intel/neural_compressor/trainer.py @@ -141,10 +141,14 @@ def __init__( self.deepspeed = None # Attach dtype and architecture to the config - self.dtype = "int8" if quantization_config is not None else str(get_parameter_dtype(self.model)).split(".")[1] + if quantization_config is not None: + self.dtype = "int8" + self.model.config.backend = quantization_config.backend + else: + self.dtype = str(get_parameter_dtype(self.model)).split(".")[1] + self.model.config.backend = "default" self.model.config.torch_dtype = self.dtype self.model.config.framework = "pytorch_fx" - self.model.config.backend = "default" self.model.config.architectures = [self.model.__class__.__name__] self._set_signature_columns_if_needed() diff --git a/optimum/intel/neural_compressor/utils.py b/optimum/intel/neural_compressor/utils.py index aff8035a00..dd77011c04 100644 --- a/optimum/intel/neural_compressor/utils.py +++ b/optimum/intel/neural_compressor/utils.py @@ -31,6 +31,7 @@ CONFIG_NAME = "best_configure.yaml" + _HEAD_TO_AUTOMODELS = { "fill-mask": "INCModelForMaskedLM", "text-generation": "INCModelForCausalLM", diff --git a/setup.py b/setup.py index c8cdce500f..30ba6e462a 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,11 @@ QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] EXTRAS_REQUIRE = { - "neural-compressor": ["neural-compressor>=2.2.0", "onnx", "onnxruntime<1.15.0"], + "neural-compressor": [ + "neural-compressor>=2.2.0", + "onnx", + "onnxruntime<1.15.0", + ], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index a885590e93..b9989fbe69 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -35,6 +35,7 @@ from onnx import load as onnx_load from parameterized import parameterized from transformers import ( + AutoModelForCausalLM, AutoModelForQuestionAnswering, AutoModelForSequenceClassification, AutoTokenizer, @@ -116,6 +117,11 @@ class OptimizationTest(unittest.TestCase): ("token-classification", "hf-internal-testing/tiny-random-AlbertForTokenClassification", 30), ) + TEXT_GENERATION_SUPPORTED_ARCHITECTURES = ( + "hf-internal-testing/tiny-random-BloomForCausalLM", + "hf-internal-testing/tiny-random-GPTNeoForCausalLM", + ) + @parameterized.expand(SUPPORTED_ARCHITECTURES_DYNAMIC) def test_dynamic_quantization(self, task, model_name, expected_quantized_matmuls): quantization_config = PostTrainingQuantConfig(approach="dynamic") @@ -278,6 +284,34 @@ def test_dynamic_diffusion_model(self): # Compare model outputs self.assertTrue(np.allclose(loaded_pipe_outputs, outputs, atol=1e-4)) + @parameterized.expand(TEXT_GENERATION_SUPPORTED_ARCHITECTURES) + def test_quantize_text_generate_model(self, model_id): + set_seed(42) + model = AutoModelForCausalLM.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) + tokens = tokenizer("This is a sample", return_tensors="pt") + + def calibration_fn(p_model): + tmp_model = INCModelForCausalLM(p_model, model.config) + tmp_model.generate(**tokens, max_new_tokens=32, do_sample=False) + + quantization_config = PostTrainingQuantConfig(approach="static") + model.config.return_dict = False + quantizer = INCQuantizer.from_pretrained(model, calibration_fn=calibration_fn) + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + save_directory=tmp_dir, + save_onnx_model=False, + ) + model = INCModelForCausalLM.from_pretrained(tmp_dir, export=True) + + pre_outputs = quantizer._quantized_model.generate( + **tokens, do_sample=False, num_beams=1, temperature=0.9, min_length=20, max_length=20 + ) + outputs = model.generate(**tokens, do_sample=False, num_beams=1, temperature=0.9, min_length=20, max_length=20) + self.assertTrue(torch.equal(pre_outputs, outputs)) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) def test_aware_training_quantization(self, task, model_name, expected_quantized_matmuls): quantization_config = QuantizationAwareTrainingConfig() From 1acbc1ce834e664e477aed2754f8eaba9388d400 Mon Sep 17 00:00:00 2001 From: Chen Peter Date: Thu, 6 Jul 2023 15:53:50 +0800 Subject: [PATCH 055/134] Allow user to set the CACHE_DIR (#362) * Allow user to set the CACHE_DIR user can set CACHE_DIR to empty and disable the model cache or user can set the CACHE_DIR to another folder Signed-off-by: Peter Chen * Set default CACHE_DIR in __init__ Signed-off-by: Peter Chen * Set default CACHE_DIR in compile since users may set ov_config after initilization. Signed-off-by: Peter Chen * make style Signed-off-by: Peter Chen --------- Signed-off-by: Peter Chen --- optimum/intel/openvino/modeling_base.py | 8 +++++-- optimum/intel/openvino/modeling_diffusion.py | 24 ++++++++++++++------ optimum/intel/openvino/modeling_seq2seq.py | 18 ++++++++++++--- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 8955a03870..6f7fe69bfe 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -302,8 +302,12 @@ def _from_transformers( def compile(self): if self.request is None: logger.info("Compiling the model...") - cache_dir = Path(self.model_save_dir).joinpath("model_cache") - ov_config = {**self.ov_config, "CACHE_DIR": str(cache_dir)} + ov_config = {**self.ov_config} + if "CACHE_DIR" not in self.ov_config.keys(): + # Set default CACHE_DIR only if it is not set. + cache_dir = Path(self.model_save_dir).joinpath("model_cache") + ov_config["CACHE_DIR"] = str(cache_dir) + logger.info(f"Set CACHE_DIR to {str(cache_dir)}") self.request = core.compile_model(self.model, self._device, ov_config) def _reshape( diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 4ac701d443..e911a9f10d 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -79,22 +79,32 @@ def __init__( self.vae_decoder = OVModelVaeDecoder( vae_decoder, self, - {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER)}, + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER)}, ) self.text_encoder = OVModelTextEncoder( text_encoder, self, - {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER)}, + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER)}, ) self.unet = OVModelUnet( unet, self, - {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_UNET_SUBFOLDER)}, + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_UNET_SUBFOLDER)}, + ) + vae_ov_config = ( + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else { + **self.ov_config, + "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER), + } ) - vae_ov_config = { - **self.ov_config, - "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER), - } self.vae_encoder = OVModelVaeEncoder(vae_encoder, self, vae_ov_config) if vae_encoder is not None else None self.tokenizer = tokenizer self.scheduler = scheduler diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index c56f608daf..0f52335639 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -152,16 +152,28 @@ def __init__( enable_compilation = kwargs.get("compile", True) encoder_cache_dir = Path(self.model_save_dir).joinpath("encoder_cache") encoder_cache_dir.mkdir(parents=True, exist_ok=True) - ov_encoder_config = {**self.ov_config, "CACHE_DIR": str(encoder_cache_dir)} + ov_encoder_config = ( + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": str(encoder_cache_dir)} + ) self.encoder = OVEncoder(self.encoder_model, self._device, ov_encoder_config) decoder_cache_dir = Path(self.model_save_dir).joinpath("decoder_cache") decoder_cache_dir.mkdir(parents=True, exist_ok=True) - ov_decoder_config = {**self.ov_config, "CACHE_DIR": str(decoder_cache_dir)} + ov_decoder_config = ( + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": str(decoder_cache_dir)} + ) self.decoder = OVDecoder(self.decoder_model, self._device, ov_decoder_config) if self.use_cache: decoder_past_cache_dir = Path(self.model_save_dir).joinpath("decoder_past_cache") decoder_past_cache_dir.mkdir(parents=True, exist_ok=True) - ov_decoder_past_config = {**self.ov_config, "CACHE_DIR": str(decoder_past_cache_dir)} + ov_decoder_past_config = ( + {**self.ov_config} + if "CACHE_DIR" in self.ov_config.keys() + else {**self.ov_config, "CACHE_DIR": str(decoder_past_cache_dir)} + ) self.decoder_with_past = OVDecoder(self.decoder_with_past_model, self._device, ov_decoder_past_config) if enable_compilation: self.compile() From 3a123d999dd0e029b252cc1613614ddc80d35fe1 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 7 Jul 2023 17:38:35 +0200 Subject: [PATCH 056/134] deprecate python 3.7 support (#366) * deprecate python 3.7 support * Update setup.py Co-authored-by: Helena Kloosterman * Update .github/workflows/test_openvino_basic.yml Co-authored-by: Helena Kloosterman --------- Co-authored-by: Helena Kloosterman --- .github/workflows/test_openvino_basic.yml | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_openvino_basic.yml b/.github/workflows/test_openvino_basic.yml index 92471358b0..effb99a84d 100644 --- a/.github/workflows/test_openvino_basic.yml +++ b/.github/workflows/test_openvino_basic.yml @@ -24,7 +24,7 @@ jobs: matrix: # Testing lower and upper bound of supported Python versions # This also ensures that the test fails if dependencies break for Python 3.7 - python-version: ["3.7", "3.10"] + python-version: ["3.8", "3.11"] transformers: ['transformers', 'git+https://github.com/huggingface/transformers.git'] optimum: ['optimum', 'git+https://github.com/huggingface/optimum.git'] diff --git a/setup.py b/setup.py index 30ba6e462a..d0580e494d 100644 --- a/setup.py +++ b/setup.py @@ -63,9 +63,10 @@ "Intended Audience :: Education", "Intended Audience :: Science/Research", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering :: Artificial Intelligence", ], keywords="transformers, quantization, pruning, knowledge distillation, optimization, training", From 735bae1a0179974e9e96df82ab18b6570c99a945 Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Wed, 19 Jul 2023 20:32:12 +0200 Subject: [PATCH 057/134] Cap transformers to <4.31 for [nncf] (#374) NNCF does not yet support transformers 4.31, this PR caps the transformers version for now for users who install the NNCF extra. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0580e494d..8bc849c566 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ "onnxruntime<1.15.0", ], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], + "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0", "transformers<4.31"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From 84a38b0b211e04ab3b4915b4422f161fdf255130 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 20 Jul 2023 11:40:21 +0200 Subject: [PATCH 058/134] Update model for test loading static quantized squad model (#375) --- tests/neural_compressor/test_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/neural_compressor/test_modeling.py b/tests/neural_compressor/test_modeling.py index f6aca1c671..5514e3d036 100644 --- a/tests/neural_compressor/test_modeling.py +++ b/tests/neural_compressor/test_modeling.py @@ -44,7 +44,7 @@ ("echarlaix/distilbert-base-uncased-finetuned-sst-2-english-int8-dynamic", "text-classification"), ("echarlaix/distilbert-sst2-inc-dynamic-quantization-magnitude-pruning-0.1", "text-classification"), ("hf-internal-testing/tiny-random-bert", "fill-mask"), - ("Intel/bert-base-uncased-squad-int8-static", "question-answering"), + ("Intel/distilbert-base-uncased-distilled-squad-int8-static", "question-answering"), ("hf-internal-testing/tiny-random-gpt2", "text-generation"), ("Intel/t5-small-xsum-int8-dynamic", "text2text-generation"), # ("echarlaix/stable-diffusion-v1-5-inc-int8-dynamic", "stable-diffusion") From 13294f44a5a356c3d8c5d5fae695fd3f0d17642a Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:16:57 +0200 Subject: [PATCH 059/134] Fix OV dataloader (#376) * Fix OV dataloder * remove transformers version constraint --- optimum/intel/openvino/quantization.py | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index bda901e5d2..29e16736bb 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -22,6 +22,7 @@ import openvino import torch import transformers +from accelerate.data_loader import DataLoaderStateMixin from datasets import Dataset, load_dataset from nncf import NNCFConfig from nncf.torch import create_compressed_model, register_default_init_args @@ -56,6 +57,13 @@ class OVDataLoader(PTInitializingDataLoader): def get_inputs(self, dataloader_output) -> Tuple[Tuple, Dict]: return (), dataloader_output + @property + def batch_size(self): + batch_size = self._data_loader.batch_size + if batch_size is None and isinstance(self._data_loader, DataLoaderStateMixin): + batch_size = self._data_loader.total_batch_size + return batch_size + class OVQuantizer(OptimumQuantizer): """ diff --git a/setup.py b/setup.py index 8bc849c566..d0580e494d 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ "onnxruntime<1.15.0", ], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0", "transformers<4.31"], + "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, From 42ab8ccc7b82a1506b5afbbecf465a647ce71aed Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Wed, 5 Jul 2023 16:29:40 +0800 Subject: [PATCH 060/134] OVModelForCausalLM pastkv uses bf16 precision --- optimum/intel/openvino/modeling_decoder.py | 72 ++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 2c5520b6f3..e484aabb0e 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import logging from pathlib import Path from tempfile import TemporaryDirectory @@ -20,7 +21,9 @@ import numpy as np import openvino import torch -from openvino.runtime import Core, Tensor +from openvino.runtime import Core, Tensor, Type +from openvino.preprocess import PrePostProcessor + from transformers import AutoModelForCausalLM, PretrainedConfig from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import CausalLMOutputWithPast @@ -248,6 +251,59 @@ def compile(self): class OVModelForCausalLM(OVBaseDecoderModel, GenerationMixin): export_feature = "text-generation" auto_model_class = AutoModelForCausalLM + + def __init__(self, + model: openvino.runtime.Model, + config: PretrainedConfig = None, + device: str = "CPU", + dynamic_shapes: bool = True, + ov_config: Dict[str, str] | None = None, + model_save_dir: str | Path | TemporaryDirectory | None = None, + **kwargs): + model = self._try_modify_model_io_to_bf16(model, ov_config, device, **kwargs) + super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) + + def _try_modify_model_io_to_bf16(self, + model: openvino.runtime.Model, + ov_config: Dict[str, str] | None = None, + device: str = "CPU", + **kwargs): + if device == 'CPU': + # if INFERENCE_PRECISION_HINT in ov_config equals to bf16, pastkv will use bf16 + pastkv_will_use = Type.bf16 if ov_config and ov_config.get("INFERENCE_PRECISION_HINT", "") == "bf16" else Type.f32 + + # for batch testsing can set INFERENCE_PRECISION_HINT in enviroment, pastkv will try to use the specific precision + if "INFERENCE_PRECISION_HINT" in os.environ: + hint = os.environ["INFERENCE_PRECISION_HINT"] + if hint == "bf16": + pastkv_will_use = Type.bf16 + else: + logger.warning( + f"Unknown precision type {hint} in INFERENCE_PRECISION_HINT, will be ignored." + ) + use_cache = kwargs.get("use_cache", True) + has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) + if pastkv_will_use != Type.f32 and use_cache and has_pastkv: + ppp = PrePostProcessor(model) + need_gen = False + for key in model.inputs: + if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + for key in model.outputs: + if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + if need_gen: + model = ppp.build() + + self.pastkv_will_use = Type.f32 + for key in model.inputs: + if "past_key_values" in key.get_any_name(): + self.pastkv_will_use = key.get_element_type() + break + + return model @add_start_docstrings_to_model_forward( INPUTS_DOCSTRING.format("batch_size, sequence_length") @@ -271,10 +327,16 @@ def forward( inputs = {} if past_key_values is not None: - # Flatten the past_key_values - past_key_values = tuple( - past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer - ) + if self.pastkv_will_use == Type.bf16: + # output is u16, should change to bf16 + past_key_values = tuple( + Tensor(past_key_value, past_key_value.shape, Type.bf16) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + ) + else: + # Flatten the past_key_values + past_key_values = tuple( + past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + ) # Add the past_key_values to the decoder inputs inputs = dict(zip(self.key_value_input_names, past_key_values)) From 2fc7266989f1a401c64ad48dcdee76e2f34e7cd5 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Thu, 6 Jul 2023 18:12:14 +0800 Subject: [PATCH 061/134] use core.get_property to infer the runtime precision --- optimum/intel/openvino/modeling_decoder.py | 44 +++++++++++++--------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index e484aabb0e..577f92f177 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -251,7 +251,7 @@ def compile(self): class OVModelForCausalLM(OVBaseDecoderModel, GenerationMixin): export_feature = "text-generation" auto_model_class = AutoModelForCausalLM - + def __init__(self, model: openvino.runtime.Model, config: PretrainedConfig = None, @@ -260,27 +260,37 @@ def __init__(self, ov_config: Dict[str, str] | None = None, model_save_dir: str | Path | TemporaryDirectory | None = None, **kwargs): - model = self._try_modify_model_io_to_bf16(model, ov_config, device, **kwargs) + model = self._try_modify_model_io_to_lowprecision(model, ov_config, device, **kwargs) super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) - def _try_modify_model_io_to_bf16(self, + def _try_modify_model_io_to_lowprecision(self, model: openvino.runtime.Model, ov_config: Dict[str, str] | None = None, device: str = "CPU", **kwargs): if device == 'CPU': - # if INFERENCE_PRECISION_HINT in ov_config equals to bf16, pastkv will use bf16 - pastkv_will_use = Type.bf16 if ov_config and ov_config.get("INFERENCE_PRECISION_HINT", "") == "bf16" else Type.f32 - - # for batch testsing can set INFERENCE_PRECISION_HINT in enviroment, pastkv will try to use the specific precision - if "INFERENCE_PRECISION_HINT" in os.environ: - hint = os.environ["INFERENCE_PRECISION_HINT"] - if hint == "bf16": - pastkv_will_use = Type.bf16 - else: - logger.warning( - f"Unknown precision type {hint} in INFERENCE_PRECISION_HINT, will be ignored." - ) + pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") + # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision + if ov_config: + user_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") + openvino_types_str_map = { + "boolean": Type.boolean, + "f16": Type.f16, + "f32": Type.f32, + "f64": Type.f64, + "i8": Type.i8, + "i16": Type.i16, + "i32": Type.i32, + "i64": Type.i64, + "u8": Type.u8, + "u16": Type.u16, + "u32": Type.u32, + "u64": Type.u64, + "bf16": Type.bf16 + } + if user_hint in openvino_types_str_map: + pastkv_will_use = openvino_types_str_map[user_hint] + use_cache = kwargs.get("use_cache", True) has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) if pastkv_will_use != Type.f32 and use_cache and has_pastkv: @@ -327,10 +337,10 @@ def forward( inputs = {} if past_key_values is not None: - if self.pastkv_will_use == Type.bf16: + if self.pastkv_will_use != Type.f32: # output is u16, should change to bf16 past_key_values = tuple( - Tensor(past_key_value, past_key_value.shape, Type.bf16) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + Tensor(past_key_value, past_key_value.shape, self.pastkv_will_use) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer ) else: # Flatten the past_key_values From 5e6ad675ef667a0ce7553388c37fddb889337038 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Sat, 8 Jul 2023 00:38:33 +0800 Subject: [PATCH 062/134] fix ci failure --- optimum/intel/openvino/modeling_decoder.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 577f92f177..8d735ef347 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -257,17 +257,19 @@ def __init__(self, config: PretrainedConfig = None, device: str = "CPU", dynamic_shapes: bool = True, - ov_config: Dict[str, str] | None = None, - model_save_dir: str | Path | TemporaryDirectory | None = None, + ov_config: Dict[str, str] = None, + model_save_dir: str | Path | TemporaryDirectory = None, **kwargs): - model = self._try_modify_model_io_to_lowprecision(model, ov_config, device, **kwargs) + model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, **kwargs) super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) - def _try_modify_model_io_to_lowprecision(self, + def _try_modify_pastkv_to_lowprecision(self, model: openvino.runtime.Model, - ov_config: Dict[str, str] | None = None, + ov_config: Dict[str, str] = None, device: str = "CPU", **kwargs): + self.pastkv_will_use = Type.f32 + device = device.upper() if device == 'CPU': pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision @@ -307,7 +309,6 @@ def _try_modify_model_io_to_lowprecision(self, if need_gen: model = ppp.build() - self.pastkv_will_use = Type.f32 for key in model.inputs: if "past_key_values" in key.get_any_name(): self.pastkv_will_use = key.get_element_type() From 16167700476a9132322020c18b8c80c04b21f531 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Sat, 8 Jul 2023 04:01:44 +0800 Subject: [PATCH 063/134] fix ci failure --- optimum/intel/openvino/modeling_decoder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 8d735ef347..92111a870b 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -257,19 +257,19 @@ def __init__(self, config: PretrainedConfig = None, device: str = "CPU", dynamic_shapes: bool = True, - ov_config: Dict[str, str] = None, - model_save_dir: str | Path | TemporaryDirectory = None, + ov_config: Optional[Dict[str, str]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, **kwargs): model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, **kwargs) super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) def _try_modify_pastkv_to_lowprecision(self, model: openvino.runtime.Model, - ov_config: Dict[str, str] = None, + ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs): self.pastkv_will_use = Type.f32 - device = device.upper() + device = device.upper() if device else device if device == 'CPU': pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision @@ -338,10 +338,10 @@ def forward( inputs = {} if past_key_values is not None: - if self.pastkv_will_use != Type.f32: - # output is u16, should change to bf16 + if self.pastkv_will_use == Type.bf16: + # numpy does not support bf16, pretending u16, should change to bf16 past_key_values = tuple( - Tensor(past_key_value, past_key_value.shape, self.pastkv_will_use) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + Tensor(past_key_value, past_key_value.shape, Type.bf16) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer ) else: # Flatten the past_key_values From 00ba904b3346c072c6ca54a54a3fc7bacbbf392b Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Sat, 8 Jul 2023 04:29:19 +0800 Subject: [PATCH 064/134] fix CI style failure --- optimum/intel/openvino/modeling_decoder.py | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 92111a870b..260ca7308d 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging from pathlib import Path from tempfile import TemporaryDirectory @@ -21,9 +20,8 @@ import numpy as np import openvino import torch -from openvino.runtime import Core, Tensor, Type from openvino.preprocess import PrePostProcessor - +from openvino.runtime import Core, Tensor, Type from transformers import AutoModelForCausalLM, PretrainedConfig from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import CausalLMOutputWithPast @@ -252,25 +250,25 @@ class OVModelForCausalLM(OVBaseDecoderModel, GenerationMixin): export_feature = "text-generation" auto_model_class = AutoModelForCausalLM - def __init__(self, - model: openvino.runtime.Model, - config: PretrainedConfig = None, - device: str = "CPU", - dynamic_shapes: bool = True, - ov_config: Optional[Dict[str, str]] = None, - model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, - **kwargs): + def __init__( + self, + model: openvino.runtime.Model, + config: PretrainedConfig = None, + device: str = "CPU", + dynamic_shapes: bool = True, + ov_config: Optional[Dict[str, str]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + **kwargs, + ): model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, **kwargs) super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) - def _try_modify_pastkv_to_lowprecision(self, - model: openvino.runtime.Model, - ov_config: Optional[Dict[str, str]] = None, - device: str = "CPU", - **kwargs): + def _try_modify_pastkv_to_lowprecision( + self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs + ): self.pastkv_will_use = Type.f32 device = device.upper() if device else device - if device == 'CPU': + if device == "CPU": pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if ov_config: @@ -288,7 +286,7 @@ def _try_modify_pastkv_to_lowprecision(self, "u16": Type.u16, "u32": Type.u32, "u64": Type.u64, - "bf16": Type.bf16 + "bf16": Type.bf16, } if user_hint in openvino_types_str_map: pastkv_will_use = openvino_types_str_map[user_hint] @@ -341,7 +339,9 @@ def forward( if self.pastkv_will_use == Type.bf16: # numpy does not support bf16, pretending u16, should change to bf16 past_key_values = tuple( - Tensor(past_key_value, past_key_value.shape, Type.bf16) for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + Tensor(past_key_value, past_key_value.shape, Type.bf16) + for pkv_per_layer in past_key_values + for past_key_value in pkv_per_layer ) else: # Flatten the past_key_values From a2f579b2c7c0d92bdaca996cef936fe8befce286 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Sat, 8 Jul 2023 18:49:27 +0800 Subject: [PATCH 065/134] remove ov obj from class members --- optimum/intel/openvino/modeling_decoder.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 260ca7308d..77c0cc56ea 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -266,7 +266,6 @@ def __init__( def _try_modify_pastkv_to_lowprecision( self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs ): - self.pastkv_will_use = Type.f32 device = device.upper() if device else device if device == "CPU": pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") @@ -307,11 +306,6 @@ def _try_modify_pastkv_to_lowprecision( if need_gen: model = ppp.build() - for key in model.inputs: - if "past_key_values" in key.get_any_name(): - self.pastkv_will_use = key.get_element_type() - break - return model @add_start_docstrings_to_model_forward( @@ -336,7 +330,11 @@ def forward( inputs = {} if past_key_values is not None: - if self.pastkv_will_use == Type.bf16: + if len(self.key_value_input_names): + pastkv_will_use = self.model.input(self.key_value_input_names[0]).get_element_type() + else: + pastkv_will_use = Type.f32 + if pastkv_will_use == Type.bf16: # numpy does not support bf16, pretending u16, should change to bf16 past_key_values = tuple( Tensor(past_key_value, past_key_value.shape, Type.bf16) From 30406c222247c19e9c37da74699901e77ba26605 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Fri, 14 Jul 2023 18:15:16 +0800 Subject: [PATCH 066/134] apply review comments --- optimum/intel/openvino/modeling_decoder.py | 33 ++++++---------------- optimum/intel/openvino/utils.py | 18 ++++++++++++ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 77c0cc56ea..22a723252d 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -33,7 +33,7 @@ from ..utils.import_utils import is_transformers_version from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .utils import ONNX_WEIGHTS_NAME +from .utils import ONNX_WEIGHTS_NAME, STR_TO_OV_TYPE if is_transformers_version("<", "4.25.0"): @@ -267,28 +267,16 @@ def _try_modify_pastkv_to_lowprecision( self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs ): device = device.upper() if device else device - if device == "CPU": + if device: pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if ov_config: - user_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") - openvino_types_str_map = { - "boolean": Type.boolean, - "f16": Type.f16, - "f32": Type.f32, - "f64": Type.f64, - "i8": Type.i8, - "i16": Type.i16, - "i32": Type.i32, - "i64": Type.i64, - "u8": Type.u8, - "u16": Type.u16, - "u32": Type.u32, - "u64": Type.u64, - "bf16": Type.bf16, - } - if user_hint in openvino_types_str_map: - pastkv_will_use = openvino_types_str_map[user_hint] + user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") + user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") + if user_precision_hint in STR_TO_OV_TYPE: + pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] + if user_mode_hint.upper() == "ACCURACY": + pastkv_will_use = Type.f32 use_cache = kwargs.get("use_cache", True) has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) @@ -330,10 +318,7 @@ def forward( inputs = {} if past_key_values is not None: - if len(self.key_value_input_names): - pastkv_will_use = self.model.input(self.key_value_input_names[0]).get_element_type() - else: - pastkv_will_use = Type.f32 + pastkv_will_use = self.model.input(self.key_value_input_names[0]).get_element_type() if pastkv_will_use == Type.bf16: # numpy does not support bf16, pretending u16, should change to bf16 past_key_values = tuple( diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index c5d003ba9e..fe46d28b42 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -14,6 +14,7 @@ import numpy as np +from openvino.runtime import Type from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size @@ -50,6 +51,23 @@ } +STR_TO_OV_TYPE = { + "boolean": Type.boolean, + "f16": Type.f16, + "f32": Type.f32, + "f64": Type.f64, + "i8": Type.i8, + "i16": Type.i16, + "i32": Type.i32, + "i64": Type.i64, + "u8": Type.u8, + "u16": Type.u16, + "u32": Type.u32, + "u64": Type.u64, + "bf16": Type.bf16, +} + + _HEAD_TO_AUTOMODELS = { "fill-mask": "OVModelForMaskedLM", "text-generation": "OVModelForCausalLM", From fbbabb5c0cbf45172225979800dabcf5404b762c Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Mon, 17 Jul 2023 20:06:30 +0800 Subject: [PATCH 067/134] apply review comment --- optimum/intel/openvino/modeling_decoder.py | 53 +++++++++++----------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 22a723252d..ee6c18b6e8 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -266,33 +266,32 @@ def __init__( def _try_modify_pastkv_to_lowprecision( self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs ): - device = device.upper() if device else device - if device: - pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") - # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision - if ov_config: - user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") - user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") - if user_precision_hint in STR_TO_OV_TYPE: - pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] - if user_mode_hint.upper() == "ACCURACY": - pastkv_will_use = Type.f32 - - use_cache = kwargs.get("use_cache", True) - has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) - if pastkv_will_use != Type.f32 and use_cache and has_pastkv: - ppp = PrePostProcessor(model) - need_gen = False - for key in model.inputs: - if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): - need_gen = True - ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) - for key in model.outputs: - if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): - need_gen = True - ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) - if need_gen: - model = ppp.build() + device = device.upper() + pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") + # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision + if ov_config: + user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") + user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") + if user_precision_hint in STR_TO_OV_TYPE: + pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] + if user_mode_hint.upper() == "ACCURACY": + pastkv_will_use = Type.f32 + + use_cache = kwargs.get("use_cache", True) + has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) + if pastkv_will_use != Type.f32 and use_cache and has_pastkv: + ppp = PrePostProcessor(model) + need_gen = False + for key in model.inputs: + if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + for key in model.outputs: + if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + if need_gen: + model = ppp.build() return model From 59c72199d288a6210a590f5a3e2f82a240cc6697 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Thu, 20 Jul 2023 02:45:50 +0800 Subject: [PATCH 068/134] _save_pretrained still saves unmodified model --- optimum/intel/openvino/modeling_decoder.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index ee6c18b6e8..25a976649b 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import os from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict, Optional, Tuple, Union @@ -33,7 +34,7 @@ from ..utils.import_utils import is_transformers_version from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .utils import ONNX_WEIGHTS_NAME, STR_TO_OV_TYPE +from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME, STR_TO_OV_TYPE if is_transformers_version("<", "4.25.0"): @@ -279,8 +280,9 @@ def _try_modify_pastkv_to_lowprecision( use_cache = kwargs.get("use_cache", True) has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) + self.model_org = model if pastkv_will_use != Type.f32 and use_cache and has_pastkv: - ppp = PrePostProcessor(model) + ppp = PrePostProcessor(model.clone()) need_gen = False for key in model.inputs: if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): @@ -295,6 +297,21 @@ def _try_modify_pastkv_to_lowprecision( return model + def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): + """ + Saves the model to the OpenVINO IR format so that it can be re-loaded using the + [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `Path`): + The directory where to save the model files. + file_name(`str`, *optional*): + The model file name to use when saving the model. Overwrites the default file names. + """ + file_name = file_name if file_name is not None else OV_XML_FILE_NAME + dst_path = os.path.join(save_directory, file_name) + openvino.runtime.serialize(self.model_org, dst_path, dst_path.replace(".xml", ".bin")) + @add_start_docstrings_to_model_forward( INPUTS_DOCSTRING.format("batch_size, sequence_length") + TEXT_GENERATION_EXAMPLE.format( From c3f748a77498b8cb7e43e54f9897932fd950a274 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Thu, 20 Jul 2023 20:35:31 +0800 Subject: [PATCH 069/134] apply review comments --- optimum/intel/openvino/modeling_decoder.py | 107 +++++++++------------ 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 25a976649b..def6313137 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -110,6 +110,8 @@ def __init__( "`dynamic_shapes` was set to `False` but static shapes are not supported for causal language model. Please set `dynamic_shapes=True`." ) + model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device) + super().__init__( model, config, @@ -137,6 +139,50 @@ def __init__( "To export your model, simply set `export=True`." ) + def _try_modify_pastkv_to_lowprecision( + self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU" + ): + device = device.upper() + pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") + # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision + if ov_config: + user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") + user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") + if user_precision_hint in STR_TO_OV_TYPE: + pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] + elif user_mode_hint.upper() == "ACCURACY": + pastkv_will_use = Type.f32 + + use_cache = any("past_key_values" in key.get_any_name() for key in model.inputs) + self.model_org = model + if pastkv_will_use != Type.f32 and use_cache: + ppp = PrePostProcessor(model.clone()) + need_gen = False + for key in model.inputs: + if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + for key in model.outputs: + if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + need_gen = True + ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + if need_gen: + model = ppp.build() + + return model + + def _save_pretrained(self, save_directory: Union[str, Path]): + """ + Saves the model to the OpenVINO IR format so that it can be re-loaded using the + [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `Path`): + The directory where to save the model files. + """ + dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) + openvino.runtime.serialize(self.model_org, dst_path) + @classmethod def _from_transformers( cls, @@ -251,67 +297,6 @@ class OVModelForCausalLM(OVBaseDecoderModel, GenerationMixin): export_feature = "text-generation" auto_model_class = AutoModelForCausalLM - def __init__( - self, - model: openvino.runtime.Model, - config: PretrainedConfig = None, - device: str = "CPU", - dynamic_shapes: bool = True, - ov_config: Optional[Dict[str, str]] = None, - model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, - **kwargs, - ): - model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, **kwargs) - super().__init__(model, config, device, dynamic_shapes, ov_config, model_save_dir, **kwargs) - - def _try_modify_pastkv_to_lowprecision( - self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU", **kwargs - ): - device = device.upper() - pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") - # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision - if ov_config: - user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") - user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") - if user_precision_hint in STR_TO_OV_TYPE: - pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] - if user_mode_hint.upper() == "ACCURACY": - pastkv_will_use = Type.f32 - - use_cache = kwargs.get("use_cache", True) - has_pastkv = any("past_key_values" in key.get_any_name() for key in model.inputs) - self.model_org = model - if pastkv_will_use != Type.f32 and use_cache and has_pastkv: - ppp = PrePostProcessor(model.clone()) - need_gen = False - for key in model.inputs: - if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): - need_gen = True - ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) - for key in model.outputs: - if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): - need_gen = True - ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) - if need_gen: - model = ppp.build() - - return model - - def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): - """ - Saves the model to the OpenVINO IR format so that it can be re-loaded using the - [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. - - Arguments: - save_directory (`str` or `Path`): - The directory where to save the model files. - file_name(`str`, *optional*): - The model file name to use when saving the model. Overwrites the default file names. - """ - file_name = file_name if file_name is not None else OV_XML_FILE_NAME - dst_path = os.path.join(save_directory, file_name) - openvino.runtime.serialize(self.model_org, dst_path, dst_path.replace(".xml", ".bin")) - @add_start_docstrings_to_model_forward( INPUTS_DOCSTRING.format("batch_size, sequence_length") + TEXT_GENERATION_EXAMPLE.format( From cc304717dda8bf755e8b311261e1effc3778107d Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:18:16 +0200 Subject: [PATCH 070/134] Add stable diffusion XL OpenVINO export and inference (#377) * Add stable diffusion XL OpenVINO export and inference * fix invisible-watermark version * add num_images_per_prompt * add utils_tests * Add default value for architecture parameters when missing * fix style * fix config saving step * add documentation --- .github/workflows/test_openvino.yml | 3 +- docs/source/inference.mdx | 37 +- optimum/intel/__init__.py | 34 +- optimum/intel/openvino/__init__.py | 8 +- optimum/intel/openvino/modeling_base.py | 2 +- .../intel/openvino/modeling_base_seq2seq.py | 2 +- optimum/intel/openvino/modeling_decoder.py | 2 +- optimum/intel/openvino/modeling_diffusion.py | 432 +++++++++++------- optimum/intel/openvino/quantization.py | 2 +- optimum/intel/openvino/trainer.py | 2 +- .../dummy_openvino_and_diffusers_objects.py | 44 ++ setup.py | 2 +- tests/openvino/test_modeling.py | 227 +-------- tests/openvino/test_stable_diffusion.py | 418 +++++++++++++++++ tests/openvino/utils_tests.py | 91 ++++ 15 files changed, 919 insertions(+), 387 deletions(-) create mode 100644 tests/openvino/test_stable_diffusion.py create mode 100644 tests/openvino/utils_tests.py diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index 1761d02c83..a76a1c0ecd 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -30,7 +30,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[openvino,nncf,tests] + python -m pip install git+https://github.com/huggingface/optimum.git + pip install .[openvino,nncf,tests,diffusers] - name: Test with Pytest run: | pytest tests/openvino/ --ignore test_modeling_basic diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 792916bf7c..a73f837ead 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -11,7 +11,7 @@ specific language governing permissions and limitations under the License. Optimum Intel can be used to load optimized models from the [Hugging Face Hub](https://huggingface.co/models?library=openvino&sort=downloads) and create pipelines to run inference with OpenVINO Runtime without rewriting your APIs. -## Switching from Transformers to Optimum Inference +## Switching from Transformers to Optimum You can now easily perform inference with OpenVINO Runtime on a variety of Intel processors ([see](https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_Supported_Devices.html) the full list of supported devices). For that, just replace the `AutoModelForXxx` class with the corresponding `OVModelForXxx` class. @@ -102,7 +102,7 @@ model.half() model.compile() ``` -## Export and inference of sequence-to-sequence models +## Sequence-to-sequence models Sequence-to-sequence (Seq2Seq) models, that generate a new sequence from an input, can also be used when running inference with OpenVINO. When Seq2Seq models are exported to the OpenVINO IR, they are decomposed into two parts : the encoder and the "decoder" (which actually consists of the decoder with the language modeling head), that are later combined during inference. To leverage the pre-computed key/values hidden-states to speed up sequential decoding, simply pass `use_cache=True` to the `from_pretrained()` method. An additional model component will be exported: the "decoder" with pre-computed key/values as one of its inputs. @@ -128,7 +128,7 @@ tokenizer.save_pretrained(save_directory) [{'translation_text': "Il n'est jamais sorti sans un livre sous son bras, et il est souvent revenu avec deux."}] ``` -## Export and inference of Stable Diffusion models +## Stable Diffusion Stable Diffusion models can also be used when running inference with OpenVINO. When Stable Diffusion models are exported to the OpenVINO format, they are decomposed into three components that are later combined during inference: @@ -140,10 +140,11 @@ are exported to the OpenVINO format, they are decomposed into three components t Make sure you have 🤗 Diffusers installed. To install `diffusers`: -``` -pip install diffusers +```bash +pip install optimum[diffusers] ``` +### Text-to-Image Here is an example of how you can load an OpenVINO Stable Diffusion model and run inference using OpenVINO Runtime: ```python @@ -188,6 +189,28 @@ In case you want to change any parameters such as the outputs height or width, y ![img](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/stable_diffusion_v1_5_sail_boat_rembrandt.png) +### Image-to-Image + +```python +import requests +import torch +from PIL import Image +from io import BytesIO +from optimum.intel import OVStableDiffusionImg2ImgPipeline + +model_id = "runwayml/stable-diffusion-v1-5" +pipeline = OVStableDiffusionImg2ImgPipeline.from_pretrained(model_id, export=True) +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" +response = requests.get(url) +init_image = Image.open(BytesIO(response.content)).convert("RGB") +init_image = init_image.resize((768, 512)) +prompt = "A fantasy landscape, trending on artstation" +image = pipeline(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images[0] +image.save("fantasy_landscape.png") +``` + + + ## Supported tasks As shown in the table below, each task is associated with a class enabling to automatically load your model. @@ -205,3 +228,7 @@ As shown in the table below, each task is associated with a class enabling to au | `text-generation` | `OVModelForCausalLM` | | `text2text-generation` | `OVModelForSeq2SeqLM` | | `text-to-image` | `OVStableDiffusionPipeline` | +| `text-to-image` | `OVStableDiffusionXLPipeline` | +| `image-to-image` | `OVStableDiffusionImg2ImgPipeline` | +| `image-to-image` | `OVStableDiffusionXLImg2ImgPipeline` | +| `inpaint` | `OVStableDiffusionPipeline` | diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index e465240bb9..65e39d365f 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -56,9 +56,23 @@ if not (is_openvino_available() and is_diffusers_available()): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: - _import_structure["utils.dummy_openvino_and_diffusers_objects"] = ["OVStableDiffusionPipeline"] + _import_structure["utils.dummy_openvino_and_diffusers_objects"] = [ + "OVStableDiffusionPipeline", + "OVStableDiffusionImg2ImgPipeline", + "OVStableDiffusionInpaintPipeline", + "OVStableDiffusionXLPipeline", + "OVStableDiffusionXLImg2ImgPipeline", + ] else: - _import_structure["openvino"].append("OVStableDiffusionPipeline") + _import_structure["openvino"].extend( + [ + "OVStableDiffusionPipeline", + "OVStableDiffusionImg2ImgPipeline", + "OVStableDiffusionInpaintPipeline", + "OVStableDiffusionXLPipeline", + "OVStableDiffusionXLImg2ImgPipeline", + ] + ) try: if not is_openvino_available(): @@ -138,9 +152,21 @@ if not (is_openvino_available() and is_diffusers_available()): raise OptionalDependencyNotAvailable() except OptionalDependencyNotAvailable: - from .utils.dummy_openvino_and_diffusers_objects import OVStableDiffusionPipeline + from .utils.dummy_openvino_and_diffusers_objects import ( + OVStableDiffusionImg2ImgPipeline, + OVStableDiffusionInpaintPipeline, + OVStableDiffusionPipeline, + OVStableDiffusionXLImg2ImgPipeline, + OVStableDiffusionXLPipeline, + ) else: - from .openvino import OVStableDiffusionPipeline + from .openvino import ( + OVStableDiffusionImg2ImgPipeline, + OVStableDiffusionInpaintPipeline, + OVStableDiffusionPipeline, + OVStableDiffusionXLImg2ImgPipeline, + OVStableDiffusionXLPipeline, + ) try: if not is_openvino_available(): diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 3f8737e1b9..cfbac71fd1 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -47,4 +47,10 @@ if is_diffusers_available(): - from .modeling_diffusion import OVStableDiffusionPipeline + from .modeling_diffusion import ( + OVStableDiffusionImg2ImgPipeline, + OVStableDiffusionInpaintPipeline, + OVStableDiffusionPipeline, + OVStableDiffusionXLImg2ImgPipeline, + OVStableDiffusionXLPipeline, + ) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 6f7fe69bfe..a6087ff952 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -25,8 +25,8 @@ from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters import TasksManager from optimum.exporters.onnx import export +from optimum.exporters.tasks import TasksManager from optimum.modeling_base import OptimizedModel from ..utils.import_utils import is_transformers_version diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index a839d5c3a8..a8ce3d0bf5 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -24,8 +24,8 @@ from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters import TasksManager from optimum.exporters.onnx import export_models, get_encoder_decoder_models_for_export +from optimum.exporters.tasks import TasksManager from ..utils.import_utils import is_transformers_version from .modeling_base import OVBaseModel diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 2c5520b6f3..bf29f920a2 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -25,8 +25,8 @@ from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import CausalLMOutputWithPast -from optimum.exporters import TasksManager from optimum.exporters.onnx import export +from optimum.exporters.tasks import TasksManager from optimum.utils import NormalizedConfigManager from ..utils.import_utils import is_transformers_version diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index e911a9f10d..a0e7d7fbdc 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -15,13 +15,20 @@ import importlib import logging import os +import shutil from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Dict, List, Optional, Union import numpy as np import openvino -from diffusers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, StableDiffusionPipeline +from diffusers import ( + DDIMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionPipeline, + StableDiffusionXLPipeline, +) from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME from diffusers.utils import CONFIG_NAME from huggingface_hub import snapshot_download @@ -29,10 +36,14 @@ from openvino.runtime import Core from transformers import CLIPFeatureExtractor, CLIPTokenizer -from optimum.exporters import TasksManager -from optimum.exporters.onnx import export_models, get_stable_diffusion_models_for_export +from optimum.exporters.onnx import main_export from optimum.pipelines.diffusers.pipeline_stable_diffusion import StableDiffusionPipelineMixin +from optimum.pipelines.diffusers.pipeline_stable_diffusion_img2img import StableDiffusionImg2ImgPipelineMixin +from optimum.pipelines.diffusers.pipeline_stable_diffusion_inpaint import StableDiffusionInpaintPipelineMixin +from optimum.pipelines.diffusers.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineMixin +from optimum.pipelines.diffusers.pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipelineMixin from optimum.utils import ( + DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, DIFFUSION_MODEL_UNET_SUBFOLDER, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, @@ -48,7 +59,7 @@ logger = logging.getLogger(__name__) -class OVStableDiffusionPipeline(OVBaseModel, StableDiffusionPipelineMixin): +class OVStableDiffusionPipelineBase(OVBaseModel): auto_model_class = StableDiffusionPipeline config_name = "model_index.json" export_feature = "stable-diffusion" @@ -63,6 +74,8 @@ def __init__( scheduler: Union["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"], feature_extractor: Optional["CLIPFeatureExtractor"] = None, vae_encoder: Optional[openvino.runtime.Model] = None, + text_encoder_2: Optional[openvino.runtime.Model] = None, + tokenizer_2: Optional["CLIPTokenizer"] = None, device: str = "CPU", dynamic_shapes: bool = True, compile: bool = True, @@ -74,44 +87,30 @@ def __init__( self._device = device.upper() self.is_dynamic = dynamic_shapes self.ov_config = ov_config if ov_config is not None else {} - model_save_dir = model_save_dir.name if isinstance(model_save_dir, TemporaryDirectory) else model_save_dir - - self.vae_decoder = OVModelVaeDecoder( - vae_decoder, - self, - {**self.ov_config} - if "CACHE_DIR" in self.ov_config.keys() - else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER)}, + self._model_save_dir = ( + Path(model_save_dir.name) if isinstance(model_save_dir, TemporaryDirectory) else model_save_dir ) - self.text_encoder = OVModelTextEncoder( - text_encoder, - self, - {**self.ov_config} - if "CACHE_DIR" in self.ov_config.keys() - else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER)}, + self.vae_decoder = OVModelVaeDecoder(vae_decoder, self) + self.unet = OVModelUnet(unet, self) + self.text_encoder = OVModelTextEncoder(text_encoder, self) if text_encoder is not None else None + self.text_encoder_2 = ( + OVModelTextEncoder(text_encoder_2, self, model_name=DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER) + if text_encoder_2 is not None + else None ) - self.unet = OVModelUnet( - unet, - self, - {**self.ov_config} - if "CACHE_DIR" in self.ov_config.keys() - else {**self.ov_config, "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_UNET_SUBFOLDER)}, - ) - vae_ov_config = ( - {**self.ov_config} - if "CACHE_DIR" in self.ov_config.keys() - else { - **self.ov_config, - "CACHE_DIR": os.path.join(model_save_dir, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER), - } - ) - self.vae_encoder = OVModelVaeEncoder(vae_encoder, self, vae_ov_config) if vae_encoder is not None else None + self.vae_encoder = OVModelVaeEncoder(vae_encoder, self) if vae_encoder is not None else None + + if "block_out_channels" in self.vae_decoder.config: + self.vae_scale_factor = 2 ** (len(self.vae_decoder.config["block_out_channels"]) - 1) + else: + self.vae_scale_factor = 8 + self.tokenizer = tokenizer + self.tokenizer_2 = tokenizer_2 self.scheduler = scheduler self.feature_extractor = feature_extractor self.safety_checker = None self.preprocessors = [] - self._vae_scale_factor = 8 if self.is_dynamic: self.reshape(batch_size=-1, height=-1, width=-1, num_images_per_prompt=-1) @@ -124,6 +123,7 @@ def __init__( DIFFUSION_MODEL_UNET_SUBFOLDER: self.unet, DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER: self.vae_decoder, DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER: self.vae_encoder, + DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER: self.text_encoder_2, } for name in sub_models.keys(): self._internal_dict[name] = ( @@ -142,23 +142,32 @@ def _save_pretrained(self, save_directory: Union[str, Path]): The directory where to save the model files """ save_directory = Path(save_directory) - src_to_dst_file = { - self.vae_decoder.model: save_directory / DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER / OV_XML_FILE_NAME, - self.text_encoder.model: save_directory / DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER / OV_XML_FILE_NAME, - self.unet.model: save_directory / DIFFUSION_MODEL_UNET_SUBFOLDER / OV_XML_FILE_NAME, + + sub_models_to_save = { + self.unet: DIFFUSION_MODEL_UNET_SUBFOLDER, + self.vae_decoder: DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, + self.vae_encoder: DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, + self.text_encoder: DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, + self.text_encoder_2: DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER, } - if self.vae_encoder is not None: - src_to_dst_file[self.vae_encoder.model] = ( - save_directory / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / OV_XML_FILE_NAME - ) - for src_file, dst_path in src_to_dst_file.items(): - dst_path.parent.mkdir(parents=True, exist_ok=True) - openvino.runtime.serialize(src_file, str(dst_path)) - self.tokenizer.save_pretrained(save_directory.joinpath("tokenizer")) - self.scheduler.save_pretrained(save_directory.joinpath("scheduler")) + for ov_model, dst_path in sub_models_to_save.items(): + if ov_model is not None: + dst_path = save_directory / dst_path / OV_XML_FILE_NAME + dst_path.parent.mkdir(parents=True, exist_ok=True) + openvino.runtime.serialize(ov_model.model, dst_path) + model_dir = ov_model.config.get("_name_or_path", None) or ov_model._model_dir / ov_model._model_name + config_path = Path(model_dir) / ov_model.CONFIG_NAME + if config_path.is_file(): + shutil.copyfile(config_path, dst_path.parent / ov_model.CONFIG_NAME) + + self.scheduler.save_pretrained(save_directory / "scheduler") if self.feature_extractor is not None: - self.feature_extractor.save_pretrained(save_directory.joinpath("feature_extractor")) + self.feature_extractor.save_pretrained(save_directory / "feature_extractor") + if self.tokenizer is not None: + self.tokenizer.save_pretrained(save_directory / "tokenizer") + if self.tokenizer_2 is not None: + self.tokenizer_2.save_pretrained(save_directory / "tokenizer_2") @classmethod def _from_pretrained( @@ -172,6 +181,7 @@ def _from_pretrained( text_encoder_file_name: Optional[str] = None, unet_file_name: Optional[str] = None, vae_encoder_file_name: Optional[str] = None, + text_encoder_2_file_name: Optional[str] = None, local_files_only: bool = False, from_onnx: bool = False, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, @@ -180,24 +190,26 @@ def _from_pretrained( default_file_name = ONNX_WEIGHTS_NAME if from_onnx else OV_XML_FILE_NAME vae_decoder_file_name = vae_decoder_file_name or default_file_name text_encoder_file_name = text_encoder_file_name or default_file_name + text_encoder_2_file_name = text_encoder_2_file_name or default_file_name unet_file_name = unet_file_name or default_file_name vae_encoder_file_name = vae_encoder_file_name or default_file_name model_id = str(model_id) - sub_models_to_load, _, _ = cls.extract_init_dict(config) - sub_models_names = set(sub_models_to_load.keys()).intersection({"feature_extractor", "tokenizer", "scheduler"}) + patterns = set(config.keys()) + sub_models_names = patterns.intersection({"feature_extractor", "tokenizer", "tokenizer_2", "scheduler"}) if not os.path.isdir(model_id): - patterns = set(config.keys()) patterns.update({"vae_encoder", "vae_decoder"}) allow_patterns = {os.path.join(k, "*") for k in patterns if not k.startswith("_")} allow_patterns.update( { vae_decoder_file_name, text_encoder_file_name, + text_encoder_2_file_name, unet_file_name, vae_encoder_file_name, vae_decoder_file_name.replace(".xml", ".bin"), text_encoder_file_name.replace(".xml", ".bin"), + text_encoder_2_file_name.replace(".xml", ".bin"), unet_file_name.replace(".xml", ".bin"), vae_encoder_file_name.replace(".xml", ".bin"), SCHEDULER_CONFIG_NAME, @@ -221,7 +233,7 @@ def _from_pretrained( # Check if the subcomponent needs to be loaded if kwargs.get(name, None) is not None: continue - library_name, library_classes = sub_models_to_load[name] + library_name, library_classes = config[name] if library_classes is not None: library = importlib.import_module(library_name) class_obj = getattr(library, library_classes) @@ -232,28 +244,32 @@ def _from_pretrained( else: kwargs[name] = load_method(new_model_save_dir) - vae_decoder = cls.load_model( - new_model_save_dir / DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER / vae_decoder_file_name - ) - text_encoder = cls.load_model( - new_model_save_dir / DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER / text_encoder_file_name - ) unet = cls.load_model(new_model_save_dir / DIFFUSION_MODEL_UNET_SUBFOLDER / unet_file_name) - vae_encoder_path = new_model_save_dir / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / vae_encoder_file_name - vae_encoder = cls.load_model(vae_encoder_path) if vae_encoder_path.is_file() else None + + components = { + "vae_encoder": new_model_save_dir / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / vae_encoder_file_name, + "vae_decoder": new_model_save_dir / DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER / vae_decoder_file_name, + "text_encoder": new_model_save_dir / DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER / text_encoder_file_name, + "text_encoder_2": new_model_save_dir / DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER / text_encoder_2_file_name, + } + + for key, value in components.items(): + components[key] = cls.load_model(value) if value.is_file() else None if model_save_dir is None: model_save_dir = new_model_save_dir return cls( - vae_decoder=vae_decoder, - text_encoder=text_encoder, + vae_decoder=components["vae_decoder"], + text_encoder=components["text_encoder"], unet=unet, config=config, - tokenizer=kwargs.pop("tokenizer"), + tokenizer=kwargs.pop("tokenizer", None), scheduler=kwargs.pop("scheduler"), feature_extractor=kwargs.pop("feature_extractor", None), - vae_encoder=vae_encoder, + vae_encoder=components["vae_encoder"], + text_encoder_2=components["text_encoder_2"], + tokenizer_2=kwargs.pop("tokenizer_2", None), model_save_dir=model_save_dir, **kwargs, ) @@ -268,38 +284,25 @@ def _from_transformers( force_download: bool = False, cache_dir: Optional[str] = None, local_files_only: bool = False, - task: Optional[str] = None, tokenizer: "CLIPTokenizer" = None, scheduler: Union["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"] = None, feature_extractor: Optional["CLIPFeatureExtractor"] = None, **kwargs, ): - task = task or cls.export_feature save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) - model_kwargs = { - "revision": revision, - "use_auth_token": use_auth_token, - "cache_dir": cache_dir, - "local_files_only": local_files_only, - "force_download": force_download, - "config": config, - } - model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - - output_names = [ - os.path.join(DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, ONNX_WEIGHTS_NAME), - os.path.join(DIFFUSION_MODEL_UNET_SUBFOLDER, ONNX_WEIGHTS_NAME), - os.path.join(DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ONNX_WEIGHTS_NAME), - os.path.join(DIFFUSION_MODEL_VAE_DECODER_SUBFOLDER, ONNX_WEIGHTS_NAME), - ] - models_and_onnx_configs = get_stable_diffusion_models_for_export(model) - model.save_config(save_dir_path) - export_models( - models_and_onnx_configs=models_and_onnx_configs, - output_dir=save_dir_path, - output_names=output_names, + main_export( + model_name_or_path=model_id, + output=save_dir_path, + task=cls.export_feature, + do_validation=False, + no_post_process=True, + revision=revision, + cache_dir=cache_dir, + use_auth_token=use_auth_token, + local_files_only=local_files_only, + force_download=force_download, ) return cls._from_pretrained( @@ -312,9 +315,9 @@ def _from_transformers( cache_dir=cache_dir, local_files_only=local_files_only, model_save_dir=save_dir, - tokenizer=tokenizer or model.tokenizer, - scheduler=scheduler or model.scheduler, - feature_extractor=feature_extractor or model.feature_extractor, + tokenizer=tokenizer, + scheduler=scheduler, + feature_extractor=feature_extractor, **kwargs, ) @@ -332,14 +335,14 @@ def height(self) -> int: height = self.unet.model.inputs[0].get_partial_shape()[2] if height.is_dynamic: return -1 - return height.get_length() * self._vae_scale_factor + return height.get_length() * self.vae_scale_factor @property def width(self) -> int: width = self.unet.model.inputs[0].get_partial_shape()[3] if width.is_dynamic: return -1 - return width.get_length() * self._vae_scale_factor + return width.get_length() * self.vae_scale_factor def _reshape_unet( self, @@ -355,15 +358,29 @@ def _reshape_unet( # The factor of 2 comes from the guidance scale > 1 batch_size = 2 * batch_size * num_images_per_prompt - height = height // 8 if height > 0 else height - width = width // 8 if width > 0 else width + requires_aesthetics_score = getattr(self.config, "requires_aesthetics_score", False) + height = height // self.vae_scale_factor if height > 0 else height + width = width // self.vae_scale_factor if width > 0 else width shapes = {} for inputs in model.inputs: shapes[inputs] = inputs.get_partial_shape() - if inputs.get_any_name().startswith("timestep"): + if inputs.get_any_name() == "timestep": shapes[inputs][0] = 1 - elif inputs.get_any_name().startswith("sample"): - shapes[inputs] = [batch_size, 4, height, width] + elif inputs.get_any_name() == "sample": + in_channels = self.unet.config.get("in_channels", None) + if in_channels is None: + in_channels = shapes[inputs][1] + if in_channels.is_dynamic: + logger.warning( + "Could not identify `in_channels` from the unet configuration, to statically reshape the unet please provide a configuration." + ) + self.is_dynamic = True + + shapes[inputs] = [batch_size, in_channels, height, width] + elif inputs.get_any_name() == "text_embeds": + shapes[inputs] = [batch_size, self.text_encoder_2.config["projection_dim"]] + elif inputs.get_any_name() == "time_ids": + shapes[inputs] = [batch_size, 5 if requires_aesthetics_score else 6] else: shapes[inputs][0] = batch_size shapes[inputs][1] = self.tokenizer.model_max_length @@ -377,9 +394,32 @@ def _reshape_text_encoder(self, model: openvino.runtime.Model, batch_size: int = return model def _reshape_vae_decoder(self, model: openvino.runtime.Model, height: int = -1, width: int = -1): - height = height // 8 if height > -1 else height - width = width // 8 if width > -1 else width - shapes = {model.inputs[0]: [1, 4, height, width]} + height = height // self.vae_scale_factor if height > -1 else height + width = width // self.vae_scale_factor if width > -1 else width + latent_channels = self.vae_decoder.config.get("latent_channels", None) + if latent_channels is None: + latent_channels = model.inputs[0].get_partial_shape()[1] + if latent_channels.is_dynamic: + logger.warning( + "Could not identify `latent_channels` from the VAE decoder configuration, to statically reshape the VAE decoder please provide a configuration." + ) + self.is_dynamic = True + shapes = {model.inputs[0]: [1, latent_channels, height, width]} + model.reshape(shapes) + return model + + def _reshape_vae_encoder( + self, model: openvino.runtime.Model, batch_size: int = -1, height: int = -1, width: int = -1 + ): + in_channels = self.vae_encoder.config.get("in_channels", None) + if in_channels is None: + in_channels = model.inputs[0].get_partial_shape()[1] + if in_channels.is_dynamic: + logger.warning( + "Could not identify `in_channels` from the VAE encoder configuration, to statically reshape the VAE encoder please provide a configuration." + ) + self.is_dynamic = True + shapes = {model.inputs[0]: [batch_size, in_channels, height, width]} model.reshape(shapes) return model @@ -391,9 +431,18 @@ def reshape( num_images_per_prompt: int = -1, ): self.is_dynamic = -1 in {batch_size, height, width, num_images_per_prompt} - self.text_encoder.model = self._reshape_text_encoder(self.text_encoder.model, batch_size) self.vae_decoder.model = self._reshape_vae_decoder(self.vae_decoder.model, height, width) self.unet.model = self._reshape_unet(self.unet.model, batch_size, height, width, num_images_per_prompt) + + if self.text_encoder is not None: + self.text_encoder.model = self._reshape_text_encoder(self.text_encoder.model, batch_size) + + if self.text_encoder_2 is not None: + self.text_encoder_2.model = self._reshape_text_encoder(self.text_encoder_2.model, batch_size) + + if self.vae_encoder is not None: + self.vae_encoder.model = self._reshape_vae_encoder(self.vae_encoder.model, batch_size, height, width) + self.clear_requests() return self @@ -402,66 +451,26 @@ def half(self): Converts all the model weights to FP16 for more efficient inference on GPU. """ compress_model_transformation(self.vae_decoder.model) - compress_model_transformation(self.text_encoder.model) compress_model_transformation(self.unet.model) + for component in {self.text_encoder, self.text_encoder_2, self.vae_encoder}: + if component is not None: + compress_model_transformation(component.model) self.clear_requests() return self def clear_requests(self): - self.text_encoder.request = None self.vae_decoder.request = None self.unet.request = None + for component in {self.text_encoder, self.text_encoder_2, self.vae_encoder}: + if component is not None: + component.request = None def compile(self): - self.text_encoder._compile() self.vae_decoder._compile() self.unet._compile() - - def __call__( - self, - prompt: Union[str, List[str]], - height: Optional[int] = 512, - width: Optional[int] = 512, - num_inference_steps: Optional[int] = 50, - guidance_scale: Optional[float] = 7.5, - negative_prompt: Optional[Union[str, List[str]]] = None, - num_images_per_prompt: Optional[int] = 1, - **kwargs, - ): - _height = self.height - _width = self.width - - if _height != -1 and height != _height: - logger.warning( - f"`height` was set to {height} but the static model will output images of height {_height}." - "To fix the height, please reshape your model accordingly using the `.reshape()` method." - ) - height = _height - - if _width != -1 and width != _width: - logger.warning( - f"`width` was set to {width} but the static model will output images of width {_width}." - "To fix the width, please reshape your model accordingly using the `.reshape()` method." - ) - width = _width - - if guidance_scale is not None and guidance_scale <= 1 and not self.is_dynamic: - raise ValueError( - f"`guidance_scale` was set to {guidance_scale}, static shapes are only supported for `guidance_scale` > 1, " - "please set `dynamic_shapes` to `True` when loading the model." - ) - - return StableDiffusionPipelineMixin.__call__( - self, - prompt=prompt, - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=guidance_scale, - negative_prompt=negative_prompt, - num_images_per_prompt=num_images_per_prompt, - **kwargs, - ) + for component in {self.text_encoder, self.text_encoder_2, self.vae_encoder}: + if component is not None: + component._compile @classmethod def _load_config(cls, config_name_or_path: Union[str, os.PathLike], **kwargs): @@ -472,12 +481,15 @@ def _save_config(self, save_directory): class OVModelPart: + CONFIG_NAME = "config.json" + def __init__( self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None, model_name: str = "encoder", + model_dir: str = None, ): self.model = model self.parent_model = parent_model @@ -486,9 +498,16 @@ def __init__( inputs.get_any_name(): OV_TO_NP_TYPE[inputs.get_element_type().get_type_name()] for inputs in self.model.inputs } - self.ov_config = ov_config or self.parent_model.ov_config + self.ov_config = ov_config or {**self.parent_model.ov_config} self.request = None self._model_name = model_name + self._model_dir = Path(model_dir or parent_model._model_save_dir) + config_path = self._model_dir / model_name / self.CONFIG_NAME + self.config = self.parent_model._dict_from_json_file(config_path) if config_path.is_file() else {} + + # TODO : disable if self._model_dir tmp directory + if "CACHE_DIR" not in self.ov_config: + self.ov_config["CACHE_DIR"] = os.path.join(self._model_dir, self._model_name) def _compile(self): if self.request is None: @@ -502,9 +521,13 @@ def device(self): class OVModelTextEncoder(OVModelPart): def __init__( - self, model: openvino.runtime.Model, parent_model: OVBaseModel, ov_config: Optional[Dict[str, str]] = None + self, + model: openvino.runtime.Model, + parent_model: OVBaseModel, + ov_config: Optional[Dict[str, str]] = None, + model_name: str = "text_encoder", ): - super().__init__(model, parent_model, ov_config, "text_encoder") + super().__init__(model, parent_model, ov_config, model_name) def __call__(self, input_ids: np.ndarray): self._compile() @@ -522,7 +545,14 @@ def __init__( ): super().__init__(model, parent_model, ov_config, "unet") - def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_states: np.ndarray): + def __call__( + self, + sample: np.ndarray, + timestep: np.ndarray, + encoder_hidden_states: np.ndarray, + text_embeds: Optional[np.ndarray] = None, + time_ids: Optional[np.ndarray] = None, + ): self._compile() inputs = { @@ -531,6 +561,11 @@ def __call__(self, sample: np.ndarray, timestep: np.ndarray, encoder_hidden_stat "encoder_hidden_states": encoder_hidden_states, } + if text_embeds is not None: + inputs["text_embeds"] = text_embeds + if time_ids is not None: + inputs["time_ids"] = time_ids + outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) @@ -565,3 +600,88 @@ def __call__(self, sample: np.ndarray): } outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) + + +class OVStableDiffusionPipeline(OVStableDiffusionPipelineBase, StableDiffusionPipelineMixin): + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + **kwargs, + ): + height = height or self.unet.config.get("sample_size", 64) * self.vae_scale_factor + width = width or self.unet.config.get("sample_size", 64) * self.vae_scale_factor + _height = self.height + _width = self.width + + if _height != -1 and height != _height: + logger.warning( + f"`height` was set to {height} but the static model will output images of height {_height}." + "To fix the height, please reshape your model accordingly using the `.reshape()` method." + ) + height = _height + + if _width != -1 and width != _width: + logger.warning( + f"`width` was set to {width} but the static model will output images of width {_width}." + "To fix the width, please reshape your model accordingly using the `.reshape()` method." + ) + width = _width + + if guidance_scale is not None and guidance_scale <= 1 and not self.is_dynamic: + raise ValueError( + f"`guidance_scale` was set to {guidance_scale}, static shapes are only supported for `guidance_scale` > 1, " + "please set `dynamic_shapes` to `True` when loading the model." + ) + + return StableDiffusionPipelineMixin.__call__( + self, + prompt=prompt, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) + + +class OVStableDiffusionImg2ImgPipeline(OVStableDiffusionPipelineBase, StableDiffusionImg2ImgPipelineMixin): + def __call__(self, *args, **kwargs): + # TODO : add default height and width if model statically reshaped + # resize image if doesn't match height and width given during reshaping + return StableDiffusionImg2ImgPipelineMixin.__call__(self, *args, **kwargs) + + +class OVStableDiffusionInpaintPipeline(OVStableDiffusionPipelineBase, StableDiffusionInpaintPipelineMixin): + def __call__(self, *args, **kwargs): + # TODO : add default height and width if model statically reshaped + return StableDiffusionInpaintPipelineMixin.__call__(self, *args, **kwargs) + + +class OVStableDiffusionXLPipelineBase(OVStableDiffusionPipelineBase): + auto_model_class = StableDiffusionXLPipeline + export_feature = "stable-diffusion-xl" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # additional invisible-watermark dependency for SD XL + from optimum.pipelines.diffusers.watermark import StableDiffusionXLWatermarker + + self.watermark = StableDiffusionXLWatermarker() + + +class OVStableDiffusionXLPipeline(OVStableDiffusionXLPipelineBase, StableDiffusionXLPipelineMixin): + def __call__(self, *args, **kwargs): + return StableDiffusionXLPipelineMixin.__call__(self, *args, **kwargs) + + +class OVStableDiffusionXLImg2ImgPipeline(OVStableDiffusionXLPipelineBase, StableDiffusionXLImg2ImgPipelineMixin): + def __call__(self, *args, **kwargs): + return StableDiffusionXLImg2ImgPipelineMixin.__call__(self, *args, **kwargs) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 29e16736bb..80adccd171 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -33,8 +33,8 @@ from torch.utils.data import DataLoader, RandomSampler, TensorDataset from transformers import DataCollator, PreTrainedModel, default_data_collator -from optimum.exporters import TasksManager from optimum.exporters.onnx import export +from optimum.exporters.tasks import TasksManager from optimum.quantization_base import OptimumQuantizer from ..utils.constant import _TASK_ALIASES diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index b9e89fcc85..811309806a 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -78,8 +78,8 @@ logging, ) -from optimum.exporters import TasksManager from optimum.exporters.onnx import OnnxConfig +from optimum.exporters.tasks import TasksManager from ..utils.constant import _TASK_ALIASES from ..utils.import_utils import is_transformers_version diff --git a/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py b/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py index c5e92d7125..72c2b8de10 100644 --- a/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py +++ b/optimum/intel/utils/dummy_openvino_and_diffusers_objects.py @@ -24,3 +24,47 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["openvino", "diffusers"]) + + +class OVStableDiffusionImg2ImgPipeline(metaclass=DummyObject): + _backends = ["openvino", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino", "diffusers"]) + + +class OVStableDiffusionInpaintPipeline(metaclass=DummyObject): + _backends = ["openvino", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino", "diffusers"]) + + +class OVStableDiffusionXLPipeline(metaclass=DummyObject): + _backends = ["openvino", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino", "diffusers"]) + + +class OVStableDiffusionXLImg2ImgPipeline(metaclass=DummyObject): + _backends = ["openvino", "diffusers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino", "diffusers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino", "diffusers"]) diff --git a/setup.py b/setup.py index d0580e494d..6b07a48896 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], - "diffusers": ["diffusers"], + "diffusers": ["diffusers", "invisible-watermark>=0.2.0"], "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, } diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 1a8e35d55d..7fe47ecb24 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -43,6 +43,7 @@ pipeline, set_seed, ) +from utils_tests import MODEL_NAMES from optimum.intel.openvino import ( OV_DECODER_NAME, @@ -60,12 +61,6 @@ OVModelForTokenClassification, OVStableDiffusionPipeline, ) -from optimum.intel.openvino.modeling_diffusion import ( - OVModelTextEncoder, - OVModelUnet, - OVModelVaeDecoder, - OVModelVaeEncoder, -) from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, @@ -76,72 +71,6 @@ from optimum.utils.testing_utils import require_diffusers -MODEL_NAMES = { - "albert": "hf-internal-testing/tiny-random-albert", - "audio_spectrogram_transformer": "Ericwang/tiny-random-ast", - "beit": "hf-internal-testing/tiny-random-BeitForImageClassification", - "bert": "hf-internal-testing/tiny-random-bert", - "bart": "hf-internal-testing/tiny-random-bart", - "bigbird_pegasus": "hf-internal-testing/tiny-random-bigbird_pegasus", - "blenderbot-small": "hf-internal-testing/tiny-random-BlenderbotModel", - "blenderbot": "hf-internal-testing/tiny-random-blenderbot", - "bloom": "hf-internal-testing/tiny-random-BloomModel", - "camembert": "hf-internal-testing/tiny-random-camembert", - "convbert": "hf-internal-testing/tiny-random-ConvBertForSequenceClassification", - "codegen": "hf-internal-testing/tiny-random-CodeGenForCausalLM", - "data2vec_text": "hf-internal-testing/tiny-random-Data2VecTextModel", - "data2vec_vision": "hf-internal-testing/tiny-random-Data2VecVisionModel", - "data2vec_audio": "hf-internal-testing/tiny-random-Data2VecAudioModel", - "deberta": "hf-internal-testing/tiny-random-deberta", - "deberta_v2": "hf-internal-testing/tiny-random-DebertaV2Model", - "deit": "hf-internal-testing/tiny-random-deit", - "convnext": "hf-internal-testing/tiny-random-convnext", - "distilbert": "hf-internal-testing/tiny-random-distilbert", - "electra": "hf-internal-testing/tiny-random-electra", - "flaubert": "hf-internal-testing/tiny-random-flaubert", - # "gpt_bigcode": "bigcode/tiny_starcoder_py", - "gpt2": "hf-internal-testing/tiny-random-gpt2", - "gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel", - "gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", - "gptj": "hf-internal-testing/tiny-random-GPTJModel", - "hubert": "hf-internal-testing/tiny-random-HubertModel", - "ibert": "hf-internal-testing/tiny-random-ibert", - "levit": "hf-internal-testing/tiny-random-LevitModel", - "longt5": "hf-internal-testing/tiny-random-longt5", - "llama": "fxmarty/tiny-llama-fast-tokenizer", - "m2m_100": "hf-internal-testing/tiny-random-m2m_100", - "opt": "hf-internal-testing/tiny-random-OPTModel", - "marian": "sshleifer/tiny-marian-en-de", # hf-internal-testing ones are broken - "mbart": "hf-internal-testing/tiny-random-mbart", - "mobilebert": "hf-internal-testing/tiny-random-MobileBertModel", - "mobilenet_v1": "google/mobilenet_v1_0.75_192", - "mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model", - "mobilevit": "hf-internal-testing/tiny-random-mobilevit", - "mt5": "stas/mt5-tiny-random", - "nystromformer": "hf-internal-testing/tiny-random-NystromformerModel", - "pegasus": "hf-internal-testing/tiny-random-pegasus", - "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel", - "resnet": "hf-internal-testing/tiny-random-resnet", - "roberta": "hf-internal-testing/tiny-random-roberta", - "roformer": "hf-internal-testing/tiny-random-roformer", - "segformer": "hf-internal-testing/tiny-random-SegformerModel", - "squeezebert": "hf-internal-testing/tiny-random-squeezebert", - "stable-diffusion": "hf-internal-testing/tiny-stable-diffusion-torch", - "sew": "hf-internal-testing/tiny-random-SEWModel", - "sew_d": "hf-internal-testing/tiny-random-SEWDModel", - "swin": "hf-internal-testing/tiny-random-SwinModel", - "t5": "hf-internal-testing/tiny-random-t5", - "unispeech": "hf-internal-testing/tiny-random-unispeech", - "unispeech_sat": "hf-internal-testing/tiny-random-UnispeechSatModel", - "vit": "hf-internal-testing/tiny-random-vit", - "wavlm": "hf-internal-testing/tiny-random-WavlmModel", - "wav2vec2": "anton-l/wav2vec2-random-tiny-classifier", - "wav2vec2-conformer": "hf-internal-testing/tiny-random-wav2vec2-conformer", - "xlm": "hf-internal-testing/tiny-random-xlm", - "xlm_roberta": "hf-internal-testing/tiny-xlm-roberta", -} - - TENSOR_ALIAS_TO_TYPE = { "pt": torch.Tensor, "np": np.ndarray, @@ -224,13 +153,17 @@ def test_load_from_hub_and_save_seq2seq_model(self): def test_load_from_hub_and_save_stable_diffusion_model(self): loaded_pipeline = OVStableDiffusionPipeline.from_pretrained(self.OV_DIFFUSION_MODEL_ID, compile=False) self.assertIsInstance(loaded_pipeline.config, Dict) - prompt = "sailing ship in storm by Leonardo da Vinci" - height = 16 - width = 16 - vae_scale_factor = 4 # needed for dummy stable diffusion model + batch_size, height, width = 2, 16, 16 np.random.seed(0) - pipeline_outputs = loaded_pipeline(prompt, num_inference_steps=1, height=height, width=width, output_type="np") - self.assertEqual(pipeline_outputs.images.shape, (1, height // vae_scale_factor, width // vae_scale_factor, 3)) + inputs = { + "prompt": ["sailing ship in storm by Leonardo da Vinci"] * batch_size, + "height": height, + "width": width, + "num_inference_steps": 2, + "output_type": "np", + } + pipeline_outputs = loaded_pipeline(**inputs).images + self.assertEqual(pipeline_outputs.shape, (batch_size, height, width, 3)) with tempfile.TemporaryDirectory() as tmpdirname: loaded_pipeline.save_pretrained(tmpdirname) pipeline = OVStableDiffusionPipeline.from_pretrained(tmpdirname) @@ -246,8 +179,8 @@ def test_load_from_hub_and_save_stable_diffusion_model(self): self.assertIn(OV_XML_FILE_NAME, folder_contents) self.assertIn(OV_XML_FILE_NAME.replace(".xml", ".bin"), folder_contents) np.random.seed(0) - outputs = pipeline(prompt, num_inference_steps=1, height=height, width=width, output_type="np").images - self.assertTrue(np.array_equal(pipeline_outputs.images, outputs)) + outputs = pipeline(**inputs).images + self.assertTrue(np.array_equal(pipeline_outputs, outputs)) class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase): @@ -864,137 +797,3 @@ def test_pipeline(self, model_arch): outputs = pipe([np.random.random(16000)]) self.assertEqual(pipe.device, model.device) self.assertTrue(all(item["score"] > 0.0 for item in outputs[0])) - - -class OVStableDiffusionPipelineIntegrationTest(unittest.TestCase): - SUPPORTED_ARCHITECTURES = ("stable-diffusion",) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - @require_diffusers - def test_compare_to_diffusers(self, model_arch: str): - model_id = MODEL_NAMES[model_arch] - ov_pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True, compile=False) - self.assertIsInstance(ov_pipeline.text_encoder, OVModelTextEncoder) - self.assertIsInstance(ov_pipeline.vae_encoder, OVModelVaeEncoder) - self.assertIsInstance(ov_pipeline.vae_decoder, OVModelVaeDecoder) - self.assertIsInstance(ov_pipeline.unet, OVModelUnet) - self.assertIsInstance(ov_pipeline.config, Dict) - - from diffusers import StableDiffusionPipeline - - diffusers_pipeline = StableDiffusionPipeline.from_pretrained(model_id) - diffusers_pipeline.safety_checker = None - num_images_per_prompt, height, width, scale_factor = 1, 512, 512, 8 - latents_shape = ( - num_images_per_prompt, - diffusers_pipeline.unet.in_channels, - height // scale_factor, - width // scale_factor, - ) - latents = np.random.randn(*latents_shape).astype(np.float32) - kwargs = { - "prompt": "sailing ship in storm by Leonardo da Vinci", - "num_inference_steps": 1, - "output_type": "np", - "num_images_per_prompt": num_images_per_prompt, - "height": height, - "width": width, - } - ov_pipeline.to("cpu") - ov_pipeline.compile() - ov_outputs = ov_pipeline(latents=latents, **kwargs).images - self.assertIsInstance(ov_outputs, np.ndarray) - with torch.no_grad(): - diffusers_outputs = diffusers_pipeline(latents=torch.from_numpy(latents), **kwargs).images - # Compare model outputs - self.assertTrue(np.allclose(ov_outputs, diffusers_outputs, atol=1e-4)) - # Compare model devices - self.assertEqual(diffusers_pipeline.device.type, ov_pipeline.device) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - @require_diffusers - def test_num_images_per_prompt(self, model_arch: str): - from diffusers import DPMSolverMultistepScheduler - - model_id = MODEL_NAMES[model_arch] - scheduler = DPMSolverMultistepScheduler.from_pretrained(model_id, subfolder="scheduler") - pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True, scheduler=scheduler) - prompt = "sailing ship in storm by Leonardo da Vinci" - - for batch_size in [1, 3]: - for num_images in [1, 2]: - outputs = pipeline( - [prompt] * batch_size, num_inference_steps=2, num_images_per_prompt=num_images, output_type="np" - ) - self.assertEqual(outputs.images.shape, (batch_size * num_images, 128, 128, 3)) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - @require_diffusers - def test_num_images_per_prompt_static_model(self, model_arch: str): - model_id = MODEL_NAMES[model_arch] - batch_size = 3 - num_images_per_prompt = 4 - height = 128 - width = 64 - vae_scale_factor = 4 - prompt = "sailing ship in storm by Leonardo da Vinci" - pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True, compile=False) - pipeline.half() - pipeline.reshape( - batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images_per_prompt - ) - self.assertFalse(pipeline.is_dynamic) - pipeline.compile() - # Verify output shapes requirements not matching the static model don't impact the final outputs - outputs = pipeline( - [prompt] * batch_size, - num_inference_steps=2, - num_images_per_prompt=num_images_per_prompt, - height=width, - width=width, - output_type="np", - ).images - self.assertEqual( - outputs.shape, - (batch_size * num_images_per_prompt, height // vae_scale_factor, width // vae_scale_factor, 3), - ) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - @require_diffusers - def test_image_reproducibility(self, model_arch: str): - model_id = MODEL_NAMES[model_arch] - pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True) - - kwargs = { - "prompt": "sailing ship in storm by Leonardo da Vinci", - "output_type": "np", - "num_inference_steps": 2, - } - np.random.seed(0) - outputs_1 = pipeline(**kwargs) - np.random.seed(0) - outputs_2 = pipeline(**kwargs) - outputs_3 = pipeline(**kwargs) - - # Compare model outputs - self.assertTrue(np.array_equal(outputs_1.images[0], outputs_2.images[0])) - self.assertFalse(np.array_equal(outputs_1.images[0], outputs_3.images[0])) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - @require_diffusers - def test_height_width_properties(self, model_arch: str): - model_id = MODEL_NAMES[model_arch] - batch_size = 1 - num_images_per_prompt = 4 - height = 128 - width = 64 - pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True, compile=False, dynamic=True) - self.assertTrue(pipeline.is_dynamic) - self.assertEqual(pipeline.height, -1) - self.assertEqual(pipeline.width, -1) - pipeline.reshape( - batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images_per_prompt - ) - self.assertFalse(pipeline.is_dynamic) - self.assertEqual(pipeline.height, height) - self.assertEqual(pipeline.width, width) diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py new file mode 100644 index 0000000000..b2439c7e7c --- /dev/null +++ b/tests/openvino/test_stable_diffusion.py @@ -0,0 +1,418 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import tempfile +import unittest +from typing import Dict + +import numpy as np +import torch +from diffusers import ( + StableDiffusionPipeline, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLPipeline, +) +from diffusers.utils import floats_tensor, load_image +from parameterized import parameterized +from utils_tests import MODEL_NAMES, SEED + +from optimum.intel import ( + OVStableDiffusionImg2ImgPipeline, + OVStableDiffusionInpaintPipeline, + OVStableDiffusionPipeline, + OVStableDiffusionXLImg2ImgPipeline, + OVStableDiffusionXLPipeline, +) +from optimum.intel.openvino.modeling_diffusion import ( + OVModelTextEncoder, + OVModelUnet, + OVModelVaeDecoder, + OVModelVaeEncoder, +) +from optimum.onnxruntime import ( + ORTStableDiffusionImg2ImgPipeline, + ORTStableDiffusionInpaintPipeline, + ORTStableDiffusionXLImg2ImgPipeline, + ORTStableDiffusionXLPipeline, +) + + +def _generate_inputs(batch_size=1): + inputs = { + "prompt": ["sailing ship in storm by Leonardo da Vinci"] * batch_size, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + +def _create_image(height=128, width=128): + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ) + return image.resize((width, height)) + + +class OVStableDiffusionPipelineBaseTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = ("stable-diffusion",) + MODEL_CLASS = OVStableDiffusionPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False) + pipeline.to("cpu") + pipeline.compile() + self.assertEqual(pipeline.vae_scale_factor, 2) + self.assertEqual(pipeline.vae_decoder.config["latent_channels"], 4) + self.assertEqual(pipeline.unet.config["in_channels"], 4) + batch_size, height = 2, 128 + for width in [64, 128]: + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + for num_images in [1, 3]: + outputs = pipeline(**inputs, num_images_per_prompt=num_images).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_callback(self, model_arch: str): + MODEL_NAMES[model_arch] + + def callback_fn(step: int, timestep: int, latents: np.ndarray) -> None: + callback_fn.has_been_called = True + callback_fn.number_of_steps += 1 + + pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + callback_fn.has_been_called = False + callback_fn.number_of_steps = 0 + inputs = self.generate_inputs(height=64, width=64) + pipeline(**inputs, callback=callback_fn, callback_steps=1) + self.assertTrue(callback_fn.has_been_called) + self.assertEqual(callback_fn.number_of_steps, inputs["num_inference_steps"]) + + def generate_inputs(self, height=128, width=128, batch_size=1): + inputs = _generate_inputs(batch_size) + inputs["height"] = height + inputs["width"] = width + return inputs + + +class OVStableDiffusionImg2ImgPipelineTest(OVStableDiffusionPipelineBaseTest): + SUPPORTED_ARCHITECTURES = ("stable-diffusion",) + MODEL_CLASS = OVStableDiffusionImg2ImgPipeline + ORT_MODEL_CLASS = ORTStableDiffusionImg2ImgPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_diffusers_pipeline(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) + inputs = self.generate_inputs() + inputs["prompt"] = "A painting of a squirrel eating a burger" + + output = pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] + # https://github.com/huggingface/diffusers/blob/v0.17.1/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py#L71 + expected_slice = np.array([0.69643, 0.58484, 0.50314, 0.58760, 0.55368, 0.59643, 0.51529, 0.41217, 0.49087]) + self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-1)) + + ort_pipeline = self.ORT_MODEL_CLASS.from_pretrained(model_id, export=True) + ort_output = ort_pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] + self.assertTrue(np.allclose(output, ort_output, atol=1e-1)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt_static_model(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) + batch_size, num_images, height, width = 2, 3, 128, 64 + pipeline.half() + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + def generate_inputs(self, height=128, width=128, batch_size=1): + inputs = _generate_inputs(batch_size) + inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) + inputs["strength"] = 0.75 + return inputs + + +class OVStableDiffusionPipelineTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = ("stable-diffusion",) + MODEL_CLASS = OVStableDiffusionPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_to_diffusers(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + ov_pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) + self.assertIsInstance(ov_pipeline.text_encoder, OVModelTextEncoder) + self.assertIsInstance(ov_pipeline.vae_encoder, OVModelVaeEncoder) + self.assertIsInstance(ov_pipeline.vae_decoder, OVModelVaeDecoder) + self.assertIsInstance(ov_pipeline.unet, OVModelUnet) + self.assertIsInstance(ov_pipeline.config, Dict) + + pipeline = StableDiffusionPipeline.from_pretrained(MODEL_NAMES[model_arch]) + pipeline.safety_checker = None + batch_size, num_images_per_prompt, height, width = 1, 2, 64, 64 + + latents = ov_pipeline.prepare_latents( + batch_size * num_images_per_prompt, + ov_pipeline.unet.config["in_channels"], + height, + width, + dtype=np.float32, + generator=np.random.RandomState(0), + ) + + kwargs = { + "prompt": "sailing ship in storm by Leonardo da Vinci", + "num_inference_steps": 1, + "num_images_per_prompt": num_images_per_prompt, + "height": height, + "width": width, + "guidance_rescale": 0.1, + } + + for output_type in ["latent", "np"]: + ov_outputs = ov_pipeline(latents=latents, output_type=output_type, **kwargs).images + self.assertIsInstance(ov_outputs, np.ndarray) + with torch.no_grad(): + outputs = pipeline(latents=torch.from_numpy(latents), output_type=output_type, **kwargs).images + # Compare model outputs + self.assertTrue(np.allclose(ov_outputs, outputs, atol=1e-4)) + + # Compare model devices + self.assertEqual(pipeline.device.type, ov_pipeline.device) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_image_reproducibility(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) + inputs = _generate_inputs() + height, width = 64, 64 + np.random.seed(0) + ov_outputs_1 = pipeline(**inputs, height=height, width=width) + np.random.seed(0) + ov_outputs_2 = pipeline(**inputs, height=height, width=width) + ov_outputs_3 = pipeline(**inputs, height=height, width=width) + # Compare model outputs + self.assertTrue(np.array_equal(ov_outputs_1.images[0], ov_outputs_2.images[0])) + self.assertFalse(np.array_equal(ov_outputs_1.images[0], ov_outputs_3.images[0])) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt_static_model(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False) + batch_size, num_images, height, width = 3, 4, 128, 64 + prompt = "sailing ship in storm by Leonardo da Vinci" + pipeline.half() + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + self.assertFalse(pipeline.is_dynamic) + pipeline.compile() + # Verify output shapes requirements not matching the static model don't impact the final outputs + outputs = pipeline( + [prompt] * batch_size, + num_inference_steps=2, + num_images_per_prompt=num_images, + height=height + 8, + width=width, + output_type="np", + ).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_height_width_properties(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + batch_size, num_images, height, width = 2, 4, 128, 64 + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=True) + self.assertTrue(pipeline.is_dynamic) + self.assertEqual(pipeline.height, -1) + self.assertEqual(pipeline.width, -1) + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + self.assertFalse(pipeline.is_dynamic) + self.assertEqual(pipeline.height, height) + self.assertEqual(pipeline.width, width) + + +class OVStableDiffusionInpaintPipelineTest(OVStableDiffusionPipelineBaseTest): + SUPPORTED_ARCHITECTURES = ("stable-diffusion",) + MODEL_CLASS = OVStableDiffusionInpaintPipeline + ORT_MODEL_CLASS = ORTStableDiffusionInpaintPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_diffusers_pipeline(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) + batch_size, num_images, height, width = 1, 1, 64, 64 + latents = pipeline.prepare_latents( + batch_size * num_images, + pipeline.unet.config["in_channels"], + height, + width, + dtype=np.float32, + generator=np.random.RandomState(0), + ) + inputs = self.generate_inputs(height=height, width=width) + outputs = pipeline(**inputs, latents=latents).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + ort_pipeline = self.ORT_MODEL_CLASS.from_pretrained(model_id, export=True) + ort_outputs = ort_pipeline(**inputs, latents=latents).images + self.assertTrue(np.allclose(outputs, ort_outputs, atol=1e-1)) + + expected_slice = np.array([0.4692, 0.5260, 0.4005, 0.3609, 0.3259, 0.4676, 0.5593, 0.4728, 0.4411]) + self.assertTrue(np.allclose(outputs[0, -3:, -3:, -1].flatten(), expected_slice, atol=1e-1)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt_static_model(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) + batch_size, num_images, height, width = 1, 3, 128, 64 + pipeline.half() + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + def generate_inputs(self, height=128, width=128, batch_size=1): + inputs = super(OVStableDiffusionInpaintPipelineTest, self).generate_inputs(height, width, batch_size) + inputs["image"] = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ).resize((width, height)) + + inputs["mask_image"] = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + ).resize((width, height)) + + return inputs + + +class OVtableDiffusionXLPipelineTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = ("stable-diffusion-xl",) + MODEL_CLASS = OVStableDiffusionXLPipeline + ORT_MODEL_CLASS = ORTStableDiffusionXLPipeline + PT_MODEL_CLASS = StableDiffusionXLPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_to_diffusers(self, model_arch: str): + ov_pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + self.assertIsInstance(ov_pipeline.text_encoder, OVModelTextEncoder) + self.assertIsInstance(ov_pipeline.text_encoder_2, OVModelTextEncoder) + self.assertIsInstance(ov_pipeline.vae_encoder, OVModelVaeEncoder) + self.assertIsInstance(ov_pipeline.vae_decoder, OVModelVaeDecoder) + self.assertIsInstance(ov_pipeline.unet, OVModelUnet) + self.assertIsInstance(ov_pipeline.config, Dict) + + pipeline = self.PT_MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch]) + batch_size, num_images_per_prompt, height, width = 2, 3, 64, 128 + latents = ov_pipeline.prepare_latents( + batch_size * num_images_per_prompt, + ov_pipeline.unet.config["in_channels"], + height, + width, + dtype=np.float32, + generator=np.random.RandomState(0), + ) + + kwargs = { + "prompt": ["sailing ship in storm by Leonardo da Vinci"] * batch_size, + "num_inference_steps": 1, + "num_images_per_prompt": num_images_per_prompt, + "height": height, + "width": width, + "guidance_rescale": 0.1, + } + + for output_type in ["latent", "np"]: + ov_outputs = ov_pipeline(latents=latents, output_type=output_type, **kwargs).images + + self.assertIsInstance(ov_outputs, np.ndarray) + with torch.no_grad(): + outputs = pipeline(latents=torch.from_numpy(latents), output_type=output_type, **kwargs).images + + # Compare model outputs + self.assertTrue(np.allclose(ov_outputs, outputs, atol=1e-4)) + # Compare model devices + self.assertEqual(pipeline.device.type, ov_pipeline.device) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_image_reproducibility(self, model_arch: str): + pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + batch_size, num_images_per_prompt, height, width = 2, 3, 64, 128 + + inputs = _generate_inputs(batch_size) + np.random.seed(0) + ov_outputs_1 = pipeline(**inputs, height=height, width=width, num_images_per_prompt=num_images_per_prompt) + np.random.seed(0) + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline.save_pretrained(tmp_dir) + pipeline = self.MODEL_CLASS.from_pretrained(tmp_dir) + ov_outputs_2 = pipeline(**inputs, height=height, width=width, num_images_per_prompt=num_images_per_prompt) + ov_outputs_3 = pipeline(**inputs, height=height, width=width, num_images_per_prompt=num_images_per_prompt) + self.assertTrue(np.array_equal(ov_outputs_1.images[0], ov_outputs_2.images[0])) + self.assertFalse(np.array_equal(ov_outputs_1.images[0], ov_outputs_3.images[0])) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt_static_model(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False) + batch_size, num_images, height, width = 3, 4, 128, 64 + pipeline.half() + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + self.assertFalse(pipeline.is_dynamic) + pipeline.compile() + # Verify output shapes requirements not matching the static model don't impact the final outputs + inputs = _generate_inputs(batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, height=height, width=width).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + +class OVStableDiffusionXLImg2ImgPipelineTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = ("stable-diffusion-xl",) + MODEL_CLASS = OVStableDiffusionXLImg2ImgPipeline + ORT_MODEL_CLASS = ORTStableDiffusionXLImg2ImgPipeline + PT_MODEL_CLASS = StableDiffusionXLImg2ImgPipeline + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_inference(self, model_arch: str): + pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline.save_pretrained(tmp_dir) + pipeline = self.MODEL_CLASS.from_pretrained(tmp_dir) + + inputs = self.generate_inputs() + output = pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] + expected_slice = np.array([0.6515, 0.5405, 0.4858, 0.5632, 0.5174, 0.5681, 0.4948, 0.4253, 0.5080]) + self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-1)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_num_images_per_prompt_static_model(self, model_arch: str): + model_id = MODEL_NAMES[model_arch] + pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) + batch_size, num_images, height, width = 2, 3, 128, 64 + pipeline.half() + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + def generate_inputs(self, height=128, width=128, batch_size=1): + inputs = _generate_inputs(batch_size) + inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) + inputs["strength"] = 0.75 + return inputs diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py new file mode 100644 index 0000000000..796432271b --- /dev/null +++ b/tests/openvino/utils_tests.py @@ -0,0 +1,91 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch + + +MODEL_NAMES = { + "albert": "hf-internal-testing/tiny-random-albert", + "audio_spectrogram_transformer": "Ericwang/tiny-random-ast", + "beit": "hf-internal-testing/tiny-random-BeitForImageClassification", + "bert": "hf-internal-testing/tiny-random-bert", + "bart": "hf-internal-testing/tiny-random-bart", + "bigbird_pegasus": "hf-internal-testing/tiny-random-bigbird_pegasus", + "blenderbot-small": "hf-internal-testing/tiny-random-BlenderbotModel", + "blenderbot": "hf-internal-testing/tiny-random-blenderbot", + "bloom": "hf-internal-testing/tiny-random-BloomModel", + "camembert": "hf-internal-testing/tiny-random-camembert", + "convbert": "hf-internal-testing/tiny-random-ConvBertForSequenceClassification", + "codegen": "hf-internal-testing/tiny-random-CodeGenForCausalLM", + "data2vec_text": "hf-internal-testing/tiny-random-Data2VecTextModel", + "data2vec_vision": "hf-internal-testing/tiny-random-Data2VecVisionModel", + "data2vec_audio": "hf-internal-testing/tiny-random-Data2VecAudioModel", + "deberta": "hf-internal-testing/tiny-random-deberta", + "deberta_v2": "hf-internal-testing/tiny-random-DebertaV2Model", + "deit": "hf-internal-testing/tiny-random-deit", + "convnext": "hf-internal-testing/tiny-random-convnext", + "distilbert": "hf-internal-testing/tiny-random-distilbert", + "electra": "hf-internal-testing/tiny-random-electra", + "flaubert": "hf-internal-testing/tiny-random-flaubert", + # "gpt_bigcode": "bigcode/tiny_starcoder_py", + "gpt2": "hf-internal-testing/tiny-random-gpt2", + "gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel", + "gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "gptj": "hf-internal-testing/tiny-random-GPTJModel", + "hubert": "hf-internal-testing/tiny-random-HubertModel", + "ibert": "hf-internal-testing/tiny-random-ibert", + "levit": "hf-internal-testing/tiny-random-LevitModel", + "longt5": "hf-internal-testing/tiny-random-longt5", + "llama": "fxmarty/tiny-llama-fast-tokenizer", + "m2m_100": "hf-internal-testing/tiny-random-m2m_100", + "opt": "hf-internal-testing/tiny-random-OPTModel", + "marian": "sshleifer/tiny-marian-en-de", # hf-internal-testing ones are broken + "mbart": "hf-internal-testing/tiny-random-mbart", + "mobilebert": "hf-internal-testing/tiny-random-MobileBertModel", + "mobilenet_v1": "google/mobilenet_v1_0.75_192", + "mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model", + "mobilevit": "hf-internal-testing/tiny-random-mobilevit", + "mt5": "stas/mt5-tiny-random", + "nystromformer": "hf-internal-testing/tiny-random-NystromformerModel", + "pegasus": "hf-internal-testing/tiny-random-pegasus", + "poolformer": "hf-internal-testing/tiny-random-PoolFormerModel", + "resnet": "hf-internal-testing/tiny-random-resnet", + "roberta": "hf-internal-testing/tiny-random-roberta", + "roformer": "hf-internal-testing/tiny-random-roformer", + "segformer": "hf-internal-testing/tiny-random-SegformerModel", + "squeezebert": "hf-internal-testing/tiny-random-squeezebert", + "stable-diffusion": "hf-internal-testing/tiny-stable-diffusion-torch", + "stable-diffusion-xl": "echarlaix/tiny-random-stable-diffusion-xl", + "sew": "hf-internal-testing/tiny-random-SEWModel", + "sew_d": "hf-internal-testing/tiny-random-SEWDModel", + "swin": "hf-internal-testing/tiny-random-SwinModel", + "t5": "hf-internal-testing/tiny-random-t5", + "unispeech": "hf-internal-testing/tiny-random-unispeech", + "unispeech_sat": "hf-internal-testing/tiny-random-UnispeechSatModel", + "vit": "hf-internal-testing/tiny-random-vit", + "wavlm": "hf-internal-testing/tiny-random-WavlmModel", + "wav2vec2": "anton-l/wav2vec2-random-tiny-classifier", + "wav2vec2-conformer": "hf-internal-testing/tiny-random-wav2vec2-conformer", + "xlm": "hf-internal-testing/tiny-random-xlm", + "xlm_roberta": "hf-internal-testing/tiny-xlm-roberta", +} + + +TENSOR_ALIAS_TO_TYPE = { + "pt": torch.Tensor, + "np": np.ndarray, +} + +SEED = 42 From bd19cbe4df9c4b9c02c86d88bafa9cd50c2378af Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:39:11 +0200 Subject: [PATCH 071/134] Add SD XL documentation (#380) --- docs/source/inference.mdx | 84 ++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index a73f837ead..7611308386 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -38,7 +38,7 @@ To easily save the resulting model, you can use the `save_pretrained()` method, ```python # Save the exported model -save_directory = "a_local_path" +save_directory = "openvino_distilbert" model.save_pretrained(save_directory) tokenizer.save_pretrained(save_directory) ``` @@ -121,7 +121,7 @@ text = "He never went out without a book under his arm, and he often came back w result = translation_pipe(text) # Save the exported model -save_directory = "a_local_path" +save_directory = "openvino_t5" model.save_pretrained(save_directory) tokenizer.save_pretrained(save_directory) @@ -151,18 +151,18 @@ Here is an example of how you can load an OpenVINO Stable Diffusion model and ru from optimum.intel import OVStableDiffusionPipeline model_id = "echarlaix/stable-diffusion-v1-5-openvino" -stable_diffusion = OVStableDiffusionPipeline.from_pretrained(model_id) +pipeline = OVStableDiffusionPipeline.from_pretrained(model_id) prompt = "sailing ship in storm by Rembrandt" -images = stable_diffusion(prompt).images +images = pipeline(prompt).images ``` To load your PyTorch model and convert it to OpenVINO on-the-fly, you can set `export=True`. ```python model_id = "runwayml/stable-diffusion-v1-5" -stable_diffusion = OVStableDiffusionPipeline.from_pretrained(model_id, export=True) +pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=True) # Don't forget to save the exported model -stable_diffusion.save_pretrained("a_local_path") +pipeline.save_pretrained("openvino-sd-v1-5") ``` To further speed up inference, the model can be statically reshaped : @@ -175,12 +175,12 @@ height = 512 width = 512 # Statically reshape the model -stable_diffusion.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images_per_prompt) +pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images_per_prompt) # Compile the model before the first inference -stable_diffusion.compile() +pipeline.compile() # Run inference -images = stable_diffusion(prompt, height=height, width=width, num_images_per_prompt=num_images_per_prompt).images +images = pipeline(prompt, height=height, width=width, num_images_per_prompt=num_images_per_prompt).images ``` In case you want to change any parameters such as the outputs height or width, you'll need to statically reshape your model once again. @@ -200,6 +200,7 @@ from optimum.intel import OVStableDiffusionImg2ImgPipeline model_id = "runwayml/stable-diffusion-v1-5" pipeline = OVStableDiffusionImg2ImgPipeline.from_pretrained(model_id, export=True) + url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" response = requests.get(url) init_image = Image.open(BytesIO(response.content)).convert("RGB") @@ -210,6 +211,71 @@ image.save("fantasy_landscape.png") ``` +## Stable Diffusion XL + +Before using `OVtableDiffusionXLPipeline` make sure to have `diffusers` and `invisible_watermark` installed. You can install the libraries as follows: + +```bash +pip install diffusers +pip install invisible-watermark>=2.0 +``` + +### Text-to-Image + +Here is an example of how you can load a PyTorch SDXL model, convert it to the adapted format on-the-fly and run inference with OpenVINO Runtime: + +```python +from optimum.intel import OVtableDiffusionXLPipeline + +model_id = "stabilityai/stable-diffusion-xl-base-0.9" +base = OVtableDiffusionXLPipeline.from_pretrained(model_id, export=True) +prompt = "train station by Caspar David Friedrich" +image = base(prompt).images[0] + +# Don't forget to save your OpenVINO model +base.save_pretrained("openvino-sd-xl-base-0.9") +``` + +| | | +|---|---| +| ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/train_station_friedrich.png) | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/train_station_friedrich_2.png) | + + +### Image-to-Image + +You can use SDXL as follows for *image-to-image*: + +```python +from optimum.intel import OVStableDiffusionXLImg2ImgPipeline +from diffusers.utils import load_image + +model_id = "stabilityai/stable-diffusion-xl-refiner-0.9" +pipeline = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(model_id, export=True) + +url = "https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/castle_friedrich.png" +image = load_image(url).convert("RGB") +prompt = "medieval castle by Caspar David Friedrich" +image = pipeline(prompt, image=image).images[0] +image.save("medieval_castle.png") +``` + + +### Refining the image output + +The image can be refined by making use of a model like [stabilityai/stable-diffusion-xl-refiner-0.9](https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-0.9). In this case, you only have to output the latents from the base model. + + +```python +from optimum.intel import OVStableDiffusionXLImg2ImgPipeline + +model_id = "stabilityai/stable-diffusion-xl-refiner-0.9" +refiner = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(model_id, export=True) + +image = base(prompt=prompt, output_type="latent").images[0] +image = refiner(prompt=prompt, image=image[None, :]).images[0] +``` + + ## Supported tasks From c3895e4ebbe059227c6e1c3f55bb92ea97c6f7a0 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Tue, 25 Jul 2023 18:14:18 +0200 Subject: [PATCH 072/134] Dev version --- optimum/intel/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/version.py b/optimum/intel/version.py index 65db31ad88..44992771bc 100644 --- a/optimum/intel/version.py +++ b/optimum/intel/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.9.1.dev0" +__version__ = "1.10.1.dev0" From 9f1cd585acf121734f0e77a37e74a229c547ab5d Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 25 Jul 2023 18:18:26 +0200 Subject: [PATCH 073/134] Fix min optimum version (#382) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b07a48896..357ebd3b66 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ assert False, "Error: Could not open '%s' due %s\n" % (filepath, error) INSTALL_REQUIRE = [ - "optimum>=1.8.8", + "optimum>=1.10.0", "transformers>=4.20.0", "datasets>=1.4.0", "sentencepiece", From a3efab348ac85c9ac1540b575cff05a866f93cb3 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:57:52 +0200 Subject: [PATCH 074/134] Fix compilation openvino stable diffusion (#384) --- docs/source/inference.mdx | 2 +- optimum/intel/openvino/modeling_diffusion.py | 2 +- tests/openvino/test_stable_diffusion.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 7611308386..ed611402b7 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -297,4 +297,4 @@ As shown in the table below, each task is associated with a class enabling to au | `text-to-image` | `OVStableDiffusionXLPipeline` | | `image-to-image` | `OVStableDiffusionImg2ImgPipeline` | | `image-to-image` | `OVStableDiffusionXLImg2ImgPipeline` | -| `inpaint` | `OVStableDiffusionPipeline` | +| `inpaint` | `OVStableDiffusionInpaintPipeline` | diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index a0e7d7fbdc..04a16699b3 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -470,7 +470,7 @@ def compile(self): self.unet._compile() for component in {self.text_encoder, self.text_encoder_2, self.vae_encoder}: if component is not None: - component._compile + component._compile() @classmethod def _load_config(cls, config_name_or_path: Union[str, os.PathLike], **kwargs): diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py index b2439c7e7c..99195bc1cb 100644 --- a/tests/openvino/test_stable_diffusion.py +++ b/tests/openvino/test_stable_diffusion.py @@ -25,6 +25,7 @@ StableDiffusionXLPipeline, ) from diffusers.utils import floats_tensor, load_image +from openvino.runtime.ie_api import CompiledModel from parameterized import parameterized from utils_tests import MODEL_NAMES, SEED @@ -352,8 +353,12 @@ def test_compare_to_diffusers(self, model_arch: str): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_image_reproducibility(self, model_arch: str): pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) - batch_size, num_images_per_prompt, height, width = 2, 3, 64, 128 + # Verify every subcomponent is compiled by default + for component in {"unet", "vae_encoder", "vae_decoder", "text_encoder", "text_encoder_2"}: + self.assertIsInstance(getattr(pipeline, component).request, CompiledModel) + + batch_size, num_images_per_prompt, height, width = 2, 3, 64, 128 inputs = _generate_inputs(batch_size) np.random.seed(0) ov_outputs_1 = pipeline(**inputs, height=height, width=width, num_images_per_prompt=num_images_per_prompt) From d679ab842fd51956553451dd45d2ee173045d50b Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Wed, 26 Jul 2023 16:59:36 +0200 Subject: [PATCH 075/134] Dev version --- optimum/intel/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/version.py b/optimum/intel/version.py index 44992771bc..09c74e091a 100644 --- a/optimum/intel/version.py +++ b/optimum/intel/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.10.1.dev0" +__version__ = "1.10.2.dev0" From b68abe8e8d9dd8a12dddfa9c733c66d517379513 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:51:51 +0200 Subject: [PATCH 076/134] remove optimum install from source in tests (#381) --- .github/workflows/test_openvino.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index a76a1c0ecd..c8a7516797 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -30,7 +30,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install git+https://github.com/huggingface/optimum.git pip install .[openvino,nncf,tests,diffusers] - name: Test with Pytest run: | From 94fc0dcb14cf7e65ac4e58d1f31af94fcbbc6c7c Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:52:07 +0200 Subject: [PATCH 077/134] Add upgrade strategy installation instruction in doc (#383) --- README.md | 6 ++++-- docs/source/installation.mdx | 16 ++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cf183c4a4a..93cd3cf76c 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,10 @@ To install the latest release of 🤗 Optimum Intel with the corresponding requi | Accelerator | Installation | |:-----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------| -| [Intel Neural Compressor](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html) | `python -m pip install "optimum[neural-compressor]"` | -| [OpenVINO](https://docs.openvino.ai/latest/index.html) | `python -m pip install "optimum[openvino,nncf]"` | +| [Intel Neural Compressor](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html) | `pip install --upgrade-strategy eager "optimum[neural-compressor]"` | +| [OpenVINO](https://docs.openvino.ai/latest/index.html) | `pip install --upgrade-strategy eager "optimum[openvino,nncf]"` | + +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. We recommend creating a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) and upgrading pip with `python -m pip install --upgrade pip`. diff --git a/docs/source/installation.mdx b/docs/source/installation.mdx index 3bad5e6886..cf8688d105 100644 --- a/docs/source/installation.mdx +++ b/docs/source/installation.mdx @@ -18,13 +18,17 @@ limitations under the License. To install the latest release of 🤗 Optimum Intel with the corresponding required dependencies, you can do respectively: -| Accelerator | Installation | -|:-----------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------| -| [Intel Neural Compressor (INC)](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html) | `python -m pip install "optimum[neural-compressor]"`| -| [Intel OpenVINO](https://docs.openvino.ai/latest/index.html) | `python -m pip install "optimum[openvino,nncf]"` | +| Accelerator | Installation | +|:-----------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------| +| [Intel Neural Compressor (INC)](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html) | `pip install --upgrade-strategy eager "optimum[neural-compressor]"`| +| [Intel OpenVINO](https://docs.openvino.ai/latest/index.html) | `pip install --upgrade-strategy eager "optimum[openvino,nncf]"` | -We recommend creating a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) and upgrading -pip with `python -m pip install --upgrade pip`. +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. + +We recommend creating a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) and upgrading pip with : +```bash +python -m pip install --upgrade pip +``` Optimum Intel is a fast-moving project, and you may want to install from source with the following command: From 1db9bf4f1658c65a5aea5eb1a062ee7bd7d3e9d0 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:44:34 +0200 Subject: [PATCH 078/134] Modify model example documentation (#387) --- docs/source/inference.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index ed611402b7..9b181d0ad9 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -227,13 +227,13 @@ Here is an example of how you can load a PyTorch SDXL model, convert it to the a ```python from optimum.intel import OVtableDiffusionXLPipeline -model_id = "stabilityai/stable-diffusion-xl-base-0.9" +model_id = "stabilityai/stable-diffusion-xl-base-1.0" base = OVtableDiffusionXLPipeline.from_pretrained(model_id, export=True) prompt = "train station by Caspar David Friedrich" image = base(prompt).images[0] # Don't forget to save your OpenVINO model -base.save_pretrained("openvino-sd-xl-base-0.9") +base.save_pretrained("openvino-sd-xl-base-1.0") ``` | | | @@ -249,7 +249,7 @@ You can use SDXL as follows for *image-to-image*: from optimum.intel import OVStableDiffusionXLImg2ImgPipeline from diffusers.utils import load_image -model_id = "stabilityai/stable-diffusion-xl-refiner-0.9" +model_id = "stabilityai/stable-diffusion-xl-refiner-1.0" pipeline = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(model_id, export=True) url = "https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/castle_friedrich.png" @@ -262,13 +262,13 @@ image.save("medieval_castle.png") ### Refining the image output -The image can be refined by making use of a model like [stabilityai/stable-diffusion-xl-refiner-0.9](https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-0.9). In this case, you only have to output the latents from the base model. +The image can be refined by making use of a model like [stabilityai/stable-diffusion-xl-refiner-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0). In this case, you only have to output the latents from the base model. ```python from optimum.intel import OVStableDiffusionXLImg2ImgPipeline -model_id = "stabilityai/stable-diffusion-xl-refiner-0.9" +model_id = "stabilityai/stable-diffusion-xl-refiner-1.0" refiner = OVStableDiffusionXLImg2ImgPipeline.from_pretrained(model_id, export=True) image = base(prompt=prompt, output_type="latent").images[0] From c238d0290b545f3e9688a53dedc61da790937630 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Thu, 27 Jul 2023 22:08:11 +0800 Subject: [PATCH 079/134] quantization will access the modified model; disable the function when quantization --- optimum/intel/__init__.py | 2 ++ optimum/intel/openvino/__init__.py | 2 +- optimum/intel/openvino/modeling_decoder.py | 23 ++++++++++++++++++---- tests/openvino/test_quantization.py | 6 +++--- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index e465240bb9..bc50c12f11 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -74,6 +74,7 @@ [ "OVModelForAudioClassification", "OVModelForCausalLM", + "OVModelForCausalLMDisablePastKVOpt", "OVModelForFeatureExtraction", "OVModelForImageClassification", "OVModelForMaskedLM", @@ -151,6 +152,7 @@ from .openvino import ( OVModelForAudioClassification, OVModelForCausalLM, + OVModelForCausalLMDisablePastKVOpt, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 3f8737e1b9..1bcdf03c56 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -42,7 +42,7 @@ OVModelForSequenceClassification, OVModelForTokenClassification, ) -from .modeling_decoder import OVModelForCausalLM +from .modeling_decoder import OVModelForCausalLM, OVModelForCausalLMDisablePastKVOpt from .modeling_seq2seq import OVModelForSeq2SeqLM diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index def6313137..d2cf61ae49 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -95,6 +95,8 @@ """, ) class OVBaseDecoderModel(OVModel): + disable_pastkv_lp_opt = False + def __init__( self, model: openvino.runtime.Model, @@ -110,7 +112,7 @@ def __init__( "`dynamic_shapes` was set to `False` but static shapes are not supported for causal language model. Please set `dynamic_shapes=True`." ) - model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device) + model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, dynamic_shapes) super().__init__( model, @@ -140,7 +142,11 @@ def __init__( ) def _try_modify_pastkv_to_lowprecision( - self, model: openvino.runtime.Model, ov_config: Optional[Dict[str, str]] = None, device: str = "CPU" + self, + model: openvino.runtime.Model, + ov_config: Optional[Dict[str, str]] = None, + device: str = "CPU", + dynamic_shapes: bool = True, ): device = device.upper() pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") @@ -155,7 +161,7 @@ def _try_modify_pastkv_to_lowprecision( use_cache = any("past_key_values" in key.get_any_name() for key in model.inputs) self.model_org = model - if pastkv_will_use != Type.f32 and use_cache: + if pastkv_will_use != Type.f32 and use_cache and self.disable_pastkv_lp_opt is False: ppp = PrePostProcessor(model.clone()) need_gen = False for key in model.inputs: @@ -168,6 +174,10 @@ def _try_modify_pastkv_to_lowprecision( ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) if need_gen: model = ppp.build() + if dynamic_shapes: + height = -1 if self.export_feature == "image-classification" else None + width = -1 if self.export_feature == "image-classification" else None + self.model_org = self._reshape(self.model_org, -1, -1, height, width) return model @@ -181,7 +191,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): The directory where to save the model files. """ dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(self.model_org, dst_path) + openvino.runtime.serialize(self.model if self.disable_pastkv_lp_opt else self.model_org, dst_path) @classmethod def _from_transformers( @@ -473,3 +483,8 @@ def _convert_to_standard_cache( def can_generate(self): """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" return True + + +# bf16 may be in used when quantization, disable pastkv lower precision optimization +class OVModelForCausalLMDisablePastKVOpt(OVModelForCausalLM): + disable_pastkv_lp_opt = True diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 18fbd2107e..3b3d5f640e 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -36,7 +36,7 @@ OVConfig, OVModelForQuestionAnswering, OVModelForSequenceClassification, - OVModelForCausalLM, + OVModelForCausalLMDisablePastKVOpt, OVModelForTokenClassification, OVQuantizer, OVTrainer, @@ -65,7 +65,7 @@ class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 42, 32), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 41, 21), + (OVModelForCausalLMDisablePastKVOpt, "hf-internal-testing/tiny-random-gpt2", 41, 21), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) @@ -147,7 +147,7 @@ class OVWeightCompressionTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 39), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 5), + (OVModelForCausalLMDisablePastKVOpt, "hf-internal-testing/tiny-random-gpt2", 5), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) From cf1a87660dc17e6c0551d66fab0522300d952a78 Mon Sep 17 00:00:00 2001 From: Luo Cheng Date: Thu, 27 Jul 2023 22:37:01 +0800 Subject: [PATCH 080/134] apply review comments --- optimum/intel/__init__.py | 4 +- optimum/intel/openvino/__init__.py | 2 +- optimum/intel/openvino/modeling_decoder.py | 43 +++++++++++----------- tests/openvino/test_quantization.py | 6 +-- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index bc50c12f11..edfa4d0342 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -74,7 +74,7 @@ [ "OVModelForAudioClassification", "OVModelForCausalLM", - "OVModelForCausalLMDisablePastKVOpt", + "OVModelForCausalLMDisablePKVOpt", "OVModelForFeatureExtraction", "OVModelForImageClassification", "OVModelForMaskedLM", @@ -152,7 +152,7 @@ from .openvino import ( OVModelForAudioClassification, OVModelForCausalLM, - OVModelForCausalLMDisablePastKVOpt, + OVModelForCausalLMDisablePKVOpt, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 1bcdf03c56..1b056930e8 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -42,7 +42,7 @@ OVModelForSequenceClassification, OVModelForTokenClassification, ) -from .modeling_decoder import OVModelForCausalLM, OVModelForCausalLMDisablePastKVOpt +from .modeling_decoder import OVModelForCausalLM, OVModelForCausalLMDisablePKVOpt from .modeling_seq2seq import OVModelForSeq2SeqLM diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index d2cf61ae49..aecb629dec 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -95,7 +95,7 @@ """, ) class OVBaseDecoderModel(OVModel): - disable_pastkv_lp_opt = False + disable_pkv_lp_opt = False def __init__( self, @@ -149,35 +149,37 @@ def _try_modify_pastkv_to_lowprecision( dynamic_shapes: bool = True, ): device = device.upper() - pastkv_will_use = core.get_property(device, "INFERENCE_PRECISION_HINT") + pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if ov_config: - user_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") - user_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") - if user_precision_hint in STR_TO_OV_TYPE: - pastkv_will_use = STR_TO_OV_TYPE[user_precision_hint] - elif user_mode_hint.upper() == "ACCURACY": - pastkv_will_use = Type.f32 + config_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") + config_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") + if config_precision_hint in STR_TO_OV_TYPE: + pkv_precision = STR_TO_OV_TYPE[config_precision_hint] + elif config_mode_hint.upper() == "ACCURACY": + pkv_precision = Type.f32 use_cache = any("past_key_values" in key.get_any_name() for key in model.inputs) - self.model_org = model - if pastkv_will_use != Type.f32 and use_cache and self.disable_pastkv_lp_opt is False: + self._original_model = model + self._pkv_precision = Type.f32 + if pkv_precision != Type.f32 and use_cache and self.disable_pkv_lp_opt is False: ppp = PrePostProcessor(model.clone()) need_gen = False for key in model.inputs: - if "past_key_values" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): need_gen = True - ppp.input(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + ppp.input(key.get_any_name()).tensor().set_element_type(pkv_precision) for key in model.outputs: - if "present" in key.get_any_name() and pastkv_will_use != key.get_element_type(): + if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): need_gen = True - ppp.output(key.get_any_name()).tensor().set_element_type(pastkv_will_use) + ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) if need_gen: model = ppp.build() + self._pkv_precision = pkv_precision if dynamic_shapes: height = -1 if self.export_feature == "image-classification" else None width = -1 if self.export_feature == "image-classification" else None - self.model_org = self._reshape(self.model_org, -1, -1, height, width) + self._original_model = self._reshape(self._original_model, -1, -1, height, width) return model @@ -191,7 +193,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): The directory where to save the model files. """ dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(self.model if self.disable_pastkv_lp_opt else self.model_org, dst_path) + openvino.runtime.serialize(self.model if self.disable_pkv_lp_opt else self._original_model, dst_path) @classmethod def _from_transformers( @@ -329,9 +331,8 @@ def forward( inputs = {} if past_key_values is not None: - pastkv_will_use = self.model.input(self.key_value_input_names[0]).get_element_type() - if pastkv_will_use == Type.bf16: - # numpy does not support bf16, pretending u16, should change to bf16 + if self._pkv_precision == Type.bf16: + # numpy does not support bf16, pretending f16, should change to bf16 past_key_values = tuple( Tensor(past_key_value, past_key_value.shape, Type.bf16) for pkv_per_layer in past_key_values @@ -486,5 +487,5 @@ def can_generate(self): # bf16 may be in used when quantization, disable pastkv lower precision optimization -class OVModelForCausalLMDisablePastKVOpt(OVModelForCausalLM): - disable_pastkv_lp_opt = True +class OVModelForCausalLMDisablePKVOpt(OVModelForCausalLM): + disable_pkv_lp_opt = True diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 3b3d5f640e..0696642e0c 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -36,7 +36,7 @@ OVConfig, OVModelForQuestionAnswering, OVModelForSequenceClassification, - OVModelForCausalLMDisablePastKVOpt, + OVModelForCausalLMDisablePKVOpt, OVModelForTokenClassification, OVQuantizer, OVTrainer, @@ -65,7 +65,7 @@ class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 42, 32), - (OVModelForCausalLMDisablePastKVOpt, "hf-internal-testing/tiny-random-gpt2", 41, 21), + (OVModelForCausalLMDisablePKVOpt, "hf-internal-testing/tiny-random-gpt2", 41, 21), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) @@ -147,7 +147,7 @@ class OVWeightCompressionTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 39), - (OVModelForCausalLMDisablePastKVOpt, "hf-internal-testing/tiny-random-gpt2", 5), + (OVModelForCausalLMDisablePKVOpt, "hf-internal-testing/tiny-random-gpt2", 5), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) From 38cc327529943bc5e91ba0cb7b2c3d3667bc6cb7 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Fri, 28 Jul 2023 12:22:41 +0400 Subject: [PATCH 081/134] fix typos in SDXL inference docs (#389) --- docs/source/inference.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 9b181d0ad9..34253716e8 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -217,7 +217,7 @@ Before using `OVtableDiffusionXLPipeline` make sure to have `diffusers` and `inv ```bash pip install diffusers -pip install invisible-watermark>=2.0 +pip install invisible-watermark>=0.2.0 ``` ### Text-to-Image @@ -225,10 +225,10 @@ pip install invisible-watermark>=2.0 Here is an example of how you can load a PyTorch SDXL model, convert it to the adapted format on-the-fly and run inference with OpenVINO Runtime: ```python -from optimum.intel import OVtableDiffusionXLPipeline +from optimum.intel import OVStableDiffusionXLPipeline model_id = "stabilityai/stable-diffusion-xl-base-1.0" -base = OVtableDiffusionXLPipeline.from_pretrained(model_id, export=True) +base = OVStableDiffusionXLPipeline.from_pretrained(model_id, export=True) prompt = "train station by Caspar David Friedrich" image = base(prompt).images[0] From 097bd9f2d54845905335467b966d49dd4c764674 Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Fri, 28 Jul 2023 10:25:38 +0200 Subject: [PATCH 082/134] OpenVINO documentation updates (#386) --- docs/source/inference.mdx | 51 ++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 34253716e8..c4b15abe2d 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -34,7 +34,8 @@ outputs = cls_pipe("He's a dreadful magician.") [{'label': 'NEGATIVE', 'score': 0.9919503927230835}] ``` -To easily save the resulting model, you can use the `save_pretrained()` method, which will save both the BIN and XML files describing the graph. +To easily save the resulting model, you can use the `save_pretrained()` method, which will save both the BIN and XML files describing the graph. It is useful to save the tokenizer to the same directory, to enable easy loading of the tokenizer for the model. + ```python # Save the exported model @@ -52,17 +53,6 @@ model.reshape(1, 9) model.compile() ``` -Currently, OpenVINO only supports static shapes when running inference on Intel GPUs. FP16 precision can also be enabled in order to further decrease latency. - -```python -# Fix the batch size to 1 and the sequence length to 9 -model.reshape(1, 9) -# Enable FP16 precision -model.half() -model.to("gpu") -# Compile the model before the first inference -model.compile() -``` When fixing the shapes with the `reshape()` method, inference cannot be performed with an input of a different shape. When instantiating your pipeline, you can specify the maximum total input sequence length after tokenization in order for shorter sequences to be padded and for longer sequences to be truncated. @@ -89,7 +79,18 @@ qa_pipe = pipeline( metric = task_evaluator.compute(model_or_pipeline=qa_pipe, data=eval_dataset, metric="squad") ``` -By default the model will be compiled when instantiating our `OVModel`. In the case where the model is reshaped, placed to an other device or if FP16 precision is enabled, the model will need to be recompiled again, which will happen by default before the first inference (thus inflating the latency of the first inference). To avoid an unnecessary compilation, you can disable the first compilation by setting `compile=False`. The model should also be compiled before the first inference with `model.compile()`. + +To run inference on Intel integrated or discrete GPU, use `.to("gpu")`. On GPU, models run in FP16 precision by default. (See [OpenVINO documentation](https://docs.openvino.ai/nightly/openvino_docs_install_guides_configurations_for_intel_gpu.html) about installing drivers for GPU inference). + +```python +# Static shapes speed up inference +model.reshape(1, 9) +model.to("gpu") +# Compile the model before the first inference +model.compile() +``` + +By default the model will be compiled when instantiating our `OVModel`. In the case where the model is reshaped or placed to another device, the model will need to be recompiled again, which will happen by default before the first inference (thus inflating the latency of the first inference). To avoid an unnecessary compilation, you can disable the first compilation by setting `compile=False`. The model can be compiled before the first inference with `model.compile()`. ```python from optimum.intel import OVModelForSequenceClassification @@ -97,17 +98,33 @@ from optimum.intel import OVModelForSequenceClassification model_id = "distilbert-base-uncased-finetuned-sst-2-english" # Load the model and disable the model compilation model = OVModelForSequenceClassification.from_pretrained(model_id, export=True, compile=False) -model.half() +# Reshape to a static sequence length of 128 +model.reshape(1,128) # Compile the model before the first inference model.compile() ``` +It is possible to pass an `ov_config` parameter to `from_pretrained()` with custom OpenVINO configuration values. This can be used for example to enable full precision inference on devices where FP16 or BF16 inference precision is used by default. + + +```python +model = OVModelForSequenceClassification.from_pretrained(model_id, ov_config={"INFERENCE_PRECISION_HINT":"f32"}) +``` + +Optimum Intel leverages OpenVINO's model caching to speed up model compiling. By default a `model_cache` directory is created in the model's directory in the [Hugging Face Hub cache](https://huggingface.co/docs/huggingface_hub/main/en/guides/manage-cache). To override this, use the ov_config parameter and set `CACHE_DIR` to a different value. To disable model caching, set `CACHE_DIR` to an empty string. + + +```python +model = OVModelForSequenceClassification.from_pretrained(model_id, ov_config={"CACHE_DIR":""}) +``` + ## Sequence-to-sequence models Sequence-to-sequence (Seq2Seq) models, that generate a new sequence from an input, can also be used when running inference with OpenVINO. When Seq2Seq models are exported to the OpenVINO IR, they are decomposed into two parts : the encoder and the "decoder" (which actually consists of the decoder with the language modeling head), that are later combined during inference. -To leverage the pre-computed key/values hidden-states to speed up sequential decoding, simply pass `use_cache=True` to the `from_pretrained()` method. An additional model component will be exported: the "decoder" with pre-computed key/values as one of its inputs. -This specific export comes from the fact that during the first pass, the decoder has no pre-computed key/values hidden-states, while during the rest of the generation past key/values will be used to speed up sequential decoding. -Here is an example on how you can run inference for a translation task using an MarianMT model and then export it to the OpenVINO IR: +To speed up sequential decoding, a cache with pre-computed key/values hidden-states will be used by default. An additional model component will be exported: the "decoder" with pre-computed key/values as one of its inputs. This specific export comes from the fact that during the first pass, the decoder has no pre-computed key/values hidden-states, while during the rest of the generation past key/values will be used to speed up sequential decoding. To disable this cache, set `use_cache=False` in the `from_pretrained()` method. + +Here is an example on how you can run inference for a translation task using a T5 model and then export it to OpenVINO IR: + ```python from transformers import AutoTokenizer, pipeline From e4a81e587353a92503cdf4dbf16accc61ad4d5cc Mon Sep 17 00:00:00 2001 From: Dina Suehiro Jones Date: Fri, 28 Jul 2023 01:28:46 -0700 Subject: [PATCH 083/134] Fix typos, broken documentation links, and clarify instructions for building docs (#388) * Fix broken links * Update instructions for building docs * Fix typos --------- Co-authored-by: Dina Jones --- docs/README.md | 7 +++++-- docs/source/optimization_inc.mdx | 10 +++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index b4c6293b4e..a96a46edfc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,14 +22,17 @@ To generate the documentation for 🤗 Optimum Intel, simply run the following command from the root of the `optimum-intel` repository: ```bash -make doc BUILD_DIR=intel-doc-build +make doc BUILD_DIR=intel-doc-build VERSION=main ``` This command will generate the HTML files that will be rendered as the documentation on the [Hugging Face website](https://huggingface.co/docs/optimum/index). You can inspect them in your favorite browser. You can also adapt the `BUILD_DIR` argument to any -temporary folder that you prefer. +temporary folder that you prefer. By default, the comamnd builds a Docker container +with the latest files from the main branch. To build documentation for a different +commit or a fork, use the `DEFAULT_CLONE_URL` and `COMMIT_SHA_SUBPACKAGE` +environment variables. --- **NOTE** diff --git a/docs/source/optimization_inc.mdx b/docs/source/optimization_inc.mdx index 27e7f703ad..de3be5f9ec 100644 --- a/docs/source/optimization_inc.mdx +++ b/docs/source/optimization_inc.mdx @@ -61,7 +61,7 @@ quantizer.quantize(quantization_config=quantization_config, save_directory="dyna ### Static quantization -In the same maneer we can apply static quantization, for which we also need to generate the calibration dataset in order to perform the calibration step. +In the same manner we can apply static quantization, for which we also need to generate the calibration dataset in order to perform the calibration step. ```python from functools import partial @@ -129,7 +129,7 @@ Please refer to INC [documentation](https://github.com/intel/neural-compressor/b ## During training optimization -The [`INCTrainer`] class provides an API to train your model while combining different compression techniques such as knowledge distillation, pruning and quantization. +The [`INCTrainer`](https://huggingface.co/docs/optimum/main/intel/reference_inc#optimum.intel.INCTrainer) class provides an API to train your model while combining different compression techniques such as knowledge distillation, pruning and quantization. The `INCTrainer` is very similar to the 🤗 Transformers [`Trainer`](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#trainer), which can be replaced with minimal changes in your code. ### Quantization @@ -181,8 +181,8 @@ trainer.save_model() ### Pruning -In the same maneer, pruning can be applied by specifiying the pruning configuration detailing the desired pruning process. -To know more about the different supported methodologies, you can refer to the Neural Compressor [documentation](https://github.com/intel/neural-compressor/tree/master/neural_compressor/pruner#pruning-types). +In the same manner, pruning can be applied by specifying the pruning configuration detailing the desired pruning process. +To know more about the different supported methodologies, you can refer to the Neural Compressor [documentation](https://github.com/intel/neural-compressor/tree/master/neural_compressor/compression/pruner#pruning-types). At the moment, pruning is applied on both the linear and the convolutional layers, and not on other layers such as the embeddings. It's important to mention that the pruning sparsity defined in the configuration will be applied on these layers, and thus will not results in the global model sparsity. ```diff @@ -219,7 +219,7 @@ model = AutoModelForSequenceClassification.from_pretrained(save_dir) ``` ### Knowledge distillation -Knowledge distillation can also be applied in the same maneer. +Knowledge distillation can also be applied in the same manner. To know more about the different supported methodologies, you can refer to the Neural Compressor [documentation](https://github.com/intel/neural-compressor/blob/master/docs/source/distillation.md) ```diff From f43787379e40eb3120c4b99f84726578adf0f60a Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Fri, 28 Jul 2023 19:36:29 +0400 Subject: [PATCH 084/134] fix sdxl model reshape (#390) * fix sdxl model reshape * one more fix for time_ids --- optimum/intel/openvino/modeling_diffusion.py | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 04a16699b3..6d263fdd50 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -351,6 +351,7 @@ def _reshape_unet( height: int = -1, width: int = -1, num_images_per_prompt: int = -1, + tokenizer_max_length: int = -1, ): if batch_size == -1 or num_images_per_prompt == -1: batch_size = -1 @@ -358,7 +359,6 @@ def _reshape_unet( # The factor of 2 comes from the guidance scale > 1 batch_size = 2 * batch_size * num_images_per_prompt - requires_aesthetics_score = getattr(self.config, "requires_aesthetics_score", False) height = height // self.vae_scale_factor if height > 0 else height width = width // self.vae_scale_factor if width > 0 else width shapes = {} @@ -380,16 +380,18 @@ def _reshape_unet( elif inputs.get_any_name() == "text_embeds": shapes[inputs] = [batch_size, self.text_encoder_2.config["projection_dim"]] elif inputs.get_any_name() == "time_ids": - shapes[inputs] = [batch_size, 5 if requires_aesthetics_score else 6] + shapes[inputs] = [batch_size, inputs.get_partial_shape()[1]] else: shapes[inputs][0] = batch_size - shapes[inputs][1] = self.tokenizer.model_max_length + shapes[inputs][1] = tokenizer_max_length model.reshape(shapes) return model - def _reshape_text_encoder(self, model: openvino.runtime.Model, batch_size: int = -1): + def _reshape_text_encoder( + self, model: openvino.runtime.Model, batch_size: int = -1, tokenizer_max_length: int = -1 + ): if batch_size != -1: - shapes = {model.inputs[0]: [batch_size, self.tokenizer.model_max_length]} + shapes = {model.inputs[0]: [batch_size, tokenizer_max_length]} model.reshape(shapes) return model @@ -432,13 +434,25 @@ def reshape( ): self.is_dynamic = -1 in {batch_size, height, width, num_images_per_prompt} self.vae_decoder.model = self._reshape_vae_decoder(self.vae_decoder.model, height, width) - self.unet.model = self._reshape_unet(self.unet.model, batch_size, height, width, num_images_per_prompt) + if self.tokenizer is None and self.tokenizer_2 is None: + tokenizer_max_len = -1 + else: + tokenizer_max_len = ( + self.tokenizer.model_max_length if self.tokenizer is not None else self.tokenizer_2.model_max_length + ) + self.unet.model = self._reshape_unet( + self.unet.model, batch_size, height, width, num_images_per_prompt, tokenizer_max_len + ) if self.text_encoder is not None: - self.text_encoder.model = self._reshape_text_encoder(self.text_encoder.model, batch_size) + self.text_encoder.model = self._reshape_text_encoder( + self.text_encoder.model, batch_size, self.tokenizer.model_max_length + ) if self.text_encoder_2 is not None: - self.text_encoder_2.model = self._reshape_text_encoder(self.text_encoder_2.model, batch_size) + self.text_encoder_2.model = self._reshape_text_encoder( + self.text_encoder_2.model, batch_size, self.tokenizer_2.model_max_length + ) if self.vae_encoder is not None: self.vae_encoder.model = self._reshape_vae_encoder(self.vae_encoder.model, batch_size, height, width) From 6b4c571bbdf93d420c6e467fafb7520bd6a9efe6 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 12:05:18 +0400 Subject: [PATCH 085/134] Revised the code --- optimum/intel/openvino/modeling_decoder.py | 86 +++++++--------------- optimum/intel/openvino/quantization.py | 4 +- tests/openvino/test_quantization.py | 3 +- 3 files changed, 30 insertions(+), 63 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index aecb629dec..61359644b1 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -95,8 +95,6 @@ """, ) class OVBaseDecoderModel(OVModel): - disable_pkv_lp_opt = False - def __init__( self, model: openvino.runtime.Model, @@ -112,8 +110,6 @@ def __init__( "`dynamic_shapes` was set to `False` but static shapes are not supported for causal language model. Please set `dynamic_shapes=True`." ) - model = self._try_modify_pastkv_to_lowprecision(model, ov_config, device, dynamic_shapes) - super().__init__( model, config, @@ -132,6 +128,7 @@ def __init__( self.output_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.outputs)} self.key_value_input_names = [key for key in self.input_names if "key_values" in key] self.key_value_output_names = [key for key in self.output_names if "present" in key] + self.update_pkv_precision() if use_cache ^ self.use_cache: raise ValueError( @@ -141,59 +138,31 @@ def __init__( "To export your model, simply set `export=True`." ) - def _try_modify_pastkv_to_lowprecision( - self, - model: openvino.runtime.Model, - ov_config: Optional[Dict[str, str]] = None, - device: str = "CPU", - dynamic_shapes: bool = True, - ): - device = device.upper() - pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") + def update_pkv_precision(self): + if not self.use_cache: + return + device = self._device.upper() + inference_precision_hint = core.get_property(device, "INFERENCE_PRECISION_HINT") + pkv_precision = Type.f32 # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision - if ov_config: - config_precision_hint = ov_config.get("INFERENCE_PRECISION_HINT", "") - config_mode_hint = ov_config.get("EXECUTION_MODE_HINT", "") - if config_precision_hint in STR_TO_OV_TYPE: - pkv_precision = STR_TO_OV_TYPE[config_precision_hint] - elif config_mode_hint.upper() == "ACCURACY": - pkv_precision = Type.f32 - - use_cache = any("past_key_values" in key.get_any_name() for key in model.inputs) - self._original_model = model - self._pkv_precision = Type.f32 - if pkv_precision != Type.f32 and use_cache and self.disable_pkv_lp_opt is False: - ppp = PrePostProcessor(model.clone()) - need_gen = False - for key in model.inputs: - if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): - need_gen = True - ppp.input(key.get_any_name()).tensor().set_element_type(pkv_precision) - for key in model.outputs: - if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): - need_gen = True - ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) - if need_gen: - model = ppp.build() - self._pkv_precision = pkv_precision - if dynamic_shapes: - height = -1 if self.export_feature == "image-classification" else None - width = -1 if self.export_feature == "image-classification" else None - self._original_model = self._reshape(self._original_model, -1, -1, height, width) + if self.ov_config: + inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") + if inference_precision_hint in STR_TO_OV_TYPE: + pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] - return model - - def _save_pretrained(self, save_directory: Union[str, Path]): - """ - Saves the model to the OpenVINO IR format so that it can be re-loaded using the - [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. - - Arguments: - save_directory (`str` or `Path`): - The directory where to save the model files. - """ - dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(self.model if self.disable_pkv_lp_opt else self._original_model, dst_path) + self._pkv_precision = Type.f32 + ppp = PrePostProcessor(self.model) + for key in self.model.inputs: + if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): + ppp.input(key.get_any_name()).tensor().set_element_type(pkv_precision) + for key in self.model.outputs: + if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): + ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) + + self.model = ppp.build() + self._pkv_precision = pkv_precision + if self.is_dynamic: + self.model = self._reshape(self.model, -1, -1) @classmethod def _from_transformers( @@ -483,9 +452,4 @@ def _convert_to_standard_cache( def can_generate(self): """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" - return True - - -# bf16 may be in used when quantization, disable pastkv lower precision optimization -class OVModelForCausalLMDisablePKVOpt(OVModelForCausalLM): - disable_pkv_lp_opt = True + return True \ No newline at end of file diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 29e16736bb..ef19861476 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -257,9 +257,11 @@ def _quantize_ovcausallm( batch_size=batch_size, remove_unused_columns=remove_unused_columns, data_collator=data_collator, - ) + ) # Prefeth past_key_values + self.model.ov_config["INFERENCE_PRECISION_HINT"] = "f32" + self.model.update_pkv_precision() self.model.compile() subset_size = kwargs.get("subset_size", 300) data_cache = [] diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 0696642e0c..86bbf9fec4 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -38,6 +38,7 @@ OVModelForSequenceClassification, OVModelForCausalLMDisablePKVOpt, OVModelForTokenClassification, + OVModelForCausalLM, OVQuantizer, OVTrainer, ) @@ -65,7 +66,7 @@ class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 42, 32), - (OVModelForCausalLMDisablePKVOpt, "hf-internal-testing/tiny-random-gpt2", 41, 21), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 41, 21), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) From 02c9acd83de5f3c6835e3f8048fda1283e0be35c Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 12:06:10 +0400 Subject: [PATCH 086/134] Style --- optimum/intel/openvino/modeling_decoder.py | 7 +++---- optimum/intel/openvino/quantization.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 61359644b1..296d70d3b6 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -import os from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict, Optional, Tuple, Union @@ -34,7 +33,7 @@ from ..utils.import_utils import is_transformers_version from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME, STR_TO_OV_TYPE +from .utils import ONNX_WEIGHTS_NAME, STR_TO_OV_TYPE if is_transformers_version("<", "4.25.0"): @@ -158,7 +157,7 @@ def update_pkv_precision(self): for key in self.model.outputs: if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) - + self.model = ppp.build() self._pkv_precision = pkv_precision if self.is_dynamic: @@ -452,4 +451,4 @@ def _convert_to_standard_cache( def can_generate(self): """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" - return True \ No newline at end of file + return True diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index ef19861476..dad55156db 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -257,7 +257,7 @@ def _quantize_ovcausallm( batch_size=batch_size, remove_unused_columns=remove_unused_columns, data_collator=data_collator, - ) + ) # Prefeth past_key_values self.model.ov_config["INFERENCE_PRECISION_HINT"] = "f32" From a34033ba42ebdc5e26f30229034d9a9eb9dcd4fa Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:06:42 +0200 Subject: [PATCH 087/134] Update documentation (#392) --- docs/source/inference.mdx | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index c4b15abe2d..c0360322ea 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -202,8 +202,9 @@ images = pipeline(prompt, height=height, width=width, num_images_per_prompt=num_ In case you want to change any parameters such as the outputs height or width, you'll need to statically reshape your model once again. - -![img](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/stable_diffusion_v1_5_sail_boat_rembrandt.png) +
+ +
### Image-to-Image @@ -227,7 +228,6 @@ image = pipeline(prompt=prompt, image=init_image, strength=0.75, guidance_scale= image.save("fantasy_landscape.png") ``` - ## Stable Diffusion XL Before using `OVtableDiffusionXLPipeline` make sure to have `diffusers` and `invisible_watermark` installed. You can install the libraries as follows: @@ -239,18 +239,16 @@ pip install invisible-watermark>=0.2.0 ### Text-to-Image -Here is an example of how you can load a PyTorch SDXL model, convert it to the adapted format on-the-fly and run inference with OpenVINO Runtime: +Here is an example of how you can load a SDXL OpenVINO model from [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) and run inference using OpenVINO Runtime: ```python from optimum.intel import OVStableDiffusionXLPipeline model_id = "stabilityai/stable-diffusion-xl-base-1.0" -base = OVStableDiffusionXLPipeline.from_pretrained(model_id, export=True) +base = OVStableDiffusionXLPipeline.from_pretrained(model_id) prompt = "train station by Caspar David Friedrich" image = base(prompt).images[0] - -# Don't forget to save your OpenVINO model -base.save_pretrained("openvino-sd-xl-base-1.0") +image.save("train_station.png") ``` | | | @@ -260,7 +258,7 @@ base.save_pretrained("openvino-sd-xl-base-1.0") ### Image-to-Image -You can use SDXL as follows for *image-to-image*: +Here is an example of how you can load a PyTorch SDXL model, convert it to OpenVINO on-the-fly and run inference using OpenVINO Runtime for *image-to-image*: ```python from optimum.intel import OVStableDiffusionXLImg2ImgPipeline @@ -273,7 +271,8 @@ url = "https://huggingface.co/datasets/optimum/documentation-images/resolve/main image = load_image(url).convert("RGB") prompt = "medieval castle by Caspar David Friedrich" image = pipeline(prompt, image=image).images[0] -image.save("medieval_castle.png") +# Don't forget to save your OpenVINO model so that you can load it without exporting it with `export=True` +pipeline.save_pretrained("openvino-sd-xl-refiner-1.0") ``` From fd3d752c30f4c5e23f7c201b42b6e069c0ea850d Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 12:15:12 +0400 Subject: [PATCH 088/134] Fixed import issues --- optimum/intel/__init__.py | 2 -- tests/openvino/test_quantization.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index edfa4d0342..e465240bb9 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -74,7 +74,6 @@ [ "OVModelForAudioClassification", "OVModelForCausalLM", - "OVModelForCausalLMDisablePKVOpt", "OVModelForFeatureExtraction", "OVModelForImageClassification", "OVModelForMaskedLM", @@ -152,7 +151,6 @@ from .openvino import ( OVModelForAudioClassification, OVModelForCausalLM, - OVModelForCausalLMDisablePKVOpt, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 86bbf9fec4..da9ba3b25a 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -36,7 +36,6 @@ OVConfig, OVModelForQuestionAnswering, OVModelForSequenceClassification, - OVModelForCausalLMDisablePKVOpt, OVModelForTokenClassification, OVModelForCausalLM, OVQuantizer, @@ -148,7 +147,7 @@ class OVWeightCompressionTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 39), - (OVModelForCausalLMDisablePKVOpt, "hf-internal-testing/tiny-random-gpt2", 5), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 5), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) From c054ef271e705ba69c43c2257f2b448cdd4705ff Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 12:17:11 +0400 Subject: [PATCH 089/134] Fixed import issues --- optimum/intel/openvino/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 1b056930e8..3f8737e1b9 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -42,7 +42,7 @@ OVModelForSequenceClassification, OVModelForTokenClassification, ) -from .modeling_decoder import OVModelForCausalLM, OVModelForCausalLMDisablePKVOpt +from .modeling_decoder import OVModelForCausalLM from .modeling_seq2seq import OVModelForSeq2SeqLM From d43d103f294bedcc0cf8d522e699217c6b9b6912 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 13:32:23 +0400 Subject: [PATCH 090/134] Fixed saving decoder models and update request after changing PKV precision --- optimum/intel/openvino/modeling_decoder.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 296d70d3b6..92f48be35f 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import os from pathlib import Path from tempfile import TemporaryDirectory from typing import Dict, Optional, Tuple, Union @@ -33,7 +34,7 @@ from ..utils.import_utils import is_transformers_version from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .utils import ONNX_WEIGHTS_NAME, STR_TO_OV_TYPE +from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME, STR_TO_OV_TYPE if is_transformers_version("<", "4.25.0"): @@ -127,6 +128,7 @@ def __init__( self.output_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.outputs)} self.key_value_input_names = [key for key in self.input_names if "key_values" in key] self.key_value_output_names = [key for key in self.output_names if "present" in key] + self._original_model = self.model.clone() # keep original model for serialization self.update_pkv_precision() if use_cache ^ self.use_cache: @@ -162,6 +164,19 @@ def update_pkv_precision(self): self._pkv_precision = pkv_precision if self.is_dynamic: self.model = self._reshape(self.model, -1, -1) + self.request = None + + def _save_pretrained(self, save_directory: Union[str, Path]): + """ + Saves the model to the OpenVINO IR format so that it can be re-loaded using the + [`~optimum.intel.openvino.modeling.OVModel.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `Path`): + The directory where to save the model files. + """ + dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) + openvino.runtime.serialize(self._original_model, dst_path) @classmethod def _from_transformers( From 272de1f5400534dbc8f85015189d4490665ee577 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 14:21:10 +0400 Subject: [PATCH 091/134] Fixed issue --- optimum/intel/openvino/modeling_decoder.py | 21 ++++++++++++--------- optimum/intel/openvino/quantization.py | 3 +-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 92f48be35f..037103c783 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -139,17 +139,19 @@ def __init__( "To export your model, simply set `export=True`." ) - def update_pkv_precision(self): + def update_pkv_precision(self, force_fp32=False): if not self.use_cache: return - device = self._device.upper() - inference_precision_hint = core.get_property(device, "INFERENCE_PRECISION_HINT") + pkv_precision = Type.f32 - # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision - if self.ov_config: - inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") - if inference_precision_hint in STR_TO_OV_TYPE: - pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] + if not force_fp32: + device = self._device.upper() + inference_precision_hint = core.get_property(device, "INFERENCE_PRECISION_HINT") + # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision + if self.ov_config: + inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") + if inference_precision_hint in STR_TO_OV_TYPE: + pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] self._pkv_precision = Type.f32 ppp = PrePostProcessor(self.model) @@ -175,8 +177,9 @@ def _save_pretrained(self, save_directory: Union[str, Path]): save_directory (`str` or `Path`): The directory where to save the model files. """ + model_to_save = self.model if self._pkv_precision == Type.f32 else self._original_model dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(self._original_model, dst_path) + openvino.runtime.serialize(model_to_save, dst_path) @classmethod def _from_transformers( diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index dad55156db..e5441ca39b 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -260,8 +260,7 @@ def _quantize_ovcausallm( ) # Prefeth past_key_values - self.model.ov_config["INFERENCE_PRECISION_HINT"] = "f32" - self.model.update_pkv_precision() + self.model.update_pkv_precision(True) self.model.compile() subset_size = kwargs.get("subset_size", 300) data_cache = [] From 49d197ba20c74fdd27dc96190a171e9f89a64b2c Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 15:28:29 +0400 Subject: [PATCH 092/134] Fixed double compilation --- optimum/intel/openvino/modeling_decoder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 037103c783..2cf2b749d5 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -117,6 +117,7 @@ def __init__( dynamic_shapes=True, ov_config=ov_config, model_save_dir=model_save_dir, + compile=False, **kwargs, ) @@ -130,6 +131,9 @@ def __init__( self.key_value_output_names = [key for key in self.output_names if "present" in key] self._original_model = self.model.clone() # keep original model for serialization self.update_pkv_precision() + enable_compilation = kwargs.get("compile", True) + if enable_compilation: + self.compile() if use_cache ^ self.use_cache: raise ValueError( From 88fdddff586b2f814fb1b44cce2902e45c1126f4 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 16:05:57 +0400 Subject: [PATCH 093/134] Fixed issue --- optimum/intel/openvino/modeling_decoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 2cf2b749d5..f8c110dc98 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -110,6 +110,9 @@ def __init__( "`dynamic_shapes` was set to `False` but static shapes are not supported for causal language model. Please set `dynamic_shapes=True`." ) + enable_compilation = kwargs.get("compile", True) + kwargs["compile"] = False # avoid extra compilation in the base class + super().__init__( model, config, @@ -117,7 +120,6 @@ def __init__( dynamic_shapes=True, ov_config=ov_config, model_save_dir=model_save_dir, - compile=False, **kwargs, ) @@ -131,7 +133,6 @@ def __init__( self.key_value_output_names = [key for key in self.output_names if "present" in key] self._original_model = self.model.clone() # keep original model for serialization self.update_pkv_precision() - enable_compilation = kwargs.get("compile", True) if enable_compilation: self.compile() From c8147e5b89072562cdb0d87ccbbb296caa0ab3c8 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jul 2023 19:54:28 +0400 Subject: [PATCH 094/134] Applied comments --- optimum/intel/openvino/modeling_decoder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index f8c110dc98..1c2adb8ce9 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -117,12 +117,13 @@ def __init__( model, config, device=device, - dynamic_shapes=True, + dynamic_shapes=False, ov_config=ov_config, model_save_dir=model_save_dir, **kwargs, ) + self.is_dynamic = dynamic_shapes use_cache = kwargs.pop("use_cache", True) self.use_cache = any("past_key_values" in key.get_any_name() for key in model.inputs) self.main_input_name = "input_ids" @@ -133,6 +134,8 @@ def __init__( self.key_value_output_names = [key for key in self.output_names if "present" in key] self._original_model = self.model.clone() # keep original model for serialization self.update_pkv_precision() + if self.is_dynamic: + self.model = self._reshape(self.model, -1, -1) if enable_compilation: self.compile() @@ -155,10 +158,9 @@ def update_pkv_precision(self, force_fp32=False): # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if self.ov_config: inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") - if inference_precision_hint in STR_TO_OV_TYPE: - pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] + if inference_precision_hint in STR_TO_OV_TYPE: + pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] - self._pkv_precision = Type.f32 ppp = PrePostProcessor(self.model) for key in self.model.inputs: if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): @@ -169,8 +171,6 @@ def update_pkv_precision(self, force_fp32=False): self.model = ppp.build() self._pkv_precision = pkv_precision - if self.is_dynamic: - self.model = self._reshape(self.model, -1, -1) self.request = None def _save_pretrained(self, save_directory: Union[str, Path]): From 6e1ea065c13e6caab1a6b6f5b2a532a005e607a8 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 1 Aug 2023 10:25:26 +0400 Subject: [PATCH 095/134] Updated PKV precision selection logic --- optimum/intel/openvino/modeling_decoder.py | 36 +++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 1c2adb8ce9..b06e2c94d8 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -154,24 +154,30 @@ def update_pkv_precision(self, force_fp32=False): pkv_precision = Type.f32 if not force_fp32: device = self._device.upper() - inference_precision_hint = core.get_property(device, "INFERENCE_PRECISION_HINT") + pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if self.ov_config: inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") - if inference_precision_hint in STR_TO_OV_TYPE: - pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] - - ppp = PrePostProcessor(self.model) - for key in self.model.inputs: - if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): - ppp.input(key.get_any_name()).tensor().set_element_type(pkv_precision) - for key in self.model.outputs: - if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): - ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) - - self.model = ppp.build() - self._pkv_precision = pkv_precision - self.request = None + if inference_precision_hint in STR_TO_OV_TYPE: + pkv_precision = STR_TO_OV_TYPE[inference_precision_hint] + + ppp = PrePostProcessor(self.model) + for key in self.model.inputs: + if "past_key_values" in key.get_any_name() and pkv_precision != key.get_element_type(): + ppp.input(key.get_any_name()).tensor().set_element_type(pkv_precision) + for key in self.model.outputs: + if "present" in key.get_any_name() and pkv_precision != key.get_element_type(): + ppp.output(key.get_any_name()).tensor().set_element_type(pkv_precision) + + self.model = ppp.build() + self._pkv_precision = pkv_precision + else: + if hasattr(self, '_pkv_precision') and self._pkv_precision != Type.f32: + self._pkv_precision = Type.f32 + self.model = self._original_model.clone() + if self.is_dynamic: + self.model = self._reshape(self.model, -1, -1) + self.request = None def _save_pretrained(self, save_directory: Union[str, Path]): """ From e0e42d2bc03729976444a47e5b6451f4199ba671 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 1 Aug 2023 11:32:29 +0400 Subject: [PATCH 096/134] Style --- optimum/intel/openvino/modeling_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index b06e2c94d8..f446e46b48 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -172,7 +172,7 @@ def update_pkv_precision(self, force_fp32=False): self.model = ppp.build() self._pkv_precision = pkv_precision else: - if hasattr(self, '_pkv_precision') and self._pkv_precision != Type.f32: + if hasattr(self, "_pkv_precision") and self._pkv_precision != Type.f32: self._pkv_precision = Type.f32 self.model = self._original_model.clone() if self.is_dynamic: From f2b189546bd5c81c2deef85175f81982ecc26c10 Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Tue, 1 Aug 2023 12:54:52 +0400 Subject: [PATCH 097/134] Remove pkv history from quantization statistics (#394) --- optimum/intel/openvino/quantization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index acedad7b34..99e22e72f5 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -299,7 +299,7 @@ def __getattr__(self, attr): self.model.request = InferRequestWrapper(self.model.request) for _, data in enumerate(calibration_dataloader): - self.model.generate(**data, max_new_tokens=10) + self.model.generate(**data, max_new_tokens=1) if len(data_cache) >= subset_size: break self.model.request = self.model.request.request From 7cb47dcf3279f946afc82bcedc57013b9be8a080 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:19:47 +0200 Subject: [PATCH 098/134] Add SDXL refiner test (#395) --- tests/openvino/test_stable_diffusion.py | 23 ++++++++++------------- tests/openvino/utils_tests.py | 1 + 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py index 99195bc1cb..35e85eeaa7 100644 --- a/tests/openvino/test_stable_diffusion.py +++ b/tests/openvino/test_stable_diffusion.py @@ -122,16 +122,12 @@ def test_compare_diffusers_pipeline(self, model_arch: str): pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) inputs = self.generate_inputs() inputs["prompt"] = "A painting of a squirrel eating a burger" - - output = pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] + np.random.seed(0) + output = pipeline(**inputs).images[0, -3:, -3:, -1] # https://github.com/huggingface/diffusers/blob/v0.17.1/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py#L71 expected_slice = np.array([0.69643, 0.58484, 0.50314, 0.58760, 0.55368, 0.59643, 0.51529, 0.41217, 0.49087]) self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-1)) - ort_pipeline = self.ORT_MODEL_CLASS.from_pretrained(model_id, export=True) - ort_output = ort_pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] - self.assertTrue(np.allclose(output, ort_output, atol=1e-1)) - @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_num_images_per_prompt_static_model(self, model_arch: str): model_id = MODEL_NAMES[model_arch] @@ -387,23 +383,24 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): class OVStableDiffusionXLImg2ImgPipelineTest(unittest.TestCase): - SUPPORTED_ARCHITECTURES = ("stable-diffusion-xl",) + SUPPORTED_ARCHITECTURES = ("stable-diffusion-xl", "stable-diffusion-xl-refiner") MODEL_CLASS = OVStableDiffusionXLImg2ImgPipeline ORT_MODEL_CLASS = ORTStableDiffusionXLImg2ImgPipeline PT_MODEL_CLASS = StableDiffusionXLImg2ImgPipeline - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_inference(self, model_arch: str): - pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + def test_inference(self): + model_id = "hf-internal-testing/tiny-stable-diffusion-xl-pipe" + pipeline = self.MODEL_CLASS.from_pretrained(model_id) with tempfile.TemporaryDirectory() as tmp_dir: pipeline.save_pretrained(tmp_dir) pipeline = self.MODEL_CLASS.from_pretrained(tmp_dir) inputs = self.generate_inputs() - output = pipeline(**inputs, generator=np.random.RandomState(0)).images[0, -3:, -3:, -1] - expected_slice = np.array([0.6515, 0.5405, 0.4858, 0.5632, 0.5174, 0.5681, 0.4948, 0.4253, 0.5080]) - self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-1)) + np.random.seed(0) + output = pipeline(**inputs).images[0, -3:, -3:, -1] + expected_slice = np.array([0.5675, 0.5108, 0.4758, 0.5280, 0.5080, 0.5473, 0.4789, 0.4286, 0.4861]) + self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-3)) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_num_images_per_prompt_static_model(self, model_arch: str): diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 796432271b..6a14e96796 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -68,6 +68,7 @@ "squeezebert": "hf-internal-testing/tiny-random-squeezebert", "stable-diffusion": "hf-internal-testing/tiny-stable-diffusion-torch", "stable-diffusion-xl": "echarlaix/tiny-random-stable-diffusion-xl", + "stable-diffusion-xl-refiner": "echarlaix/tiny-random-stable-diffusion-xl-refiner", "sew": "hf-internal-testing/tiny-random-SEWModel", "sew_d": "hf-internal-testing/tiny-random-SEWDModel", "swin": "hf-internal-testing/tiny-random-SwinModel", From f2fdce2fd4c85a2d0ce53d3f4c653aa49120fdbe Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 1 Aug 2023 19:01:10 +0200 Subject: [PATCH 099/134] enable onnx export for INC PTQ model (#373) * enable onnx export for PTQ * fix output loading quantized model check * tests refactorization * fix style --- .../intel/neural_compressor/quantization.py | 19 +- tests/neural_compressor/test_onnx.py | 80 +++++++++ tests/neural_compressor/test_optimization.py | 160 ++--------------- tests/neural_compressor/utils_tests.py | 169 ++++++++++++++++++ 4 files changed, 279 insertions(+), 149 deletions(-) create mode 100644 tests/neural_compressor/test_onnx.py create mode 100644 tests/neural_compressor/utils_tests.py diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index ee1dbc978e..6b1af7abed 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -184,14 +184,19 @@ def quantize( remove_unused_columns=remove_unused_columns, data_collator=data_collator, ) + op_type_dict = getattr(quantization_config, "op_type_dict", None) + if op_type_dict is None or "Embedding" not in op_type_dict: + logger.warning("ONNX export is no supported for model with quantized embeddings") + save_onnx_model = False - # Disable ONNX export for post-training quantized model as deprecated in neural-compressor>=2.2.0 - if save_onnx_model: - logger.warning( - "ONNX export for post-training quantized model is no longer supported by neural-compressor>=2.2.0. " - "To apply quantization on an ONNX model, check out optimum.onnxruntime.ORTQuantizer" - ) - save_onnx_model = False + else: + # Disable ONNX export for dynamically quantized model as deprecated in neural-compressor>=2.2.0 + if save_onnx_model: + logger.warning( + "ONNX export for dynamic quantized model is no longer supported by neural-compressor>=2.2.0. " + "To apply dynamic quantization on an ONNX model, you can use optimum.onnxruntime.ORTQuantizer" + ) + save_onnx_model = False if ( quantization_config.backend == "ipex" diff --git a/tests/neural_compressor/test_onnx.py b/tests/neural_compressor/test_onnx.py new file mode 100644 index 0000000000..69f608953a --- /dev/null +++ b/tests/neural_compressor/test_onnx.py @@ -0,0 +1,80 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa + + +import os +import tempfile + +from neural_compressor.config import PostTrainingQuantConfig +from parameterized import parameterized +from transformers import AutoTokenizer, set_seed +from utils_tests import SEED, INCTestMixin, _generate_dataset + +from optimum.intel import ( + INCConfig, + INCModelForCausalLM, + INCModelForSeq2SeqLM, + INCModelForQuestionAnswering, + INCModelForSequenceClassification, + INCModelForMaskedLM, + INCModelForTokenClassification, + INCQuantizer, +) +from optimum.onnxruntime import ORTModelForCausalLM, ORTModelForSequenceClassification +from optimum.pipelines import ORT_SUPPORTED_TASKS + +os.environ["CUDA_VISIBLE_DEVICES"] = "" +set_seed(SEED) + + +class OptimizationTest(INCTestMixin): + SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( + ("text-classification", "hf-internal-testing/tiny-random-bert", 34), + ) + + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) + def test_static_quantization(self, task, model_name, expected_quantized_matmuls): + num_samples = 10 + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + quantizer = INCQuantizer.from_pretrained(model, task=task) + calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=num_samples) + save_onnx_model = True + op_type_dict = ( + {"Embedding": {"weight": {"dtype": ["fp32"]}, "activation": {"dtype": ["fp32"]}}} + if save_onnx_model + else None + ) + quantization_config = PostTrainingQuantConfig(approach="static", op_type_dict=op_type_dict) + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + calibration_dataset=calibration_dataset, + save_directory=tmp_dir, + save_onnx_model=save_onnx_model, + ) + self.check_model_outputs( + q_model=quantizer._quantized_model, + task=task, + tokenizer=tokenizer, + save_directory=tmp_dir, + expected_quantized_matmuls=expected_quantized_matmuls, + is_static=True, + num_samples=num_samples, + load_onnx_model=save_onnx_model, + ) diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index b9989fbe69..578f556153 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -1,4 +1,4 @@ -# Copyright 2021 The HuggingFace Team. All rights reserved. +# Copyright 2023 The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ # ruff: noqa + import os import tempfile -import unittest -from functools import partial import evaluate import numpy as np @@ -32,22 +31,18 @@ TuningCriterion, WeightPruningConfig, ) -from onnx import load as onnx_load from parameterized import parameterized from transformers import ( AutoModelForCausalLM, AutoModelForQuestionAnswering, - AutoModelForSequenceClassification, AutoTokenizer, - EvalPrediction, - TrainingArguments, - Seq2SeqTrainingArguments, - default_data_collator, - pipeline, BertTokenizer, EncoderDecoderModel, + Seq2SeqTrainingArguments, + pipeline, set_seed, ) +from utils_tests import SEED, INCTestMixin, _generate_dataset from optimum.intel import ( INCConfig, @@ -58,63 +53,27 @@ INCModelForMaskedLM, INCModelForTokenClassification, INCQuantizer, - INCStableDiffusionPipeline, - INCTrainer, INCSeq2SeqTrainer, + INCStableDiffusionPipeline, ) -from optimum.intel.neural_compressor.utils import _HEAD_TO_AUTOMODELS -from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME, ONNX_WEIGHTS_NAME +from optimum.intel.utils.constant import DIFFUSION_WEIGHTS_NAME from optimum.onnxruntime import ORTModelForCausalLM, ORTModelForSequenceClassification from optimum.pipelines import ORT_SUPPORTED_TASKS os.environ["CUDA_VISIBLE_DEVICES"] = "" -set_seed(1009) - -_TASK_TO_DATASET = { - "text-classification": ("glue", "sst2", "sentence"), - "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), - "text2text-generation": ("cnn_dailymail", "3.0.0", ("article", "highlights")), -} - - -def num_quantized_matmul_onnx_model(onnx_model): - num_quantized_matmul = 0 - for initializer in onnx_model.graph.initializer: - if "MatMul" in initializer.name and "quantized" in initializer.name: - num_quantized_matmul += 1 - return num_quantized_matmul +set_seed(SEED) -def _preprocess_function(examples, tokenizer, column_name): - return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) - - -def _compute_metrics(outputs, metric): - return metric.compute(predictions=np.argmax(outputs.predictions, axis=1), references=outputs.label_ids) - - -def _generate_dataset(quantizer, tokenizer, num_samples=10): - dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[quantizer.task] - dataset = quantizer.get_calibration_dataset( - dataset_name, - dataset_config_name=dataset_config_name, - preprocess_function=partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), - num_samples=num_samples, - dataset_split="train", - ) - return dataset - - -class OptimizationTest(unittest.TestCase): +class OptimizationTest(INCTestMixin): SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( - ("text-classification", "hf-internal-testing/tiny-random-bert", 30), + ("text-classification", "hf-internal-testing/tiny-random-bert", 34), # ("text-generation", "hf-internal-testing/tiny-random-BloomForCausalLM", 1), # TODO : enable causal lm task once INC ONNX export fixed ) SUPPORTED_ARCHITECTURES_DYNAMIC = SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS + ( - ("fill-mask", "hf-internal-testing/tiny-random-DistilBertForMaskedLM", 30), - ("token-classification", "hf-internal-testing/tiny-random-AlbertForTokenClassification", 30), + ("fill-mask", "hf-internal-testing/tiny-random-DistilBertForMaskedLM", 34), + ("token-classification", "hf-internal-testing/tiny-random-AlbertForTokenClassification", 34), ) TEXT_GENERATION_SUPPORTED_ARCHITECTURES = ( @@ -148,16 +107,19 @@ def test_dynamic_quantization(self, task, model_name, expected_quantized_matmuls @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) def test_static_quantization(self, task, model_name, expected_quantized_matmuls): num_samples = 10 - quantization_config = PostTrainingQuantConfig(approach="static") model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token - quantizer = INCQuantizer.from_pretrained(model, task=task) calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=num_samples) save_onnx_model = False - + op_type_dict = ( + {"Embedding": {"weight": {"dtype": ["fp32"]}, "activation": {"dtype": ["fp32"]}}} + if save_onnx_model + else None + ) + quantization_config = PostTrainingQuantConfig(approach="static", op_type_dict=op_type_dict) with tempfile.TemporaryDirectory() as tmp_dir: quantizer.quantize( quantization_config=quantization_config, @@ -530,89 +492,3 @@ def _compute_metrics(pred): self.assertIsInstance(loaded_model_outputs.logits, torch.Tensor) # Compare tensor outputs self.assertTrue(torch.allclose(loaded_model_outputs.logits, model_outputs.logits, atol=1e-4)) - - def check_model_outputs( - self, - q_model, - task, - tokenizer, - save_directory, - expected_quantized_matmuls, - is_static=True, - load_onnx_model=True, - num_samples=None, - file_name=ONNX_WEIGHTS_NAME, - ): - tokens = tokenizer("This is a sample input", return_tensors="pt") - inc_model = eval(_HEAD_TO_AUTOMODELS[task]).from_pretrained(save_directory) - model_kwargs = ( - {"decoder_file_name": file_name, "use_cache": False} - if task == "text-generation" - else {"file_name": file_name} - ) - inc_config = INCConfig.from_pretrained(save_directory) - self.assertEqual(inc_config.save_onnx_model, load_onnx_model) - - if num_samples is not None: - self.assertEqual(inc_config.quantization["dataset_num_samples"], num_samples) - - if load_onnx_model: - onnx_model = onnx_load(os.path.join(save_directory, file_name)) - num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) - - if num_quantized_matmul > 0: - self.assertEqual(inc_config.quantization["is_static"], is_static) - - self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) - ort_model = ORT_SUPPORTED_TASKS[task]["class"][0].from_pretrained(save_directory, **model_kwargs) - ort_outputs = ort_model(**tokens) - self.assertTrue("logits" in ort_outputs) - - with torch.no_grad(): - model_outputs = q_model(**tokens) - inc_model_outputs = inc_model(**tokens) - self.assertTrue(torch.equal(model_outputs["logits"], inc_model_outputs["logits"])) - # self.assertTrue(torch.allclose(ort_outputs.logits, inc_model_outputs.logits, atol=1e-4)) - - @staticmethod - def get_trainer( - model_name, - task, - save_directory, - q_config=None, - p_config=None, - d_config=None, - save_onnx_model=True, - num_train_samples=8, - num_eval_samples=8, - ): - model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) - tokenizer = AutoTokenizer.from_pretrained(model_name) - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - - metric = evaluate.load("accuracy") - dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] - dataset = load_dataset(dataset_name, dataset_config_name) - dataset = dataset.map( - partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), batched=True - ) - - trainer = INCTrainer( - model=model, - quantization_config=q_config, - pruning_config=p_config, - distillation_config=d_config, - task=task, - args=TrainingArguments(save_directory, num_train_epochs=2.0, do_train=True, do_eval=True), - train_dataset=dataset["train"].select(range(num_train_samples)), - eval_dataset=dataset["validation"].select(range(num_eval_samples)), - compute_metrics=partial(_compute_metrics, metric=metric), - tokenizer=tokenizer, - data_collator=default_data_collator, - ) - trainer.train() - trainer.evaluate() - trainer.save_model(save_onnx_model=save_onnx_model) - trainer.model.eval() - return trainer diff --git a/tests/neural_compressor/utils_tests.py b/tests/neural_compressor/utils_tests.py new file mode 100644 index 0000000000..60606f146f --- /dev/null +++ b/tests/neural_compressor/utils_tests.py @@ -0,0 +1,169 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ruff: noqa + + +import os +import unittest +from functools import partial + +import evaluate +import numpy as np +import torch +from datasets import load_dataset +from onnx import load as onnx_load +from transformers import AutoTokenizer, TrainingArguments, default_data_collator + +from optimum.intel import ( + INCConfig, + INCModelForCausalLM, + INCModelForSeq2SeqLM, + INCModelForQuestionAnswering, + INCModelForSequenceClassification, + INCModelForMaskedLM, + INCModelForTokenClassification, + INCQuantizer, + INCTrainer, + INCSeq2SeqTrainer, + INCStableDiffusionPipeline, +) +from optimum.intel.neural_compressor.utils import _HEAD_TO_AUTOMODELS +from optimum.intel.utils.constant import ONNX_WEIGHTS_NAME +from optimum.onnxruntime import ORTModelForCausalLM, ORTModelForSequenceClassification +from optimum.pipelines import ORT_SUPPORTED_TASKS + +SEED = 1009 +_TASK_TO_DATASET = { + "text-classification": ("glue", "sst2", "sentence"), + "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), + "text2text-generation": ("cnn_dailymail", "3.0.0", ("article", "highlights")), +} + + +def num_quantized_matmul_onnx_model(onnx_model): + num_quantized_matmul = 0 + for initializer in onnx_model.graph.initializer: + if "QuantizeLinear" in initializer.name: + num_quantized_matmul += 1 + return num_quantized_matmul + + +def _preprocess_function(examples, tokenizer, column_name): + return tokenizer(examples[column_name], padding="max_length", max_length=128, truncation=True) + + +def _compute_metrics(outputs, metric): + return metric.compute(predictions=np.argmax(outputs.predictions, axis=1), references=outputs.label_ids) + + +def _generate_dataset(quantizer, tokenizer, num_samples=10): + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[quantizer.task] + dataset = quantizer.get_calibration_dataset( + dataset_name, + dataset_config_name=dataset_config_name, + preprocess_function=partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), + num_samples=num_samples, + dataset_split="train", + ) + return dataset + + +class INCTestMixin(unittest.TestCase): + def check_model_outputs( + self, + q_model, + task, + tokenizer, + save_directory, + expected_quantized_matmuls, + is_static=True, + load_onnx_model=True, + num_samples=None, + file_name=ONNX_WEIGHTS_NAME, + ): + tokens = tokenizer("This is a sample input", return_tensors="pt") + inc_model = eval(_HEAD_TO_AUTOMODELS[task]).from_pretrained(save_directory) + model_kwargs = ( + {"decoder_file_name": file_name, "use_cache": False} + if task == "text-generation" + else {"file_name": file_name} + ) + inc_config = INCConfig.from_pretrained(save_directory) + self.assertEqual(inc_config.save_onnx_model, load_onnx_model) + + if num_samples is not None: + self.assertEqual(inc_config.quantization["dataset_num_samples"], num_samples) + + if load_onnx_model: + onnx_model = onnx_load(os.path.join(save_directory, file_name)) + num_quantized_matmul = num_quantized_matmul_onnx_model(onnx_model) + + if num_quantized_matmul > 0: + self.assertEqual(inc_config.quantization["is_static"], is_static) + + self.assertEqual(expected_quantized_matmuls, num_quantized_matmul) + ort_model = ORT_SUPPORTED_TASKS[task]["class"][0].from_pretrained(save_directory, **model_kwargs) + ort_outputs = ort_model(**tokens) + self.assertTrue("logits" in ort_outputs) + + with torch.no_grad(): + model_outputs = q_model(**tokens) + inc_model_outputs = inc_model(**tokens) + outputs = model_outputs["logits"] if isinstance(model_outputs, dict) else model_outputs[0] + self.assertTrue(torch.equal(outputs, inc_model_outputs["logits"])) + # self.assertTrue(torch.allclose(ort_outputs.logits, inc_model_outputs.logits, atol=1e-4)) + + @staticmethod + def get_trainer( + model_name, + task, + save_directory, + q_config=None, + p_config=None, + d_config=None, + save_onnx_model=True, + num_train_samples=8, + num_eval_samples=8, + ): + model = ORT_SUPPORTED_TASKS[task]["class"][0].auto_model_class.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + metric = evaluate.load("accuracy") + dataset_name, dataset_config_name, column_name = _TASK_TO_DATASET[task] + dataset = load_dataset(dataset_name, dataset_config_name) + dataset = dataset.map( + partial(_preprocess_function, tokenizer=tokenizer, column_name=column_name), batched=True + ) + + trainer = INCTrainer( + model=model, + quantization_config=q_config, + pruning_config=p_config, + distillation_config=d_config, + task=task, + args=TrainingArguments(save_directory, num_train_epochs=2.0, do_train=True, do_eval=True), + train_dataset=dataset["train"].select(range(num_train_samples)), + eval_dataset=dataset["validation"].select(range(num_eval_samples)), + compute_metrics=partial(_compute_metrics, metric=metric), + tokenizer=tokenizer, + data_collator=default_data_collator, + ) + trainer.train() + trainer.evaluate() + trainer.save_model(save_onnx_model=save_onnx_model) + trainer.model.eval() + return trainer From 44cebca1c9a7af29f07a4334f9ffa026685d7aa6 Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Wed, 2 Aug 2023 16:53:03 +0200 Subject: [PATCH 100/134] Tiny test updates (#398) --- .github/workflows/test_openvino.yml | 2 ++ tests/openvino/test_modeling.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index c8a7516797..cb58f412a6 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -30,6 +30,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + # install PyTorch CPU version to avoid installing CUDA packages on GitHub runner without GPU + pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu pip install .[openvino,nncf,tests,diffusers] - name: Test with Pytest run: | diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 7fe47ecb24..2461481f59 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -422,7 +422,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "pegasus", ) GENERATION_LENGTH = 100 - SPEEDUP_CACHE = 1.2 + SPEEDUP_CACHE = 1.1 @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): From 0a80e2053b26d72869710c5f2c14b9ee06d5141e Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Wed, 2 Aug 2023 18:10:29 +0200 Subject: [PATCH 101/134] Add more audio tasks for OpenVINO inference (#396) * Add more audio tasks for OpenVINO inference * Fix style * Add reference docs for new audio tasks * Add reference link to top of OpenVINO inference docs * Allow import from optimum.intel directly * Add relative link to reference docs * Add more imports to optimum.intel * Use optimum.intel imports in test --- docs/source/inference.mdx | 2 + docs/source/reference_ov.mdx | 12 +- optimum/intel/__init__.py | 6 + optimum/intel/openvino/__init__.py | 3 + optimum/intel/openvino/modeling.py | 245 ++++++++++++++++++ optimum/intel/utils/dummy_openvino_objects.py | 33 +++ tests/openvino/test_modeling.py | 168 +++++++++++- tests/openvino/utils_tests.py | 1 + 8 files changed, 464 insertions(+), 6 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index c0360322ea..0060dfabbf 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -34,6 +34,8 @@ outputs = cls_pipe("He's a dreadful magician.") [{'label': 'NEGATIVE', 'score': 0.9919503927230835}] ``` +See the [reference documentation](reference_ov) for more information about parameters, and examples for different tasks. + To easily save the resulting model, you can use the `save_pretrained()` method, which will save both the BIN and XML files describing the graph. It is useful to save the tokenizer to the same directory, to enable easy loading of the tokenizer for the model. diff --git a/docs/source/reference_ov.mdx b/docs/source/reference_ov.mdx index 8bc111d594..4c5ede653e 100644 --- a/docs/source/reference_ov.mdx +++ b/docs/source/reference_ov.mdx @@ -36,11 +36,21 @@ limitations under the License. [[autodoc]] openvino.modeling.OVModelForTokenClassification - ## OVModelForAudioClassification [[autodoc]] openvino.modeling.OVModelForAudioClassification +## OVModelForAudioFrameClassification + +[[autodoc]] openvino.modeling.OVModelForAudioFrameClassification + +## OVModelForCTC + +[[autodoc]] openvino.modeling.OVModelForCTC + +## OVModelForAudioXVector + +[[autodoc]] openvino.modeling.OVModelForAudioXVector ## OVModelForImageClassification diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index 65e39d365f..8e3f7619a2 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -87,6 +87,9 @@ _import_structure["openvino"].extend( [ "OVModelForAudioClassification", + "OVModelForAudioFrameClassification", + "OVModelForAudioXVector", + "OVModelForCTC", "OVModelForCausalLM", "OVModelForFeatureExtraction", "OVModelForImageClassification", @@ -176,7 +179,10 @@ else: from .openvino import ( OVModelForAudioClassification, + OVModelForAudioFrameClassification, + OVModelForAudioXVector, OVModelForCausalLM, + OVModelForCTC, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index cfbac71fd1..fd0806cbcc 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -35,6 +35,9 @@ from .modeling import ( OVModelForAudioClassification, + OVModelForAudioFrameClassification, + OVModelForAudioXVector, + OVModelForCTC, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index b999a4116e..eb1f7a410f 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -23,6 +23,9 @@ AutoConfig, AutoModel, AutoModelForAudioClassification, + AutoModelForAudioFrameClassification, + AutoModelForAudioXVector, + AutoModelForCTC, AutoModelForImageClassification, AutoModelForMaskedLM, AutoModelForQuestionAnswering, @@ -32,13 +35,17 @@ from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import ( BaseModelOutput, + CausalLMOutput, ImageClassifierOutput, MaskedLMOutput, QuestionAnsweringModelOutput, SequenceClassifierOutput, TokenClassifierOutput, + XVectorOutput, ) +from optimum.exporters import TasksManager + from .modeling_base import OVBaseModel @@ -93,6 +100,13 @@ Pixel values can be obtained from encoded images using [`AutoFeatureExtractor`](https://huggingface.co/docs/transformers/autoclass_tutorial#autofeatureextractor). """ +AUDIO_INPUTS_DOCSTRING = r""" + Args: + input_values (`torch.Tensor` of shape `({0})`): + Float values of input raw speech waveform.. + Input values can be obtained from audio file loaded into an array using [`AutoFeatureExtractor`](https://huggingface.co/docs/transformers/autoclass_tutorial#autofeatureextractor). +""" + class OVModel(OVBaseModel): base_model_prefix = "openvino_model" @@ -575,3 +589,234 @@ def forward( outputs = self.request(inputs) logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] return SequenceClassifierOutput(logits=logits) + + +CTC_EXAMPLE = r""" + Example of CTC: + + ```python + >>> from transformers import {processor_class} + >>> from optimum.intel import {model_class} + >>> from datasets import load_dataset + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}", export=True) + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="np") + >>> logits = model(**inputs).logits + >>> predicted_ids = np.argmax(logits, axis=-1) + + >>> transcription = processor.batch_decode(predicted_ids) + ``` +""" + + +@add_start_docstrings( + """ + Onnx Model with a language modeling head on top for Connectionist Temporal Classification (CTC). + """, + MODEL_START_DOCSTRING, +) +class OVModelForCTC(OVModel): + """ + CTC model for OpenVINO. + """ + + auto_model_class = AutoModelForCTC + export_feature = TasksManager.infer_task_from_model(auto_model_class) + + @add_start_docstrings_to_model_forward( + AUDIO_INPUTS_DOCSTRING.format("batch_size, sequence_length") + + CTC_EXAMPLE.format( + processor_class=_FEATURE_EXTRACTOR_FOR_DOC, + model_class="OVModelForCTC", + checkpoint="facebook/hubert-large-ls960-ft", + ) + ) + def forward( + self, + input_values: Optional[torch.Tensor] = None, + attention_mask: Optional[Union[torch.Tensor, np.ndarray]] = None, + **kwargs, + ): + np_inputs = isinstance(input_values, np.ndarray) + if not np_inputs: + input_values = np.array(input_values) + attention_mask = np.array(attention_mask) if attention_mask is not None else attention_mask + + inputs = { + "input_values": input_values, + } + + # Add the attention_mask when needed + if "attention_mask" in self.input_names: + inputs["attention_mask"] = attention_mask + + # Run inference + outputs = self.request(inputs) + logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] + return CausalLMOutput(logits=logits) + + +AUDIO_XVECTOR_EXAMPLE = r""" + Example of Audio XVector: + + ```python + >>> from transformers import {processor_class} + >>> from optimum.intel import {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> feature_extractor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}", export=True) + + >>> # audio file is decoded on the fly + >>> inputs = feature_extractor( + ... [d["array"] for d in dataset[:2]["audio"]], sampling_rate=sampling_rate, return_tensors="pt", padding=True + ... ) + >>> embeddings = model(**inputs).embeddings + + >>> embeddings = torch.nn.functional.normalize(embeddings, dim=-1).cpu() + + >>> cosine_sim = torch.nn.CosineSimilarity(dim=-1) + >>> similarity = cosine_sim(embeddings[0], embeddings[1]) + >>> threshold = 0.7 + >>> if similarity < threshold: + ... print("Speakers are not the same!") + >>> round(similarity.item(), 2) + ``` +""" + + +@add_start_docstrings( + """ + Onnx Model with an XVector feature extraction head on top for tasks like Speaker Verification. + """, + MODEL_START_DOCSTRING, +) +class OVModelForAudioXVector(OVModel): + """ + Audio XVector model for OpenVINO. + """ + + auto_model_class = AutoModelForAudioXVector + export_feature = TasksManager.infer_task_from_model(auto_model_class) + + @add_start_docstrings_to_model_forward( + AUDIO_INPUTS_DOCSTRING.format("batch_size, sequence_length") + + AUDIO_XVECTOR_EXAMPLE.format( + processor_class=_FEATURE_EXTRACTOR_FOR_DOC, + model_class="OVModelForAudioXVector", + checkpoint="anton-l/wav2vec2-base-superb-sv", + ) + ) + def forward( + self, + input_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + **kwargs, + ): + np_inputs = isinstance(input_values, np.ndarray) + if not np_inputs: + input_values = np.array(input_values) + attention_mask = np.array(attention_mask) if attention_mask is not None else attention_mask + + inputs = { + "input_values": input_values, + } + + # Add the attention_mask when needed + if "attention_mask" in self.input_names: + inputs["attention_mask"] = attention_mask + + # Run inference + outputs = self.request(inputs) + logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] + embeddings = ( + torch.from_numpy(outputs["embeddings"]).to(self.device) if not np_inputs else outputs["embeddings"] + ) + + return XVectorOutput(logits=logits, embeddings=embeddings) + + +AUDIO_FRAME_CLASSIFICATION_EXAMPLE = r""" + Example of audio frame classification: + + ```python + >>> from transformers import {processor_class} + >>> from optimum.intel import {model_class} + >>> from datasets import load_dataset + >>> import torch + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> feature_extractor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}", export=True) + + >>> inputs = feature_extractor(dataset[0]["audio"]["array"], return_tensors="pt", sampling_rate=sampling_rate) + >>> logits = model(**inputs).logits + + >>> probabilities = torch.sigmoid(torch.as_tensor(logits)[0]) + >>> labels = (probabilities > 0.5).long() + >>> labels[0].tolist() + ``` +""" + + +@add_start_docstrings( + """ + OpenVINO Model for with a frame classification head on top for tasks like Speaker Diarization. + """, + MODEL_START_DOCSTRING, +) +class OVModelForAudioFrameClassification(OVModel): + """ + Audio Frame Classification model for OpenVINO. + """ + + auto_model_class = AutoModelForAudioFrameClassification + export_feature = TasksManager.infer_task_from_model(auto_model_class) + + @add_start_docstrings_to_model_forward( + AUDIO_INPUTS_DOCSTRING.format("batch_size, sequence_length") + + AUDIO_FRAME_CLASSIFICATION_EXAMPLE.format( + processor_class=_FEATURE_EXTRACTOR_FOR_DOC, + model_class="OVModelForAudioFrameClassification", + checkpoint="anton-l/wav2vec2-base-superb-sd", + ) + ) + def forward( + self, + input_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + **kwargs, + ): + np_inputs = isinstance(input_values, np.ndarray) + if not np_inputs: + input_values = np.array(input_values) + attention_mask = np.array(attention_mask) if attention_mask is not None else attention_mask + + inputs = { + "input_values": input_values, + } + + # Add the attention_mask when needed + if "attention_mask" in self.input_names: + inputs["attention_mask"] = attention_mask + + # Run inference + outputs = self.request(inputs) + logits = torch.from_numpy(outputs["logits"]).to(self.device) if not np_inputs else outputs["logits"] + + return TokenClassifierOutput(logits=logits) diff --git a/optimum/intel/utils/dummy_openvino_objects.py b/optimum/intel/utils/dummy_openvino_objects.py index ff5be62360..b7c4939a72 100644 --- a/optimum/intel/utils/dummy_openvino_objects.py +++ b/optimum/intel/utils/dummy_openvino_objects.py @@ -26,6 +26,39 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["openvino"]) +class OVModelForAudioFrameClassification(metaclass=DummyObject): + _backends = ["openvino"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino"]) + + +class OVModelForAudioXVector(metaclass=DummyObject): + _backends = ["openvino"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino"]) + + +class OVModelForCTC(metaclass=DummyObject): + _backends = ["openvino"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["openvino"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["openvino"]) + + class OVModelForCausalLM(metaclass=DummyObject): _backends = ["openvino"] diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 2461481f59..e1833ff23a 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -30,7 +30,10 @@ AutoFeatureExtractor, AutoModel, AutoModelForAudioClassification, + AutoModelForAudioFrameClassification, + AutoModelForAudioXVector, AutoModelForCausalLM, + AutoModelForCTC, AutoModelForImageClassification, AutoModelForMaskedLM, AutoModelForQuestionAnswering, @@ -45,13 +48,12 @@ ) from utils_tests import MODEL_NAMES -from optimum.intel.openvino import ( - OV_DECODER_NAME, - OV_DECODER_WITH_PAST_NAME, - OV_ENCODER_NAME, - OV_XML_FILE_NAME, +from optimum.intel import ( OVModelForAudioClassification, + OVModelForAudioFrameClassification, + OVModelForAudioXVector, OVModelForCausalLM, + OVModelForCTC, OVModelForFeatureExtraction, OVModelForImageClassification, OVModelForMaskedLM, @@ -61,6 +63,7 @@ OVModelForTokenClassification, OVStableDiffusionPipeline, ) +from optimum.intel.openvino import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, @@ -797,3 +800,158 @@ def test_pipeline(self, model_arch): outputs = pipe([np.random.random(16000)]) self.assertEqual(pipe.device, model.device) self.assertTrue(all(item["score"] > 0.0 for item in outputs[0])) + + +class OVModelForCTCIntegrationTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = [ + "data2vec_audio", + "hubert", + "sew", + "sew_d", + "unispeech", + "unispeech_sat", + "wavlm", + "wav2vec2-hf", + "wav2vec2-conformer", + ] + + def _generate_random_audio_data(self): + np.random.seed(10) + t = np.linspace(0, 5.0, int(5.0 * 22050), endpoint=False) + # generate pure sine wave at 220 Hz + audio_data = 0.5 * np.sin(2 * np.pi * 220 * t) + return audio_data + + def test_load_vanilla_transformers_which_is_not_supported(self): + with self.assertRaises(Exception) as context: + _ = OVModelForCTC.from_pretrained(MODEL_NAMES["t5"], export=True) + + self.assertIn("Unrecognized configuration class", str(context.exception)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_to_transformers(self, model_arch): + model_id = MODEL_NAMES[model_arch] + set_seed(SEED) + ov_model = OVModelForCTC.from_pretrained(model_id, export=True) + self.assertIsInstance(ov_model.config, PretrainedConfig) + + set_seed(SEED) + transformers_model = AutoModelForCTC.from_pretrained(model_id) + processor = AutoFeatureExtractor.from_pretrained(model_id) + input_values = processor(self._generate_random_audio_data(), return_tensors="pt") + + with torch.no_grad(): + transformers_outputs = transformers_model(**input_values) + + for input_type in ["pt", "np"]: + input_values = processor(self._generate_random_audio_data(), return_tensors=input_type) + ov_outputs = ov_model(**input_values) + + self.assertTrue("logits" in ov_outputs) + self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) + + # compare tensor outputs + self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + + gc.collect() + + +class OVModelForAudioXVectorIntegrationTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = [ + "data2vec_audio", + "unispeech_sat", + "wavlm", + "wav2vec2-hf", + "wav2vec2-conformer", + ] + + def _generate_random_audio_data(self): + np.random.seed(10) + t = np.linspace(0, 5.0, int(5.0 * 22050), endpoint=False) + # generate pure sine wave at 220 Hz + audio_data = 0.5 * np.sin(2 * np.pi * 220 * t) + return audio_data + + def test_load_vanilla_transformers_which_is_not_supported(self): + with self.assertRaises(Exception) as context: + _ = OVModelForAudioXVector.from_pretrained(MODEL_NAMES["t5"], export=True) + + self.assertIn("Unrecognized configuration class", str(context.exception)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_to_transformers(self, model_arch): + model_id = MODEL_NAMES[model_arch] + set_seed(SEED) + ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True) + self.assertIsInstance(ov_model.config, PretrainedConfig) + + set_seed(SEED) + transformers_model = AutoModelForAudioXVector.from_pretrained(model_id) + processor = AutoFeatureExtractor.from_pretrained(model_id) + input_values = processor(self._generate_random_audio_data(), return_tensors="pt") + + with torch.no_grad(): + transformers_outputs = transformers_model(**input_values) + for input_type in ["pt", "np"]: + input_values = processor(self._generate_random_audio_data(), return_tensors=input_type) + ov_outputs = ov_model(**input_values) + + self.assertTrue("logits" in ov_outputs) + self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) + + # compare tensor outputs + self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + self.assertTrue( + torch.allclose(torch.Tensor(ov_outputs.embeddings), transformers_outputs.embeddings, atol=1e-4) + ) + + gc.collect() + + +class OVModelForAudioFrameClassificationIntegrationTest(unittest.TestCase): + SUPPORTED_ARCHITECTURES = [ + "data2vec_audio", + "unispeech_sat", + "wavlm", + "wav2vec2-hf", + "wav2vec2-conformer", + ] + + def _generate_random_audio_data(self): + np.random.seed(10) + t = np.linspace(0, 5.0, int(5.0 * 22050), endpoint=False) + # generate pure sine wave at 220 Hz + audio_data = 0.5 * np.sin(2 * np.pi * 220 * t) + return audio_data + + def test_load_vanilla_transformers_which_is_not_supported(self): + with self.assertRaises(Exception) as context: + _ = OVModelForAudioFrameClassification.from_pretrained(MODEL_NAMES["t5"], export=True) + + self.assertIn("Unrecognized configuration class", str(context.exception)) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_compare_to_transformers(self, model_arch): + model_id = MODEL_NAMES[model_arch] + set_seed(SEED) + ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True) + self.assertIsInstance(ov_model.config, PretrainedConfig) + + set_seed(SEED) + transformers_model = AutoModelForAudioFrameClassification.from_pretrained(model_id) + processor = AutoFeatureExtractor.from_pretrained(model_id) + input_values = processor(self._generate_random_audio_data(), return_tensors="pt") + + with torch.no_grad(): + transformers_outputs = transformers_model(**input_values) + for input_type in ["pt", "np"]: + input_values = processor(self._generate_random_audio_data(), return_tensors=input_type) + ov_outputs = ov_model(**input_values) + + self.assertTrue("logits" in ov_outputs) + self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) + + # compare tensor outputs + self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + + gc.collect() diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 6a14e96796..eeb751153a 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -78,6 +78,7 @@ "vit": "hf-internal-testing/tiny-random-vit", "wavlm": "hf-internal-testing/tiny-random-WavlmModel", "wav2vec2": "anton-l/wav2vec2-random-tiny-classifier", + "wav2vec2-hf": "hf-internal-testing/tiny-random-Wav2Vec2Model", "wav2vec2-conformer": "hf-internal-testing/tiny-random-wav2vec2-conformer", "xlm": "hf-internal-testing/tiny-random-xlm", "xlm_roberta": "hf-internal-testing/tiny-xlm-roberta", From 54d5d79677041382c623457742c98de5b351feb3 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Wed, 9 Aug 2023 11:43:08 +0300 Subject: [PATCH 102/134] Do not download onnx model in sd pipeline if from_onnx=False (#402) --- optimum/intel/openvino/modeling_diffusion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 6d263fdd50..970afc1283 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -217,6 +217,9 @@ def _from_pretrained( cls.config_name, } ) + ignore_patterns = ["*.msgpack", "*.safetensors", "*pytorch_model.bin"] + if not from_onnx: + ignore_patterns.extend(["*.onnx", "*.onnx_data"]) # Downloads all repo's files matching the allowed patterns model_id = snapshot_download( model_id, @@ -225,7 +228,7 @@ def _from_pretrained( use_auth_token=use_auth_token, revision=revision, allow_patterns=allow_patterns, - ignore_patterns=["*.msgpack", "*.safetensors", "*pytorch_model.bin"], + ignore_patterns=ignore_patterns, ) new_model_save_dir = Path(model_id) From cb7c5b268c97191c4ff9000c979efa0b8642b285 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Fri, 11 Aug 2023 08:27:55 -0700 Subject: [PATCH 103/134] Support OVStableDiffusionPipelineBase to Load Textual Inversion in Runtime (#400) * Enable OVStableDiffusionPipelineBase to load textual inversion embeddings in runtime * Move OVTextualInversionLoaderMixin in loaders.py * Reformat with black * Fix format via make style * Fix notebook format via make style * Add docs for textual inversion --- docs/source/inference.mdx | 79 ++++ .../openvino/optimum_openvino_inference.ipynb | 4 +- .../stable_diffusion_optimization.ipynb | 8 +- optimum/intel/openvino/loaders.py | 404 ++++++++++++++++++ optimum/intel/openvino/modeling_diffusion.py | 3 +- optimum/intel/openvino/utils.py | 4 + 6 files changed, 498 insertions(+), 4 deletions(-) create mode 100644 optimum/intel/openvino/loaders.py diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 0060dfabbf..785031bdde 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -208,6 +208,44 @@ In case you want to change any parameters such as the outputs height or width, y +### Text-to-Image with Textual Inversion +Here is an example of how you can load an OpenVINO Stable Diffusion model with pre-trained textual inversion embeddings and run inference using OpenVINO Runtime: + + +First, you can run original pipeline without textual inversion +```python +from optimum.intel import OVStableDiffusionPipeline +import numpy as np + +model_id = "echarlaix/stable-diffusion-v1-5-openvino" +prompt = "A back-pack" +# Set a random seed for better comparison +np.random.seed(42) + +pipeline = OVStableDiffusionPipeline.from_pretrained(model_id, export=False, compile=False) +pipeline.compile() +image1 = pipeline(prompt, num_inference_steps=50).images[0] +image1.save("stable_diffusion_v1_5_without_textual_inversion.png") +``` + +Then, you can load [sd-concepts-library/cat-toy](https://huggingface.co/sd-concepts-library/cat-toy) textual inversion embedding and run pipeline with same prompt again +```python +# Reset stable diffusion pipeline +pipeline.clear_requests() + +# Load textual inversion into stable diffusion pipeline +pipeline.load_textual_inversion("sd-concepts-library/cat-toy", "") + +# Compile the model before the first inference +pipeline.compile() +image2 = pipeline(prompt, num_inference_steps=50).images[0] +image2.save("stable_diffusion_v1_5_with_textual_inversion.png") +``` +The left image shows the generation result of original stable diffusion v1.5, the right image shows the generation result of stable diffusion v1.5 with textual inversion. +| | | +|---|---| +| ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/stable_diffusion_v1_5_without_textual_inversion.png) | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/stable_diffusion_v1_5_with_textual_inversion.png) | + ### Image-to-Image @@ -257,6 +295,47 @@ image.save("train_station.png") |---|---| | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/train_station_friedrich.png) | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/train_station_friedrich_2.png) | +### Text-to-Image with Textual Inversion + +Here is an example of how you can load an SDXL OpenVINO model from [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) with pre-trained textual inversion embeddings and run inference using OpenVINO Runtime: + + +First, you can run original pipeline without textual inversion +```python +from optimum.intel import OVStableDiffusionXLPipeline +import numpy as np + +model_id = "stabilityai/stable-diffusion-xl-base-1.0" +prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a beautiful woman wearing a black jacket and red shirt, best quality, intricate details." +# Set a random seed for better comparison +np.random.seed(0) + +base = OVStableDiffusionXLPipeline.from_pretrained(model_id, export=False, compile=False) +base.compile() +image1 = base(prompt, num_inference_steps=50).images[0] +image1.save("sdxl_without_textual_inversion.png") +``` + +Then, you can load [charturnerv2](https://civitai.com/models/3036/charturner-character-turnaround-helper-for-15-and-21) textual inversion embedding and run pipeline with same prompt again +```python +# Reset stable diffusion pipeline +base.clear_requests() + +# Load textual inversion into stable diffusion pipeline +base.load_textual_inversion("./charturnerv2.pt", "charturnerv2") + +# Compile the model before the first inference +base.compile() +image2 = base(prompt, num_inference_steps=50).images[0] +image2.save("sdxl_with_textual_inversion.png") + +The left image shows the generation result of the original SDXL base 1.0, the right image shows the generation result of SDXL base 1.0 with textual inversion. +``` + +| | | +|---|---| +| ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/sdxl_without_textual_inversion.png) | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/sdxl_with_textual_inversion.png) | + ### Image-to-Image diff --git a/notebooks/openvino/optimum_openvino_inference.ipynb b/notebooks/openvino/optimum_openvino_inference.ipynb index bec8a970f2..446e668911 100644 --- a/notebooks/openvino/optimum_openvino_inference.ipynb +++ b/notebooks/openvino/optimum_openvino_inference.ipynb @@ -344,7 +344,9 @@ "from optimum.intel.openvino import OVModelForQuestionAnswering\n", "from transformers import AutoTokenizer, pipeline\n", "\n", - "model = OVModelForQuestionAnswering.from_pretrained(\"helenai/distilbert-base-uncased-distilled-squad-ov-fp32\", compile=False)\n", + "model = OVModelForQuestionAnswering.from_pretrained(\n", + " \"helenai/distilbert-base-uncased-distilled-squad-ov-fp32\", compile=False\n", + ")\n", "tokenizer = AutoTokenizer.from_pretrained(\"helenai/distilbert-base-uncased-distilled-squad-ov-fp32\")\n", "\n", "max_length = 128\n", diff --git a/notebooks/openvino/stable_diffusion_optimization.ipynb b/notebooks/openvino/stable_diffusion_optimization.ipynb index 6c79bc5df0..b44c00f78c 100644 --- a/notebooks/openvino/stable_diffusion_optimization.ipynb +++ b/notebooks/openvino/stable_diffusion_optimization.ipynb @@ -69,7 +69,9 @@ "metadata": {}, "outputs": [], "source": [ - "quantized_pipe = OVStableDiffusionPipeline.from_pretrained(\"OpenVINO/Stable-Diffusion-Pokemon-en-quantized\", compile=False)\n", + "quantized_pipe = OVStableDiffusionPipeline.from_pretrained(\n", + " \"OpenVINO/Stable-Diffusion-Pokemon-en-quantized\", compile=False\n", + ")\n", "quantized_pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1)\n", "quantized_pipe.compile()" ] @@ -102,7 +104,9 @@ "metadata": {}, "outputs": [], "source": [ - "optimized_pipe = OVStableDiffusionPipeline.from_pretrained(\"OpenVINO/stable-diffusion-pokemons-tome-quantized\", compile=False)\n", + "optimized_pipe = OVStableDiffusionPipeline.from_pretrained(\n", + " \"OpenVINO/stable-diffusion-pokemons-tome-quantized\", compile=False\n", + ")\n", "optimized_pipe.reshape(batch_size=1, height=512, width=512, num_images_per_prompt=1)\n", "optimized_pipe.compile()" ] diff --git a/optimum/intel/openvino/loaders.py b/optimum/intel/openvino/loaders.py new file mode 100644 index 0000000000..be0c03fb41 --- /dev/null +++ b/optimum/intel/openvino/loaders.py @@ -0,0 +1,404 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Dict, List, Optional, Union + +import torch +from diffusers.utils import ( + DIFFUSERS_CACHE, + HF_HUB_OFFLINE, + _get_model_file, + is_safetensors_available, +) + + +if is_safetensors_available(): + import safetensors + +import openvino +from openvino.runtime import Type +from openvino.runtime import opset11 as ops +from openvino.runtime.passes import Manager, Matcher, MatcherPass, WrapType +from transformers import PreTrainedTokenizer + +from .utils import TEXTUAL_INVERSION_EMBEDDING_KEY, TEXTUAL_INVERSION_NAME, TEXTUAL_INVERSION_NAME_SAFE + + +logger = logging.getLogger(__name__) + + +class InsertTextEmbedding(MatcherPass): + r""" + OpenVINO ngraph transformation for inserting pre-trained texual inversion embedding to text encoder + """ + + def __init__(self, token_ids_and_embeddings): + MatcherPass.__init__(self) + self.model_changed = False + param = WrapType("opset1.Constant") + + def callback(matcher: Matcher) -> bool: + root = matcher.get_match_root() + if root.get_friendly_name() == TEXTUAL_INVERSION_EMBEDDING_KEY: + add_ti = root + consumers = matcher.get_match_value().get_target_inputs() + for token_id, embedding in token_ids_and_embeddings: + ti_weights = ops.constant(embedding, Type.f32, name=str(token_id)) + ti_weights_unsqueeze = ops.unsqueeze(ti_weights, axes=0) + add_ti = ops.concat( + nodes=[add_ti, ti_weights_unsqueeze], + axis=0, + name=f"{TEXTUAL_INVERSION_EMBEDDING_KEY}.textual_inversion_{token_id}", + ) + + for consumer in consumers: + consumer.replace_source_output(add_ti.output(0)) + + # Use new operation for additional matching + self.register_new_node(add_ti) + + # Root node wasn't replaced or changed + return False + + self.register_matcher(Matcher(param, "InsertTextEmbedding"), callback) + + +# Adapted from diffusers.loaders.TextualInversionLoaderMixin +class OVTextualInversionLoaderMixin: + r""" + Load textual inversion tokens and embeddings to the tokenizer and text encoder. + """ + + def maybe_convert_prompt(self, prompt: Union[str, List[str]], tokenizer: "PreTrainedTokenizer"): + r""" + Processes prompts that include a special token corresponding to a multi-vector textual inversion embedding to + be replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual + inversion token or if the textual inversion token is a single vector, the input prompt is returned. + + Parameters: + prompt (`str` or list of `str`): + The prompt or prompts to guide the image generation. + tokenizer (`PreTrainedTokenizer`): + The tokenizer responsible for encoding the prompt into input tokens. + + Returns: + `str` or list of `str`: The converted prompt + """ + if not isinstance(prompt, List): + prompts = [prompt] + else: + prompts = prompt + + prompts = [self._maybe_convert_prompt(p, tokenizer) for p in prompts] + + if not isinstance(prompt, List): + return prompts[0] + + return prompts + + def _maybe_convert_prompt(self, prompt: str, tokenizer: "PreTrainedTokenizer"): + r""" + Maybe convert a prompt into a "multi vector"-compatible prompt. If the prompt includes a token that corresponds + to a multi-vector textual inversion embedding, this function will process the prompt so that the special token + is replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual + inversion token or a textual inversion token that is a single vector, the input prompt is simply returned. + + Parameters: + prompt (`str`): + The prompt to guide the image generation. + tokenizer (`PreTrainedTokenizer`): + The tokenizer responsible for encoding the prompt into input tokens. + + Returns: + `str`: The converted prompt + """ + tokens = tokenizer.tokenize(prompt) + unique_tokens = set(tokens) + for token in unique_tokens: + if token in tokenizer.added_tokens_encoder: + replacement = token + i = 1 + while f"{token}_{i}" in tokenizer.added_tokens_encoder: + replacement += f" {token}_{i}" + i += 1 + + prompt = prompt.replace(token, replacement) + + return prompt + + def load_textual_inversion( + self, + pretrained_model_name_or_path: Union[str, List[str], Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]], + token: Optional[Union[str, List[str]]] = None, + **kwargs, + ): + r""" + Load textual inversion embeddings into the text encoder of [`StableDiffusionPipeline`] (both 🤗 Diffusers and + Automatic1111 formats are supported). + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike` or `List[str or os.PathLike]` or `Dict` or `List[Dict]`): + Can be either one of the following or a list of them: + + - A string, the *model id* (for example `sd-concepts-library/low-poly-hd-logos-icons`) of a + pretrained model hosted on the Hub. + - A path to a *directory* (for example `./my_text_inversion_directory/`) containing the textual + inversion weights. + - A path to a *file* (for example `./my_text_inversions.pt`) containing textual inversion weights. + - A [torch state + dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). + + token (`str` or `List[str]`, *optional*): + Override the token to use for the textual inversion weights. If `pretrained_model_name_or_path` is a + list, then `token` must also be a list of equal length. + weight_name (`str`, *optional*): + Name of a custom weight file. This should be used when: + + - The saved textual inversion file is in 🤗 Diffusers format, but was saved under a specific weight + name such as `text_inv.bin`. + - The saved textual inversion file is in the Automatic1111 format. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + + Example: + + To load a textual inversion embedding vector in 🤗 Diffusers format: + + ```py + from optimum.intel import OVStableDiffusionPipeline + + model_id = "runwayml/stable-diffusion-v1-5" + pipe = OVStableDiffusionPipeline.from_pretrained(model_id, compile=False) + + pipe.load_textual_inversion("sd-concepts-library/cat-toy") + pipe.compile() + + prompt = "A backpack" + + image = pipe(prompt, num_inference_steps=50).images[0] + image.save("cat-backpack.png") + ``` + + To load a textual inversion embedding vector in Automatic1111 format, make sure to download the vector first + (for example from [civitAI](https://civitai.com/models/3036?modelVersionId=9857)) and then load the vector + locally: + + ```py + from optimum.intel import OVStableDiffusionPipeline + + model_id = "runwayml/stable-diffusion-v1-5" + pipe = StableDiffusionPipeline.from_pretrained(model_id, compile=False) + + pipe.load_textual_inversion("./charturnerv2.pt", token="charturnerv2") + pipe.compile() + + prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a woman wearing a black jacket and red shirt, best quality, intricate details." + + image = pipe(prompt, num_inference_steps=50).images[0] + image.save("character.png") + ``` + """ + + if not hasattr(self, "tokenizer") or not isinstance(self.tokenizer, PreTrainedTokenizer): + raise ValueError( + f"{self.__class__.__name__} requires `self.tokenizer` of type `PreTrainedTokenizer` for calling" + f" `{self.load_textual_inversion.__name__}`" + ) + + if not hasattr(self, "text_encoder") or not isinstance(self.text_encoder.model, openvino.runtime.Model): + raise ValueError( + f"{self.__class__.__name__} requires `self.text_encoder.model` of type `openvino.runtime.Model` for calling" + f" `{self.load_textual_inversion.__name__}`" + ) + + cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE) + use_auth_token = kwargs.pop("use_auth_token", None) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", None) + weight_name = kwargs.pop("weight_name", None) + use_safetensors = kwargs.pop("use_safetensors", None) + + if use_safetensors and not is_safetensors_available(): + raise ValueError( + "`use_safetensors`=True but safetensors is not installed. Please install safetensors with `pip install safetensors" + ) + + allow_pickle = False + if use_safetensors is None: + use_safetensors = is_safetensors_available() + allow_pickle = True + + user_agent = { + "file_type": "text_inversion", + "framework": "pytorch", + } + + if not isinstance(pretrained_model_name_or_path, list): + pretrained_model_name_or_paths = [pretrained_model_name_or_path] + else: + pretrained_model_name_or_paths = pretrained_model_name_or_path + + if isinstance(token, str): + tokens = [token] + elif token is None: + tokens = [None] * len(pretrained_model_name_or_paths) + else: + tokens = token + + if len(pretrained_model_name_or_paths) != len(tokens): + raise ValueError( + f"You have passed a list of models of length {len(pretrained_model_name_or_paths)}, and list of tokens of length {len(tokens)}" + f"Make sure both lists have the same length." + ) + + valid_tokens = [t for t in tokens if t is not None] + if len(set(valid_tokens)) < len(valid_tokens): + raise ValueError(f"You have passed a list of tokens that contains duplicates: {tokens}") + + token_ids_and_embeddings = [] + + for pretrained_model_name_or_path, token in zip(pretrained_model_name_or_paths, tokens): + if not isinstance(pretrained_model_name_or_path, dict): + # 1. Load textual inversion file + model_file = None + # Let's first try to load .safetensors weights + if (use_safetensors and weight_name is None) or ( + weight_name is not None and weight_name.endswith(".safetensors") + ): + try: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=weight_name or TEXTUAL_INVERSION_NAME_SAFE, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = safetensors.torch.load_file(model_file, device="cpu") + except Exception as e: + if not allow_pickle: + raise e + + model_file = None + + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=weight_name or TEXTUAL_INVERSION_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + use_auth_token=use_auth_token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = torch.load(model_file, map_location="cpu") + else: + state_dict = pretrained_model_name_or_path + + # 2. Load token and embedding correcly from file + loaded_token = None + if isinstance(state_dict, torch.Tensor): + if token is None: + raise ValueError( + "You are trying to load a textual inversion embedding that has been saved as a PyTorch tensor. Make sure to pass the name of the corresponding token in this case: `token=...`." + ) + embedding = state_dict + elif len(state_dict) == 1: + # diffusers + loaded_token, embedding = next(iter(state_dict.items())) + elif "string_to_param" in state_dict: + # A1111 + loaded_token = state_dict["name"] + embedding = state_dict["string_to_param"]["*"] + + if token is not None and loaded_token != token: + logger.info(f"The loaded token: {loaded_token} is overwritten by the passed token {token}.") + else: + token = loaded_token + + embedding = embedding.detach().cpu().numpy() + + # 3. Make sure we don't mess up the tokenizer or text encoder + vocab = self.tokenizer.get_vocab() + if token in vocab: + raise ValueError( + f"Token {token} already in tokenizer vocabulary. Please choose a different token name or remove {token} and embedding from the tokenizer and text encoder." + ) + elif f"{token}_1" in vocab: + multi_vector_tokens = [token] + i = 1 + while f"{token}_{i}" in self.tokenizer.added_tokens_encoder: + multi_vector_tokens.append(f"{token}_{i}") + i += 1 + + raise ValueError( + f"Multi-vector Token {multi_vector_tokens} already in tokenizer vocabulary. Please choose a different token name or remove the {multi_vector_tokens} and embedding from the tokenizer and text encoder." + ) + is_multi_vector = len(embedding.shape) > 1 and embedding.shape[0] > 1 + if is_multi_vector: + tokens = [token] + [f"{token}_{i}" for i in range(1, embedding.shape[0])] + embeddings = [e for e in embedding] # noqa: C416 + else: + tokens = [token] + embeddings = [embedding[0]] if len(embedding.shape) > 1 else [embedding] + # add tokens and get ids + self.tokenizer.add_tokens(tokens) + token_ids = self.tokenizer.convert_tokens_to_ids(tokens) + token_ids_and_embeddings += zip(token_ids, embeddings) + + logger.info(f"Loaded textual inversion embedding for {token}.") + + # Insert textual inversion embeddings to text encoder with OpenVINO ngraph transformation + manager = Manager() + manager.register_pass(InsertTextEmbedding(token_ids_and_embeddings)) + manager.run_passes(self.text_encoder.model) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 970afc1283..73ec66d473 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -50,6 +50,7 @@ DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ) +from .loaders import OVTextualInversionLoaderMixin from .modeling_base import OVBaseModel from .utils import ONNX_WEIGHTS_NAME, OV_TO_NP_TYPE, OV_XML_FILE_NAME @@ -59,7 +60,7 @@ logger = logging.getLogger(__name__) -class OVStableDiffusionPipelineBase(OVBaseModel): +class OVStableDiffusionPipelineBase(OVBaseModel, OVTextualInversionLoaderMixin): auto_model_class = StableDiffusionPipeline config_name = "model_index.json" export_feature = "stable-diffusion" diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index fe46d28b42..f611ab3526 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -34,6 +34,10 @@ EXTERNAL_DATA_FORMAT_SIZE_LIMIT = 2 * 1024 * 1024 * 1024 +TEXTUAL_INVERSION_NAME = "learned_embeds.bin" +TEXTUAL_INVERSION_NAME_SAFE = "learned_embeds.safetensors" +TEXTUAL_INVERSION_EMBEDDING_KEY = "text_model.embeddings.token_embedding.weight" + OV_TO_NP_TYPE = { "boolean": np.bool_, From 1f9acd39a42a5902e9fc112eae3a9258e76a6192 Mon Sep 17 00:00:00 2001 From: Xiake Sun Date: Mon, 14 Aug 2023 00:48:40 -0700 Subject: [PATCH 104/134] Fix textual inversion docs and update sample image. (#406) --- docs/source/inference.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 785031bdde..6a884dba3e 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -242,6 +242,7 @@ image2 = pipeline(prompt, num_inference_steps=50).images[0] image2.save("stable_diffusion_v1_5_with_textual_inversion.png") ``` The left image shows the generation result of original stable diffusion v1.5, the right image shows the generation result of stable diffusion v1.5 with textual inversion. + | | | |---|---| | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/stable_diffusion_v1_5_without_textual_inversion.png) | ![](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/textual_inversion/stable_diffusion_v1_5_with_textual_inversion.png) | @@ -306,9 +307,9 @@ from optimum.intel import OVStableDiffusionXLPipeline import numpy as np model_id = "stabilityai/stable-diffusion-xl-base-1.0" -prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a beautiful woman wearing a black jacket and red shirt, best quality, intricate details." +prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a beautiful woman wearing a red jacket and black shirt, best quality, intricate details." # Set a random seed for better comparison -np.random.seed(0) +np.random.seed(112) base = OVStableDiffusionXLPipeline.from_pretrained(model_id, export=False, compile=False) base.compile() From c70ad82dbc265d9609e31de6323b976dec99e16e Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Fri, 18 Aug 2023 16:05:26 +0300 Subject: [PATCH 105/134] Fix import error is_safetensors_available (#407) --- optimum/intel/openvino/loaders.py | 3 ++- optimum/intel/utils/import_utils.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/loaders.py b/optimum/intel/openvino/loaders.py index be0c03fb41..ea6c06e65d 100644 --- a/optimum/intel/openvino/loaders.py +++ b/optimum/intel/openvino/loaders.py @@ -20,9 +20,10 @@ DIFFUSERS_CACHE, HF_HUB_OFFLINE, _get_model_file, - is_safetensors_available, ) +from ..utils.import_utils import is_safetensors_available + if is_safetensors_available(): import safetensors diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index 36e2229174..e804353997 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -91,6 +91,14 @@ except importlib_metadata.PackageNotFoundError: _diffusers_available = False +_safetensors_version = "N/A" +_safetensors_available = importlib.util.find_spec("safetensors") is not None +if _safetensors_available: + try: + _safetensors_version = importlib_metadata.version("safetensors") + except importlib_metadata.PackageNotFoundError: + _safetensors_available = False + def is_transformers_available(): return _transformers_available @@ -116,6 +124,10 @@ def is_diffusers_available(): return _diffusers_available +def is_safetensors_available(): + return _safetensors_available + + # This function was copied from: https://github.com/huggingface/accelerate/blob/874c4967d94badd24f893064cc3bef45f57cadf7/src/accelerate/utils/versions.py#L319 def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str): """ From c0ff71f214af99060a01871c5b3e1ef7f9efa380 Mon Sep 17 00:00:00 2001 From: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com> Date: Wed, 23 Aug 2023 15:09:40 +0200 Subject: [PATCH 106/134] Added repo_name argument to doc-builder (#410) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 84955e6d66..90a2827bed 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,7 @@ doc: build_doc_docker_image @test -n "$(VERSION)" || (echo "VERSION is empty." ; exit 1) docker run -v $(CURRENT_DIR):/doc_folder --workdir=/doc_folder doc_maker \ doc-builder build optimum.intel /optimum-intel/docs/source/ \ + --repo_name optimum-intel \ --build_dir $(BUILD_DIR) \ --version $(VERSION) \ --version_tag_suffix "" \ From f0b9ef489877858655c4f95f4770e46f8e2d5a36 Mon Sep 17 00:00:00 2001 From: "Cheng, Penghui" Date: Wed, 23 Aug 2023 21:10:22 +0800 Subject: [PATCH 107/134] Fixed export onnx on neural-compressor 2.2.2 (#409) Signed-off-by: Cheng, Penghui --- .../intel/neural_compressor/quantization.py | 33 +++++++++++++------ tests/neural_compressor/test_onnx.py | 2 +- tests/neural_compressor/utils_tests.py | 6 ++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index 6b1af7abed..de5a5d5727 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -284,16 +284,29 @@ def _onnx_export( device = model.model.device inputs = {k: v.to(device) for k, v in inputs.items()} - torch_to_int8_onnx( - model.model, - q_config=model.q_config, - save_path=str(output_path), - example_inputs=inputs, - opset_version=opset, - dynamic_axes=dynamic_axes, - input_names=list(config.inputs.keys()), - output_names=list(config.outputs.keys()), - ) + if is_neural_compressor_version(">", "2.2.1"): + torch_to_int8_onnx( + self._original_model, + model.model, + q_config=model.q_config, + save_path=str(output_path), + example_inputs=inputs, + opset_version=opset, + dynamic_axes=dynamic_axes, + input_names=list(config.inputs.keys()), + output_names=list(config.outputs.keys()), + ) + else: + torch_to_int8_onnx( + model.model, + q_config=model.q_config, + save_path=str(output_path), + example_inputs=inputs, + opset_version=opset, + dynamic_axes=dynamic_axes, + input_names=list(config.inputs.keys()), + output_names=list(config.outputs.keys()), + ) def _set_task(self): if self.task is None: diff --git a/tests/neural_compressor/test_onnx.py b/tests/neural_compressor/test_onnx.py index 69f608953a..5f82b60046 100644 --- a/tests/neural_compressor/test_onnx.py +++ b/tests/neural_compressor/test_onnx.py @@ -42,7 +42,7 @@ class OptimizationTest(INCTestMixin): SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( - ("text-classification", "hf-internal-testing/tiny-random-bert", 34), + ("text-classification", "hf-internal-testing/tiny-random-bert", 32), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) diff --git a/tests/neural_compressor/utils_tests.py b/tests/neural_compressor/utils_tests.py index 60606f146f..34e699c186 100644 --- a/tests/neural_compressor/utils_tests.py +++ b/tests/neural_compressor/utils_tests.py @@ -54,10 +54,10 @@ def num_quantized_matmul_onnx_model(onnx_model): num_quantized_matmul = 0 - for initializer in onnx_model.graph.initializer: - if "QuantizeLinear" in initializer.name: + for node in onnx_model.graph.node: + if "quantizelinear" == node.op_type.lower(): num_quantized_matmul += 1 - return num_quantized_matmul + return num_quantized_matmul // 2 def _preprocess_function(examples, tokenizer, column_name): From 0760d9a438877286cb425884ad1bfe753541f7c3 Mon Sep 17 00:00:00 2001 From: Sawradip Saha <67541368+sawradip@users.noreply.github.com> Date: Wed, 23 Aug 2023 20:50:58 +0600 Subject: [PATCH 108/134] Timm Models integration to Optimum-intel (#404) * Relevant file changes * Added test for timm * Fixed Style * Added style changes * Reduces abstractions * Fixed styles * Added Feature & text for Timm model saving * Added Timm to dependency * Added Image Processor source reference * Fixed Divergence in modeling_diffusion.py * Fixed divergence in modeling_diffusion.py * Adde Timm example to docs * Fixed style --- optimum/intel/openvino/modeling.py | 69 ++++- optimum/intel/openvino/modeling_base.py | 27 +- optimum/intel/openvino/modeling_timm.py | 339 ++++++++++++++++++++++++ setup.py | 1 + tests/openvino/test_modeling.py | 39 +++ 5 files changed, 473 insertions(+), 2 deletions(-) create mode 100644 optimum/intel/openvino/modeling_timm.py diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index eb1f7a410f..7f2fbd0fd3 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -11,14 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging +from pathlib import Path from typing import Optional, Union import numpy as np import openvino import torch import transformers +from huggingface_hub import model_info from transformers import ( AutoConfig, AutoModel, @@ -31,6 +32,7 @@ AutoModelForQuestionAnswering, AutoModelForSequenceClassification, AutoModelForTokenClassification, + PretrainedConfig, ) from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import ( @@ -47,6 +49,7 @@ from optimum.exporters import TasksManager from .modeling_base import OVBaseModel +from .modeling_timm import TimmConfig, TimmForImageClassification, TimmOnnxConfig, is_timm_ov_dir logger = logging.getLogger(__name__) @@ -481,6 +484,20 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> outputs = pipe(url) ``` + This class can also be used with [timm](https://github.com/huggingface/pytorch-image-models) + models hosted on [HuggingFaceHub](https://huggingface.co/timm). Example: + ```python + >>> from transformers import pipeline + >>> from optimum.intel.openvino.modeling_timm import TimmImageProcessor + >>> from optimum.intel import OVModelForImageClassification + + >>> model_id = "timm/vit_tiny_patch16_224.augreg_in21k" + >>> preprocessor = TimmImageProcessor.from_pretrained(model_id) + >>> model = OVModelForImageClassification.from_pretrained(model_id, export=True) + >>> pipe = pipeline("image-classification", model=model, feature_extractor=preprocessor) + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> outputs = pipe(url) + ``` """ @@ -497,6 +514,56 @@ class OVModelForImageClassification(OVModel): def __init__(self, model=None, config=None, **kwargs): super().__init__(model, config, **kwargs) + @classmethod + def from_pretrained( + cls, + model_id: Union[str, Path], + export: bool = False, + config: Optional["PretrainedConfig"] = None, + use_auth_token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + subfolder: str = "", + local_files_only: bool = False, + task: Optional[str] = None, + trust_remote_code: bool = False, + **kwargs, + ): + # Fix the mismatch between timm_config and huggingface_config + local_timm_model = is_timm_ov_dir(model_id) + if local_timm_model or model_info(model_id).library_name == "timm": + config = TimmConfig.from_pretrained(model_id, **kwargs) + # If locally saved timm model, dirrectly load + if local_timm_model: + return super()._from_pretrained( + model_id=model_id, + config=config, + ) + model = TimmForImageClassification.from_pretrained(model_id, **kwargs) + onnx_config = TimmOnnxConfig(model.config) + + return cls._to_onnx_to_load( + model=model, + config=config, + onnx_config=onnx_config, + ) + else: + return super().from_pretrained( + model_id=model_id, + config=config, + export=export, + use_auth_token=use_auth_token, + revision=revision, + force_download=force_download, + cache_dir=cache_dir, + subfolder=subfolder, + local_files_only=local_files_only, + task=task, + trust_remote_code=trust_remote_code, + **kwargs, + ) + @add_start_docstrings_to_model_forward( IMAGE_INPUTS_DOCSTRING.format("batch_size, num_channels, height, width") + IMAGE_CLASSIFICATION_EXAMPLE.format( diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index a6087ff952..14ac76137f 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -25,7 +25,7 @@ from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters.onnx import export +from optimum.exporters.onnx import OnnxConfig, export from optimum.exporters.tasks import TasksManager from optimum.modeling_base import OptimizedModel @@ -276,6 +276,31 @@ def _from_transformers( ) onnx_config = onnx_config_class(model.config) + + return cls._to_onnx_to_load( + model=model, + config=config, + onnx_config=onnx_config, + use_auth_token=use_auth_token, + revision=revision, + force_download=force_download, + cache_dir=cache_dir, + local_files_only=local_files_only, + ) + + @classmethod + def _to_onnx_to_load( + cls, + model: PreTrainedModel, + config: PretrainedConfig, + onnx_config: OnnxConfig, + use_auth_token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + local_files_only: bool = False, + **kwargs, + ): save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) diff --git a/optimum/intel/openvino/modeling_timm.py b/optimum/intel/openvino/modeling_timm.py new file mode 100644 index 0000000000..e0d4cb7a56 --- /dev/null +++ b/optimum/intel/openvino/modeling_timm.py @@ -0,0 +1,339 @@ +import json +import os +from collections import OrderedDict +from glob import glob +from typing import Dict, List, Optional, Union + +import numpy as np +import timm +import torch +from huggingface_hub import model_info +from packaging import version +from timm.layers.config import set_fused_attn +from timm.models._hub import load_model_config_from_hf +from transformers import PretrainedConfig, PreTrainedModel +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict +from transformers.image_transforms import resize, to_channel_dimension_format +from transformers.image_utils import ( + IMAGENET_STANDARD_MEAN, + IMAGENET_STANDARD_STD, + ChannelDimension, + ImageFeatureExtractionMixin, + ImageInput, + PILImageResampling, + make_list_of_images, + to_numpy_array, + valid_images, +) +from transformers.modeling_outputs import ImageClassifierOutput +from transformers.utils import TensorType + +from optimum.exporters.onnx.config import VisionOnnxConfig +from optimum.utils import NormalizedVisionConfig + + +set_fused_attn(False, False) + + +def is_timm_ov_dir(model_dir): + config_file = None + has_xml = False + has_bin = False + if os.path.isdir(model_dir): + for filename in glob(os.path.join(model_dir, "*")): + if filename.endswith(".xml"): + has_xml = True + if filename.endswith(".bin"): + has_bin = True + if filename.endswith("config.json"): + config_file = filename + if config_file and has_xml and has_bin: + with open(config_file) as conf: + hf_hub_id = json.load(conf).get("hf_hub_id", None) + if hf_hub_id and model_info(hf_hub_id).library_name == "timm": + return True + return False + + +class TimmConfig(PretrainedConfig): + model_type = "timm" + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + local_files_only: bool = False, + token: Optional[Union[str, bool]] = None, + revision: str = "main", + **kwargs, + ) -> "PretrainedConfig": + if is_timm_ov_dir(pretrained_model_name_or_path): + config_path = os.path.join(pretrained_model_name_or_path, "config.json") + return cls.from_json_file(config_path) + + kwargs["cache_dir"] = cache_dir + kwargs["force_download"] = force_download + kwargs["local_files_only"] = local_files_only + kwargs["revision"] = revision + + config_dict = load_model_config_from_hf(pretrained_model_name_or_path)[0] + config_dict["num_labels"] = config_dict.pop("num_classes") + config_dict["image_size"] = config_dict.get("input_size")[-1] + + return cls.from_dict(config_dict, **kwargs) + + +class TimmOnnxConfig(VisionOnnxConfig): + DEFAULT_TIMM_ONNX_OPSET = 13 + outputs = OrderedDict([("logits", {0: "batch_size"})]) + NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig + MIN_TORCH_VERSION = version.parse("1.11") + + @property + def inputs(self) -> Dict[str, Dict[int, str]]: + return {"pixel_values": {0: "batch_size", 1: "num_channels", 2: "height", 3: "width"}} + + +class TimmForImageClassification(PreTrainedModel): + def __init__(self, config: TimmConfig, num_labels: int = None, **kwargs) -> None: + super().__init__(config, **kwargs) + if num_labels: + config.num_labels = num_labels + self.model = timm.create_model( + "hf-hub:" + self.config.hf_hub_id, + num_classes=self.config.num_labels, + pretrained=True, + in_chans=3, + ) + self.model.eval() + + @classmethod + def from_pretrained(cls, model_name_or_path, **kwargs): + config = TimmConfig.from_pretrained(model_name_or_path, **kwargs) + return cls(config, **kwargs) + + def forward(self, pixel_values: Optional[torch.Tensor] = None): + logits = self.model( + pixel_values, + ) + + return ImageClassifierOutput( + logits=logits, + ) + + +# Adapted from ViTImageProcessor - https://github.com/huggingface/transformers/blob/main/src/transformers/models/vit/image_processing_vit.py +class TimmImageProcessor(BaseImageProcessor, ImageFeatureExtractionMixin): + r""" + Constructs a ViT image processor. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to resize the image's (height, width) dimensions to the specified `(size["height"], + size["width"])`. Can be overridden by the `do_resize` parameter in the `preprocess` method. + size (`dict`, *optional*, defaults to `{"height": 224, "width": 224}`): + Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` + method. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): + Resampling filter to use if resizing the image. Can be overridden by the `resample` parameter in the + `preprocess` method. + do_rescale (`bool`, *optional*, defaults to `True`): + Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the `do_rescale` + parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. Can be overridden by the `rescale_factor` parameter in the + `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` + method. + image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + do_resize: bool = True, + size: Optional[Dict[str, int]] = None, + resample: PILImageResampling = PILImageResampling.BILINEAR, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + size = size if size is not None else {"height": 224, "width": 224} + size = get_size_dict(size) + self.do_resize = do_resize + self.do_rescale = do_rescale + self.do_normalize = do_normalize + self.size = size + self.resample = resample + self.rescale_factor = rescale_factor + self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN + self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + **kwargs, + ): + timm_config_dict, _ = load_model_config_from_hf(pretrained_model_name_or_path) + + _, im_h, im_w = timm_config_dict.get("input_size", [3, 224, 224]) + + image_preprocess_config_dict = { + "crop_size": {"height": im_h, "width": im_w}, + "do_center_crop": True if timm_config_dict.get("crop_mode") == "center" else False, + "do_normalize": True, + "do_reduce_labels": False, + "do_rescale": True, + "do_resize": True, + "image_mean": timm_config_dict.get("mean", IMAGENET_STANDARD_MEAN), + "image_processor_type": "TimmImageProcessor", + "image_std": timm_config_dict.get("std", IMAGENET_STANDARD_STD), + "resample": 3, + "rescale_factor": 0.00392156862745098, + "size": {"height": im_h, "width": im_w}, + } + + return cls.from_dict(image_preprocess_config_dict, **kwargs) + + def resize( + self, + image: np.ndarray, + size: Dict[str, int], + resample: PILImageResampling = PILImageResampling.BILINEAR, + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Resize an image to `(size["height"], size["width"])`. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. + resample: + `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BILINEAR`. + data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The resized image. + """ + size = get_size_dict(size) + if "height" not in size or "width" not in size: + raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") + if image.ndim == 2: + image = np.stack([image] * 3, axis=-1) + return resize( + image, size=(size["height"], size["width"]), resample=resample, data_format=data_format, **kwargs + ) + + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Dict[str, int] = None, + resample: PILImageResampling = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Union[str, ChannelDimension] = ChannelDimension.FIRST, + **kwargs, + ): + """ + Preprocess an image or batch of images. + + Args: + images (`ImageInput`): + Image to preprocess. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Dictionary in the format `{"height": h, "width": w}` specifying the size of the output image after + resizing. + resample (`PILImageResampling` filter, *optional*, defaults to `self.resample`): + `PILImageResampling` filter to use if resizing the image e.g. `PILImageResampling.BILINEAR`. Only has + an effect if `do_resize` is set to `True`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image values between [0 - 1]. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use if `do_normalize` is set to `True`. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + """ + do_resize = do_resize if do_resize is not None else self.do_resize + do_rescale = do_rescale if do_rescale is not None else self.do_rescale + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + resample = resample if resample is not None else self.resample + rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + + size = size if size is not None else self.size + size_dict = get_size_dict(size) + + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + if do_resize and size is None: + raise ValueError("Size must be specified if do_resize is True.") + + if do_rescale and rescale_factor is None: + raise ValueError("Rescale factor must be specified if do_rescale is True.") + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if do_resize: + images = [self.resize(image=image, size=size_dict, resample=resample) for image in images] + + if do_rescale: + images = [self.rescale(image=image, scale=rescale_factor) for image in images] + + if do_normalize: + images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + + images = [to_channel_dimension_format(image, data_format) for image in images] + data = {"pixel_values": images} + return BatchFeature(data=data, tensor_type=return_tensors) diff --git a/setup.py b/setup.py index 357ebd3b66..a100ffec8e 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ "datasets>=1.4.0", "sentencepiece", "scipy", + "timm", "accelerate", # transformers 4.29 require accelerate for PyTorch ] diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index e1833ff23a..5fe3354732 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -21,6 +21,7 @@ import numpy as np import requests +import timm import torch from datasets import load_dataset from evaluate import evaluator @@ -65,6 +66,7 @@ ) from optimum.intel.openvino import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder +from optimum.intel.openvino.modeling_timm import TimmImageProcessor from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, DIFFUSION_MODEL_UNET_SUBFOLDER, @@ -585,6 +587,8 @@ class OVModelForImageClassificationIntegrationTest(unittest.TestCase): "vit", ) + TIMM_MODELS = ("timm/pit_s_distilled_224.in1k", "timm/vit_tiny_patch16_224.augreg_in21k") + @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] @@ -619,6 +623,41 @@ def test_pipeline(self, model_arch): self.assertTrue(isinstance(outputs[0]["label"], str)) gc.collect() + @parameterized.expand(TIMM_MODELS) + def test_compare_to_timm(self, model_id): + ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True) + self.assertIsInstance(ov_model.config, PretrainedConfig) + timm_model = timm.create_model(model_id, pretrained=True) + preprocessor = TimmImageProcessor.from_pretrained(model_id) + url = "http://images.cocodataset.org/val2017/000000039769.jpg" + image = Image.open(requests.get(url, stream=True).raw) + inputs = preprocessor(images=image, return_tensors="pt") + with torch.no_grad(): + timm_model.eval() + timm_outputs = timm_model(inputs["pixel_values"].float()) + for input_type in ["pt", "np"]: + inputs = preprocessor(images=image, return_tensors=input_type) + ov_outputs = ov_model(**inputs) + self.assertIn("logits", ov_outputs) + self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) + # Compare tensor outputs + self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), timm_outputs, atol=1e-4)) + gc.collect() + + @parameterized.expand(TIMM_MODELS) + def test_timm_save_and_infer(self, model_id): + ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True) + with tempfile.TemporaryDirectory() as tmpdirname: + model_save_path = os.path.join(tmpdirname, "timm_ov_model") + ov_model.save_pretrained(model_save_path) + new_ov_model = OVModelForImageClassification.from_pretrained( + model_save_path, + ) + new_ov_model( + pixel_values=torch.zeros((5, 3, new_ov_model.config.image_size, new_ov_model.config.image_size)) + ) + gc.collect() + class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( From 437f328e2e8ec272543d14f0114a3b478978602a Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 23 Aug 2023 20:07:26 +0200 Subject: [PATCH 109/134] Fix openvino test as trainer training args are now immutable (#412) --- tests/openvino/test_training.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/openvino/test_training.py b/tests/openvino/test_training.py index c9e529c55a..6699687c69 100644 --- a/tests/openvino/test_training.py +++ b/tests/openvino/test_training.py @@ -208,9 +208,7 @@ def override_movement_sparsifier_initialization(self, trainer: OVTrainer, sparsi # make sure the binary masks will have many zeros initialize_movement_sparsifier_parameters_by_sparsity(movement_controller, sparsity=sparsity) - def get_training_args(self) -> OVTrainingArguments: - num_train_epochs = 3 - train_batch_size = 4 + def get_training_args(self, train_batch_size=4, eval_batch_size=1, num_train_epochs=3) -> OVTrainingArguments: args = OVTrainingArguments( output_dir=self.output_dir, num_train_epochs=num_train_epochs, @@ -219,7 +217,7 @@ def get_training_args(self) -> OVTrainingArguments: do_eval=True, logging_steps=1, per_device_train_batch_size=train_batch_size, - per_device_eval_batch_size=1, + per_device_eval_batch_size=eval_batch_size, no_cuda=True, full_determinism=True, remove_unused_columns=False, @@ -485,8 +483,9 @@ def data_transform(examples, max_length: int = 128): def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model): torch_model = torch_model.eval() for batch_size in [1, 4]: + self.trainer.args = self.get_training_args(eval_batch_size=batch_size) + self.trainer.create_accelerator_and_postprocess() for seq_length in [16, 89, 128]: - self.trainer.args.per_device_eval_batch_size = batch_size dataset = deepcopy(self.eval_dataset) dataset.set_transform(partial(self.data_transform, max_length=seq_length)) for inputs in self.trainer.get_eval_dataloader(dataset): @@ -640,7 +639,8 @@ def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model): torch_model = torch_model.eval() batch_sizes = [1] if self.is_swin else [1, 4] for batch_size in batch_sizes: - self.trainer.args.per_device_eval_batch_size = batch_size + self.trainer.args = self.get_training_args(eval_batch_size=batch_size) + self.trainer.create_accelerator_and_postprocess() for inputs in self.trainer.get_eval_dataloader(): self.assertEqual(inputs["pixel_values"].shape[0], batch_size) ovmodel_outputs = ovmodel(**inputs) @@ -822,8 +822,9 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model): torch_model = torch_model.eval() for batch_size in [1, 4]: + self.trainer.args = self.get_training_args(eval_batch_size=batch_size) + self.trainer.create_accelerator_and_postprocess() for seq_length in [12345, 16000]: - self.trainer.args.per_device_eval_batch_size = batch_size dataset = deepcopy(self.eval_dataset) dataset.set_transform(partial(self.data_transform, max_length=seq_length)) for inputs in self.trainer.get_eval_dataloader(dataset): From 499c5625f0f3ce728d95569e08dbe920bb8d23b9 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 24 Aug 2023 17:17:38 +0200 Subject: [PATCH 110/134] Fix timm ov model (#413) * Fix ov timm model loading * increase tolerance --- optimum/intel/openvino/modeling.py | 10 ++++------ tests/openvino/test_modeling.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index 7f2fbd0fd3..4f5927df01 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os from pathlib import Path from typing import Optional, Union @@ -532,14 +533,11 @@ def from_pretrained( ): # Fix the mismatch between timm_config and huggingface_config local_timm_model = is_timm_ov_dir(model_id) - if local_timm_model or model_info(model_id).library_name == "timm": + if local_timm_model or (not os.path.isdir(model_id) and model_info(model_id).library_name == "timm"): config = TimmConfig.from_pretrained(model_id, **kwargs) - # If locally saved timm model, dirrectly load + # If locally saved timm model, directly load if local_timm_model: - return super()._from_pretrained( - model_id=model_id, - config=config, - ) + return super()._from_pretrained(model_id=model_id, config=config) model = TimmForImageClassification.from_pretrained(model_id, **kwargs) onnx_config = TimmOnnxConfig(model.config) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 5fe3354732..b56b7766e7 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -641,7 +641,7 @@ def test_compare_to_timm(self, model_id): self.assertIn("logits", ov_outputs) self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) # Compare tensor outputs - self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), timm_outputs, atol=1e-4)) + self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), timm_outputs, atol=1e-3)) gc.collect() @parameterized.expand(TIMM_MODELS) From 3497d616a3b780f7a746a7a4ed500c80c6d33166 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:42:53 +0200 Subject: [PATCH 111/134] Fix transformers version for ipex (#416) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a100ffec8e..985bbad46c 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ ], "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], - "ipex": ["intel-extension-for-pytorch", "onnx"], + "ipex": ["transformers<4.32.0", "intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers", "invisible-watermark>=0.2.0"], "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, From f39a84de49f1df5602dcc6c9fcb4f34f44a3330e Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:42:24 +0200 Subject: [PATCH 112/134] Remove timm from being a hard dependency (#414) * Fix ov timm model loading * increase tolerance * change timm from hard to soft dependency * add missing import --- optimum/intel/openvino/modeling.py | 12 +++++++++-- optimum/intel/openvino/modeling_timm.py | 27 +++---------------------- optimum/intel/openvino/utils.py | 25 +++++++++++++++++++++++ optimum/intel/utils/import_utils.py | 14 +++++++++++++ setup.py | 2 +- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index 4f5927df01..1cea230429 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -49,8 +49,9 @@ from optimum.exporters import TasksManager +from ..utils.import_utils import is_timm_available from .modeling_base import OVBaseModel -from .modeling_timm import TimmConfig, TimmForImageClassification, TimmOnnxConfig, is_timm_ov_dir +from .utils import _is_timm_ov_dir logger = logging.getLogger(__name__) @@ -532,8 +533,15 @@ def from_pretrained( **kwargs, ): # Fix the mismatch between timm_config and huggingface_config - local_timm_model = is_timm_ov_dir(model_id) + local_timm_model = _is_timm_ov_dir(model_id) if local_timm_model or (not os.path.isdir(model_id) and model_info(model_id).library_name == "timm"): + if not is_timm_available(): + raise ImportError( + "To load a timm model, timm needs to be installed. Please install it with `pip install timm`." + ) + + from .modeling_timm import TimmConfig, TimmForImageClassification, TimmOnnxConfig + config = TimmConfig.from_pretrained(model_id, **kwargs) # If locally saved timm model, directly load if local_timm_model: diff --git a/optimum/intel/openvino/modeling_timm.py b/optimum/intel/openvino/modeling_timm.py index e0d4cb7a56..d7f6302f7a 100644 --- a/optimum/intel/openvino/modeling_timm.py +++ b/optimum/intel/openvino/modeling_timm.py @@ -1,13 +1,10 @@ -import json import os from collections import OrderedDict -from glob import glob from typing import Dict, List, Optional, Union import numpy as np import timm import torch -from huggingface_hub import model_info from packaging import version from timm.layers.config import set_fused_attn from timm.models._hub import load_model_config_from_hf @@ -31,28 +28,10 @@ from optimum.exporters.onnx.config import VisionOnnxConfig from optimum.utils import NormalizedVisionConfig - -set_fused_attn(False, False) +from .utils import _is_timm_ov_dir -def is_timm_ov_dir(model_dir): - config_file = None - has_xml = False - has_bin = False - if os.path.isdir(model_dir): - for filename in glob(os.path.join(model_dir, "*")): - if filename.endswith(".xml"): - has_xml = True - if filename.endswith(".bin"): - has_bin = True - if filename.endswith("config.json"): - config_file = filename - if config_file and has_xml and has_bin: - with open(config_file) as conf: - hf_hub_id = json.load(conf).get("hf_hub_id", None) - if hf_hub_id and model_info(hf_hub_id).library_name == "timm": - return True - return False +set_fused_attn(False, False) class TimmConfig(PretrainedConfig): @@ -69,7 +48,7 @@ def from_pretrained( revision: str = "main", **kwargs, ) -> "PretrainedConfig": - if is_timm_ov_dir(pretrained_model_name_or_path): + if _is_timm_ov_dir(pretrained_model_name_or_path): config_path = os.path.join(pretrained_model_name_or_path, "config.json") return cls.from_json_file(config_path) diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index f611ab3526..cf0f8dee20 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -13,7 +13,12 @@ # limitations under the License. +import json +import os +from glob import glob + import numpy as np +from huggingface_hub import model_info from openvino.runtime import Type from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size @@ -95,3 +100,23 @@ def use_external_data_format(num_parameters: int) -> bool: """ return compute_serialized_parameters_size(num_parameters, ParameterFormat.Float) >= EXTERNAL_DATA_FORMAT_SIZE_LIMIT + + +def _is_timm_ov_dir(model_dir): + config_file = None + has_xml = False + has_bin = False + if os.path.isdir(model_dir): + for filename in glob(os.path.join(model_dir, "*")): + if filename.endswith(".xml"): + has_xml = True + if filename.endswith(".bin"): + has_bin = True + if filename.endswith("config.json"): + config_file = filename + if config_file and has_xml and has_bin: + with open(config_file) as conf: + hf_hub_id = json.load(conf).get("hf_hub_id", None) + if hf_hub_id and model_info(hf_hub_id).library_name == "timm": + return True + return False diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index e804353997..7b2eeb540e 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -91,6 +91,7 @@ except importlib_metadata.PackageNotFoundError: _diffusers_available = False + _safetensors_version = "N/A" _safetensors_available = importlib.util.find_spec("safetensors") is not None if _safetensors_available: @@ -100,6 +101,15 @@ _safetensors_available = False +_timm_available = importlib.util.find_spec("timm") is not None +_timm_version = "N/A" +if _timm_available: + try: + _timm_version = importlib_metadata.version("timm") + except importlib_metadata.PackageNotFoundError: + _timm_available = False + + def is_transformers_available(): return _transformers_available @@ -128,6 +138,10 @@ def is_safetensors_available(): return _safetensors_available +def is_timm_available(): + return _timm_available + + # This function was copied from: https://github.com/huggingface/accelerate/blob/874c4967d94badd24f893064cc3bef45f57cadf7/src/accelerate/utils/versions.py#L319 def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str): """ diff --git a/setup.py b/setup.py index 985bbad46c..c35640226d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ "datasets>=1.4.0", "sentencepiece", "scipy", - "timm", "accelerate", # transformers 4.29 require accelerate for PyTorch ] @@ -31,6 +30,7 @@ "sacremoses", "torchaudio", "rjieba", + "timm", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] From d78140794e0fb46ee2deccf8e48e733111b255f4 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:10:05 +0200 Subject: [PATCH 113/134] Add OpenVINO llama test (#419) --- tests/openvino/test_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index b56b7766e7..1c2fc534d4 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -421,7 +421,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "gpt2", "gpt_neo", "gpt_neox", - # "llama", + "llama", "marian", "opt", "pegasus", From 375942f558988a6544284697525e86f7d314f314 Mon Sep 17 00:00:00 2001 From: fxmarty <9808326+fxmarty@users.noreply.github.com> Date: Thu, 7 Sep 2023 23:00:53 +0900 Subject: [PATCH 114/134] pin npm version (#424) --- docs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index e081afeed3..7cfa6cd514 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \ software-properties-common \ npm -RUN npm install npm@latest -g && \ +RUN npm install npm@9.8.1 -g && \ npm install n -g && \ n latest From f3bb7f2c30b6d9d6adb737e318e8e9b9996325d8 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 8 Sep 2023 11:50:39 +0200 Subject: [PATCH 115/134] Add vae image processor (#421) * add vae image processor * add test * add optimum min version --- optimum/intel/openvino/modeling_base.py | 8 -- optimum/intel/openvino/modeling_diffusion.py | 22 +++- setup.py | 5 +- tests/openvino/test_stable_diffusion.py | 104 +++++++++++++++---- 4 files changed, 101 insertions(+), 38 deletions(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 14ac76137f..59fc89649a 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -65,7 +65,6 @@ class PreTrainedModel(OptimizedModel): """, ) class OVBaseModel(PreTrainedModel): - _AUTOMODELS_TO_TASKS = {cls_name: task for task, cls_name in TasksManager._TASKS_TO_AUTOMODELS.items()} auto_model_class = None export_feature = None @@ -391,13 +390,6 @@ def _ensure_supported_device(self, device: str = None): def forward(self, *args, **kwargs): raise NotImplementedError - @classmethod - def _auto_model_to_task(cls, auto_model_class): - """ - Get the task corresponding to a class (for example AutoModelForXXX in transformers). - """ - return cls._AUTOMODELS_TO_TASKS[auto_model_class.__name__] - def can_generate(self) -> bool: """ Returns whether this model can generate sequences with `.generate()`. diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 73ec66d473..1085c9e81c 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -30,7 +30,7 @@ StableDiffusionXLPipeline, ) from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME -from diffusers.utils import CONFIG_NAME +from diffusers.utils import CONFIG_NAME, is_invisible_watermark_available from huggingface_hub import snapshot_download from openvino._offline_transformations import compress_model_transformation from openvino.runtime import Core @@ -42,6 +42,7 @@ from optimum.pipelines.diffusers.pipeline_stable_diffusion_inpaint import StableDiffusionInpaintPipelineMixin from optimum.pipelines.diffusers.pipeline_stable_diffusion_xl import StableDiffusionXLPipelineMixin from optimum.pipelines.diffusers.pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipelineMixin +from optimum.pipelines.diffusers.pipeline_utils import VaeImageProcessor from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER, DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, @@ -106,6 +107,8 @@ def __init__( else: self.vae_scale_factor = 8 + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.tokenizer = tokenizer self.tokenizer_2 = tokenizer_2 self.scheduler = scheduler @@ -687,12 +690,21 @@ class OVStableDiffusionXLPipelineBase(OVStableDiffusionPipelineBase): auto_model_class = StableDiffusionXLPipeline export_feature = "stable-diffusion-xl" - def __init__(self, *args, **kwargs): + def __init__(self, *args, add_watermarker: Optional[bool] = None, **kwargs): super().__init__(*args, **kwargs) - # additional invisible-watermark dependency for SD XL - from optimum.pipelines.diffusers.watermark import StableDiffusionXLWatermarker - self.watermark = StableDiffusionXLWatermarker() + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + if not is_invisible_watermark_available(): + raise ImportError( + "`add_watermarker` requires invisible-watermark to be installed, which can be installed with `pip install invisible-watermark`." + ) + from optimum.pipelines.diffusers.watermark import StableDiffusionXLWatermarker + + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None class OVStableDiffusionXLPipeline(OVStableDiffusionXLPipelineBase, StableDiffusionXLPipelineMixin): diff --git a/setup.py b/setup.py index c35640226d..769431c31c 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ assert False, "Error: Could not open '%s' due %s\n" % (filepath, error) INSTALL_REQUIRE = [ - "optimum>=1.10.0", + "optimum>=1.13.0", "transformers>=4.20.0", "datasets>=1.4.0", "sentencepiece", @@ -31,6 +31,7 @@ "torchaudio", "rjieba", "timm", + "invisible-watermark>=0.2.0", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] @@ -44,7 +45,7 @@ "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], "ipex": ["transformers<4.32.0", "intel-extension-for-pytorch", "onnx"], - "diffusers": ["diffusers", "invisible-watermark>=0.2.0"], + "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, } diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py index 35e85eeaa7..e04e2d6fd3 100644 --- a/tests/openvino/test_stable_diffusion.py +++ b/tests/openvino/test_stable_diffusion.py @@ -18,6 +18,7 @@ from typing import Dict import numpy as np +import PIL import torch from diffusers import ( StableDiffusionPipeline, @@ -60,17 +61,32 @@ def _generate_inputs(batch_size=1): return inputs -def _create_image(height=128, width=128): - image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ) - return image.resize((width, height)) +def _create_image(height=128, width=128, batch_size=1, channel=3, input_type="pil"): + if input_type == "pil": + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ).resize((width, height)) + elif input_type == "np": + image = np.random.rand(height, width, channel) + elif input_type == "pt": + image = torch.rand((channel, height, width)) + + return [image] * batch_size + + +def to_np(image): + if isinstance(image[0], PIL.Image.Image): + return np.stack([np.array(i) for i in image], axis=0) + elif isinstance(image, torch.Tensor): + return image.cpu().numpy().transpose(0, 2, 3, 1) + return image class OVStableDiffusionPipelineBaseTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ("stable-diffusion",) MODEL_CLASS = OVStableDiffusionPipeline + TASK = "text-to-image" @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_num_images_per_prompt(self, model_arch: str): @@ -104,6 +120,36 @@ def callback_fn(step: int, timestep: int, latents: np.ndarray) -> None: self.assertTrue(callback_fn.has_been_called) self.assertEqual(callback_fn.number_of_steps, inputs["num_inference_steps"]) + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_shape(self, model_arch: str): + height, width, batch_size = 128, 64, 1 + pipeline = self.MODEL_CLASS.from_pretrained(MODEL_NAMES[model_arch], export=True) + + if self.TASK == "image-to-image": + input_types = ["np", "pil", "pt"] + elif self.TASK == "text-to-image": + input_types = ["np"] + else: + input_types = ["pil"] + + for input_type in input_types: + if self.TASK == "image-to-image": + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size, input_type=input_type) + else: + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + for output_type in ["np", "pil", "latent"]: + inputs["output_type"] = output_type + outputs = pipeline(**inputs).images + if output_type == "pil": + self.assertEqual((len(outputs), outputs[0].height, outputs[0].width), (batch_size, height, width)) + elif output_type == "np": + self.assertEqual(outputs.shape, (batch_size, height, width, 3)) + else: + self.assertEqual( + outputs.shape, + (batch_size, 4, height // pipeline.vae_scale_factor, width // pipeline.vae_scale_factor), + ) + def generate_inputs(self, height=128, width=128, batch_size=1): inputs = _generate_inputs(batch_size) inputs["height"] = height @@ -115,13 +161,16 @@ class OVStableDiffusionImg2ImgPipelineTest(OVStableDiffusionPipelineBaseTest): SUPPORTED_ARCHITECTURES = ("stable-diffusion",) MODEL_CLASS = OVStableDiffusionImg2ImgPipeline ORT_MODEL_CLASS = ORTStableDiffusionImg2ImgPipeline + TASK = "image-to-image" @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_diffusers_pipeline(self, model_arch: str): model_id = MODEL_NAMES[model_arch] pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True) - inputs = self.generate_inputs() + height, width, batch_size = 128, 128, 1 + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) inputs["prompt"] = "A painting of a squirrel eating a burger" + inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) np.random.seed(0) output = pipeline(**inputs).images[0, -3:, -3:, -1] # https://github.com/huggingface/diffusers/blob/v0.17.1/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py#L71 @@ -139,9 +188,9 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) - def generate_inputs(self, height=128, width=128, batch_size=1): + def generate_inputs(self, height=128, width=128, batch_size=1, input_type="np"): inputs = _generate_inputs(batch_size) - inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) + inputs["image"] = _create_image(height=height, width=width, batch_size=batch_size, input_type=input_type) inputs["strength"] = 0.75 return inputs @@ -149,6 +198,7 @@ def generate_inputs(self, height=128, width=128, batch_size=1): class OVStableDiffusionPipelineTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ("stable-diffusion",) MODEL_CLASS = OVStableDiffusionPipeline + TASK = "text-to-image" @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_diffusers(self, model_arch: str): @@ -247,6 +297,7 @@ class OVStableDiffusionInpaintPipelineTest(OVStableDiffusionPipelineBaseTest): SUPPORTED_ARCHITECTURES = ("stable-diffusion",) MODEL_CLASS = OVStableDiffusionInpaintPipeline ORT_MODEL_CLASS = ORTStableDiffusionInpaintPipeline + TASK = "inpaint" @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_diffusers_pipeline(self, model_arch: str): @@ -262,6 +313,17 @@ def test_compare_diffusers_pipeline(self, model_arch: str): generator=np.random.RandomState(0), ) inputs = self.generate_inputs(height=height, width=width) + + inputs["image"] = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ).resize((width, height)) + + inputs["mask_image"] = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + ).resize((width, height)) + outputs = pipeline(**inputs, latents=latents).images self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) @@ -285,16 +347,8 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): def generate_inputs(self, height=128, width=128, batch_size=1): inputs = super(OVStableDiffusionInpaintPipelineTest, self).generate_inputs(height, width, batch_size) - inputs["image"] = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo.png" - ).resize((width, height)) - - inputs["mask_image"] = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" - "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" - ).resize((width, height)) - + inputs["image"] = _create_image(height=height, width=width, batch_size=1, input_type="pil")[0] + inputs["mask_image"] = _create_image(height=height, width=width, batch_size=1, input_type="pil")[0] return inputs @@ -303,6 +357,7 @@ class OVtableDiffusionXLPipelineTest(unittest.TestCase): MODEL_CLASS = OVStableDiffusionXLPipeline ORT_MODEL_CLASS = ORTStableDiffusionXLPipeline PT_MODEL_CLASS = StableDiffusionXLPipeline + TASK = "text-to-image" @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_diffusers(self, model_arch: str): @@ -387,6 +442,7 @@ class OVStableDiffusionXLImg2ImgPipelineTest(unittest.TestCase): MODEL_CLASS = OVStableDiffusionXLImg2ImgPipeline ORT_MODEL_CLASS = ORTStableDiffusionXLImg2ImgPipeline PT_MODEL_CLASS = StableDiffusionXLImg2ImgPipeline + TASK = "image-to-image" def test_inference(self): model_id = "hf-internal-testing/tiny-stable-diffusion-xl-pipe" @@ -396,10 +452,12 @@ def test_inference(self): pipeline.save_pretrained(tmp_dir) pipeline = self.MODEL_CLASS.from_pretrained(tmp_dir) - inputs = self.generate_inputs() + batch_size, height, width = 1, 128, 128 + inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) + inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) np.random.seed(0) output = pipeline(**inputs).images[0, -3:, -3:, -1] - expected_slice = np.array([0.5675, 0.5108, 0.4758, 0.5280, 0.5080, 0.5473, 0.4789, 0.4286, 0.4861]) + expected_slice = np.array([0.5683, 0.5121, 0.4767, 0.5253, 0.5072, 0.5462, 0.4766, 0.4279, 0.4855]) self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-3)) @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -413,8 +471,8 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) - def generate_inputs(self, height=128, width=128, batch_size=1): + def generate_inputs(self, height=128, width=128, batch_size=1, input_type="np"): inputs = _generate_inputs(batch_size) - inputs["image"] = floats_tensor((batch_size, 3, height, width), rng=random.Random(SEED)) + inputs["image"] = _create_image(height=height, width=width, batch_size=batch_size, input_type=input_type) inputs["strength"] = 0.75 return inputs From a07a2f5a07cbaacd9d69d350159768886fe5b2ac Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 8 Sep 2023 12:14:48 +0200 Subject: [PATCH 116/134] add MPT support (#425) * add MPT support * remove deprecated class after optimum v1.13 * set version optimum * patch model during export * update optimum latest compatible version --- optimum/commands/neural_compressor/quantize.py | 2 +- optimum/intel/neural_compressor/modeling_base.py | 8 -------- optimum/intel/openvino/modeling_decoder.py | 2 +- tests/openvino/test_modeling.py | 1 + tests/openvino/utils_tests.py | 1 + 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/optimum/commands/neural_compressor/quantize.py b/optimum/commands/neural_compressor/quantize.py index 72248ed277..c822b1a195 100644 --- a/optimum/commands/neural_compressor/quantize.py +++ b/optimum/commands/neural_compressor/quantize.py @@ -46,7 +46,7 @@ def parse_args_inc_quantize(parser: "ArgumentParser"): default="auto", help=( "The task to export the model for. If not specified, the task will be auto-inferred based on the model. Available tasks depend on the model, but are among:" - f" {str(list(TasksManager._TASKS_TO_AUTOMODELS.keys()))}." + f" {str(TasksManager.get_all_tasks())}." ), ) diff --git a/optimum/intel/neural_compressor/modeling_base.py b/optimum/intel/neural_compressor/modeling_base.py index 0eaed698e8..5cba8c4095 100644 --- a/optimum/intel/neural_compressor/modeling_base.py +++ b/optimum/intel/neural_compressor/modeling_base.py @@ -54,7 +54,6 @@ """, ) class INCBaseModel: - _AUTOMODELS_TO_TASKS = {cls_name: task for task, cls_name in TasksManager._TASKS_TO_AUTOMODELS.items()} base_model_prefix = "inc_model" def __init__( @@ -256,12 +255,5 @@ def _from_transformers( **kwargs, ) - @classmethod - def _auto_model_to_task(cls, auto_model_class): - """ - Get the task corresponding to a class (for example AutoModelForXXX in transformers). - """ - return cls._AUTOMODELS_TO_TASKS[auto_model_class.__name__] - def eval(self): self.model.eval() diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 16cf6c20d3..b5d4f0be5d 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -232,7 +232,7 @@ def _from_transformers( onnx_config = onnx_config_constructor(model.config, use_past=use_cache) # TODO : create ModelPatcher to patch each architecture - if config.model_type == "bloom": + if config.model_type in {"bloom", "mpt"}: model.transformer._prepare_attn_mask = _prepare_attn_mask elif config.model_type == "llama": model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 1c2fc534d4..4b11435e0e 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -423,6 +423,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "gpt_neox", "llama", "marian", + "mpt", "opt", "pegasus", ) diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index eeb751153a..94643b02f4 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -57,6 +57,7 @@ "mobilenet_v1": "google/mobilenet_v1_0.75_192", "mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model", "mobilevit": "hf-internal-testing/tiny-random-mobilevit", + "mpt": "hf-internal-testing/tiny-random-MptForCausalLM", "mt5": "stas/mt5-tiny-random", "nystromformer": "hf-internal-testing/tiny-random-NystromformerModel", "pegasus": "hf-internal-testing/tiny-random-pegasus", From 0e572fe441508f4995d4e286d70edcbda5790265 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 8 Sep 2023 12:21:37 +0200 Subject: [PATCH 117/134] Fix INC CLI (#426) --- optimum/commands/neural_compressor/quantize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/commands/neural_compressor/quantize.py b/optimum/commands/neural_compressor/quantize.py index c822b1a195..e95eb3ce83 100644 --- a/optimum/commands/neural_compressor/quantize.py +++ b/optimum/commands/neural_compressor/quantize.py @@ -28,7 +28,7 @@ def parse_args_inc_quantize(parser: "ArgumentParser"): required_group = parser.add_argument_group("Required arguments") required_group.add_argument( "--model", - type=Path, + type=str, required=True, help="Path to the repository where the model to quantize is located.", ) @@ -83,7 +83,7 @@ def run(self): if task == "auto": try: - task = TasksManager.infer_task_from_model(str(model_id)) + task = TasksManager.infer_task_from_model(model_id) except Exception as e: return ( f"### Error: {e}. Please pass explicitely the task as it could not be infered.", From 6923c93119ea09149ddc941e9ce969bd4b306950 Mon Sep 17 00:00:00 2001 From: Ella Charlaix Date: Fri, 8 Sep 2023 14:15:18 +0200 Subject: [PATCH 118/134] Dev version --- optimum/intel/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/version.py b/optimum/intel/version.py index 09c74e091a..f8f59092d8 100644 --- a/optimum/intel/version.py +++ b/optimum/intel/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.10.2.dev0" +__version__ = "1.12.0.dev0" From 77142fd98cbb0ed892c2d004bbee14cf4d21eb5a Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:19:40 +0200 Subject: [PATCH 119/134] Fix import from diffusers latest release (#431) --- tests/openvino/test_stable_diffusion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py index e04e2d6fd3..781fbe0ec6 100644 --- a/tests/openvino/test_stable_diffusion.py +++ b/tests/openvino/test_stable_diffusion.py @@ -25,7 +25,8 @@ StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPipeline, ) -from diffusers.utils import floats_tensor, load_image +from diffusers.utils import load_image +from diffusers.utils.testing_utils import floats_tensor from openvino.runtime.ie_api import CompiledModel from parameterized import parameterized from utils_tests import MODEL_NAMES, SEED From a7782aecb0e1eb6dd7511885b651bac6b377256a Mon Sep 17 00:00:00 2001 From: "Wang, Mengni" Date: Fri, 15 Sep 2023 21:52:53 +0800 Subject: [PATCH 120/134] Integrate INC weight-only quantization (#417) * Integrate weight-only quantizaion of INC Signed-off-by: Mengni Wang * add ut Signed-off-by: Mengni Wang * reformat files Signed-off-by: Mengni Wang * update files * Update quantization.py * Update run_clm.py * Update test_optimization.py * Update README.md * Update quantization.py * Update test_optimization.py --------- Signed-off-by: Mengni Wang --- .../language-modeling/README.md | 9 +- .../language-modeling/run_clm.py | 63 ++++++++++++-- .../intel/neural_compressor/configuration.py | 1 + .../intel/neural_compressor/quantization.py | 44 +++++++++- optimum/intel/neural_compressor/utils.py | 10 ++- tests/neural_compressor/test_optimization.py | 85 +++++++++++++++++++ 6 files changed, 198 insertions(+), 14 deletions(-) diff --git a/examples/neural_compressor/language-modeling/README.md b/examples/neural_compressor/language-modeling/README.md index 1c8e98b9ee..b005bb78a4 100644 --- a/examples/neural_compressor/language-modeling/README.md +++ b/examples/neural_compressor/language-modeling/README.md @@ -18,7 +18,7 @@ limitations under the License. The scripts [`run_clm.py`](https://github.com/huggingface/optimum-intel/blob/main/examples/neural_compressor/language-modeling/run_clm.py) and [`run_mlm.py`](https://github.com/huggingface/optimum-intel/blob/main/examples/neural_compressor/language-modeling/run_mlm.py) -allow us to apply different quantization approaches (such as dynamic, static and aware-training quantization) as well as pruning +allow us to apply different quantization approaches (such as dynamic, static, weight-only and aware-training quantization) as well as pruning using the [Intel Neural Compressor ](https://github.com/intel/neural-compressor) library for language modeling tasks. The SmoothQuant methodology is also available for post-training quantization. @@ -67,6 +67,7 @@ python run_clm.py \ --do_eval \ --verify_loading \ --output_dir /tmp/clm_output +``` ### RoBERTa/BERT/DistilBERT and masked language modeling @@ -91,7 +92,9 @@ python run_mlm.py \ --output_dir /tmp/mlm_output ``` -In order to apply dynamic, static or aware-training quantization, `quantization_approach` must be set to -respectively `dynamic`, `static` or `aware_training`. +In order to apply dynamic, static, weight-only or aware-training quantization, `quantization_approach` must be set to +respectively `dynamic`, `static`, `weight_only` or `aware_training`. The flag `--verify_loading` can be passed along to verify that the resulting quantized model can be loaded correctly. + +> **_Note:_** `weight_only` quantization_approach requires neural-compressor >= 2.3 diff --git a/examples/neural_compressor/language-modeling/run_clm.py b/examples/neural_compressor/language-modeling/run_clm.py index 54f1e7b617..cbc523b663 100644 --- a/examples/neural_compressor/language-modeling/run_clm.py +++ b/examples/neural_compressor/language-modeling/run_clm.py @@ -196,6 +196,28 @@ class OptimizationArguments: default=False, metadata={"help": "Whether or not to verify the loading of the quantized model."}, ) + bits: int = field( + default=8, + metadata={"help": "Bits for weight only quantization, 1-8 bits."}, + ) + group_size: int = field( + default=-1, + metadata={ + "help": "Group size for weight only quantization. Group_size=[1-N] indicates " + "splitting the input channel elements per group_size. -1 indicates " + "the per-channel quantization per output channel." + }, + ) + weight_only_scheme: str = field( + default="sym", + metadata={"help": "Scheme for weight only quantization. Choose from 'sym' and 'asym'."}, + ) + quantization_methodology: str = field( + default="RTN", + metadata={ + "help": "Quantization methodology for weight only quantization. Choose from 'RTN', 'AWQ' and 'GPTQ'." + }, + ) @dataclass @@ -539,7 +561,9 @@ def group_texts(examples): desc=f"Grouping texts in chunks of {block_size}", ) - if training_args.do_train or (optim_args.apply_quantization and optim_args.quantization_approach == "static"): + if training_args.do_train or ( + optim_args.apply_quantization and optim_args.quantization_approach in ["static", "weight_only"] + ): if "train" not in tokenized_datasets: raise ValueError("--do_train requires a train dataset") train_dataset = lm_datasets["train"] @@ -587,7 +611,7 @@ def compute_metrics(eval_preds): raise ValueError("`do_train` must be set to True.") if optim_args.apply_quantization: - supported_approach = {"static", "dynamic", "aware_training"} + supported_approach = {"static", "dynamic", "aware_training", "weight_only"} if optim_args.quantization_approach not in supported_approach: raise ValueError( f"Unknown quantization approach. Supported approach are {supported_approach}." @@ -600,7 +624,27 @@ def compute_metrics(eval_preds): recipes = {"smooth_quant": True, "smooth_quant_args": {"alpha": optim_args.smooth_quant_alpha}} else: recipes = {} - quantization_config = PostTrainingQuantConfig(approach=optim_args.quantization_approach, recipes=recipes) + if optim_args.quantization_approach == "weight_only": + op_type_dict = { + ".*": { + "weight": { + "bits": optim_args.bits, + "group_size": optim_args.group_size, + "scheme": optim_args.weight_only_scheme, + "algorithm": optim_args.quantization_methodology, + }, + }, + } + if optim_args.quantization_methodology == "GPTQ": + gptq_args = { + "pad_max_length": block_size, + } + recipes.update({"gptq_args": gptq_args}) + else: + op_type_dict = {} + quantization_config = PostTrainingQuantConfig( + approach=optim_args.quantization_approach, op_type_dict=op_type_dict, recipes=recipes + ) if optim_args.apply_pruning: if optim_args.end_step is None: @@ -677,10 +721,10 @@ def compute_metrics(eval_preds): trainer.save_metrics("train", metrics) trainer.save_state() - if optim_args.apply_quantization and optim_args.quantization_approach in {"static", "dynamic"}: + if optim_args.apply_quantization and optim_args.quantization_approach in {"static", "dynamic", "weight_only"}: model = trainer.model if isinstance(trainer.model, PreTrainedModel) else trainer.model._model quantizer = INCQuantizer.from_pretrained(model) - if optim_args.quantization_approach == "static": + if optim_args.quantization_approach in ["static", "weight_only"]: num_calibration_samples = min(len(train_dataset), optim_args.num_calibration_samples) train_dataset = train_dataset.select(range(num_calibration_samples)) quantization_config.calibration_sampling_size = num_calibration_samples @@ -688,8 +732,13 @@ def compute_metrics(eval_preds): quantizer.quantize( quantization_config=quantization_config, save_directory=training_args.output_dir, - calibration_dataset=train_dataset if optim_args.quantization_approach == "static" else None, - batch_size=training_args.per_device_train_batch_size, + calibration_dataset=train_dataset + if optim_args.quantization_approach in ["static", "weight_only"] + else None, + batch_size=1 # batch_size > 1 for GPTQ is WIP + if optim_args.quantization_approach == "weight_only" and optim_args.quantization_methodology == "GPTQ" + else training_args.per_device_train_batch_size, + weight_only=True if optim_args.quantization_approach == "weight_only" else False, ) trainer.model = quantizer._quantized_model if optim_args.apply_quantization and optim_args.verify_loading: diff --git a/optimum/intel/neural_compressor/configuration.py b/optimum/intel/neural_compressor/configuration.py index 32d5e95375..7f5370e5ee 100644 --- a/optimum/intel/neural_compressor/configuration.py +++ b/optimum/intel/neural_compressor/configuration.py @@ -25,6 +25,7 @@ "post_training_dynamic_quant": "dynamic", "post_training_static_quant": "static", "quant_aware_training": "aware_training", + "post_training_weight_only": "weight_only", } diff --git a/optimum/intel/neural_compressor/quantization.py b/optimum/intel/neural_compressor/quantization.py index de5a5d5727..273d610e9d 100644 --- a/optimum/intel/neural_compressor/quantization.py +++ b/optimum/intel/neural_compressor/quantization.py @@ -74,6 +74,7 @@ logger = logging.getLogger(__name__) NEURAL_COMPRESSOR_MINIMUM_VERSION = "2.1.0" +NEURAL_COMPRESSOR_WEIGHT_ONLY_MINIMUM_VERSION = "2.3.0" IPEX_MINIMUM_VERSION = "2.1.0" if is_neural_compressor_version("<", NEURAL_COMPRESSOR_MINIMUM_VERSION): @@ -87,6 +88,7 @@ class INCQuantizationMode(Enum): DYNAMIC = "post_training_dynamic_quant" STATIC = "post_training_static_quant" AWARE_TRAINING = "quant_aware_training" + WEIGHT_ONLY = "post_training_weight_only" SUPPORTED_QUANT_MODE = {approach.value for approach in INCQuantizationMode} @@ -142,6 +144,7 @@ def quantize( data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, file_name: str = None, + weight_only: bool = False, **kwargs, ): """ @@ -160,6 +163,9 @@ def quantize( The function to use to form a batch from a list of elements of the calibration dataset. remove_unused_columns (`bool`, defaults to `True`): Whether or not to remove the columns unused by the model forward method. + weight_only (`bool`, defaults to `False`): + Whether compress weights to integer precision (4-bit by default) while keeping activations + floating-point. Fits best for LLM footprint reduction and performance acceleration. """ save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) @@ -168,7 +174,40 @@ def quantize( calibration_dataloader = None self._set_task() - if INCQuantizationMode(quantization_config.approach) == INCQuantizationMode.STATIC: + if weight_only: + # check neural-compressor version + if is_neural_compressor_version("<", NEURAL_COMPRESSOR_WEIGHT_ONLY_MINIMUM_VERSION): + raise ImportError( + f"Found an incompatible version of neural-compressor. Found version {_neural_compressor_version}, " + f"but only version {NEURAL_COMPRESSOR_WEIGHT_ONLY_MINIMUM_VERSION} or higher supports weight-only quantization." + ) + + # If op_type_dict of quantization_config is not defined, it will use default values for weight-only quantization: + # {"bits": 4, "group_size": 32, "scheme": "sym", "algorithm": "RTN"} + if isinstance(quantization_config.op_type_dict, dict) and len(quantization_config.op_type_dict) > 0: + algo = [] + for _, val in quantization_config.op_type_dict.items(): + algo += val.get("weight", {}).get("algorithm", ["RTN"]) + else: + algo = ["RTN"] + + if calibration_dataset is None and ("GPTQ" in algo or "AWQ" in algo): + raise ValueError( + "Weight-only quantization needs a calibration dataset for both GPTQ and AWQ methodologies." + ) + + if calibration_dataset is None: + calibration_dataloader = None + else: + calibration_dataloader = self._get_calibration_dataloader( + calibration_dataset=calibration_dataset, + batch_size=batch_size, + remove_unused_columns=remove_unused_columns, + data_collator=data_collator, + use_label=False if "GPTQ" in algo else True, + ) + + elif INCQuantizationMode(quantization_config.approach) == INCQuantizationMode.STATIC: # Since PyTorch fx trace does not really require an example_inputs, only need calibration_dataset or calibration_fn here. if calibration_dataset is None and self.calibration_fn is None: raise ValueError( @@ -378,6 +417,7 @@ def _get_calibration_dataloader( batch_size: int, remove_unused_columns: bool, data_collator: Optional[DataCollator] = None, + use_label: Optional[bool] = True, ) -> INCDataLoader: data_collator = data_collator if data_collator is not None else default_data_collator if remove_unused_columns: @@ -394,7 +434,7 @@ def _get_calibration_dataloader( drop_last=False, ) - return INCDataLoader.from_pytorch_dataloader(calibration_dataloader) + return INCDataLoader.from_pytorch_dataloader(calibration_dataloader, use_label) def _remove_unused_columns(self, dataset: Dataset): ignored_columns = list(set(dataset.column_names) - set(self._signature_columns)) diff --git a/optimum/intel/neural_compressor/utils.py b/optimum/intel/neural_compressor/utils.py index dd77011c04..fa21122595 100644 --- a/optimum/intel/neural_compressor/utils.py +++ b/optimum/intel/neural_compressor/utils.py @@ -49,11 +49,14 @@ class INCDataLoader(DataLoader): + use_label = True + @classmethod - def from_pytorch_dataloader(cls, dataloader: DataLoader): + def from_pytorch_dataloader(cls, dataloader: DataLoader, use_label: bool = True): if not isinstance(dataloader, DataLoader): raise TypeError(f"Expected a PyTorch DataLoader, got: {type(dataloader)}.") inc_dataloader = cls(dataloader.dataset) + cls.use_label = use_label for key, value in dataloader.__dict__.items(): inc_dataloader.__dict__[key] = value return inc_dataloader @@ -63,7 +66,10 @@ def __iter__(self): if not isinstance(input, (dict, tuple, list, UserDict)): raise TypeError(f"Model calibration cannot use input of type {type(input)}.") label = input.get("labels") if isinstance(input, dict) else None - yield input, label + if self.use_label: + yield input, label + else: + yield input def _cfgs_to_fx_cfgs(op_cfgs: Dict, observer_type: str = "post_training_static_quant") -> Dict: diff --git a/tests/neural_compressor/test_optimization.py b/tests/neural_compressor/test_optimization.py index 578f556153..e31739b943 100644 --- a/tests/neural_compressor/test_optimization.py +++ b/tests/neural_compressor/test_optimization.py @@ -168,6 +168,91 @@ def test_ipex_static_quantization_with_smoothquant(self, task, model_name, expec num_samples=num_samples, ) + def test_weight_only_quantization(self): + model_name = "hf-internal-testing/tiny-random-GPTNeoForCausalLM" + op_type_dict = { + ".*": { + "weight": { + "bits": 8, + "group_size": -1, + "scheme": "sym", + "algorithm": "RTN", + }, + }, + } + quantization_config = PostTrainingQuantConfig(approach="weight_only", op_type_dict=op_type_dict) + model = AutoModelForCausalLM.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + quantizer = INCQuantizer.from_pretrained(model, task="text-generation") + calibration_dataset = _generate_dataset(quantizer, tokenizer, num_samples=2) + + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + calibration_dataset=calibration_dataset, + save_directory=tmp_dir, + weight_only=True, + ) + q_model = AutoModelForCausalLM.from_pretrained(tmp_dir) + inp = torch.tensor([calibration_dataset[0]["input_ids"]]) + out = model(inp)[0] + q_out = q_model(inp)[0] + self.assertTrue(torch.all(torch.isclose(out, q_out, atol=5e-1))) + + op_type_dict = { + ".*": { + "weight": { + "bits": 8, + "group_size": -1, + "scheme": "sym", + "algorithm": "AWQ", + }, + }, + } + quantization_config = PostTrainingQuantConfig(approach="weight_only", op_type_dict=op_type_dict) + + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + calibration_dataset=calibration_dataset, + save_directory=tmp_dir, + weight_only=True, + ) + q_model = AutoModelForCausalLM.from_pretrained(tmp_dir) + inp = torch.tensor([calibration_dataset[0]["input_ids"]]) + out = model(inp)[0] + q_out = q_model(inp)[0] + self.assertTrue(torch.all(torch.isclose(out, q_out, atol=6e-1))) + + op_type_dict = { + ".*": { + "weight": { + "bits": 8, + "group_size": -1, + "scheme": "sym", + "algorithm": "GPTQ", + }, + }, + } + recipes = {"gptq_args": {"pad_max_length": len(calibration_dataset[0]["input_ids"])}} + quantization_config = PostTrainingQuantConfig( + approach="weight_only", op_type_dict=op_type_dict, recipes=recipes + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + quantizer.quantize( + quantization_config=quantization_config, + calibration_dataset=calibration_dataset, + save_directory=tmp_dir, + weight_only=True, + ) + q_model = AutoModelForCausalLM.from_pretrained(tmp_dir) + inp = torch.tensor([calibration_dataset[0]["input_ids"]]) + out = model(inp)[0] + q_out = q_model(inp)[0] + self.assertTrue(torch.all(torch.isclose(out, q_out, atol=5e-1))) + def test_dynamic_accuracy_strategy_quantization(self): model_name = "distilbert-base-cased-distilled-squad" model = AutoModelForQuestionAnswering.from_pretrained(model_name) From 681b946fbd8eaad9d0eb8b6a88708453f3472603 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Mon, 18 Sep 2023 20:53:13 +0400 Subject: [PATCH 121/134] move VAE decoder to fp32 execution precision on GPU (#432) --- optimum/intel/openvino/modeling_diffusion.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 1085c9e81c..da1bde6dbb 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -606,6 +606,11 @@ def __call__(self, latent_sample: np.ndarray): outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) + def _compile(self): + if "GPU" in self.device: + self.ov_config.update({"INFERENCE_PRECISION_HINT": "f32"}) + super()._compile() + class OVModelVaeEncoder(OVModelPart): def __init__( @@ -622,6 +627,11 @@ def __call__(self, sample: np.ndarray): outputs = self.request(inputs, shared_memory=True) return list(outputs.values()) + def _compile(self): + if "GPU" in self.device: + self.ov_config.update({"INFERENCE_PRECISION_HINT": "f32"}) + super()._compile() + class OVStableDiffusionPipeline(OVStableDiffusionPipelineBase, StableDiffusionPipelineMixin): def __call__( From 4b8ed2404abf4408a16d251870a8deb25a4c448b Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Tue, 19 Sep 2023 14:22:15 +0400 Subject: [PATCH 122/134] OV migrate model export on pytorch frontend (#397) * switch on pytorch frontend * fixes for seq2seq * wip * cleanup * fix style * revert changes not related to pr * clear ts registry: * remove ov dev from deps * update tests * return serialize back * switch on pytorch frontend * fixes for seq2seq * wip * cleanup * fix style * revert changes not related to pr * clear ts registry: * remove ov dev from deps * return serialize back * Added weights compression * Changed NNCF version to develop * resolve dictionary as input * fix llama export in quantization flow * rebase with fixes * update prerelease package * fix onnx name issues * experiments with tests * better workaround for nncf patch torch ops and apply review comments * remove flag from_onnx * refactoring * docstrings and typehints * small fixes * add docstring to main_export * fix timm models * fix circular imports * update ov version * revert excluding deberta * update nncf on package --------- Co-authored-by: Alexander --- optimum/exporters/openvino/__init__.py | 5 + optimum/exporters/openvino/__main__.py | 293 +++++++++++++ optimum/exporters/openvino/convert.py | 390 ++++++++++++++++++ optimum/exporters/openvino/utils.py | 142 +++++++ optimum/intel/openvino/modeling.py | 2 +- optimum/intel/openvino/modeling_base.py | 20 +- .../intel/openvino/modeling_base_seq2seq.py | 15 +- optimum/intel/openvino/modeling_decoder.py | 25 +- optimum/intel/openvino/modeling_diffusion.py | 6 +- optimum/intel/openvino/modeling_seq2seq.py | 1 - optimum/intel/openvino/quantization.py | 108 ++--- optimum/intel/openvino/trainer.py | 18 +- optimum/intel/utils/modeling_utils.py | 20 + setup.py | 4 +- tests/openvino/test_modeling.py | 74 +++- tests/openvino/test_quantization.py | 11 +- 16 files changed, 1026 insertions(+), 108 deletions(-) create mode 100644 optimum/exporters/openvino/__init__.py create mode 100644 optimum/exporters/openvino/__main__.py create mode 100644 optimum/exporters/openvino/convert.py create mode 100644 optimum/exporters/openvino/utils.py diff --git a/optimum/exporters/openvino/__init__.py b/optimum/exporters/openvino/__init__.py new file mode 100644 index 0000000000..d87d8dda9e --- /dev/null +++ b/optimum/exporters/openvino/__init__.py @@ -0,0 +1,5 @@ +from .__main__ import main_export +from .convert import export, export_models, export_pytorch_via_onnx + + +__all__ = ["main_export", "export", "export_models"] diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py new file mode 100644 index 0000000000..5cf0adb176 --- /dev/null +++ b/optimum/exporters/openvino/__main__.py @@ -0,0 +1,293 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Union + +from requests.exceptions import ConnectionError as RequestsConnectionError +from transformers import AutoTokenizer +from transformers.utils import is_torch_available + +from optimum.exporters import TasksManager +from optimum.exporters.onnx import __main__ as optimum_main +from optimum.exporters.onnx.base import OnnxConfig, OnnxConfigWithPast +from optimum.utils import DEFAULT_DUMMY_SHAPES +from optimum.utils.save_utils import maybe_save_preprocessors + +from .convert import export_models + + +OV_XML_FILE_NAME = "openvino_model.xml" + +logger = logging.getLogger(__name__) + +if is_torch_available(): + import torch + + +def main_export( + model_name_or_path: str, + output: Union[str, Path], + task: str = "auto", + device: str = "cpu", + fp16: Optional[bool] = False, + framework: Optional[str] = None, + cache_dir: Optional[str] = None, + trust_remote_code: bool = False, + pad_token_id: Optional[int] = None, + subfolder: str = "", + revision: str = "main", + force_download: bool = False, + local_files_only: bool = False, + use_auth_token: Optional[Union[bool, str]] = None, + model_kwargs: Optional[Dict[str, Any]] = None, + custom_onnx_configs: Optional[Dict[str, "OnnxConfig"]] = None, + fn_get_submodels: Optional[Callable] = None, + **kwargs_shapes, +): + """ + Full-suite OpenVINO export. + + Args: + > Required parameters + + model_name_or_path (`str`): + Model ID on huggingface.co or path on disk to the model repository to export. + output (`Union[str, Path]`): + Path indicating the directory where to store the generated ONNX model. + + > Optional parameters + + task (`Optional[str]`, defaults to `None`): + The task to export the model for. If not specified, the task will be auto-inferred based on the model. For decoder models, + use `xxx-with-past` to export the model using past key values in the decoder. + device (`str`, defaults to `"cpu"`): + The device to use to do the export. Defaults to "cpu". + fp16 (`Optional[bool]`, defaults to `"False"`): + Use half precision during the export. PyTorch-only, requires `device="cuda"`. + framework (`Optional[str]`, defaults to `None`): + The framework to use for the ONNX export (`"pt"` or `"tf"`). If not provided, will attempt to automatically detect + the framework for the checkpoint. + cache_dir (`Optional[str]`, defaults to `None`): + Path indicating where to store cache. The default Hugging Face cache path will be used by default. + trust_remote_code (`bool`, defaults to `False`): + Allows to use custom code for the modeling hosted in the model repository. This option should only be set for repositories + you trust and in which you have read the code, as it will execute on your local machine arbitrary code present in the + model repository. + pad_token_id (`Optional[int]`, defaults to `None`): + This is needed by some models, for some tasks. If not provided, will attempt to use the tokenizer to guess it. + subfolder (`str`, defaults to `""`): + In case the relevant files are located inside a subfolder of the model repo either locally or on huggingface.co, you can + specify the folder name here. + revision (`str`, defaults to `"main"`): + Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id. + force_download (`bool`, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + local_files_only (`Optional[bool]`, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`Optional[str]`, defaults to `None`): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + model_kwargs (`Optional[Dict[str, Any]]`, defaults to `None`): + Experimental usage: keyword arguments to pass to the model during + the export. This argument should be used along the `custom_onnx_configs` argument + in case, for example, the model inputs/outputs are changed (for example, if + `model_kwargs={"output_attentions": True}` is passed). + custom_onnx_configs (`Optional[Dict[str, OnnxConfig]]`, defaults to `None`): + Experimental usage: override the default ONNX config used for the given model. This argument may be useful for advanced users that desire a finer-grained control on the export. An example is available [here](https://huggingface.co/docs/optimum/main/en/exporters/onnx/usage_guides/export_a_model). + fn_get_submodels (`Optional[Callable]`, defaults to `None`): + Experimental usage: Override the default submodels that are used at the export. This is + especially useful when exporting a custom architecture that needs to split the ONNX (e.g. encoder-decoder). If unspecified with custom models, optimum will try to use the default submodels used for the given task, with no guarantee of success. + **kwargs_shapes (`Dict`): + Shapes to use during inference. This argument allows to override the default shapes used during the ONNX export. + + Example usage: + ```python + >>> from optimum.exporters.openvino import main_export + + >>> main_export("gpt2", output="gpt2_onnx/") + ``` + """ + output = Path(output) + if not output.exists(): + output.mkdir(parents=True) + + original_task = task + task = TasksManager.map_from_synonym(task) + + framework = TasksManager.determine_framework(model_name_or_path, subfolder=subfolder, framework=framework) + + # get the shapes to be used to generate dummy inputs + input_shapes = {} + for input_name in DEFAULT_DUMMY_SHAPES.keys(): + input_shapes[input_name] = ( + kwargs_shapes[input_name] if input_name in kwargs_shapes else DEFAULT_DUMMY_SHAPES[input_name] + ) + + torch_dtype = None if fp16 is False else torch.float16 + + if task == "auto": + try: + task = TasksManager.infer_task_from_model(model_name_or_path) + except KeyError as e: + raise KeyError( + f"The task could not be automatically inferred. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}" + ) + except RequestsConnectionError as e: + raise RequestsConnectionError( + f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}" + ) + + model = TasksManager.get_model_from_task( + task, + model_name_or_path, + subfolder=subfolder, + revision=revision, + cache_dir=cache_dir, + use_auth_token=use_auth_token, + local_files_only=local_files_only, + force_download=force_download, + trust_remote_code=trust_remote_code, + framework=framework, + torch_dtype=torch_dtype, + device=device, + ) + + custom_architecture = False + is_stable_diffusion = "stable-diffusion" in task + model_type = "stable-diffusion" if is_stable_diffusion else model.config.model_type.replace("_", "-") + + if not is_stable_diffusion: + if model_type in TasksManager._UNSUPPORTED_CLI_MODEL_TYPE: + raise ValueError( + f"{model_type} is not supported yet. Only {TasksManager._SUPPORTED_CLI_MODEL_TYPE} are supported. " + f"If you want to support {model_type} please propose a PR or open up an issue." + ) + if model.config.model_type.replace("-", "_") not in TasksManager.get_supported_model_type_for_task( + task, exporter="onnx" + ): + custom_architecture = True + + if custom_architecture and custom_onnx_configs is None: + raise ValueError( + "Trying to export a model with a custom architecture, but no custom onnx configuration was passed as `custom_onnx_configs`. Please refer to https://huggingface.co/docs/optimum/main/en/exporters/onnx/usage_guides/export_a_model#custom-export-of-transformers-models for an example on how to export custom models." + ) + + if custom_architecture and original_task == "auto": + raise ValueError( + f'Automatic task detection is not supported with custom architectures. Please specify the `task` argument. Suggestion: task="{task}" (or task="{task}-with-past" if the model is decoder-based and supports KV cache)' + ) + + if ( + not custom_architecture + and not is_stable_diffusion + and task + "-with-past" in TasksManager.get_supported_tasks_for_model_type(model_type, "onnx") + ): + if original_task == "auto": # Make -with-past the default if --task was not explicitely specified + task = task + "-with-past" + else: + logger.info( + f"The task `{task}` was manually specified, and past key values will not be reused in the decoding." + f" if needed, please pass `--task {task}-with-past` to export using the past key values." + ) + + if original_task == "auto": + synonyms_for_task = sorted(TasksManager.synonyms_for_task(task)) + if synonyms_for_task: + synonyms_for_task = ", ".join(synonyms_for_task) + possible_synonyms = f" (possible synonyms are: {synonyms_for_task})" + else: + possible_synonyms = "" + logger.info(f"Automatic task detection to {task}{possible_synonyms}.") + onnx_config, models_and_onnx_configs = optimum_main._get_submodels_and_onnx_configs( + model=model, + task=task, + monolith=False, + custom_onnx_configs=custom_onnx_configs if custom_onnx_configs is not None else {}, + custom_architecture=custom_architecture, + fn_get_submodels=fn_get_submodels, + _variant="default", + ) + + if not is_stable_diffusion: + needs_pad_token_id = ( + isinstance(onnx_config, OnnxConfigWithPast) + and getattr(model.config, "pad_token_id", None) is None + and task in ["text-classification"] + ) + if needs_pad_token_id: + if pad_token_id is not None: + model.config.pad_token_id = pad_token_id + else: + try: + tok = AutoTokenizer.from_pretrained(model_name_or_path) + model.config.pad_token_id = tok.pad_token_id + except Exception: + raise ValueError( + "Could not infer the pad token id, which is needed in this case, please provide it with the --pad_token_id argument" + ) + # Saving the model config and preprocessor as this is needed sometimes. + model.config.save_pretrained(output) + generation_config = getattr(model, "generation_config", None) + if generation_config is not None: + generation_config.save_pretrained(output) + maybe_save_preprocessors(model_name_or_path, output) + + if model.config.is_encoder_decoder and task.startswith("text-generation"): + raise ValueError( + f"model.config.is_encoder_decoder is True and task is `{task}`, which are incompatible. If the task was auto-inferred, please fill a bug report" + f"at https://github.com/huggingface/optimum, if --task was explicitely passed, make sure you selected the right task for the model," + f" referring to `optimum.exporters.tasks.TaskManager`'s `_TASKS_TO_AUTOMODELS`." + ) + + files_subpaths = None + else: + # save the subcomponent configuration + for model_name in models_and_onnx_configs: + subcomponent = models_and_onnx_configs[model_name][0] + if hasattr(subcomponent, "save_config"): + subcomponent.save_config(output / model_name) + elif hasattr(subcomponent, "config") and hasattr(subcomponent.config, "save_pretrained"): + subcomponent.config.save_pretrained(output / model_name) + + files_subpaths = [os.path.join(name_dir, OV_XML_FILE_NAME) for name_dir in models_and_onnx_configs] + + # Saving the additional components needed to perform inference. + model.scheduler.save_pretrained(output.joinpath("scheduler")) + + feature_extractor = getattr(model, "feature_extractor", None) + if feature_extractor is not None: + feature_extractor.save_pretrained(output.joinpath("feature_extractor")) + + tokenizer = getattr(model, "tokenizer", None) + if tokenizer is not None: + tokenizer.save_pretrained(output.joinpath("tokenizer")) + + tokenizer_2 = getattr(model, "tokenizer_2", None) + if tokenizer_2 is not None: + tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) + + model.save_config(output) + + export_models( + models_and_onnx_configs=models_and_onnx_configs, + output_dir=output, + output_names=files_subpaths, + input_shapes=input_shapes, + device=device, + model_kwargs=model_kwargs, + ) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py new file mode 100644 index 0000000000..ab688f92fa --- /dev/null +++ b/optimum/exporters/openvino/convert.py @@ -0,0 +1,390 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import gc +import inspect +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from transformers.utils import is_tf_available, is_torch_available + +from openvino.runtime import PartialShape, save_model +from openvino.runtime.utils.types import get_element_type +from openvino.tools.ovc import convert_model +from optimum.exporters.onnx.base import OnnxConfig +from optimum.exporters.onnx.convert import check_dummy_inputs_are_allowed +from optimum.exporters.onnx.convert import export_pytorch as export_pytorch_to_onnx +from optimum.exporters.onnx.convert import export_tensorflow as export_tensorflow_onnx +from optimum.utils import is_diffusers_available + +from .utils import ( + OV_XML_FILE_NAME, + clear_class_registry, + flattenize_inputs, + get_input_shapes, + remove_none_from_dummy_inputs, +) + + +logger = logging.getLogger(__name__) + +if is_torch_available(): + import torch.nn as nn + from transformers.modeling_utils import PreTrainedModel + +if is_diffusers_available(): + from diffusers import ModelMixin + +if is_tf_available(): + from transformers.modeling_tf_utils import TFPreTrainedModel + + +def export( + model: Union["PreTrainedModel", "TFPreTrainedModel", "ModelMixin"], + config: OnnxConfig, + output: Path, + opset: Optional[int] = None, + device: str = "cpu", + input_shapes: Optional[Dict] = None, + model_kwargs: Optional[Dict[str, Any]] = None, +) -> Tuple[List[str], List[str]]: + """ + Exports a Pytorch or TensorFlow model to an OpenVINO Intermediate Representation. + + Args: + model ([`PreTrainedModel`] or [`TFPreTrainedModel`]): + The model to export. + config ([`~exporters.onnx.config.OnnxConfig`]): + The ONNX configuration associated with the exported model. + output (`Path`): + Directory to store the exported model. + opset (`Optional[int]`, defaults to `None`): + The version of the ONNX operator set to use. + device (`str`, *optional*, defaults to `cpu`): + The device on which the model will be exported. Either `cpu` or `cuda`. Only PyTorch is supported for + export on CUDA devices. + input_shapes (`Optional[Dict]`, defaults to `None`): + If specified, allows to use specific shapes for the example input provided to the exporter. + + Returns: + `Tuple[List[str], List[str]]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration. + """ + if not (is_torch_available() or is_tf_available()): + raise ImportError( + "Cannot convert because neither PyTorch nor TensorFlow are installed. " + "Please install torch or tensorflow first." + ) + + if "diffusers" in str(model.__class__) and not is_diffusers_available(): + raise ImportError("The pip package `diffusers` is required to export stable diffusion models to ONNX.") + + if is_torch_available() and isinstance(model, nn.Module): + return export_pytorch( + model, + config, + opset, + output, + device=device, + input_shapes=input_shapes, + model_kwargs=model_kwargs, + ) + + elif is_tf_available() and issubclass(type(model), TFPreTrainedModel): + output.parent.mkdir(parents=True, exist_ok=True) + if opset is None: + opset = config.DEFAULT_ONNX_OPSET + if device == "cuda": + raise RuntimeError("`tf2onnx` does not support export on CUDA device.") + if input_shapes is not None: + logger.info("`input_shapes` argument is not supported by the Tensorflow ONNX export and will be ignored.") + return export_tensorflow(model, config, opset, output) + + else: + raise RuntimeError( + "You either provided a PyTorch model with only TensorFlow installed, or a TensorFlow model with only PyTorch installed." + ) + + +def export_tensorflow(model: Union["PreTrainedModel", "ModelMixin"], config: OnnxConfig, opset: int, output: Path): + """ + Export the TensorFlow model to OpenVINO format. + + Args: + model (Union[): The model to export. + config (OnnxConfig): The configuration of the model. + opset (int): The ONNX opset version to use. + output (Path): The path to save the model. + + Returns: + input_names: list of input names from ONNX configuration + output_names: list of output names from ONNX configuration + bool: True if the model was exported successfully. + """ + onnx_path = Path(output).with_suffix(".onnx") + input_names, output_names = export_tensorflow_onnx(model, config, opset, onnx_path) + ov_model = convert_model(str(onnx_path)) + save_model( + ov_model, + output.parent / output, + compress_to_fp16=False, + ) + return input_names, output_names, True + + +def export_pytorch_via_onnx( + model: Union["PreTrainedModel", "ModelMixin"], + config: OnnxConfig, + opset: int, + output: Path, + device: str = "cpu", + input_shapes: Optional[Dict] = None, + model_kwargs: Optional[Dict[str, Any]] = None, +): + """ + Exports a PyTorch model to an OpenVINO Intermediate Representation via ONNX export. + + Args: + model ([`PreTrainedModel`]): + The model to export. + config ([`~exporters.onnx.config.OnnxConfig`]): + The configuration associated with the exported model. + opset (`int`): + The version of the ONNX operator set to use. + output (`Path`): + Directory to store the exported model. + device (`str`, defaults to `"cpu"`): + The device on which the model will be exported. Either `cpu` or `cuda`. Only PyTorch is supported for + export on CUDA devices. + input_shapes (`optional[Dict]`, defaults to `None`): + If specified, allows to use specific shapes for the example input provided to the exporter. + model_kwargs (optional[Dict[str, Any]], defaults to `None`): + Additional kwargs for model export + + Returns: + `Tuple[List[str], List[str], bool]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration and boolean flag - was legacy ONNX path were applied to model or not. + """ + import torch + + output = Path(output) + orig_torch_onnx_export = torch.onnx.export + torch.onnx.export = functools.partial(orig_torch_onnx_export, do_constant_folding=False) + model.config.torchscript = False + model.config.return_dict = True + onnx_output = output.with_suffix(".onnx") + input_names, output_names = export_pytorch_to_onnx( + model, config, opset, onnx_output, device, input_shapes, model_kwargs + ) + torch.onnx.export = orig_torch_onnx_export + ov_model = convert_model(str(onnx_output)) + save_model( + ov_model, + output.parent / OV_XML_FILE_NAME if output.suffix != ".xml" else output, + compress_to_fp16=False, + ) + return input_names, output_names, True + + +def export_pytorch( + model: Union["PreTrainedModel", "ModelMixin"], + config: OnnxConfig, + opset: int, + output: Path, + device: str = "cpu", + input_shapes: Optional[Dict] = None, + model_kwargs: Optional[Dict[str, Any]] = None, +) -> Tuple[List[str], List[str]]: + """ + Exports a PyTorch model to an OpenVINO Intermediate Representation. + + Args: + model ([`PreTrainedModel`]): + The model to export. + config ([`~exporters.onnx.config.OnnxConfig`]): + The configuration associated with the exported model. + opset (`int`): + The version of the ONNX operator set to use. + output (`Path`): + Directory to store the exported model. + device (`str`, defaults to `"cpu"`): + The device on which the model will be exported. Either `cpu` or `cuda`. Only PyTorch is supported for + export on CUDA devices. + input_shapes (`optional[Dict]`, defaults to `None`): + If specified, allows to use specific shapes for the example input provided to the exporter. + model_kwargs (optional[Dict[str, Any]], defaults to `None`): + Additional kwargs for model export + + Returns: + `Tuple[List[str], List[str], bool]`: A tuple with an ordered list of the model's inputs, and the named inputs from + the ONNX configuration and boolean flag - was legacy ONNX path were applied to model or not. + """ + import torch + from torch.utils._pytree import tree_map + + logger.info(f"Using framework PyTorch: {torch.__version__}") + output = Path(output) + + with torch.no_grad(): + model.config.torchscript = False + model.config.return_dict = True + model.eval() + + # Check if we need to override certain configuration item + if config.values_override is not None: + logger.info(f"Overriding {len(config.values_override)} configuration item(s)") + for override_config_key, override_config_value in config.values_override.items(): + logger.info(f"\t- {override_config_key} -> {override_config_value}") + setattr(model.config, override_config_key, override_config_value) + + if input_shapes is None: + input_shapes = {} # will use the defaults from DEFAULT_DUMMY_SHAPES + + # Check that inputs match, and order them properly + dummy_inputs = config.generate_dummy_inputs(framework="pt", **input_shapes) + device = torch.device(device) + if device.type == "cuda" and torch.cuda.is_available(): + model.to(device) + dummy_inputs = tree_map( + lambda value: value.to(device) if isinstance(value, torch.Tensor) else value, dummy_inputs + ) + check_dummy_inputs_are_allowed(model, dummy_inputs) + inputs = config.ordered_inputs(model) + input_names = list(inputs.keys()) + output_names = list(config.outputs.keys()) + if hasattr(model, "forward"): + sig = inspect.signature(model.forward) + else: + sig = inspect.signature(model.call) + + dummy_inputs, dict_inputs = remove_none_from_dummy_inputs(dummy_inputs) + input_info = get_input_shapes(dummy_inputs, inputs) + custom_patcher = type(config).patch_model_for_export != OnnxConfig.patch_model_for_export + try: + # TorchScript used behind OpenVINO conversion. Optimum supports only return_dict=True models for patching, + # while TorchScript do not support dictionary with values of mixed types (e.g. Tensor and None) in model input/output + # To handle it, additional wrapper on patcher forward applied. + # model.config.torchscript = True can not be used for patching, because it overrides return_dict to Flase + if custom_patcher or dict_inputs: + patcher = config.patch_model_for_export(model, model_kwargs=model_kwargs) + patched_forward = patcher.patched_forward + + @functools.wraps(patched_forward) + def ts_patched_forward(*args, **kwargs): + for i in range(len(dict_inputs)): + input_name = dict_inputs[i][0] + keys = dict_inputs[i][1] + tuple_input = kwargs[input_name] + input_dict = dict(zip(keys, tuple_input)) + kwargs[input_name] = input_dict + outputs = patched_forward(*args, **kwargs) + return tuple(outputs.values()) + + patcher.patched_forward = ts_patched_forward + with patcher: + ov_model = convert_model(model, example_input=dummy_inputs, input=input_info) + else: + model.config.torchscript = True + ov_model = convert_model(model, example_input=dummy_inputs, input=input_info) + except Exception as ex: + logger.warning(f"Export model to OpenVINO directly failed with: \n{ex}.\nModel will be exported to ONNX") + return export_pytorch_via_onnx(model, config, opset, output, device, input_shapes, model_kwargs) + ordered_dummy_inputs = {param: dummy_inputs[param] for param in sig.parameters if param in dummy_inputs} + ordered_input_names = list(inputs) + flatten_inputs = flattenize_inputs(ordered_dummy_inputs.values()) + ov_model.validate_nodes_and_infer_types() + for idx, out_tensor in enumerate(ov_model.outputs): + if idx < len(output_names): + out_tensor.get_tensor().set_names({output_names[idx]}) + + for idx, inp_tensor in enumerate(ov_model.inputs): + input_name = ordered_input_names[idx] + inp_tensor.get_tensor().set_names({input_name}) + inp_data = flatten_inputs[idx] + static_shape = PartialShape(inp_data.shape) + dims = inputs[input_name] + + for dim in dims: + static_shape[dim] = -1 + inp_tensor.get_node().set_partial_shape(static_shape) + inp_tensor.get_node().set_element_type(get_element_type(inp_data.cpu().numpy().dtype)) + ov_model.validate_nodes_and_infer_types() + save_model(ov_model, output, compress_to_fp16=False) + clear_class_registry() + del model + gc.collect() + return input_names, output_names, False + + +def export_models( + models_and_onnx_configs: Dict[ + str, Tuple[Union["PreTrainedModel", "TFPreTrainedModel", "ModelMixin"], "OnnxConfig"] + ], + output_dir: Path, + opset: Optional[int] = None, + output_names: Optional[List[str]] = None, + device: str = "cpu", + input_shapes: Optional[Dict] = None, + model_kwargs: Optional[Dict[str, Any]] = None, +) -> Tuple[List[List[str]], List[List[str]]]: + """ + Export the models to OpenVINO IR format + + Args: + models_and_onnx_configs (Dict[ str, Tuple[Union["PreTrainedModel", "TFPreTrainedModel", "ModelMixin"], "OnnxConfig"]): + output_dir (Path): output directory for saving models + opset (Optional[int], optional, Default to None): ONNX export opset + output_names (Optional[List[str]], optional, Defaults to None): model output names + device (str, optional, Defaults to "cpu"): + The device on which the model will be exported. Either `cpu` or `cuda`. Only PyTorch is supported for + export on CUDA devices. + input_shapes (Optional[Dict], optional, Defaults to None): + If specified, allows to use specific shapes for the example input provided to the exporter. + model_kwargs (Optional[Dict[str, Any]], optional): + Additional kwargs for model export + + Raises: + ValueError: if custom names set not equal of number of models + + Returns: + list of input_names and output_names from ONNX configuration + """ + outputs = [] + + if output_names is not None and len(output_names) != len(models_and_onnx_configs): + raise ValueError( + f"Provided custom names {output_names} for the export of {len(models_and_onnx_configs)} models. Please provide the same number of names as models to export." + ) + + for i, model_name in enumerate(models_and_onnx_configs.keys()): + submodel, sub_onnx_config = models_and_onnx_configs[model_name] + output_name = output_names[i] if output_names is not None else Path(model_name + ".xml") + output_path = output_dir / output_name + output_path.parent.mkdir(parents=True, exist_ok=True) + outputs.append( + export( + model=submodel, + config=sub_onnx_config, + output=output_path, + opset=opset, + device=device, + input_shapes=input_shapes, + model_kwargs=model_kwargs, + ) + ) + + outputs = list(map(list, zip(*outputs))) + return outputs diff --git a/optimum/exporters/openvino/utils.py b/optimum/exporters/openvino/utils.py new file mode 100644 index 0000000000..f0d5366526 --- /dev/null +++ b/optimum/exporters/openvino/utils.py @@ -0,0 +1,142 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Tuple, Union + +from transformers.utils import is_torch_available + +from openvino.runtime import PartialShape +from optimum.utils import is_diffusers_available + + +if is_torch_available(): + import torch + import torch.nn as nn + from transformers.modeling_utils import PreTrainedModel + +if is_diffusers_available(): + from diffusers import ModelMixin + + +OV_XML_FILE_NAME = "openvino_model.xml" + + +def is_torch_model(model: Union["PreTrainedModel", "ModelMixin"]): + """ + Checks whether the model is a torch model. + + Args: + model (Union[PretrainedModel, ModelMixin]): The model to check. + + Returns: + bool: True if the model is a torch model. + """ + if not is_torch_available(): + return False + return isinstance(model, nn.Module) + + +def flattenize_inputs(inputs: List[Any]): + """ + Flatten the inputs into a list. + + Args: + inputs (List[Any]): The inputs to flatten. + + Returns: + List[Any]: The flattened inputs. + """ + flatten_inputs = [] + for input_data in inputs: + if input_data is None: + continue + if isinstance(input_data, (list, tuple)): + flatten_inputs.extend(flattenize_inputs(input_data)) + else: + flatten_inputs.append(input_data) + return flatten_inputs + + +def remove_none_from_dummy_inputs(dummy_inputs: Dict[str, Any]): + """ + Removes None values from the dictionary. + + Args: + dummy_inputs (Dict[str, Any]): Dictionary with None values. + Returns: + upd_dummy (Dict[str, Any]): updated dictionary with removed None values + dict_dummy (List[Tuple[str, List[str]]]): list of inputs represented as dictionary provided as pair name and list of nested keys + """ + + def remove_none_from_list_tuple(item: Union[List[Any], Tuple[Any]]): + """ + Removes None values from a list or tuple. + + Args: + item (list or tuple): The list or tuple to remove None values from. + + Returns: + list or tuple: The list or tuple with None values removed. + """ + new_item = [i for i in item if i is not None] + return type(item)(new_item) + + upd_dummy = {} + dict_dummy = [] + for k, v in dummy_inputs.items(): + if v is None: + continue + if isinstance(v, dict): + dict_dummy.append((k, list(v.keys()))) + upd_dummy[k] = remove_none_from_list_tuple(tuple(v.values())) + continue + if isinstance(v, (tuple, list)): + upd_dummy[k] = remove_none_from_list_tuple(v) + continue + upd_dummy[k] = v + return upd_dummy, dict_dummy + + +def get_input_shapes(dummy_inputs: Dict[str, Any], inputs: Dict[str, Any]): + """ + Resolves input shapes based on dynamic axes from input config and dummy input shapes + + Args: + dummy_inputs (Dict[str, Any]): A dictionary of dummy inputs. + inputs (Dict[str, Any]): A dictionary of input tensors. + + Returns: + input_info: List of input info for conversion + + """ + input_info = [] + for input_name, data in dummy_inputs.items(): + if isinstance(data, (tuple, list, dict)): + return None + static_shape = PartialShape(data.shape) + if input_name in inputs: + dynamic_dims = inputs[input_name] + for dim in dynamic_dims: + static_shape[dim] = -1 + input_info.append((input_name, static_shape)) + return input_info + + +def clear_class_registry(): + """ + Removes Torchscript cached modules + """ + torch._C._jit_clear_class_registry() + torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() + torch.jit._state._clear_class_state() diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index 1cea230429..95fb0aca8b 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -549,7 +549,7 @@ def from_pretrained( model = TimmForImageClassification.from_pretrained(model_id, **kwargs) onnx_config = TimmOnnxConfig(model.config) - return cls._to_onnx_to_load( + return cls._to_load( model=model, config=config, onnx_config=onnx_config, diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 59fc89649a..42bdb8edba 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -20,15 +20,16 @@ import openvino from huggingface_hub import hf_hub_download +from openvino import Core, convert_model from openvino._offline_transformations import apply_moc_transformations, compress_model_transformation -from openvino.runtime import Core from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters.onnx import OnnxConfig, export +from optimum.exporters.onnx import OnnxConfig from optimum.exporters.tasks import TasksManager from optimum.modeling_base import OptimizedModel +from ...exporters.openvino import export from ..utils.import_utils import is_transformers_version from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME @@ -127,9 +128,7 @@ def fix_op_names_duplicates(model: openvino.runtime.Model): if isinstance(file_name, str): file_name = Path(file_name) - bin_file_name = file_name.with_suffix(".bin") if file_name.suffix == ".xml" else None - - model = core.read_model(file_name, bin_file_name) + model = core.read_model(file_name) if not file_name.suffix == ".onnx" else convert_model(file_name) if file_name.suffix == ".onnx": model = fix_op_names_duplicates(model) # should be called during model conversion to IR @@ -145,7 +144,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): The directory where to save the model files. """ dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(self.model, dst_path) + openvino.save_model(self.model, dst_path, compress_to_fp16=False) @classmethod def _from_pretrained( @@ -198,6 +197,7 @@ def _from_pretrained( else: model_file_names = [file_name] # If not ONNX then OpenVINO IR + if not from_onnx: model_file_names.append(file_name.replace(".xml", ".bin")) file_names = [] @@ -276,7 +276,7 @@ def _from_transformers( onnx_config = onnx_config_class(model.config) - return cls._to_onnx_to_load( + return cls._to_load( model=model, config=config, onnx_config=onnx_config, @@ -288,7 +288,7 @@ def _from_transformers( ) @classmethod - def _to_onnx_to_load( + def _to_load( cls, model: PreTrainedModel, config: PretrainedConfig, @@ -308,13 +308,13 @@ def _to_onnx_to_load( model=model, config=onnx_config, opset=onnx_config.DEFAULT_ONNX_OPSET, - output=save_dir_path / ONNX_WEIGHTS_NAME, + output=save_dir_path / OV_XML_FILE_NAME, ) return cls._from_pretrained( model_id=save_dir_path, config=config, - from_onnx=True, + from_onnx=False, use_auth_token=use_auth_token, revision=revision, force_download=force_download, diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index a8ce3d0bf5..f8e09b2c91 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -24,9 +24,10 @@ from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters.onnx import export_models, get_encoder_decoder_models_for_export -from optimum.exporters.tasks import TasksManager +from optimum.exporters import TasksManager +from optimum.exporters.onnx import get_encoder_decoder_models_for_export +from ...exporters.openvino import export_models from ..utils.import_utils import is_transformers_version from .modeling_base import OVBaseModel from .utils import ( @@ -104,7 +105,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): for src_file, dst_file_name in zip(src_files, dst_file_names): dst_path = os.path.join(save_directory, dst_file_name) - openvino.runtime.serialize(src_file, dst_path) + openvino.save_model(src_file, dst_path, compress_to_fp16=False) @classmethod def _from_pretrained( @@ -243,9 +244,6 @@ def _from_transformers( kwargs (`Dict`, *optional*): kwargs will be passed to the model during initialization """ - encoder_file_name = os.path.join("encoder", ONNX_ENCODER_NAME) - decoder_file_name = os.path.join("decoder", ONNX_DECODER_NAME) - decoder_with_past_file_name = os.path.join("decoder_with_past", ONNX_DECODER_WITH_PAST_NAME) task = task or cls.export_feature save_dir = TemporaryDirectory() @@ -265,6 +263,9 @@ def _from_transformers( onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config, use_past=use_cache) models_and_onnx_configs = get_encoder_decoder_models_for_export(model, onnx_config) + encoder_file_name = os.path.join("encoder", OV_ENCODER_NAME) + decoder_file_name = os.path.join("decoder", OV_DECODER_NAME) + decoder_with_past_file_name = os.path.join("decoder_with_past", OV_DECODER_WITH_PAST_NAME) output_names = [encoder_file_name, decoder_file_name] if use_cache is True: @@ -281,7 +282,7 @@ def _from_transformers( model_id=save_dir_path, config=config, use_cache=use_cache, - from_onnx=True, + from_onnx=False, use_auth_token=use_auth_token, revision=revision, force_download=force_download, diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index b5d4f0be5d..a9cd8e309b 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -27,14 +27,14 @@ from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import CausalLMOutputWithPast -from optimum.exporters.onnx import export -from optimum.exporters.tasks import TasksManager +from optimum.exporters import TasksManager from optimum.utils import NormalizedConfigManager +from ...exporters.openvino import export from ..utils.import_utils import is_transformers_version -from ..utils.modeling_utils import _prepare_attn_mask, _prepare_decoder_attention_mask +from ..utils.modeling_utils import patch_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel -from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME, STR_TO_OV_TYPE +from .utils import OV_XML_FILE_NAME, STR_TO_OV_TYPE if is_transformers_version("<", "4.25.0"): @@ -190,7 +190,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): """ model_to_save = self.model if self._pkv_precision == Type.f32 else self._original_model dst_path = os.path.join(save_directory, OV_XML_FILE_NAME) - openvino.runtime.serialize(model_to_save, dst_path) + openvino.save_model(model_to_save, dst_path, compress_to_fp16=False) @classmethod def _from_transformers( @@ -232,25 +232,20 @@ def _from_transformers( onnx_config = onnx_config_constructor(model.config, use_past=use_cache) # TODO : create ModelPatcher to patch each architecture - if config.model_type in {"bloom", "mpt"}: - model.transformer._prepare_attn_mask = _prepare_attn_mask - elif config.model_type == "llama": - model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask - elif config.model_type in {"blenderbot-small", "blenderbot", "opt", "pegasus", "bart"}: - model.model.decoder._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + model = patch_decoder_attention_mask(model) - # Export the model to the ONNX format - export(model=model, config=onnx_config, output=save_dir_path / ONNX_WEIGHTS_NAME) + # Export the model to the OpenVINO IR format + export(model=model, config=onnx_config, output=save_dir_path / OV_XML_FILE_NAME) return cls._from_pretrained( model_id=save_dir_path, config=config, - from_onnx=True, + from_onnx=False, use_auth_token=use_auth_token, revision=revision, force_download=force_download, cache_dir=cache_dir, - file_name=ONNX_WEIGHTS_NAME, + file_name=OV_XML_FILE_NAME, local_files_only=local_files_only, use_cache=use_cache, **kwargs, diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index da1bde6dbb..c2884ee57e 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -36,7 +36,6 @@ from openvino.runtime import Core from transformers import CLIPFeatureExtractor, CLIPTokenizer -from optimum.exporters.onnx import main_export from optimum.pipelines.diffusers.pipeline_stable_diffusion import StableDiffusionPipelineMixin from optimum.pipelines.diffusers.pipeline_stable_diffusion_img2img import StableDiffusionImg2ImgPipelineMixin from optimum.pipelines.diffusers.pipeline_stable_diffusion_inpaint import StableDiffusionInpaintPipelineMixin @@ -51,6 +50,7 @@ DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER, ) +from ...exporters.openvino import main_export from .loaders import OVTextualInversionLoaderMixin from .modeling_base import OVBaseModel from .utils import ONNX_WEIGHTS_NAME, OV_TO_NP_TYPE, OV_XML_FILE_NAME @@ -159,7 +159,7 @@ def _save_pretrained(self, save_directory: Union[str, Path]): if ov_model is not None: dst_path = save_directory / dst_path / OV_XML_FILE_NAME dst_path.parent.mkdir(parents=True, exist_ok=True) - openvino.runtime.serialize(ov_model.model, dst_path) + openvino.save_model(ov_model.model, dst_path, compress_to_fp16=False) model_dir = ov_model.config.get("_name_or_path", None) or ov_model._model_dir / ov_model._model_name config_path = Path(model_dir) / ov_model.CONFIG_NAME if config_path.is_file(): @@ -315,7 +315,7 @@ def _from_transformers( return cls._from_pretrained( model_id=save_dir_path, config=config, - from_onnx=True, + from_onnx=False, use_auth_token=use_auth_token, revision=revision, force_download=force_download, diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index 0f52335639..4d5f4e2934 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -412,7 +412,6 @@ def forward( # Add the encoder_hidden_states inputs when needed if "encoder_hidden_states" in self.input_names and encoder_hidden_states is not None: inputs["encoder_hidden_states"] = encoder_hidden_states - # Run inference self.request.start_async(inputs, shared_memory=True) self.request.wait() diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 99e22e72f5..3349ce142f 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -24,21 +24,23 @@ import transformers from accelerate.data_loader import DataLoaderStateMixin from datasets import Dataset, load_dataset -from nncf import NNCFConfig -from nncf.torch import create_compressed_model, register_default_init_args +from nncf import NNCFConfig, compress_weights +from nncf.torch import create_compressed_model, register_default_init_args, register_module from nncf.torch.dynamic_graph.io_handling import wrap_nncf_model_inputs_with_objwalk from nncf.torch.initialization import PTInitializingDataLoader from openvino._offline_transformations import compress_quantize_weights_transformation from openvino.runtime import Core, Tensor -from torch.utils.data import DataLoader, RandomSampler, TensorDataset +from torch.utils.data import DataLoader, RandomSampler from transformers import DataCollator, PreTrainedModel, default_data_collator +from transformers.pytorch_utils import Conv1D -from optimum.exporters.onnx import export from optimum.exporters.tasks import TasksManager from optimum.quantization_base import OptimumQuantizer +from ...exporters.openvino import export, export_pytorch_via_onnx from ..utils.constant import _TASK_ALIASES -from .configuration import INT8_WEIGHT_COMPRESSION_CONFIG, OVConfig +from ..utils.modeling_utils import patch_decoder_attention_mask +from .configuration import OVConfig from .modeling_base import OVBaseModel from .modeling_decoder import OVBaseDecoderModel from .utils import ( @@ -49,6 +51,8 @@ ) +register_module(ignored_algorithms=[])(Conv1D) + core = Core() logger = logging.getLogger(__name__) @@ -332,8 +336,8 @@ def _quantize_torchmodel( self._set_task() save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) - file_name = file_name if file_name is not None else OV_XML_FILE_NAME - output_path = save_directory.joinpath(file_name) + ov_file_name = file_name if file_name is not None else OV_XML_FILE_NAME + output_path = save_directory.joinpath(ov_file_name) output_path = output_path.with_suffix(".xml").as_posix() model_type = self.model.config.model_type.replace("_", "-") @@ -344,73 +348,73 @@ def _quantize_torchmodel( model_type=model_type, ) - if weights_only: - calibration_dataset = TensorDataset(torch.tensor([0.0, 1.0])) - calibration_dataset.column_names = [] - remove_unused_columns = False - onnx_config = onnx_config_class(self.model.config) - - def data_collator(batch): - return onnx_config.generate_dummy_inputs(framework="pt") - - calibration_dataloader = self._get_calibration_dataloader( - calibration_dataset=calibration_dataset, - batch_size=batch_size, - remove_unused_columns=remove_unused_columns, - data_collator=data_collator, - ) - if quantization_config is None: logger.info( "No configuration describing the quantization process was provided, a default OVConfig will be generated." ) - quantization_config = OVConfig(compression=INT8_WEIGHT_COMPRESSION_CONFIG) if weights_only else OVConfig() - - model_inputs = next(iter(calibration_dataloader)) - quantization_config.add_input_info(model_inputs) - nncf_config = NNCFConfig.from_dict(quantization_config.__dict__) - nncf_config = register_default_init_args(nncf_config, calibration_dataloader) - controller, compressed_model = create_compressed_model( - self.model, nncf_config, wrap_inputs_fn=wrap_nncf_model_inputs_with_objwalk + quantization_config = OVConfig() + onnx_file_name = ( + ONNX_WEIGHTS_NAME + if file_name is None and quantization_config.save_onnx_model + else Path(ov_file_name).with_suffix(".onnx") ) - compressed_model = controller.strip(do_copy=False) + if weights_only: + if getattr(self.model.config, "tie_word_embeddings", True): + # to fix problem with shared embedding weights in nncf compress_weights() + self.model.tie_weights() + compressed_model = compress_weights(self.model) + self.model = compressed_model + else: + calibration_dataloader = self._get_calibration_dataloader( + calibration_dataset=calibration_dataset, + batch_size=batch_size, + remove_unused_columns=remove_unused_columns, + data_collator=data_collator, + ) + + model_inputs = next(iter(calibration_dataloader)) + quantization_config.add_input_info(model_inputs) + nncf_config = NNCFConfig.from_dict(quantization_config.__dict__) + nncf_config = register_default_init_args(nncf_config, calibration_dataloader) + controller, compressed_model = create_compressed_model( + self.model, nncf_config, wrap_inputs_fn=wrap_nncf_model_inputs_with_objwalk + ) + compressed_model = controller.strip(do_copy=False) task = self.task model = self.model self.model.config.save_pretrained(save_directory) - + model = patch_decoder_attention_mask(model) if task == "text-generation": onnx_config = onnx_config_class(model.config, use_past=model.config.use_cache) else: onnx_config = onnx_config_class(model.config) - onnx_path = save_directory / ONNX_WEIGHTS_NAME - - # Export the model to the ONNX format + model_path = save_directory / (onnx_file_name if quantization_config.save_onnx_model else ov_file_name) + onnx_path = save_directory / onnx_file_name + export_fn = export if not quantization_config.save_onnx_model else export_pytorch_via_onnx opset = min(onnx_config.DEFAULT_ONNX_OPSET, MAX_ONNX_OPSET) opset = max(opset, MIN_ONNX_QDQ_OPSET) - export( - model=compressed_model, - config=onnx_config, - opset=opset, - output=onnx_path, - ) + _, _, is_onnx = export_fn(model=model, config=onnx_config, output=model_path, opset=opset) + if is_onnx: + # Load and save the compressed model + model = core.read_model(onnx_path) + # Model required second saving for appling weights compression transformations + self._save_pretrained(model, output_path) + # if onnx conversion happens as fallback for pytorch conversion, remove onnx model + if not quantization_config.save_onnx_model: + os.remove(onnx_path) + try: + os.remove(f"{onnx_path}_data") + except FileNotFoundError: + pass - # Load and save the compressed model - model = core.read_model(onnx_path) - self._save_pretrained(model, output_path) quantization_config.save_pretrained(save_directory) - if not quantization_config.save_onnx_model: - os.remove(onnx_path) - try: - os.remove(f"{onnx_path}_data") - except FileNotFoundError: - pass @staticmethod def _save_pretrained(model: openvino.runtime.Model, output_path: str): compress_quantize_weights_transformation(model) - openvino.runtime.serialize(model, output_path) + openvino.save_model(model, output_path, compress_to_fp16=False) def _set_task(self): if self.task is None: diff --git a/optimum/intel/openvino/trainer.py b/optimum/intel/openvino/trainer.py index 811309806a..0bba054ad3 100644 --- a/optimum/intel/openvino/trainer.py +++ b/optimum/intel/openvino/trainer.py @@ -39,13 +39,13 @@ from nncf.torch.compression_method_api import PTCompressionAlgorithmController from nncf.torch.nncf_network import NNCFNetwork from nncf.torch.quantization.algo import QuantizationController -from openvino._offline_transformations import compress_quantize_weights_transformation -from openvino.runtime import Core, PartialShape, serialize -from openvino.tools.mo.back.offline_transformations import ( +from openvino._offline_transformations import ( apply_fused_names_cleanup, apply_moc_transformations, - apply_user_transformations, + apply_pruning_transformation, + compress_quantize_weights_transformation, ) +from openvino.runtime import Core, PartialShape, save_model from torch.onnx import export as onnx_export from torch.utils._pytree import tree_map from torch.utils.data import DataLoader, Dataset, RandomSampler @@ -134,7 +134,7 @@ def remap(value): with torch.no_grad(): model.eval() # Disable node additions to be exported in the graph - model.disable_dynamic_graph_building() + model.nncf.disable_dynamic_graph_building() onnx_export( model, model_inputs, @@ -145,7 +145,7 @@ def remap(value): do_constant_folding=True, opset_version=opset, ) - model.enable_dynamic_graph_building() + model.nncf.enable_dynamic_graph_building() class OVTrainer(Trainer): @@ -752,10 +752,10 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): try: # OpenVINO IR pruning requires static-shaped input ov_model = self._reshape_ir(ov_model, static_shape=True) - apply_moc_transformations(ov_model) + apply_moc_transformations(ov_model, cf=False) if self._get_compression_controller_by_cls(QuantizationController) is not None: compress_quantize_weights_transformation(ov_model) - apply_user_transformations(ov_model, [("Pruning", {})]) + apply_pruning_transformation(ov_model) apply_fused_names_cleanup(ov_model) # Reshape back to dynamic shape IR ov_model = self._reshape_ir(ov_model, static_shape=False) @@ -772,7 +772,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): compress_quantize_weights_transformation(ov_model) # Serialize IR xml and bin - serialize(ov_model, output_path) + save_model(ov_model, output_path, compress_to_fp16=False) def _get_compression_controller_by_cls( self, controller_cls: Type[PTCompressionAlgorithmController] diff --git a/optimum/intel/utils/modeling_utils.py b/optimum/intel/utils/modeling_utils.py index c7be049990..17abf1059e 100644 --- a/optimum/intel/utils/modeling_utils.py +++ b/optimum/intel/utils/modeling_utils.py @@ -15,6 +15,7 @@ from typing import Tuple import torch +from transformers.modeling_utils import PreTrainedModel # Modified from transformers.models.bloom.modeling_bloom._make_causal_mask @@ -89,3 +90,22 @@ def _prepare_decoder_attention_mask(attention_mask, input_shape, inputs_embeds, ) return combined_attention_mask + + +def patch_decoder_attention_mask(model: "PreTrainedModel"): + """ + Apply patch on decoder with past model forward to resolve first inference based on model architecture + + Args: + model (PretrainedModel): The model to patch. + + Returns: + model with applied patch + """ + if model.config.model_type in {"bloom", "mpt"}: + model.transformer._prepare_attn_mask = _prepare_attn_mask + elif model.config.model_type == "llama": + model.model._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + elif model.config.model_type in {"blenderbot-small", "blenderbot", "opt", "pegasus", "bart"}: + model.model.decoder._prepare_decoder_attention_mask = _prepare_decoder_attention_mask + return model diff --git a/setup.py b/setup.py index 769431c31c..6d81b98b2a 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,8 @@ "onnx", "onnxruntime<1.15.0", ], - "openvino": ["openvino>=2023.0.0", "onnx", "onnxruntime"], - "nncf": ["nncf>=2.5.0", "openvino-dev>=2023.0.0"], + "openvino": ["openvino>=2023.1.0", "onnx", "onnxruntime"], + "nncf": ["nncf>=2.6.0"], "ipex": ["transformers<4.32.0", "intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], "quality": QUALITY_REQUIRE, diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 4b11435e0e..a4bf9b38e0 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -117,6 +117,9 @@ def test_load_from_hub_and_save_model(self): outputs = model(**tokens) self.assertTrue(torch.equal(loaded_model_outputs.logits, outputs.logits)) + del loaded_model + del model + gc.collect() def test_load_from_hub_and_save_decoder_model(self): tokenizer = AutoTokenizer.from_pretrained(self.OV_DECODER_MODEL_ID) @@ -134,6 +137,9 @@ def test_load_from_hub_and_save_decoder_model(self): outputs = model(**tokens) self.assertTrue(torch.equal(loaded_model_outputs.logits, outputs.logits)) + del loaded_model + del model + gc.collect() def test_load_from_hub_and_save_seq2seq_model(self): tokenizer = AutoTokenizer.from_pretrained(self.OV_SEQ2SEQ_MODEL_ID) @@ -153,6 +159,9 @@ def test_load_from_hub_and_save_seq2seq_model(self): outputs = model.generate(**tokens) self.assertTrue(torch.equal(loaded_model_outputs, outputs)) + del loaded_model + del model + gc.collect() @require_diffusers def test_load_from_hub_and_save_stable_diffusion_model(self): @@ -186,6 +195,8 @@ def test_load_from_hub_and_save_stable_diffusion_model(self): np.random.seed(0) outputs = pipeline(**inputs).images self.assertTrue(np.array_equal(pipeline_outputs, outputs)) + del pipeline + gc.collect() class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase): @@ -228,6 +239,9 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) # Compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model + gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): @@ -257,6 +271,8 @@ def test_pipeline(self, model_arch): self.assertTrue(not model.is_dynamic) self.assertGreaterEqual(outputs[0]["score"], 0.0) self.assertIsInstance(outputs[0]["label"], str) + del model + del pipe gc.collect() @@ -293,6 +309,8 @@ def test_compare_to_transformers(self, model_arch): self.assertTrue( torch.allclose(torch.Tensor(ov_outputs.end_logits), transformers_outputs.end_logits, atol=1e-4) ) + del ov_model + del transformers_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -307,6 +325,7 @@ def test_pipeline(self, model_arch): self.assertEqual(pipe.device, model.device) self.assertGreaterEqual(outputs["score"], 0.0) self.assertIsInstance(outputs["answer"], str) + del model gc.collect() def test_metric(self): @@ -323,6 +342,10 @@ def test_metric(self): ov_metric = task_evaluator.compute(model_or_pipeline=ov_pipe, data=data, metric="squad") self.assertEqual(ov_metric["exact_match"], transformers_metric["exact_match"]) self.assertEqual(ov_metric["f1"], transformers_metric["f1"]) + del transformers_pipe + del transformers_model + del ov_pipe + del ov_model gc.collect() @@ -352,6 +375,8 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) # Compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -363,6 +388,8 @@ def test_pipeline(self, model_arch): outputs = pipe("My Name is Arthur and I live in Lyon.") self.assertEqual(pipe.device, model.device) self.assertTrue(all(item["score"] > 0.0 for item in outputs)) + del model + del pipe gc.collect() @@ -396,6 +423,8 @@ def test_compare_to_transformers(self, model_arch): torch.Tensor(ov_outputs.last_hidden_state), transformers_outputs.last_hidden_state, atol=1e-4 ) ) + del transformers_model + del ov_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -407,6 +436,8 @@ def test_pipeline(self, model_arch): outputs = pipe("My Name is Arthur and I live in Lyon.") self.assertEqual(pipe.device, model.device) self.assertTrue(all(all(isinstance(item, float) for item in row) for row in outputs[0])) + del pipe + del model gc.collect() @@ -448,6 +479,8 @@ def test_compare_to_transformers(self, model_arch): transformers_outputs = transformers_model(**tokens) # Compare tensor outputs self.assertTrue(torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -463,6 +496,8 @@ def test_pipeline(self, model_arch): outputs = pipe("This is a sample", max_length=10) self.assertEqual(pipe.device, model.device) self.assertTrue(all("This is a sample" in item["generated_text"] for item in outputs)) + del pipe + del model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -478,6 +513,8 @@ def test_multiple_inputs(self, model_arch): outputs = model.generate(**tokens, generation_config=generation_config) self.assertIsInstance(outputs, torch.Tensor) self.assertEqual(outputs.shape[0], 3) + del model + gc.collect() def test_model_and_decoder_same_device(self): model_id = MODEL_NAMES["gpt2"] @@ -486,6 +523,8 @@ def test_model_and_decoder_same_device(self): self.assertEqual(model._device, "TEST") # Verify that request is being reset self.assertEqual(model.request, None) + del model + gc.collect() def test_compare_with_and_without_past_key_values(self): model_id = MODEL_NAMES["gpt2"] @@ -515,6 +554,9 @@ def test_compare_with_and_without_past_key_values(self): f"With pkv latency: {with_pkv_timer.elapsed:.3f} ms, without pkv latency: {without_pkv_timer.elapsed:.3f} ms," f" speedup: {without_pkv_timer.elapsed / with_pkv_timer.elapsed:.3f}", ) + del model_with_pkv + del model_without_pkv + gc.collect() class OVModelForMaskedLMIntegrationTest(unittest.TestCase): @@ -535,7 +577,7 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): "roformer", "squeezebert", "xlm", - # "xlm_roberta", + "xlm_roberta", ) @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -557,6 +599,8 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) # Compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -568,6 +612,8 @@ def test_pipeline(self, model_arch): outputs = pipe(f"This is a {tokenizer.mask_token}.") self.assertEqual(pipe.device, model.device) self.assertTrue(all(item["score"] > 0.0 for item in outputs)) + del pipe + del model gc.collect() @@ -610,6 +656,8 @@ def test_compare_to_transformers(self, model_arch): self.assertIsInstance(ov_outputs.logits, TENSOR_ALIAS_TO_TYPE[input_type]) # Compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -622,6 +670,8 @@ def test_pipeline(self, model_arch): self.assertEqual(pipe.device, model.device) self.assertGreaterEqual(outputs[0]["score"], 0.0) self.assertTrue(isinstance(outputs[0]["label"], str)) + del model + del pipe gc.collect() @parameterized.expand(TIMM_MODELS) @@ -703,6 +753,8 @@ def test_compare_to_transformers(self, model_arch): transformers_outputs = transformers_model(**tokens, **decoder_inputs) # Compare tensor outputs self.assertTrue(torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @@ -735,7 +787,8 @@ def test_pipeline(self, model_arch): outputs = pipe(text) self.assertEqual(pipe.device, model.device) self.assertIsInstance(outputs[0]["translation_text"], str) - + del pipe + del model gc.collect() @parameterized.expand(SUPPORTED_ARCHITECTURES) @@ -755,6 +808,7 @@ def test_generate_utils(self, model_arch): outputs = model.generate(input_ids=tokens["input_ids"]) outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True) self.assertIsInstance(outputs[0], str) + del model gc.collect() @@ -786,6 +840,9 @@ def test_compare_with_and_without_past_key_values(self): f"With pkv latency: {with_pkv_timer.elapsed:.3f} ms, without pkv latency: {without_pkv_timer.elapsed:.3f} ms," f" speedup: {without_pkv_timer.elapsed / with_pkv_timer.elapsed:.3f}", ) + del model_with_pkv + del model_without_pkv + gc.collect() class OVModelForAudioClassificationIntegrationTest(unittest.TestCase): @@ -831,6 +888,10 @@ def test_compare_to_transformers(self, model_arch): # Compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-3)) + del transformers_model + del ov_model + gc.collect() + @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] @@ -840,6 +901,9 @@ def test_pipeline(self, model_arch): outputs = pipe([np.random.random(16000)]) self.assertEqual(pipe.device, model.device) self.assertTrue(all(item["score"] > 0.0 for item in outputs[0])) + del pipe + del model + gc.collect() class OVModelForCTCIntegrationTest(unittest.TestCase): @@ -893,6 +957,8 @@ def test_compare_to_transformers(self, model_arch): # compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() @@ -945,6 +1011,8 @@ def test_compare_to_transformers(self, model_arch): torch.allclose(torch.Tensor(ov_outputs.embeddings), transformers_outputs.embeddings, atol=1e-4) ) + del transformers_model + del ov_model gc.collect() @@ -994,4 +1062,6 @@ def test_compare_to_transformers(self, model_arch): # compare tensor outputs self.assertTrue(torch.allclose(torch.Tensor(ov_outputs.logits), transformers_outputs.logits, atol=1e-4)) + del transformers_model + del ov_model gc.collect() diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index da9ba3b25a..369ad0f836 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -64,8 +64,8 @@ def get_num_quantized_nodes(ov_model): class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( - (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 42, 32), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 41, 21), + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 32, 35), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 41, 22), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS) @@ -146,8 +146,8 @@ def preprocess_function(examples, tokenizer): class OVWeightCompressionTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( - (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 39), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 5), + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 70), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 45), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) @@ -173,9 +173,8 @@ def test_automodel_weight_compression(self, model_cls, model_name, expected_int8 self.assertTrue("logits" in outputs) # Verify that that the configuration is correctly saved and loaded - expected_config = OVConfig(compression=INT8_WEIGHT_COMPRESSION_CONFIG) loaded_config = OVConfig.from_pretrained(tmp_dir) - self.assertEqual(expected_config.to_dict()["compression"], loaded_config.to_dict()["compression"]) + self.assertIsNotNone(loaded_config) class OVQuantizerQATest(unittest.TestCase): From 673484b85c31bd2939923c1ec7a79b7a8bcda023 Mon Sep 17 00:00:00 2001 From: Liubov Talamanova Date: Wed, 20 Sep 2023 17:31:25 +0100 Subject: [PATCH 123/134] Added 8-bit weight compression for OVModel (#415) * Added 8-bit weight compression for OVModel * fix test --- optimum/intel/openvino/quantization.py | 18 ++++++++++++---- tests/openvino/test_quantization.py | 29 ++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 3349ce142f..da33eca733 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -167,10 +167,6 @@ def quantize( raise ValueError("`save_directory` needs to be specified") if weights_only: - if isinstance(self.model, OVBaseModel): - raise ValueError( - "`weights_only` currently not supported for `OVModels`, only available for torch.nn.Module." - ) if calibration_dataset is not None: logger.warning( "`calibration_dataset` was provided but will not be used as `weights_only` is set to `True`." @@ -189,6 +185,7 @@ def quantize( batch_size, data_collator, remove_unused_columns, + weights_only, **kwargs, ) elif isinstance(self.model, OVBaseModel): @@ -198,6 +195,7 @@ def quantize( batch_size, data_collator, remove_unused_columns, + weights_only, **kwargs, ) elif isinstance(self.model, torch.nn.Module): @@ -221,11 +219,17 @@ def _quantize_ovbasemodel( batch_size: int = 1, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + weights_only: bool = False, **kwargs, ): save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) + if weights_only: + self.model.model = nncf.compress_weights(self.model.model) + self.model.save_pretrained(save_directory) + return + calibration_dataloader = self._get_calibration_dataloader( calibration_dataset=calibration_dataset, batch_size=batch_size, @@ -251,11 +255,17 @@ def _quantize_ovcausallm( batch_size: int = 1, data_collator: Optional[DataCollator] = None, remove_unused_columns: bool = True, + weights_only: bool = False, **kwargs, ): save_directory = Path(save_directory) save_directory.mkdir(parents=True, exist_ok=True) + if weights_only: + self.model.model = nncf.compress_weights(self.model.model) + self.model.save_pretrained(save_directory) + return + calibration_dataloader = self._get_calibration_dataloader( calibration_dataset=calibration_dataset, batch_size=batch_size, diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 369ad0f836..55758b6683 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -146,12 +146,12 @@ def preprocess_function(examples, tokenizer): class OVWeightCompressionTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS = ( - (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 70), - (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 45), + (OVModelForSequenceClassification, "hf-internal-testing/tiny-random-bert", 70, 35), + (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 45, 22), ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) - def test_automodel_weight_compression(self, model_cls, model_name, expected_int8): + def test_automodel_weight_compression(self, model_cls, model_name, expected_pt_int8, expected_ov_int8): task = model_cls.export_feature with tempfile.TemporaryDirectory() as tmp_dir: @@ -166,7 +166,7 @@ def test_automodel_weight_compression(self, model_cls, model_name, expected_int8 # TODO: uncomment once move to a newer version of NNCF which has some fixes _, num_int8 = get_num_quantized_nodes(model) - self.assertEqual(expected_int8, num_int8) + self.assertEqual(expected_pt_int8, num_int8) tokens = tokenizer("This is a sample input", return_tensors="pt") outputs = model(**tokens) @@ -176,6 +176,27 @@ def test_automodel_weight_compression(self, model_cls, model_name, expected_int8 loaded_config = OVConfig.from_pretrained(tmp_dir) self.assertIsNotNone(loaded_config) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) + def test_ovmodel_weight_compression(self, model_cls, model_name, expected_pt_int8, expected_ov_int8): + task = model_cls.export_feature + + with tempfile.TemporaryDirectory() as tmp_dir: + transformers_model = model_cls.from_pretrained(model_name, export=True) + tokenizer = AutoTokenizer.from_pretrained(model_name) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + quantizer = OVQuantizer.from_pretrained(transformers_model, task=task) + quantizer.quantize(save_directory=tmp_dir, weights_only=True) + model = model_cls.from_pretrained(tmp_dir) + + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_ov_int8, num_int8) + + tokens = tokenizer("This is a sample input", return_tensors="pt") + outputs = model(**tokens) + self.assertTrue("logits" in outputs) + class OVQuantizerQATest(unittest.TestCase): SUPPORTED_ARCHITECTURES = (("hf-internal-testing/tiny-random-BertForQuestionAnswering",),) From 99f6008d659e09fb53c47112363e66fde46f8d76 Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Thu, 21 Sep 2023 17:40:12 +0400 Subject: [PATCH 124/134] fix OVModelForCausalLM for auto device (#433) --- optimum/intel/openvino/modeling_decoder.py | 3 ++- tests/openvino/test_modeling.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index a9cd8e309b..a94496c3bd 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -154,7 +154,8 @@ def update_pkv_precision(self, force_fp32=False): pkv_precision = Type.f32 if not force_fp32: device = self._device.upper() - pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") + if "INFERENCE_PRECISION_HINT" in core.get_property(device, "SUPPORTED_PROPERTIES"): + pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if self.ov_config: inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index a4bf9b38e0..1de402bc22 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -558,6 +558,14 @@ def test_compare_with_and_without_past_key_values(self): del model_without_pkv gc.collect() + def test_auto_device_loading(self): + model_id = MODEL_NAMES["gpt2"] + model = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True, device="AUTO") + model.half() + self.assertEqual(model._device, "AUTO") + del model + gc.collect() + class OVModelForMaskedLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES = ( From 1db2651240277b85687b9f1511872421e62406c2 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 22 Sep 2023 09:51:48 +0200 Subject: [PATCH 125/134] Add image reshaping for statically reshaped OpenVINO model (#428) * add vae image processor * add image reshaping for statically reshaped model * add test * format * fix pipeline * fix reshaping * disable reshaping for inpaint SD models * add reshaping for inpaint --- optimum/intel/openvino/modeling_diffusion.py | 250 +++++++++++++++++-- tests/openvino/test_stable_diffusion.py | 46 ++-- 2 files changed, 256 insertions(+), 40 deletions(-) diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index c2884ee57e..cb9d92a15a 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -22,6 +22,7 @@ import numpy as np import openvino +import PIL from diffusers import ( DDIMScheduler, LMSDiscreteScheduler, @@ -351,6 +352,13 @@ def width(self) -> int: return -1 return width.get_length() * self.vae_scale_factor + @property + def _batch_size(self) -> int: + batch_size = self.unet.model.inputs[0].get_partial_shape()[0] + if batch_size.is_dynamic: + return -1 + return batch_size.get_length() + def _reshape_unet( self, model: openvino.runtime.Model, @@ -649,6 +657,7 @@ def __call__( width = width or self.unet.config.get("sample_size", 64) * self.vae_scale_factor _height = self.height _width = self.width + expected_batch_size = self._batch_size if _height != -1 and height != _height: logger.warning( @@ -664,11 +673,15 @@ def __call__( ) width = _width - if guidance_scale is not None and guidance_scale <= 1 and not self.is_dynamic: - raise ValueError( - f"`guidance_scale` was set to {guidance_scale}, static shapes are only supported for `guidance_scale` > 1, " - "please set `dynamic_shapes` to `True` when loading the model." - ) + if expected_batch_size != -1: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = kwargs.get("prompt_embeds").shape[0] + + _raise_invalid_batch_size(expected_batch_size, batch_size, num_images_per_prompt, guidance_scale) return StableDiffusionPipelineMixin.__call__( self, @@ -684,16 +697,115 @@ def __call__( class OVStableDiffusionImg2ImgPipeline(OVStableDiffusionPipelineBase, StableDiffusionImg2ImgPipelineMixin): - def __call__(self, *args, **kwargs): - # TODO : add default height and width if model statically reshaped - # resize image if doesn't match height and width given during reshaping - return StableDiffusionImg2ImgPipelineMixin.__call__(self, *args, **kwargs) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + image: Union[np.ndarray, PIL.Image.Image] = None, + strength: float = 0.8, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + **kwargs, + ): + _height = self.height + _width = self.width + expected_batch_size = self._batch_size + + if _height != -1 and _width != -1: + image = self.image_processor.preprocess(image, height=_height, width=_width).transpose(0, 2, 3, 1) + + if expected_batch_size != -1: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = kwargs.get("prompt_embeds").shape[0] + + _raise_invalid_batch_size(expected_batch_size, batch_size, num_images_per_prompt, guidance_scale) + + return StableDiffusionImg2ImgPipelineMixin.__call__( + self, + prompt=prompt, + image=image, + strength=strength, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) class OVStableDiffusionInpaintPipeline(OVStableDiffusionPipelineBase, StableDiffusionInpaintPipelineMixin): - def __call__(self, *args, **kwargs): - # TODO : add default height and width if model statically reshaped - return StableDiffusionInpaintPipelineMixin.__call__(self, *args, **kwargs) + def __call__( + self, + prompt: Optional[Union[str, List[str]]], + image: PIL.Image.Image, + mask_image: PIL.Image.Image, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + **kwargs, + ): + height = height or self.unet.config.get("sample_size", 64) * self.vae_scale_factor + width = width or self.unet.config.get("sample_size", 64) * self.vae_scale_factor + _height = self.height + _width = self.width + expected_batch_size = self._batch_size + + if _height != -1 and _width != -1: + if height != _height: + logger.warning( + f"`height` was set to {height} but the static model will output images of height {_height}." + "To fix the height, please reshape your model accordingly using the `.reshape()` method." + ) + height = _height + + if width != _width: + logger.warning( + f"`width` was set to {width} but the static model will output images of width {_width}." + "To fix the width, please reshape your model accordingly using the `.reshape()` method." + ) + width = _width + + if isinstance(image, list): + image = [self.image_processor.resize(i, _height, _width) for i in image] + else: + image = self.image_processor.resize(image, _height, _width) + + if isinstance(mask_image, list): + mask_image = [self.image_processor.resize(i, _height, _width) for i in mask_image] + else: + mask_image = self.image_processor.resize(mask_image, _height, _width) + + if expected_batch_size != -1: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = kwargs.get("prompt_embeds").shape[0] + + _raise_invalid_batch_size(expected_batch_size, batch_size, num_images_per_prompt, guidance_scale) + + return StableDiffusionInpaintPipelineMixin.__call__( + self, + prompt=prompt, + image=image, + mask_image=mask_image, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) class OVStableDiffusionXLPipelineBase(OVStableDiffusionPipelineBase): @@ -718,10 +830,116 @@ def __init__(self, *args, add_watermarker: Optional[bool] = None, **kwargs): class OVStableDiffusionXLPipeline(OVStableDiffusionXLPipelineBase, StableDiffusionXLPipelineMixin): - def __call__(self, *args, **kwargs): - return StableDiffusionXLPipelineMixin.__call__(self, *args, **kwargs) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + **kwargs, + ): + height = height or self.unet.config["sample_size"] * self.vae_scale_factor + width = width or self.unet.config["sample_size"] * self.vae_scale_factor + _height = self.height + _width = self.width + expected_batch_size = self._batch_size + + if _height != -1 and height != _height: + logger.warning( + f"`height` was set to {height} but the static model will output images of height {_height}." + "To fix the height, please reshape your model accordingly using the `.reshape()` method." + ) + height = _height + + if _width != -1 and width != _width: + logger.warning( + f"`width` was set to {width} but the static model will output images of width {_width}." + "To fix the width, please reshape your model accordingly using the `.reshape()` method." + ) + width = _width + + if expected_batch_size != -1: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = kwargs.get("prompt_embeds").shape[0] + + _raise_invalid_batch_size(expected_batch_size, batch_size, num_images_per_prompt, guidance_scale) + + return StableDiffusionXLPipelineMixin.__call__( + self, + prompt=prompt, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) class OVStableDiffusionXLImg2ImgPipeline(OVStableDiffusionXLPipelineBase, StableDiffusionXLImg2ImgPipelineMixin): - def __call__(self, *args, **kwargs): - return StableDiffusionXLImg2ImgPipelineMixin.__call__(self, *args, **kwargs) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + image: Union[np.ndarray, PIL.Image.Image] = None, + strength: float = 0.3, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + **kwargs, + ): + _height = self.height + _width = self.width + expected_batch_size = self._batch_size + + if _height != -1 and _width != -1: + image = self.image_processor.preprocess(image, height=_height, width=_width).transpose(0, 2, 3, 1) + + if expected_batch_size != -1: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = kwargs.get("prompt_embeds").shape[0] + + _raise_invalid_batch_size(expected_batch_size, batch_size, num_images_per_prompt, guidance_scale) + + return StableDiffusionXLImg2ImgPipelineMixin.__call__( + self, + prompt=prompt, + image=image, + strength=strength, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + **kwargs, + ) + + +def _raise_invalid_batch_size( + expected_batch_size: int, batch_size: int, num_images_per_prompt: int, guidance_scale: float +): + current_batch_size = batch_size * num_images_per_prompt * (1 if guidance_scale <= 1 else 2) + + if expected_batch_size != current_batch_size: + msg = "" + if guidance_scale is not None and guidance_scale <= 1: + msg = f"`guidance_scale` was set to {guidance_scale}, static shapes are currently only supported for `guidance_scale` > 1 " + + raise ValueError( + "The model was statically reshaped and the pipeline inputs do not match the expected shapes. " + f"The `batch_size`, `num_images_per_prompt` and `guidance_scale` were respectively set to {batch_size}, {num_images_per_prompt} and {guidance_scale}. " + f"The static model expects an input of size equal to {expected_batch_size} and got the following value instead : {current_batch_size}. " + f"To fix this, please either provide a different inputs to your model so that `batch_size` * `num_images_per_prompt` * 2 is equal to {expected_batch_size} " + "or reshape it again accordingly using the `.reshape()` method by setting `batch_size` to -1. " + msg + ) diff --git a/tests/openvino/test_stable_diffusion.py b/tests/openvino/test_stable_diffusion.py index 781fbe0ec6..0e2ea91e4c 100644 --- a/tests/openvino/test_stable_diffusion.py +++ b/tests/openvino/test_stable_diffusion.py @@ -184,10 +184,11 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) batch_size, num_images, height, width = 2, 3, 128, 64 pipeline.half() - inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) - outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images - self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + for _height in [height, height + 16]: + inputs = self.generate_inputs(height=_height, width=width, batch_size=batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) def generate_inputs(self, height=128, width=128, batch_size=1, input_type="np"): inputs = _generate_inputs(batch_size) @@ -264,21 +265,15 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): model_id = MODEL_NAMES[model_arch] pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False) batch_size, num_images, height, width = 3, 4, 128, 64 - prompt = "sailing ship in storm by Leonardo da Vinci" pipeline.half() pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) self.assertFalse(pipeline.is_dynamic) pipeline.compile() - # Verify output shapes requirements not matching the static model don't impact the final outputs - outputs = pipeline( - [prompt] * batch_size, - num_inference_steps=2, - num_images_per_prompt=num_images, - height=height + 8, - width=width, - output_type="np", - ).images - self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + # Verify output shapes requirements not matching the static model doesn't impact the final outputs + for _height in [height, height + 16]: + inputs = _generate_inputs(batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, height=_height, width=width).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_height_width_properties(self, model_arch: str): @@ -341,10 +336,11 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) batch_size, num_images, height, width = 1, 3, 128, 64 pipeline.half() - inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) - outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images - self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + for _height in [height, height + 16]: + inputs = self.generate_inputs(height=_height, width=width, batch_size=batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) def generate_inputs(self, height=128, width=128, batch_size=1): inputs = super(OVStableDiffusionInpaintPipelineTest, self).generate_inputs(height, width, batch_size) @@ -432,10 +428,11 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) self.assertFalse(pipeline.is_dynamic) pipeline.compile() - # Verify output shapes requirements not matching the static model don't impact the final outputs - inputs = _generate_inputs(batch_size) - outputs = pipeline(**inputs, num_images_per_prompt=num_images, height=height, width=width).images - self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + + for _height in [height, height + 16]: + inputs = _generate_inputs(batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images, height=_height, width=width).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) class OVStableDiffusionXLImg2ImgPipelineTest(unittest.TestCase): @@ -467,10 +464,11 @@ def test_num_images_per_prompt_static_model(self, model_arch: str): pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False) batch_size, num_images, height, width = 2, 3, 128, 64 pipeline.half() - inputs = self.generate_inputs(height=height, width=width, batch_size=batch_size) pipeline.reshape(batch_size=batch_size, height=height, width=width, num_images_per_prompt=num_images) - outputs = pipeline(**inputs, num_images_per_prompt=num_images, generator=np.random.RandomState(0)).images - self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) + for _height in [height, height + 16]: + inputs = self.generate_inputs(height=_height, width=width, batch_size=batch_size) + outputs = pipeline(**inputs, num_images_per_prompt=num_images).images + self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3)) def generate_inputs(self, height=128, width=128, batch_size=1, input_type="np"): inputs = _generate_inputs(batch_size) From 985d0d1799340c0b28a5dc119078cfb56ac6f724 Mon Sep 17 00:00:00 2001 From: Helena Kloosterman Date: Mon, 25 Sep 2023 11:03:47 +0200 Subject: [PATCH 126/134] Fix for AUTO:device; add device to compile info msg (#434) Also remove _SUPPORTED_DEVICES since they are not used --- optimum/intel/openvino/modeling_base.py | 18 +----------------- optimum/intel/openvino/modeling_decoder.py | 8 ++++++-- optimum/intel/openvino/modeling_diffusion.py | 2 +- optimum/intel/openvino/modeling_seq2seq.py | 4 ++-- tests/openvino/test_modeling.py | 11 ++++++----- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 42bdb8edba..4d9dc5651d 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -43,17 +43,6 @@ logger = logging.getLogger(__name__) -_SUPPORTED_DEVICES = { - "CPU", - "GPU", - "AUTO", - "AUTO:CPU,GPU", - "AUTO:GPU,CPU", - "MULTI", - "MULTI:CPU,GPU", - "MULTI:GPU,CPU", -} - # workaround to enable compatibility between openvino models and transformers pipelines class PreTrainedModel(OptimizedModel): @@ -325,7 +314,7 @@ def _to_load( def compile(self): if self.request is None: - logger.info("Compiling the model...") + logger.info(f"Compiling the model to {self._device} ...") ov_config = {**self.ov_config} if "CACHE_DIR" not in self.ov_config.keys(): # Set default CACHE_DIR only if it is not set. @@ -382,11 +371,6 @@ def half(self): self.request = None return self - def _ensure_supported_device(self, device: str = None): - device = device if device is not None else self._device - if device not in _SUPPORTED_DEVICES: - raise ValueError(f"Unknown device: {device}. Expected one of {_SUPPORTED_DEVICES}.") - def forward(self, *args, **kwargs): raise NotImplementedError diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index a94496c3bd..d9ebbf5c87 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -154,8 +154,12 @@ def update_pkv_precision(self, force_fp32=False): pkv_precision = Type.f32 if not force_fp32: device = self._device.upper() - if "INFERENCE_PRECISION_HINT" in core.get_property(device, "SUPPORTED_PROPERTIES"): - pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") + try: + if "INFERENCE_PRECISION_HINT" in core.get_property(device, "SUPPORTED_PROPERTIES"): + pkv_precision = core.get_property(device, "INFERENCE_PRECISION_HINT") + except RuntimeError: # use default precision when get_property fails, e.g. when device is "AUTO:GPU" + pass + # ov_config["INFERENCE_PRECISION_HINT"] may override the prefer precision if self.ov_config: inference_precision_hint = self.ov_config.get("INFERENCE_PRECISION_HINT", "") diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index cb9d92a15a..1ca517397d 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -540,7 +540,7 @@ def __init__( def _compile(self): if self.request is None: - logger.info(f"Compiling the {self._model_name}...") + logger.info(f"Compiling the {self._model_name} to {self.device} ...") self.request = core.compile_model(self.model, self.device, self.ov_config) @property diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index 4d5f4e2934..7a82c3a304 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -347,7 +347,7 @@ def __call__(self, *args, **kwargs): def _compile(self): if self.request is None: - logger.info("Compiling the encoder...") + logger.info(f"Compiling the encoder to {self._device} ...") self.request = core.compile_model(self.model, self._device, self.ov_config) @@ -442,5 +442,5 @@ def __call__(self, *args, **kwargs): def _compile(self): if self.request is None: - logger.info("Compiling the decoder...") + logger.info(f"Compiling the decoder to {self._device} ...") self.request = core.compile_model(self.model, self._device, self.ov_config).create_infer_request() diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 1de402bc22..65e1d23422 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -560,11 +560,12 @@ def test_compare_with_and_without_past_key_values(self): def test_auto_device_loading(self): model_id = MODEL_NAMES["gpt2"] - model = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True, device="AUTO") - model.half() - self.assertEqual(model._device, "AUTO") - del model - gc.collect() + for device in ("AUTO", "AUTO:CPU"): + model = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=True, device=device) + model.half() + self.assertEqual(model._device, device) + del model + gc.collect() class OVModelForMaskedLMIntegrationTest(unittest.TestCase): From fc7156790c43abc21c98082a6e52370795936dc9 Mon Sep 17 00:00:00 2001 From: "Wang, Chang" Date: Tue, 26 Sep 2023 15:59:15 +0800 Subject: [PATCH 127/134] Add starcode past-kv shape for TSModelForCausal class (#371) * add starcode past-kv shape for TSModelForCausal class Signed-off-by: changwangss * improve code style and past-kv shape Signed-off-by: changwangss * fix style Signed-off-by: changwangss * support gpt_bigcode Signed-off-by: changwangss * add gptbigcode to ipex test Signed-off-by: changwangss * fix style Signed-off-by: changwangss * fix style Signed-off-by: changwangss --------- Signed-off-by: changwangss --- optimum/intel/generation/modeling.py | 19 +++++++++++++------ tests/ipex/test_inference.py | 7 ++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/optimum/intel/generation/modeling.py b/optimum/intel/generation/modeling.py index 1a93375bbe..7748159fbc 100644 --- a/optimum/intel/generation/modeling.py +++ b/optimum/intel/generation/modeling.py @@ -49,10 +49,10 @@ def prepare_jit_inputs(model: PreTrainedModel, task: str, use_cache: bool = Fals onnx_config_class = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_class(model.config) if task == "text-generation" and use_cache: - onnx_config = onnx_config_class(model.config, use_past=True) + onnx_config = onnx_config_class(model.config, use_past=True, use_past_in_inputs=True) dummy_inputs = onnx_config.generate_dummy_inputs(framework="pt") model_inputs = {key: dummy_inputs[key] for key in signature.parameters if dummy_inputs.get(key, None) is not None} - if task == "text-generation" and use_cache: + if task == "text-generation" and use_cache and model.config.model_type != "gpt_bigcode": # WA jit.trace issue of model like llama in https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L464, or else, generation output will be incorrect pkv = [] for i in range(len(model_inputs["past_key_values"])): @@ -70,6 +70,8 @@ def prepare_jit_inputs(model: PreTrainedModel, task: str, use_cache: bool = Fals def jit_trace(model: PreTrainedModel, task: str, use_cache: bool = False): model_inputs = prepare_jit_inputs(model, task, use_cache) + # check if the model_inputs is correct. + model(**model_inputs) torch._C._jit_set_texpr_fuser_enabled(False) if "past_key_values" in model_inputs.keys(): model.config.return_dict = False @@ -273,13 +275,17 @@ def forward( num_attention_heads = self.normalized_config.num_attention_heads hidden_size = self.normalized_config.hidden_size d_k = hidden_size // num_attention_heads - - if self.config.model_type != "bloom": + if self.config.model_type == "gpt_bigcode": + new_shape = [input_ids.shape[0], 0, d_k * 2] + empty_tensor = torch.empty(size=new_shape) + if self.model_dtype is not None: + empty_tensor = empty_tensor.to(self.model_dtype) + past_key_values = tuple([empty_tensor] * num_layers) + elif self.config.model_type != "bloom": new_shape = [input_ids.shape[0], num_attention_heads, 0, d_k] empty_tensor = torch.empty(size=new_shape) if self.model_dtype is not None: empty_tensor = empty_tensor.to(self.model_dtype) - past_key_values = tuple(tuple(empty_tensor for _ in range(nb_pkv)) for _ in range(num_layers)) pkv = tuple(empty_tensor for _ in range(nb_pkv)) else: pkv = () @@ -292,7 +298,8 @@ def forward( if self.model_dtype is not None: empty_tensor = empty_tensor.to(self.model_dtype) pkv = pkv + (empty_tensor,) - past_key_values = tuple(tuple(pkv) for _ in range(num_layers)) + if past_key_values is None: + past_key_values = tuple(tuple(pkv) for _ in range(num_layers)) inputs["past_key_values"] = past_key_values outputs = self.model(**inputs) diff --git a/tests/ipex/test_inference.py b/tests/ipex/test_inference.py index d5cf571150..6889e76af2 100644 --- a/tests/ipex/test_inference.py +++ b/tests/ipex/test_inference.py @@ -40,6 +40,7 @@ "gpt2": "hf-internal-testing/tiny-random-gpt2", "gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel", "gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "gpt_bigcode": "hf-internal-testing/tiny-random-GPTBigCodeModel", } _CLASSIFICATION_TASK_TO_AUTOMODELS = { @@ -55,11 +56,7 @@ class IPEXIntegrationTest(unittest.TestCase): "roberta", ) - TEXT_GENERATION_SUPPORTED_ARCHITECTURES = ( - "gptj", - "gpt2", - "gpt_neo", - ) + TEXT_GENERATION_SUPPORTED_ARCHITECTURES = ("gptj", "gpt2", "gpt_neo", "gpt_bigcode") QA_SUPPORTED_ARCHITECTURES = ( "bert", From 25fc7578c45aa4227fb4fb102ee3cd8bc4a5d082 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 28 Sep 2023 10:57:09 +0200 Subject: [PATCH 128/134] Add OpenVINO export CLI (#437) * add openvino export CLI * fix * add test * format --- optimum/commands/export/openvino.py | 106 ++++++++++++++++++ .../commands/register/register_openvino.py | 19 ++++ tests/openvino/test_exporters_cli.py | 59 ++++++++++ 3 files changed, 184 insertions(+) create mode 100644 optimum/commands/export/openvino.py create mode 100644 optimum/commands/register/register_openvino.py create mode 100644 tests/openvino/test_exporters_cli.py diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py new file mode 100644 index 0000000000..b2d33e7647 --- /dev/null +++ b/optimum/commands/export/openvino.py @@ -0,0 +1,106 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defines the command line for the export with OpenVINO.""" + +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from ...exporters import TasksManager +from ..base import BaseOptimumCLICommand, CommandInfo + + +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace, _SubParsersAction + + +def parse_args_openvino(parser: "ArgumentParser"): + required_group = parser.add_argument_group("Required arguments") + required_group.add_argument( + "-m", "--model", type=str, required=True, help="Model ID on huggingface.co or path on disk to load model from." + ) + required_group.add_argument( + "output", type=Path, help="Path indicating the directory where to store the generated OV model." + ) + optional_group = parser.add_argument_group("Optional arguments") + optional_group.add_argument( + "--task", + default="auto", + help=( + "The task to export the model for. If not specified, the task will be auto-inferred based on the model. Available tasks depend on the model, but are among:" + f" {str(TasksManager.get_all_tasks())}. For decoder models, use `xxx-with-past` to export the model using past key values in the decoder." + ), + ) + optional_group.add_argument("--cache_dir", type=str, default=None, help="Path indicating where to store cache.") + optional_group.add_argument( + "--framework", + type=str, + choices=["pt", "tf"], + default=None, + help=( + "The framework to use for the export. If not provided, will attempt to use the local checkpoint's original framework or what is available in the environment." + ), + ) + optional_group.add_argument( + "--trust-remote-code", + action="store_true", + help=( + "Allows to use custom code for the modeling hosted in the model repository. This option should only be set for repositories you trust and in which " + "you have read the code, as it will execute on your local machine arbitrary code present in the model repository." + ), + ) + optional_group.add_argument( + "--pad-token-id", + type=int, + default=None, + help=( + "This is needed by some models, for some tasks. If not provided, will attempt to use the tokenizer to guess it." + ), + ) + + +class OVExportCommand(BaseOptimumCLICommand): + COMMAND = CommandInfo(name="openvino", help="Export PyTorch models to OpenVINO IR.") + + def __init__( + self, + subparsers: "_SubParsersAction", + args: Optional["Namespace"] = None, + command: Optional["CommandInfo"] = None, + from_defaults_factory: bool = False, + parser: Optional["ArgumentParser"] = None, + ): + super().__init__( + subparsers, args=args, command=command, from_defaults_factory=from_defaults_factory, parser=parser + ) + self.args_string = " ".join(sys.argv[3:]) + + @staticmethod + def parse_args(parser: "ArgumentParser"): + return parse_args_openvino(parser) + + def run(self): + from ...exporters.openvino.__main__ import main_export + + # TODO : add input shapes + main_export( + model_name_or_path=self.args.model, + output=self.args.output, + task=self.args.task, + framework=self.args.framework, + cache_dir=self.args.cache_dir, + trust_remote_code=self.args.trust_remote_code, + pad_token_id=self.args.pad_token_id, + # **input_shapes, + ) diff --git a/optimum/commands/register/register_openvino.py b/optimum/commands/register/register_openvino.py new file mode 100644 index 0000000000..a1a74abaca --- /dev/null +++ b/optimum/commands/register/register_openvino.py @@ -0,0 +1,19 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..export import ExportCommand +from ..export.openvino import OVExportCommand + + +REGISTER_COMMANDS = [(OVExportCommand, ExportCommand)] diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py new file mode 100644 index 0000000000..6b1ba249ab --- /dev/null +++ b/tests/openvino/test_exporters_cli.py @@ -0,0 +1,59 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import subprocess +import unittest +from tempfile import TemporaryDirectory + +from parameterized import parameterized +from utils_tests import MODEL_NAMES + +from optimum.exporters.openvino.__main__ import main_export + + +class OVCLIExportTestCase(unittest.TestCase): + """ + Integration tests ensuring supported models are correctly exported. + """ + + SUPPORTED_ARCHITECTURES = ( + ["causal-lm", "gpt2"], + ["causal-lm-with-past", "gpt2"], + ["seq2seq-lm", "t5"], + ["seq2seq-lm-with-past", "t5"], + ["sequence-classification", "bert"], + ["question-answering", "distilbert"], + ["masked-lm", "bert"], + ["default", "blenderbot"], + ["default-with-past", "blenderbot"], + ["stable-diffusion", "stable-diffusion"], + ["stable-diffusion-xl", "stable-diffusion-xl"], + ["stable-diffusion-xl", "stable-diffusion-xl-refiner"], + ) + + def _openvino_export(self, model_name: str, task: str): + with TemporaryDirectory() as tmpdir: + main_export(model_name_or_path=model_name, output=tmpdir, task=task) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_export(self, task: str, model_type: str): + self._openvino_export(MODEL_NAMES[model_type], task) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdirname: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdirname}", + shell=True, + check=True, + ) From edd888b7ce401b646a16c00926fa77db7a3f550f Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 28 Sep 2023 13:51:22 +0200 Subject: [PATCH 129/134] Fix openvino decoder model without cache (#438) * Fix openvino decoder model without cache * format * add test for use_cache=False * format * modify model for tests --- optimum/intel/openvino/modeling_decoder.py | 1 + tests/openvino/test_modeling.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index d9ebbf5c87..55af53c011 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -133,6 +133,7 @@ def __init__( self.key_value_input_names = [key for key in self.input_names if "key_values" in key] self.key_value_output_names = [key for key in self.output_names if "present" in key] self._original_model = self.model.clone() # keep original model for serialization + self._pkv_precision = Type.f32 self.update_pkv_precision() if self.is_dynamic: self.model = self._reshape(self.model, -1, -1) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 65e1d23422..a2fb38e153 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -121,10 +121,12 @@ def test_load_from_hub_and_save_model(self): del model gc.collect() - def test_load_from_hub_and_save_decoder_model(self): - tokenizer = AutoTokenizer.from_pretrained(self.OV_DECODER_MODEL_ID) + @parameterized.expand((True, False)) + def test_load_from_hub_and_save_decoder_model(self, use_cache): + model_id = "vuiseng9/ov-gpt2-fp32-kv-cache" if use_cache else "vuiseng9/ov-gpt2-fp32-no-cache" + tokenizer = AutoTokenizer.from_pretrained(model_id) tokens = tokenizer("This is a sample input", return_tensors="pt") - loaded_model = OVModelForCausalLM.from_pretrained(self.OV_DECODER_MODEL_ID, use_cache=True) + loaded_model = OVModelForCausalLM.from_pretrained(model_id, use_cache=use_cache) self.assertIsInstance(loaded_model.config, PretrainedConfig) loaded_model_outputs = loaded_model(**tokens) @@ -133,7 +135,8 @@ def test_load_from_hub_and_save_decoder_model(self): folder_contents = os.listdir(tmpdirname) self.assertTrue(OV_XML_FILE_NAME in folder_contents) self.assertTrue(OV_XML_FILE_NAME.replace(".xml", ".bin") in folder_contents) - model = OVModelForCausalLM.from_pretrained(tmpdirname, use_cache=True) + model = OVModelForCausalLM.from_pretrained(tmpdirname, use_cache=use_cache) + self.assertEqual(model.use_cache, use_cache) outputs = model(**tokens) self.assertTrue(torch.equal(loaded_model_outputs.logits, outputs.logits)) @@ -540,6 +543,7 @@ def test_compare_with_and_without_past_key_values(self): ) model_without_pkv = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=False) + # Warmup _ = model_without_pkv.generate(**tokens) with Timer() as without_pkv_timer: @@ -710,12 +714,8 @@ def test_timm_save_and_infer(self, model_id): with tempfile.TemporaryDirectory() as tmpdirname: model_save_path = os.path.join(tmpdirname, "timm_ov_model") ov_model.save_pretrained(model_save_path) - new_ov_model = OVModelForImageClassification.from_pretrained( - model_save_path, - ) - new_ov_model( - pixel_values=torch.zeros((5, 3, new_ov_model.config.image_size, new_ov_model.config.image_size)) - ) + model = OVModelForImageClassification.from_pretrained(model_save_path) + model(pixel_values=torch.zeros((5, 3, model.config.image_size, model.config.image_size))) gc.collect() From cee6b5a579adea0b019808668c3429e3df55832e Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Thu, 28 Sep 2023 15:08:46 +0200 Subject: [PATCH 130/134] Fix format documentation (#441) --- README.md | 87 +++++++++++---------- docs/source/inference.mdx | 12 +-- docs/source/optimization_inc.mdx | 128 +++++++++++++++---------------- docs/source/optimization_ov.mdx | 120 ++++++++++++++--------------- 4 files changed, 173 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 93cd3cf76c..e06f91ef17 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,8 @@ To load a quantized model hosted locally or on the 🤗 hub, you can do as follo ```python from optimum.intel import INCModelForSequenceClassification -# Load the PyTorch model hosted on the hub -loaded_model_from_hub = INCModelForSequenceClassification.from_pretrained( - "Intel/distilbert-base-uncased-finetuned-sst-2-english-int8-dynamic" -) +model_id = "Intel/distilbert-base-uncased-finetuned-sst-2-english-int8-dynamic" +model = INCModelForSequenceClassification.from_pretrained(model_id) ``` You can load many more quantized models hosted on the hub under the Intel organization [`here`](https://huggingface.co/Intel). @@ -77,15 +75,16 @@ If you want to load a PyTorch checkpoint, set `export=True` to convert your mode ```diff - from transformers import AutoModelForSequenceClassification + from optimum.intel import OVModelForSequenceClassification -from transformers import AutoTokenizer, pipeline + from transformers import AutoTokenizer, pipeline -model_id = "distilbert-base-uncased-finetuned-sst-2-english" + model_id = "distilbert-base-uncased-finetuned-sst-2-english" - model = AutoModelForSequenceClassification.from_pretrained(model_id) + model = OVModelForSequenceClassification.from_pretrained(model_id, export=True) -tokenizer = AutoTokenizer.from_pretrained(model_id) -cls_pipe = pipeline("text-classification", model=model, tokenizer=tokenizer) -text = "He's a dreadful magician." -outputs = cls_pipe(text) + tokenizer = AutoTokenizer.from_pretrained(model_id) + model.save_pretrained("./distilbert") + + classifier = pipeline("text-classification", model=model, tokenizer=tokenizer) + results = classifier("He's a dreadful magician.") ``` #### Post-training static quantization: @@ -98,7 +97,7 @@ from optimum.intel import OVQuantizer, OVModelForSequenceClassification from transformers import AutoTokenizer, AutoModelForSequenceClassification model_id = "distilbert-base-uncased-finetuned-sst-2-english" -model = AutoModelForSequenceClassification.from_pretrained(model_id) +model = AutoModelForSequenceClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) def preprocess_fn(examples, tokenizer): return tokenizer( @@ -127,46 +126,46 @@ optimized_model = OVModelForSequenceClassification.from_pretrained(save_dir) Quantization aware training (QAT) is applied in order to simulate the effects of quantization during training, to alleviate its effects on the model’s accuracy. Here is an example on how to fine-tune a DistilBERT model on the sst-2 task while applying quantization aware training (QAT). ```diff -import evaluate -import numpy as np -from datasets import load_dataset -from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, default_data_collator + import evaluate + import numpy as np + from datasets import load_dataset + from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, default_data_collator - from transformers import Trainer + from optimum.intel import OVConfig, OVModelForSequenceClassification, OVTrainer -model_id = "distilbert-base-uncased-finetuned-sst-2-english" -model = AutoModelForSequenceClassification.from_pretrained(model_id) -tokenizer = AutoTokenizer.from_pretrained(model_id) -dataset = load_dataset("glue", "sst2") -dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding=True, truncation=True, max_length=128), batched=True -) -metric = evaluate.load("glue", "sst2") -compute_metrics = lambda p: metric.compute( - predictions=np.argmax(p.predictions, axis=1), references=p.label_ids -) - -# The directory where the quantized model will be saved -save_dir = "nncf_results" - -# Load the default quantization configuration detailing the quantization we wish to apply + model_id = "distilbert-base-uncased-finetuned-sst-2-english" + model = AutoModelForSequenceClassification.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) + dataset = load_dataset("glue", "sst2") + dataset = dataset.map( + lambda examples: tokenizer(examples["sentence"], padding=True, truncation=True, max_length=128), batched=True + ) + metric = evaluate.load("glue", "sst2") + compute_metrics = lambda p: metric.compute( + predictions=np.argmax(p.predictions, axis=1), references=p.label_ids + ) + + # The directory where the quantized model will be saved + save_dir = "nncf_results" + + # Load the default quantization configuration detailing the quantization we wish to apply + ov_config = OVConfig() - trainer = Trainer( + trainer = OVTrainer( - model=model, - args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -+ ov_config=ov_config, -+ task="text-classification", -) -train_result = trainer.train() -metrics = trainer.evaluate() -trainer.save_model() + model=model, + args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, ++ ov_config=ov_config, ++ task="text-classification", + ) + train_result = trainer.train() + metrics = trainer.evaluate() + trainer.save_model() + optimized_model = OVModelForSequenceClassification.from_pretrained(save_dir) ``` diff --git a/docs/source/inference.mdx b/docs/source/inference.mdx index 6a884dba3e..f0a6d2edab 100644 --- a/docs/source/inference.mdx +++ b/docs/source/inference.mdx @@ -22,16 +22,16 @@ Here is an example on how to perform inference with OpenVINO Runtime for a text ```diff - from transformers import AutoModelForSequenceClassification + from optimum.intel import OVModelForSequenceClassification -from transformers import AutoTokenizer, pipeline + from transformers import AutoTokenizer, pipeline -model_id = "distilbert-base-uncased-finetuned-sst-2-english" + model_id = "distilbert-base-uncased-finetuned-sst-2-english" - model = AutoModelForSequenceClassification.from_pretrained(model_id) + model = OVModelForSequenceClassification.from_pretrained(model_id, export=True) -tokenizer = AutoTokenizer.from_pretrained(model_id) -cls_pipe = pipeline("text-classification", model=model, tokenizer=tokenizer) -outputs = cls_pipe("He's a dreadful magician.") + tokenizer = AutoTokenizer.from_pretrained(model_id) + cls_pipe = pipeline("text-classification", model=model, tokenizer=tokenizer) + outputs = cls_pipe("He's a dreadful magician.") -[{'label': 'NEGATIVE', 'score': 0.9919503927230835}] + [{'label': 'NEGATIVE', 'score': 0.9919503927230835}] ``` See the [reference documentation](reference_ov) for more information about parameters, and examples for different tasks. diff --git a/docs/source/optimization_inc.mdx b/docs/source/optimization_inc.mdx index de3be5f9ec..0867107661 100644 --- a/docs/source/optimization_inc.mdx +++ b/docs/source/optimization_inc.mdx @@ -137,43 +137,43 @@ The `INCTrainer` is very similar to the 🤗 Transformers [`Trainer`](https://hu To apply quantization during training, you only need to create the appropriate configuration and pass it to the `INCTrainer`. ```diff -import evaluate -import numpy as np -from datasets import load_dataset -from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, default_data_collator + import evaluate + import numpy as np + from datasets import load_dataset + from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, default_data_collator - from transformers import Trainer + from optimum.intel import INCModelForSequenceClassification, INCTrainer + from neural_compressor import QuantizationAwareTrainingConfig -model_id = "distilbert-base-uncased-finetuned-sst-2-english" -model = AutoModelForSequenceClassification.from_pretrained(model_id) -tokenizer = AutoTokenizer.from_pretrained(model_id) -dataset = load_dataset("glue", "sst2") -dataset = dataset.map(lambda examples: tokenizer(examples["sentence"], padding=True, max_length=128), batched=True) -metric = evaluate.load("glue", "sst2") -compute_metrics = lambda p: metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) + model_id = "distilbert-base-uncased-finetuned-sst-2-english" + model = AutoModelForSequenceClassification.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) + dataset = load_dataset("glue", "sst2") + dataset = dataset.map(lambda examples: tokenizer(examples["sentence"], padding=True, max_length=128), batched=True) + metric = evaluate.load("glue", "sst2") + compute_metrics = lambda p: metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids) -# The directory where the quantized model will be saved -save_dir = "quantized_model" + # The directory where the quantized model will be saved + save_dir = "quantized_model" -# The configuration detailing the quantization process -+quantization_config = QuantizationAwareTrainingConfig() + # The configuration detailing the quantization process ++ quantization_config = QuantizationAwareTrainingConfig() - trainer = Trainer( + trainer = INCTrainer( - model=model, -+ quantization_config=quantization_config, - args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -) - -train_result = trainer.train() -metrics = trainer.evaluate() -trainer.save_model() + model=model, ++ quantization_config=quantization_config, + args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, + ) + + train_result = trainer.train() + metrics = trainer.evaluate() + trainer.save_model() - model = AutoModelForSequenceClassification.from_pretrained(save_dir) + model = INCModelForSequenceClassification.from_pretrained(save_dir) @@ -190,32 +190,32 @@ At the moment, pruning is applied on both the linear and the convolutional layer + from optimum.intel import INCTrainer + from neural_compressor import WeightPruningConfig -# The configuration detailing the pruning process + # The configuration detailing the pruning process + pruning_config = WeightPruningConfig( -+ pruning_type="magnitude", -+ start_step=0, -+ end_step=15, -+ target_sparsity=0.2, -+ pruning_scope="local", ++ pruning_type="magnitude", ++ start_step=0, ++ end_step=15, ++ target_sparsity=0.2, ++ pruning_scope="local", + ) - trainer = Trainer( + trainer = INCTrainer( - model=model, -+ pruning_config=pruning_config, - args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -) - -train_result = trainer.train() -metrics = trainer.evaluate() -trainer.save_model() - -model = AutoModelForSequenceClassification.from_pretrained(save_dir) + model=model, ++ pruning_config=pruning_config, + args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, + ) + + train_result = trainer.train() + metrics = trainer.evaluate() + trainer.save_model() + + model = AutoModelForSequenceClassification.from_pretrained(save_dir) ``` ### Knowledge distillation @@ -233,21 +233,21 @@ To know more about the different supported methodologies, you can refer to the N - trainer = Trainer( + trainer = INCTrainer( - model=model, -+ distillation_config=distillation_config, - args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -) - -train_result = trainer.train() -metrics = trainer.evaluate() -trainer.save_model() - -model = AutoModelForSequenceClassification.from_pretrained(save_dir) + model=model, ++ distillation_config=distillation_config, + args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=False), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, + ) + + train_result = trainer.train() + metrics = trainer.evaluate() + trainer.save_model() + + model = AutoModelForSequenceClassification.from_pretrained(save_dir) ``` ## Loading a quantized model diff --git a/docs/source/optimization_ov.mdx b/docs/source/optimization_ov.mdx index 7ee942d2f3..866573dca9 100644 --- a/docs/source/optimization_ov.mdx +++ b/docs/source/optimization_ov.mdx @@ -73,56 +73,56 @@ Apart from optimizing a model after training like post-training quantization abo QAT simulates the effects of quantization during training, in order to alleviate its effects on the model's accuracy. It is recommended in the case where post-training quantization results in high accuracy degradation. Here is an example on how to fine-tune a DistilBERT on the sst-2 task while applying quantization aware training (QAT). ```diff -import evaluate -import numpy as np -from transformers import ( - AutoModelForSequenceClassification, - AutoTokenizer, - TrainingArguments, - default_data_collator, -) -from datasets import load_dataset + import evaluate + import numpy as np + from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + TrainingArguments, + default_data_collator, + ) + from datasets import load_dataset - from transformers import Trainer + from optimum.intel import OVConfig, OVTrainer, OVModelForSequenceClassification -model_id = "distilbert-base-uncased-finetuned-sst-2-english" -model = AutoModelForSequenceClassification.from_pretrained(model_id) -tokenizer = AutoTokenizer.from_pretrained(model_id) -# The directory where the quantized model will be saved -save_dir = "qat_model" -dataset = load_dataset("glue", "sst2") -dataset = dataset.map( - lambda examples: tokenizer(examples["sentence"], padding=True), batched=True -) -metric = evaluate.load("glue", "sst2") - -def compute_metrics(eval_preds): - preds = np.argmax(eval_preds.predictions, axis=1) - return metric.compute(predictions=preds, references=eval_preds.label_ids) - -# Load the default quantization configuration detailing the quantization we wish to apply + model_id = "distilbert-base-uncased-finetuned-sst-2-english" + model = AutoModelForSequenceClassification.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) + # The directory where the quantized model will be saved + save_dir = "qat_model" + dataset = load_dataset("glue", "sst2") + dataset = dataset.map( + lambda examples: tokenizer(examples["sentence"], padding=True), batched=True + ) + metric = evaluate.load("glue", "sst2") + + def compute_metrics(eval_preds): + preds = np.argmax(eval_preds.predictions, axis=1) + return metric.compute(predictions=preds, references=eval_preds.label_ids) + + # Load the default quantization configuration detailing the quantization we wish to apply + ov_config = OVConfig() - trainer = Trainer( + trainer = OVTrainer( - model=model, - args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -+ ov_config=ov_config, -+ task="text-classification", + model=model, + args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, ++ ov_config=ov_config, ++ task="text-classification", ) -# Train the model while applying quantization -train_result = trainer.train() -metrics = trainer.evaluate() -# Export the quantized model to OpenVINO IR format and save it -trainer.save_model() + # Train the model while applying quantization + train_result = trainer.train() + metrics = trainer.evaluate() + # Export the quantized model to OpenVINO IR format and save it + trainer.save_model() -# Load the resulting quantized model + # Load the resulting quantized model - model = AutoModelForSequenceClassification.from_pretrained(save_dir) + model = OVModelForSequenceClassification.from_pretrained(save_dir) ``` @@ -170,31 +170,31 @@ Once we have the config ready, we can start develop the training pipeline like t - from transformers import Trainer, TrainingArguments + from optimum.intel import OVConfig, OVTrainer, OVTrainingArguments -# Load teacher model + # Load teacher model + teacher_model = AutoModelForSequenceClassification.from_pretrained(teacher_model_or_path) - ov_config = OVConfig() + ov_config = OVConfig(compression=compression_config) -trainer = OVTrainer( - model=model, -+ teacher_model=teacher_model, -- args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), -+ args=OVTrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True, distillation_temperature=3, distillation_weight=0.9), - train_dataset=dataset["train"].select(range(300)), - eval_dataset=dataset["validation"], - compute_metrics=compute_metrics, - tokenizer=tokenizer, - data_collator=default_data_collator, -+ ov_config=ov_config, - task="text-classification", -) - -# Train the model like usual, internally the training is applied with pruning, quantization and distillation -train_result = trainer.train() -metrics = trainer.evaluate() -# Export the quantized model to OpenVINO IR format and save it -trainer.save_model() + trainer = OVTrainer( + model=model, ++ teacher_model=teacher_model, +- args=TrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True), ++ args=OVTrainingArguments(save_dir, num_train_epochs=1.0, do_train=True, do_eval=True, distillation_temperature=3, distillation_weight=0.9), + train_dataset=dataset["train"].select(range(300)), + eval_dataset=dataset["validation"], + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=default_data_collator, ++ ov_config=ov_config, + task="text-classification", + ) + + # Train the model like usual, internally the training is applied with pruning, quantization and distillation + train_result = trainer.train() + metrics = trainer.evaluate() + # Export the quantized model to OpenVINO IR format and save it + trainer.save_model() ``` More on the description and how to configure movement sparsity, see NNCF documentation [here](https://github.com/openvinotoolkit/nncf/blob/develop/nncf/experimental/torch/sparsity/movement/MovementSparsity.md). From 72f369c197eb197eca26e28f3c6129b4c405b782 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:38:10 +0200 Subject: [PATCH 131/134] Fix openvino main export (#439) * fix main export * format * add auto model loading to test * format * rename --- optimum/exporters/openvino/__main__.py | 30 ++++++--- optimum/exporters/openvino/convert.py | 6 +- optimum/intel/openvino/modeling.py | 6 +- optimum/intel/openvino/modeling_base.py | 46 +++++--------- .../intel/openvino/modeling_base_seq2seq.py | 62 +++++-------------- optimum/intel/openvino/modeling_decoder.py | 49 ++++++--------- optimum/intel/openvino/modeling_timm.py | 8 +-- optimum/intel/openvino/utils.py | 2 + tests/openvino/test_exporters_cli.py | 38 +++++++++--- 9 files changed, 103 insertions(+), 144 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 5cf0adb176..a7d5874585 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -27,6 +27,7 @@ from optimum.utils import DEFAULT_DUMMY_SHAPES from optimum.utils.save_utils import maybe_save_preprocessors +from ...intel.utils.modeling_utils import patch_decoder_attention_mask from .convert import export_models @@ -213,15 +214,24 @@ def main_export( else: possible_synonyms = "" logger.info(f"Automatic task detection to {task}{possible_synonyms}.") - onnx_config, models_and_onnx_configs = optimum_main._get_submodels_and_onnx_configs( - model=model, - task=task, - monolith=False, - custom_onnx_configs=custom_onnx_configs if custom_onnx_configs is not None else {}, - custom_architecture=custom_architecture, - fn_get_submodels=fn_get_submodels, - _variant="default", - ) + + if not task.startswith("text-generation"): + onnx_config, models_and_onnx_configs = optimum_main._get_submodels_and_onnx_configs( + model=model, + task=task, + monolith=False, + custom_onnx_configs=custom_onnx_configs if custom_onnx_configs is not None else {}, + custom_architecture=custom_architecture, + fn_get_submodels=fn_get_submodels, + _variant="default", + ) + else: + # TODO : ModelPatcher will be added in next optimum release + model = patch_decoder_attention_mask(model) + + onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) + onnx_config = onnx_config_constructor(model.config) + models_and_onnx_configs = {"model": (model, onnx_config)} if not is_stable_diffusion: needs_pad_token_id = ( @@ -254,7 +264,7 @@ def main_export( f" referring to `optimum.exporters.tasks.TaskManager`'s `_TASKS_TO_AUTOMODELS`." ) - files_subpaths = None + files_subpaths = ["openvino_" + model_name + ".xml" for model_name in models_and_onnx_configs.keys()] else: # save the subcomponent configuration for model_name in models_and_onnx_configs: diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index ab688f92fa..9a6cbec07b 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -137,11 +137,7 @@ def export_tensorflow(model: Union["PreTrainedModel", "ModelMixin"], config: Onn onnx_path = Path(output).with_suffix(".onnx") input_names, output_names = export_tensorflow_onnx(model, config, opset, onnx_path) ov_model = convert_model(str(onnx_path)) - save_model( - ov_model, - output.parent / output, - compress_to_fp16=False, - ) + save_model(ov_model, output.parent / output, compress_to_fp16=False) return input_names, output_names, True diff --git a/optimum/intel/openvino/modeling.py b/optimum/intel/openvino/modeling.py index 95fb0aca8b..a0c753d94d 100644 --- a/optimum/intel/openvino/modeling.py +++ b/optimum/intel/openvino/modeling.py @@ -549,11 +549,7 @@ def from_pretrained( model = TimmForImageClassification.from_pretrained(model_id, **kwargs) onnx_config = TimmOnnxConfig(model.config) - return cls._to_load( - model=model, - config=config, - onnx_config=onnx_config, - ) + return cls._to_load(model=model, config=config, onnx_config=onnx_config) else: return super().from_pretrained( model_id=model_id, diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index 4d9dc5651d..c477d487a2 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -26,10 +26,9 @@ from transformers.file_utils import add_start_docstrings from optimum.exporters.onnx import OnnxConfig -from optimum.exporters.tasks import TasksManager from optimum.modeling_base import OptimizedModel -from ...exporters.openvino import export +from ...exporters.openvino import export, main_export from ..utils.import_utils import is_transformers_version from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME @@ -240,42 +239,25 @@ def _from_transformers( kwargs (`Dict`, *optional*): kwargs will be passed to the model during initialization """ - task = task or cls.export_feature - - model_kwargs = { - "revision": revision, - "use_auth_token": use_auth_token, - "cache_dir": cache_dir, - "subfolder": subfolder, - "local_files_only": local_files_only, - "force_download": force_download, - "trust_remote_code": trust_remote_code, - } - - model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - model_type = model.config.model_type.replace("_", "-") - - onnx_config_class = TasksManager.get_exporter_config_constructor( - exporter="onnx", - model=model, - task=task, - model_name=model_id, - model_type=model_type, - ) - - onnx_config = onnx_config_class(model.config) + save_dir = TemporaryDirectory() + save_dir_path = Path(save_dir.name) - return cls._to_load( - model=model, - config=config, - onnx_config=onnx_config, - use_auth_token=use_auth_token, + main_export( + model_name_or_path=model_id, + output=save_dir_path, + task=task or cls.export_feature, + subfolder=subfolder, revision=revision, - force_download=force_download, cache_dir=cache_dir, + use_auth_token=use_auth_token, local_files_only=local_files_only, + force_download=force_download, + trust_remote_code=trust_remote_code, ) + config.save_pretrained(save_dir_path) + return cls._from_pretrained(model_id=save_dir_path, config=config, **kwargs) + @classmethod def _to_load( cls, diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index f8e09b2c91..bedd63af6d 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -24,10 +24,7 @@ from transformers import PretrainedConfig from transformers.file_utils import add_start_docstrings -from optimum.exporters import TasksManager -from optimum.exporters.onnx import get_encoder_decoder_models_for_export - -from ...exporters.openvino import export_models +from ...exporters.openvino import main_export from ..utils.import_utils import is_transformers_version from .modeling_base import OVBaseModel from .utils import ( @@ -244,56 +241,31 @@ def _from_transformers( kwargs (`Dict`, *optional*): kwargs will be passed to the model during initialization """ - task = task or cls.export_feature - save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) - model_kwargs = { - "revision": revision, - "use_auth_token": use_auth_token, - "cache_dir": cache_dir, - "subfolder": subfolder, - "local_files_only": local_files_only, - "force_download": force_download, - "trust_remote_code": trust_remote_code, - } - - model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) - onnx_config = onnx_config_constructor(model.config, use_past=use_cache) - models_and_onnx_configs = get_encoder_decoder_models_for_export(model, onnx_config) - encoder_file_name = os.path.join("encoder", OV_ENCODER_NAME) - decoder_file_name = os.path.join("decoder", OV_DECODER_NAME) - decoder_with_past_file_name = os.path.join("decoder_with_past", OV_DECODER_WITH_PAST_NAME) - - output_names = [encoder_file_name, decoder_file_name] - if use_cache is True: - output_names.append(decoder_with_past_file_name) - - export_models( - models_and_onnx_configs=models_and_onnx_configs, - opset=onnx_config.DEFAULT_ONNX_OPSET, - output_dir=save_dir_path, - output_names=output_names, - ) + if task is None: + task = cls.export_feature - return cls._from_pretrained( - model_id=save_dir_path, - config=config, - use_cache=use_cache, - from_onnx=False, - use_auth_token=use_auth_token, + if use_cache: + task = task + "-with-past" + + main_export( + model_name_or_path=model_id, + output=save_dir_path, + task=task, + subfolder=subfolder, revision=revision, - force_download=force_download, cache_dir=cache_dir, - encoder_file_name=encoder_file_name, - decoder_file_name=decoder_file_name, - decoder_with_past_file_name=decoder_with_past_file_name, + use_auth_token=use_auth_token, local_files_only=local_files_only, - **kwargs, + force_download=force_download, + trust_remote_code=trust_remote_code, ) + config.save_pretrained(save_dir_path) + return cls._from_pretrained(model_id=save_dir_path, config=config, use_cache=use_cache, **kwargs) + def _reshape(self, model: openvino.runtime.Model, batch_size: int, sequence_length: int, is_decoder=True): shapes = {} for inputs in model.inputs: diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 55af53c011..6c45172652 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -27,12 +27,10 @@ from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward from transformers.modeling_outputs import CausalLMOutputWithPast -from optimum.exporters import TasksManager from optimum.utils import NormalizedConfigManager -from ...exporters.openvino import export +from ...exporters.openvino import main_export from ..utils.import_utils import is_transformers_version -from ..utils.modeling_utils import patch_decoder_attention_mask from .modeling import _TOKENIZER_FOR_DOC, INPUTS_DOCSTRING, MODEL_START_DOCSTRING, OVModel from .utils import OV_XML_FILE_NAME, STR_TO_OV_TYPE @@ -219,44 +217,33 @@ def _from_transformers( f"This architecture : {config.model_type} was not validated, only :{', '.join(_SUPPORTED_ARCHITECTURES)} architectures were " "validated, use at your own risk." ) - task = task or cls.export_feature save_dir = TemporaryDirectory() save_dir_path = Path(save_dir.name) - model_kwargs = { - "revision": revision, - "use_auth_token": use_auth_token, - "cache_dir": cache_dir, - "subfolder": subfolder, - "local_files_only": local_files_only, - "force_download": force_download, - "trust_remote_code": trust_remote_code, - } - model = TasksManager.get_model_from_task(task, model_id, **model_kwargs) - config.is_decoder = True - config.is_encoder_decoder = False - onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) - onnx_config = onnx_config_constructor(model.config, use_past=use_cache) - # TODO : create ModelPatcher to patch each architecture - model = patch_decoder_attention_mask(model) + if task is None: + task = cls.export_feature - # Export the model to the OpenVINO IR format - export(model=model, config=onnx_config, output=save_dir_path / OV_XML_FILE_NAME) + if use_cache: + task = task + "-with-past" - return cls._from_pretrained( - model_id=save_dir_path, - config=config, - from_onnx=False, - use_auth_token=use_auth_token, + main_export( + model_name_or_path=model_id, + output=save_dir_path, + task=task, + subfolder=subfolder, revision=revision, - force_download=force_download, cache_dir=cache_dir, - file_name=OV_XML_FILE_NAME, + use_auth_token=use_auth_token, local_files_only=local_files_only, - use_cache=use_cache, - **kwargs, + force_download=force_download, + trust_remote_code=trust_remote_code, ) + config.is_decoder = True + config.is_encoder_decoder = False + config.save_pretrained(save_dir_path) + return cls._from_pretrained(model_id=save_dir_path, config=config, use_cache=use_cache, **kwargs) + def _reshape( self, model: openvino.runtime.Model, diff --git a/optimum/intel/openvino/modeling_timm.py b/optimum/intel/openvino/modeling_timm.py index d7f6302f7a..044e8bd3b6 100644 --- a/optimum/intel/openvino/modeling_timm.py +++ b/optimum/intel/openvino/modeling_timm.py @@ -94,13 +94,9 @@ def from_pretrained(cls, model_name_or_path, **kwargs): return cls(config, **kwargs) def forward(self, pixel_values: Optional[torch.Tensor] = None): - logits = self.model( - pixel_values, - ) + logits = self.model(pixel_values) - return ImageClassifierOutput( - logits=logits, - ) + return ImageClassifierOutput(logits=logits) # Adapted from ViTImageProcessor - https://github.com/huggingface/transformers/blob/main/src/transformers/models/vit/image_processing_vit.py diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index cf0f8dee20..8d65eae759 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -78,6 +78,7 @@ _HEAD_TO_AUTOMODELS = { + "feature-extraction": "OVModelForFeatureExtraction", "fill-mask": "OVModelForMaskedLM", "text-generation": "OVModelForCausalLM", "text2text-generation": "OVModelForSeq2SeqLM", @@ -87,6 +88,7 @@ "image-classification": "OVModelForImageClassification", "audio-classification": "OVModelForAudioClassification", "stable-diffusion": "OVStableDiffusionPipeline", + "stable-diffusion-xl": "OVStableDiffusionXLPipeline", } diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 6b1ba249ab..d143c4c3cc 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -19,6 +19,20 @@ from utils_tests import MODEL_NAMES from optimum.exporters.openvino.__main__ import main_export +from optimum.intel import ( # noqa + OVModelForAudioClassification, + OVModelForCausalLM, + OVModelForFeatureExtraction, + OVModelForImageClassification, + OVModelForMaskedLM, + OVModelForQuestionAnswering, + OVModelForSeq2SeqLM, + OVModelForSequenceClassification, + OVModelForTokenClassification, + OVStableDiffusionPipeline, + OVStableDiffusionXLPipeline, +) +from optimum.intel.openvino.utils import _HEAD_TO_AUTOMODELS class OVCLIExportTestCase(unittest.TestCase): @@ -27,15 +41,17 @@ class OVCLIExportTestCase(unittest.TestCase): """ SUPPORTED_ARCHITECTURES = ( - ["causal-lm", "gpt2"], - ["causal-lm-with-past", "gpt2"], - ["seq2seq-lm", "t5"], - ["seq2seq-lm-with-past", "t5"], - ["sequence-classification", "bert"], + ["text-generation", "gpt2"], + ["text-generation-with-past", "gpt2"], + ["text2text-generation", "t5"], + ["text2text-generation-with-past", "t5"], + ["text-classification", "bert"], ["question-answering", "distilbert"], - ["masked-lm", "bert"], - ["default", "blenderbot"], - ["default-with-past", "blenderbot"], + ["token-classification", "roberta"], + ["image-classification", "vit"], + ["audio-classification", "wav2vec2"], + ["fill-mask", "bert"], + ["feature-extraction", "blenderbot"], ["stable-diffusion", "stable-diffusion"], ["stable-diffusion-xl", "stable-diffusion-xl"], ["stable-diffusion-xl", "stable-diffusion-xl-refiner"], @@ -51,9 +67,11 @@ def test_export(self, task: str, model_type: str): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_exporters_cli(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdirname: + with TemporaryDirectory() as tmpdir: subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdirname}", + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", shell=True, check=True, ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) From d20711095e00f53a14d20b31d24f19aa5632414f Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Tue, 3 Oct 2023 16:23:24 +0400 Subject: [PATCH 132/134] Added 8 bit weights compression by default for decoders larger than 1B (#444) * Added 8bit compression for decoders larger than 1B * Style * Fixed issue * Fixed one more issue * Added warning for nncf absense in case of default compression to 8 bits * Fixed an issue. Added warning message when NNCF is not available --- optimum/exporters/openvino/__main__.py | 16 +++++++++++++++ optimum/exporters/openvino/convert.py | 23 +++++++++++++++++++--- optimum/intel/openvino/modeling_decoder.py | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index a7d5874585..8152b92d29 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -27,12 +27,15 @@ from optimum.utils import DEFAULT_DUMMY_SHAPES from optimum.utils.save_utils import maybe_save_preprocessors +from ...intel.utils.import_utils import is_nncf_available from ...intel.utils.modeling_utils import patch_decoder_attention_mask from .convert import export_models OV_XML_FILE_NAME = "openvino_model.xml" +_MAX_UNCOMPRESSED_DECODER_SIZE = 1e9 + logger = logging.getLogger(__name__) if is_torch_available(): @@ -232,6 +235,19 @@ def main_export( onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config) models_and_onnx_configs = {"model": (model, onnx_config)} + if model_kwargs is None: + model_kwargs = {} + load_in_8bit = model_kwargs.get("load_in_8bit", None) + if load_in_8bit is None: + if model.num_parameters() >= _MAX_UNCOMPRESSED_DECODER_SIZE: + model_kwargs["load_in_8bit"] = True + else: + model_kwargs["load_in_8bit"] = False + else: + if not is_nncf_available(): + raise ImportError( + "Quantization of the weights to int8 requires nncf, please install it with `pip install nncf`" + ) if not is_stable_diffusion: needs_pad_token_id = ( diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 9a6cbec07b..b29efe253e 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -30,6 +30,7 @@ from optimum.exporters.onnx.convert import export_tensorflow as export_tensorflow_onnx from optimum.utils import is_diffusers_available +from ...intel.utils.import_utils import is_nncf_available from .utils import ( OV_XML_FILE_NAME, clear_class_registry, @@ -52,6 +53,19 @@ from transformers.modeling_tf_utils import TFPreTrainedModel +def _save_model(model, path: str, compress_to_fp16=False, load_in_8bit=False): + if load_in_8bit: + if not is_nncf_available(): + logger.warning( + "The model will be converted with no weights quantization. Quantization of the weights to int8 requires nncf." + "please install it with `pip install nncf`" + ) + import nncf + + model = nncf.compress_weights(model) + save_model(model, path, compress_to_fp16) + + def export( model: Union["PreTrainedModel", "TFPreTrainedModel", "ModelMixin"], config: OnnxConfig, @@ -137,7 +151,7 @@ def export_tensorflow(model: Union["PreTrainedModel", "ModelMixin"], config: Onn onnx_path = Path(output).with_suffix(".onnx") input_names, output_names = export_tensorflow_onnx(model, config, opset, onnx_path) ov_model = convert_model(str(onnx_path)) - save_model(ov_model, output.parent / output, compress_to_fp16=False) + _save_model(ov_model, output.parent / output, compress_to_fp16=False, load_in_8bit=False) return input_names, output_names, True @@ -187,10 +201,12 @@ def export_pytorch_via_onnx( ) torch.onnx.export = orig_torch_onnx_export ov_model = convert_model(str(onnx_output)) - save_model( + load_in_8bit = False if model_kwargs is None else model_kwargs.get("load_in_8bit", False) + _save_model( ov_model, output.parent / OV_XML_FILE_NAME if output.suffix != ".xml" else output, compress_to_fp16=False, + load_in_8bit=load_in_8bit, ) return input_names, output_names, True @@ -318,7 +334,8 @@ def ts_patched_forward(*args, **kwargs): inp_tensor.get_node().set_partial_shape(static_shape) inp_tensor.get_node().set_element_type(get_element_type(inp_data.cpu().numpy().dtype)) ov_model.validate_nodes_and_infer_types() - save_model(ov_model, output, compress_to_fp16=False) + load_in_8bit = False if model_kwargs is None else model_kwargs.get("load_in_8bit", False) + _save_model(ov_model, output, compress_to_fp16=False, load_in_8bit=load_in_8bit) clear_class_registry() del model gc.collect() diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 6c45172652..91a2c7ddc2 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -237,6 +237,7 @@ def _from_transformers( local_files_only=local_files_only, force_download=force_download, trust_remote_code=trust_remote_code, + model_kwargs=kwargs, ) config.is_decoder = True From d76100900fdc0f767feba0f4c8391d3d7a8a02dd Mon Sep 17 00:00:00 2001 From: Alexander Kozlov Date: Wed, 4 Oct 2023 12:37:59 +0400 Subject: [PATCH 133/134] Changed the logic of default 8-bit weights compression (#445) * Added 8bit compression for decoders larger than 1B * Style * Fixed issue * Fixed one more issue * Added warning for nncf absense in case of default compression to 8 bits * Fixed an issue. Added warning message when NNCF is not available * Revised logic of the default INT8 export * Added tests for auto weights compression * Updated references --- optimum/exporters/openvino/__main__.py | 18 ++++++++---------- optimum/exporters/openvino/convert.py | 6 +++--- tests/openvino/test_quantization.py | 14 ++++++++++++++ tests/openvino/test_training.py | 10 +++++----- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 8152b92d29..3baa9119a1 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -235,19 +235,17 @@ def main_export( onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config) models_and_onnx_configs = {"model": (model, onnx_config)} - if model_kwargs is None: - model_kwargs = {} + model_kwargs = model_kwargs or {} load_in_8bit = model_kwargs.get("load_in_8bit", None) if load_in_8bit is None: if model.num_parameters() >= _MAX_UNCOMPRESSED_DECODER_SIZE: - model_kwargs["load_in_8bit"] = True - else: - model_kwargs["load_in_8bit"] = False - else: - if not is_nncf_available(): - raise ImportError( - "Quantization of the weights to int8 requires nncf, please install it with `pip install nncf`" - ) + if not is_nncf_available(): + logger.warning( + "The model will be converted with no weights quantization. Quantization of the weights to int8 requires nncf." + "please install it with `pip install nncf`" + ) + else: + model_kwargs["load_in_8bit"] = True if not is_stable_diffusion: needs_pad_token_id = ( diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index b29efe253e..ab4a41e873 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -56,10 +56,10 @@ def _save_model(model, path: str, compress_to_fp16=False, load_in_8bit=False): if load_in_8bit: if not is_nncf_available(): - logger.warning( - "The model will be converted with no weights quantization. Quantization of the weights to int8 requires nncf." - "please install it with `pip install nncf`" + raise ImportError( + "Quantization of the weights to int8 requires nncf, please install it with `pip install nncf`" ) + import nncf model = nncf.compress_weights(model) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 55758b6683..6563eed7d8 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -150,6 +150,8 @@ class OVWeightCompressionTest(unittest.TestCase): (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 45, 22), ) + UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION = ((OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 22),) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) def test_automodel_weight_compression(self, model_cls, model_name, expected_pt_int8, expected_ov_int8): task = model_cls.export_feature @@ -197,6 +199,18 @@ def test_ovmodel_weight_compression(self, model_cls, model_name, expected_pt_int outputs = model(**tokens) self.assertTrue("logits" in outputs) + @parameterized.expand(UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) + def test_ovmodel_load_with_compressed_weights(self, model_cls, model_name, expected_ov_int8): + model = model_cls.from_pretrained(model_name, export=True, load_in_8bit=True) + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_ov_int8, num_int8) + + @parameterized.expand(UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) + def test_ovmodel_load_with_uncompressed_weights(self, model_cls, model_name, expected_ov_int8): + model = model_cls.from_pretrained(model_name, export=True, load_in_8bit=False) + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(0, num_int8) + class OVQuantizerQATest(unittest.TestCase): SUPPORTED_ARCHITECTURES = (("hf-internal-testing/tiny-random-BertForQuestionAnswering",),) diff --git a/tests/openvino/test_training.py b/tests/openvino/test_training.py index 6699687c69..91defbefbb 100644 --- a/tests/openvino/test_training.py +++ b/tests/openvino/test_training.py @@ -715,7 +715,7 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): model_id="hf-internal-testing/tiny-random-Wav2Vec2Model", nncf_compression_config=[QUANTIZATION_CONFIG_FOR_WAV2VEC2], expected_fake_quantize=48, - expected_int8=31, + expected_int8=30, compression_metrics=["compression_loss"], ), "structured_movement_sparsity": OVTrainerTestDescriptor( @@ -734,7 +734,7 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): model_id="hf-internal-testing/tiny-random-Wav2Vec2Model", nncf_compression_config=[QUANTIZATION_CONFIG_FOR_WAV2VEC2, STRUCTURED_MOVEMENT_SPARSITY_CONFIG_FOR_WAV2VEC2], expected_fake_quantize=48, - expected_int8=31, + expected_int8=30, expected_binary_masks=48, compression_metrics=["compression_loss"], ), @@ -742,7 +742,7 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): model_id="hf-internal-testing/tiny-random-Wav2Vec2Model", nncf_compression_config=[QUANTIZATION_CONFIG_FOR_WAV2VEC2, UNSTRUCTURED_MOVEMENT_SPARSITY_CONFIG_FOR_WAV2VEC2], expected_fake_quantize=48, - expected_int8=31, + expected_int8=30, expected_binary_masks=48, compression_metrics=["compression_loss"], ), @@ -751,7 +751,7 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): teacher_model_id="hf-internal-testing/tiny-random-Wav2Vec2Model", nncf_compression_config=[QUANTIZATION_CONFIG_FOR_WAV2VEC2, STRUCTURED_MOVEMENT_SPARSITY_CONFIG_FOR_WAV2VEC2], expected_fake_quantize=48, - expected_int8=31, + expected_int8=30, expected_binary_masks=48, compression_metrics=["compression_loss", "distillation_loss", "task_loss"], ), @@ -760,7 +760,7 @@ def check_ovmodel_reshaping(self, ovmodel: OVModel): teacher_model_id="hf-internal-testing/tiny-random-Wav2Vec2Model", nncf_compression_config=[QUANTIZATION_CONFIG_FOR_WAV2VEC2, UNSTRUCTURED_MOVEMENT_SPARSITY_CONFIG_FOR_WAV2VEC2], expected_fake_quantize=48, - expected_int8=31, + expected_int8=30, expected_binary_masks=48, compression_metrics=["compression_loss", "distillation_loss", "task_loss"], ), From ce6c6bc3bba2de86aa33be8da62290de94d9e938 Mon Sep 17 00:00:00 2001 From: Ella Charlaix <80481427+echarlaix@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:53:10 +0200 Subject: [PATCH 134/134] Add fp16 and int8 to OpenVINO models and export CLI (#443) * Add int8 and fp16 to OV export CLI * format * add tests * update description * fix * format * add int8 compression to ovmodel * format * add test --- optimum/commands/export/openvino.py | 4 + optimum/exporters/openvino/__main__.py | 43 +++++----- optimum/exporters/openvino/convert.py | 31 +++++-- optimum/intel/openvino/modeling_base.py | 21 ++++- .../intel/openvino/modeling_base_seq2seq.py | 26 +++--- optimum/intel/openvino/modeling_decoder.py | 6 +- optimum/intel/openvino/modeling_diffusion.py | 10 ++- tests/openvino/test_exporters_cli.py | 71 ++++++++++++---- tests/openvino/test_modeling.py | 27 +++--- tests/openvino/test_quantization.py | 82 +++++++++++++------ tests/openvino/utils_tests.py | 28 +++++++ 11 files changed, 249 insertions(+), 100 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index b2d33e7647..75d8db8f00 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -68,6 +68,8 @@ def parse_args_openvino(parser: "ArgumentParser"): "This is needed by some models, for some tasks. If not provided, will attempt to use the tokenizer to guess it." ), ) + optional_group.add_argument("--fp16", action="store_true", help="Compress weights to fp16"), + optional_group.add_argument("--int8", action="store_true", help="Compress weights to int8"), class OVExportCommand(BaseOptimumCLICommand): @@ -102,5 +104,7 @@ def run(self): cache_dir=self.args.cache_dir, trust_remote_code=self.args.trust_remote_code, pad_token_id=self.args.pad_token_id, + fp16=self.args.fp16, + int8=self.args.int8, # **input_shapes, ) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 3baa9119a1..bc6d942c24 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -19,7 +19,6 @@ from requests.exceptions import ConnectionError as RequestsConnectionError from transformers import AutoTokenizer -from transformers.utils import is_torch_available from optimum.exporters import TasksManager from optimum.exporters.onnx import __main__ as optimum_main @@ -34,13 +33,10 @@ OV_XML_FILE_NAME = "openvino_model.xml" -_MAX_UNCOMPRESSED_DECODER_SIZE = 1e9 +_MAX_UNCOMPRESSED_SIZE = 1e9 logger = logging.getLogger(__name__) -if is_torch_available(): - import torch - def main_export( model_name_or_path: str, @@ -60,6 +56,7 @@ def main_export( model_kwargs: Optional[Dict[str, Any]] = None, custom_onnx_configs: Optional[Dict[str, "OnnxConfig"]] = None, fn_get_submodels: Optional[Callable] = None, + int8: Optional[bool] = None, **kwargs_shapes, ): """ @@ -126,6 +123,13 @@ def main_export( >>> main_export("gpt2", output="gpt2_onnx/") ``` """ + if int8 and not is_nncf_available(): + raise ImportError( + "Quantization of the weights to int8 requires nncf, please install it with `pip install nncf`" + ) + + model_kwargs = model_kwargs or {} + output = Path(output) if not output.exists(): output.mkdir(parents=True) @@ -142,8 +146,6 @@ def main_export( kwargs_shapes[input_name] if input_name in kwargs_shapes else DEFAULT_DUMMY_SHAPES[input_name] ) - torch_dtype = None if fp16 is False else torch.float16 - if task == "auto": try: task = TasksManager.infer_task_from_model(model_name_or_path) @@ -167,7 +169,6 @@ def main_export( force_download=force_download, trust_remote_code=trust_remote_code, framework=framework, - torch_dtype=torch_dtype, device=device, ) @@ -235,17 +236,19 @@ def main_export( onnx_config_constructor = TasksManager.get_exporter_config_constructor(model=model, exporter="onnx", task=task) onnx_config = onnx_config_constructor(model.config) models_and_onnx_configs = {"model": (model, onnx_config)} - model_kwargs = model_kwargs or {} - load_in_8bit = model_kwargs.get("load_in_8bit", None) - if load_in_8bit is None: - if model.num_parameters() >= _MAX_UNCOMPRESSED_DECODER_SIZE: - if not is_nncf_available(): - logger.warning( - "The model will be converted with no weights quantization. Quantization of the weights to int8 requires nncf." - "please install it with `pip install nncf`" - ) - else: - model_kwargs["load_in_8bit"] = True + + if int8 is None: + int8 = False + num_parameters = model.num_parameters() if not is_stable_diffusion else model.unet.num_parameters() + if num_parameters >= _MAX_UNCOMPRESSED_SIZE: + if is_nncf_available(): + int8 = True + logger.info("The model weights will be quantized to int8.") + else: + logger.warning( + "The model will be converted with no weights quantization. Quantization of the weights to int8 requires nncf." + "please install it with `pip install nncf`" + ) if not is_stable_diffusion: needs_pad_token_id = ( @@ -313,5 +316,7 @@ def main_export( output_names=files_subpaths, input_shapes=input_shapes, device=device, + fp16=fp16, + int8=int8, model_kwargs=model_kwargs, ) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index ab4a41e873..14636f1f77 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -74,6 +74,8 @@ def export( device: str = "cpu", input_shapes: Optional[Dict] = None, model_kwargs: Optional[Dict[str, Any]] = None, + fp16: bool = False, + int8: bool = False, ) -> Tuple[List[str], List[str]]: """ Exports a Pytorch or TensorFlow model to an OpenVINO Intermediate Representation. @@ -115,6 +117,8 @@ def export( device=device, input_shapes=input_shapes, model_kwargs=model_kwargs, + fp16=fp16, + int8=int8, ) elif is_tf_available() and issubclass(type(model), TFPreTrainedModel): @@ -133,7 +137,12 @@ def export( ) -def export_tensorflow(model: Union["PreTrainedModel", "ModelMixin"], config: OnnxConfig, opset: int, output: Path): +def export_tensorflow( + model: Union["PreTrainedModel", "ModelMixin"], + config: OnnxConfig, + opset: int, + output: Path, +): """ Export the TensorFlow model to OpenVINO format. @@ -163,6 +172,8 @@ def export_pytorch_via_onnx( device: str = "cpu", input_shapes: Optional[Dict] = None, model_kwargs: Optional[Dict[str, Any]] = None, + fp16: bool = False, + int8: bool = False, ): """ Exports a PyTorch model to an OpenVINO Intermediate Representation via ONNX export. @@ -201,12 +212,11 @@ def export_pytorch_via_onnx( ) torch.onnx.export = orig_torch_onnx_export ov_model = convert_model(str(onnx_output)) - load_in_8bit = False if model_kwargs is None else model_kwargs.get("load_in_8bit", False) _save_model( ov_model, output.parent / OV_XML_FILE_NAME if output.suffix != ".xml" else output, - compress_to_fp16=False, - load_in_8bit=load_in_8bit, + compress_to_fp16=fp16, + load_in_8bit=int8, ) return input_names, output_names, True @@ -219,6 +229,8 @@ def export_pytorch( device: str = "cpu", input_shapes: Optional[Dict] = None, model_kwargs: Optional[Dict[str, Any]] = None, + fp16: bool = False, + int8: bool = False, ) -> Tuple[List[str], List[str]]: """ Exports a PyTorch model to an OpenVINO Intermediate Representation. @@ -313,7 +325,9 @@ def ts_patched_forward(*args, **kwargs): ov_model = convert_model(model, example_input=dummy_inputs, input=input_info) except Exception as ex: logger.warning(f"Export model to OpenVINO directly failed with: \n{ex}.\nModel will be exported to ONNX") - return export_pytorch_via_onnx(model, config, opset, output, device, input_shapes, model_kwargs) + return export_pytorch_via_onnx( + model, config, opset, output, device, input_shapes, model_kwargs, fp16=fp16, int8=int8 + ) ordered_dummy_inputs = {param: dummy_inputs[param] for param in sig.parameters if param in dummy_inputs} ordered_input_names = list(inputs) flatten_inputs = flattenize_inputs(ordered_dummy_inputs.values()) @@ -334,8 +348,7 @@ def ts_patched_forward(*args, **kwargs): inp_tensor.get_node().set_partial_shape(static_shape) inp_tensor.get_node().set_element_type(get_element_type(inp_data.cpu().numpy().dtype)) ov_model.validate_nodes_and_infer_types() - load_in_8bit = False if model_kwargs is None else model_kwargs.get("load_in_8bit", False) - _save_model(ov_model, output, compress_to_fp16=False, load_in_8bit=load_in_8bit) + _save_model(ov_model, output, compress_to_fp16=fp16, load_in_8bit=int8) clear_class_registry() del model gc.collect() @@ -352,6 +365,8 @@ def export_models( device: str = "cpu", input_shapes: Optional[Dict] = None, model_kwargs: Optional[Dict[str, Any]] = None, + fp16: bool = False, + int8: bool = False, ) -> Tuple[List[List[str]], List[List[str]]]: """ Export the models to OpenVINO IR format @@ -396,6 +411,8 @@ def export_models( device=device, input_shapes=input_shapes, model_kwargs=model_kwargs, + fp16=fp16, + int8=int8, ) ) diff --git a/optimum/intel/openvino/modeling_base.py b/optimum/intel/openvino/modeling_base.py index c477d487a2..58eb2163d0 100644 --- a/optimum/intel/openvino/modeling_base.py +++ b/optimum/intel/openvino/modeling_base.py @@ -29,7 +29,7 @@ from optimum.modeling_base import OptimizedModel from ...exporters.openvino import export, main_export -from ..utils.import_utils import is_transformers_version +from ..utils.import_utils import is_nncf_available, is_transformers_version from .utils import ONNX_WEIGHTS_NAME, OV_XML_FILE_NAME @@ -93,7 +93,7 @@ def __init__( self.generation_config = GenerationConfig.from_model_config(config) if self.can_generate() else None @staticmethod - def load_model(file_name: Union[str, Path]): + def load_model(file_name: Union[str, Path], load_in_8bit: bool = False): """ Loads the model. @@ -120,6 +120,15 @@ def fix_op_names_duplicates(model: openvino.runtime.Model): if file_name.suffix == ".onnx": model = fix_op_names_duplicates(model) # should be called during model conversion to IR + if load_in_8bit: + if not is_nncf_available(): + raise ImportError( + "Quantization of the weights to int8 requires nncf, please install it with `pip install nncf`" + ) + import nncf + + model = nncf.compress_weights(model) + return model def _save_pretrained(self, save_directory: Union[str, Path]): @@ -146,6 +155,7 @@ def _from_pretrained( file_name: Optional[str] = None, from_onnx: bool = False, local_files_only: bool = False, + load_in_8bit: bool = False, **kwargs, ): """ @@ -203,7 +213,8 @@ def _from_pretrained( model_save_dir = Path(model_cache_path).parent file_name = file_names[0] - model = cls.load_model(file_name) + model = cls.load_model(file_name, load_in_8bit=load_in_8bit) + return cls(model, config=config, model_save_dir=model_save_dir, **kwargs) @classmethod @@ -219,6 +230,7 @@ def _from_transformers( local_files_only: bool = False, task: Optional[str] = None, trust_remote_code: bool = False, + load_in_8bit: bool = False, **kwargs, ): """ @@ -253,10 +265,11 @@ def _from_transformers( local_files_only=local_files_only, force_download=force_download, trust_remote_code=trust_remote_code, + int8=load_in_8bit, ) config.save_pretrained(save_dir_path) - return cls._from_pretrained(model_id=save_dir_path, config=config, **kwargs) + return cls._from_pretrained(model_id=save_dir_path, config=config, load_in_8bit=load_in_8bit, **kwargs) @classmethod def _to_load( diff --git a/optimum/intel/openvino/modeling_base_seq2seq.py b/optimum/intel/openvino/modeling_base_seq2seq.py index bedd63af6d..527adc4347 100644 --- a/optimum/intel/openvino/modeling_base_seq2seq.py +++ b/optimum/intel/openvino/modeling_base_seq2seq.py @@ -119,6 +119,7 @@ def _from_pretrained( local_files_only: bool = False, use_cache: bool = True, from_onnx: bool = False, + load_in_8bit: bool = False, **kwargs, ): """ @@ -159,14 +160,14 @@ def _from_pretrained( encoder_file_name = encoder_file_name or default_encoder_file_name decoder_file_name = decoder_file_name or default_decoder_file_name decoder_with_past_file_name = decoder_with_past_file_name or default_decoder_with_past_file_name - + decoder_with_past = None # Load model from a local directory if os.path.isdir(model_id): - encoder = cls.load_model(os.path.join(model_id, encoder_file_name)) - decoder = cls.load_model(os.path.join(model_id, decoder_file_name)) - decoder_with_past = ( - cls.load_model(os.path.join(model_id, decoder_with_past_file_name)) if use_cache else None - ) + encoder = cls.load_model(os.path.join(model_id, encoder_file_name), load_in_8bit) + decoder = cls.load_model(os.path.join(model_id, decoder_file_name), load_in_8bit) + if use_cache: + decoder_with_past = cls.load_model(os.path.join(model_id, decoder_with_past_file_name), load_in_8bit) + model_save_dir = Path(model_id) # Load model from hub @@ -193,9 +194,10 @@ def _from_pretrained( file_names[name] = model_cache_path model_save_dir = Path(model_cache_path).parent - encoder = cls.load_model(file_names["encoder"]) - decoder = cls.load_model(file_names["decoder"]) - decoder_with_past = cls.load_model(file_names["decoder_with_past"]) if use_cache else None + encoder = cls.load_model(file_names["encoder"], load_in_8bit) + decoder = cls.load_model(file_names["decoder"], load_in_8bit) + if use_cache: + decoder_with_past = cls.load_model(file_names["decoder_with_past"], load_in_8bit) return cls( encoder=encoder, @@ -220,6 +222,7 @@ def _from_transformers( task: Optional[str] = None, use_cache: bool = True, trust_remote_code: bool = False, + load_in_8bit: bool = False, **kwargs, ): """ @@ -261,10 +264,13 @@ def _from_transformers( local_files_only=local_files_only, force_download=force_download, trust_remote_code=trust_remote_code, + int8=load_in_8bit, ) config.save_pretrained(save_dir_path) - return cls._from_pretrained(model_id=save_dir_path, config=config, use_cache=use_cache, **kwargs) + return cls._from_pretrained( + model_id=save_dir_path, config=config, use_cache=use_cache, load_in_8bit=load_in_8bit, **kwargs + ) def _reshape(self, model: openvino.runtime.Model, batch_size: int, sequence_length: int, is_decoder=True): shapes = {} diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 91a2c7ddc2..68d737fe74 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -210,6 +210,7 @@ def _from_transformers( task: Optional[str] = None, use_cache: bool = True, trust_remote_code: bool = False, + load_in_8bit: bool = False, **kwargs, ): if config.model_type not in _SUPPORTED_ARCHITECTURES: @@ -238,12 +239,15 @@ def _from_transformers( force_download=force_download, trust_remote_code=trust_remote_code, model_kwargs=kwargs, + int8=load_in_8bit, ) config.is_decoder = True config.is_encoder_decoder = False config.save_pretrained(save_dir_path) - return cls._from_pretrained(model_id=save_dir_path, config=config, use_cache=use_cache, **kwargs) + return cls._from_pretrained( + model_id=save_dir_path, config=config, use_cache=use_cache, load_in_8bit=load_in_8bit, **kwargs + ) def _reshape( self, diff --git a/optimum/intel/openvino/modeling_diffusion.py b/optimum/intel/openvino/modeling_diffusion.py index 1ca517397d..1ca0b93643 100644 --- a/optimum/intel/openvino/modeling_diffusion.py +++ b/optimum/intel/openvino/modeling_diffusion.py @@ -190,6 +190,7 @@ def _from_pretrained( local_files_only: bool = False, from_onnx: bool = False, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + load_in_8bit: bool = False, **kwargs, ): default_file_name = ONNX_WEIGHTS_NAME if from_onnx else OV_XML_FILE_NAME @@ -252,7 +253,9 @@ def _from_pretrained( else: kwargs[name] = load_method(new_model_save_dir) - unet = cls.load_model(new_model_save_dir / DIFFUSION_MODEL_UNET_SUBFOLDER / unet_file_name) + unet = cls.load_model( + new_model_save_dir / DIFFUSION_MODEL_UNET_SUBFOLDER / unet_file_name, load_in_8bit=load_in_8bit + ) components = { "vae_encoder": new_model_save_dir / DIFFUSION_MODEL_VAE_ENCODER_SUBFOLDER / vae_encoder_file_name, @@ -262,7 +265,7 @@ def _from_pretrained( } for key, value in components.items(): - components[key] = cls.load_model(value) if value.is_file() else None + components[key] = cls.load_model(value, load_in_8bit=load_in_8bit) if value.is_file() else None if model_save_dir is None: model_save_dir = new_model_save_dir @@ -295,6 +298,7 @@ def _from_transformers( tokenizer: "CLIPTokenizer" = None, scheduler: Union["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"] = None, feature_extractor: Optional["CLIPFeatureExtractor"] = None, + load_in_8bit: bool = False, **kwargs, ): save_dir = TemporaryDirectory() @@ -311,6 +315,7 @@ def _from_transformers( use_auth_token=use_auth_token, local_files_only=local_files_only, force_download=force_download, + int8=load_in_8bit, ) return cls._from_pretrained( @@ -326,6 +331,7 @@ def _from_transformers( tokenizer=tokenizer, scheduler=scheduler, feature_extractor=feature_extractor, + load_in_8bit=load_in_8bit, **kwargs, ) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index d143c4c3cc..d2b9960258 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -16,7 +16,7 @@ from tempfile import TemporaryDirectory from parameterized import parameterized -from utils_tests import MODEL_NAMES +from utils_tests import _ARCHITECTURES_TO_EXPECTED_INT8, MODEL_NAMES, get_num_quantized_nodes from optimum.exporters.openvino.__main__ import main_export from optimum.intel import ( # noqa @@ -41,25 +41,25 @@ class OVCLIExportTestCase(unittest.TestCase): """ SUPPORTED_ARCHITECTURES = ( - ["text-generation", "gpt2"], - ["text-generation-with-past", "gpt2"], - ["text2text-generation", "t5"], - ["text2text-generation-with-past", "t5"], - ["text-classification", "bert"], - ["question-answering", "distilbert"], - ["token-classification", "roberta"], - ["image-classification", "vit"], - ["audio-classification", "wav2vec2"], - ["fill-mask", "bert"], - ["feature-extraction", "blenderbot"], - ["stable-diffusion", "stable-diffusion"], - ["stable-diffusion-xl", "stable-diffusion-xl"], - ["stable-diffusion-xl", "stable-diffusion-xl-refiner"], + ("text-generation", "gpt2"), + ("text-generation-with-past", "gpt2"), + ("text2text-generation", "t5"), + ("text2text-generation-with-past", "t5"), + ("text-classification", "albert"), + ("question-answering", "distilbert"), + ("token-classification", "roberta"), + ("image-classification", "vit"), + ("audio-classification", "wav2vec2"), + ("fill-mask", "bert"), + ("feature-extraction", "blenderbot"), + ("stable-diffusion", "stable-diffusion"), + ("stable-diffusion-xl", "stable-diffusion-xl"), + ("stable-diffusion-xl", "stable-diffusion-xl-refiner"), ) - def _openvino_export(self, model_name: str, task: str): + def _openvino_export(self, model_name: str, task: str, fp16: bool = False, int8: bool = False): with TemporaryDirectory() as tmpdir: - main_export(model_name_or_path=model_name, output=tmpdir, task=task) + main_export(model_name_or_path=model_name, output=tmpdir, task=task, fp16=fp16, int8=int8) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_export(self, task: str, model_type: str): @@ -75,3 +75,40 @@ def test_exporters_cli(self, task: str, model_type: str): ) model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_fp16(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --fp16 {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_int8(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --int8 {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + if task.startswith("text2text-generation"): + models = [model.encoder, model.decoder] + if task.endswith("with-past"): + models.append(model.decoder_with_past) + elif task.startswith("stable-diffusion"): + models = [model.unet, model.vae_encoder, model.vae_decoder] + models.append(model.text_encoder if task == "stable-diffusion" else model.text_encoder_2) + else: + models = [model] + + expected_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] + for i, model in enumerate(models): + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_int8[i], num_int8) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index a2fb38e153..f09bd35acd 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -249,8 +249,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - # TODO : Replace from_transformers with export for optimum-intel v1.8 - model = OVModelForSequenceClassification.from_pretrained(model_id, from_transformers=True, compile=False) + model = OVModelForSequenceClassification.from_pretrained(model_id, export=True, compile=False) tokenizer = AutoTokenizer.from_pretrained(model_id) pipe = pipeline("text-classification", model=model, tokenizer=tokenizer) text = "This restaurant is awesome" @@ -319,7 +318,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForQuestionAnswering.from_pretrained(model_id, from_transformers=True) + model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True) tokenizer = AutoTokenizer.from_pretrained(model_id) pipe = pipeline("question-answering", model=model, tokenizer=tokenizer) question = "What's my name?" @@ -334,7 +333,7 @@ def test_pipeline(self, model_arch): def test_metric(self): model_id = "distilbert-base-cased-distilled-squad" set_seed(SEED) - ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, from_transformers=True) + ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True) transformers_model = AutoModelForQuestionAnswering.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) data = load_dataset("squad", split="validation").select(range(50)) @@ -385,7 +384,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForTokenClassification.from_pretrained(model_id, from_transformers=True) + model = OVModelForTokenClassification.from_pretrained(model_id, export=True) tokenizer = AutoTokenizer.from_pretrained(model_id) pipe = pipeline("token-classification", model=model, tokenizer=tokenizer) outputs = pipe("My Name is Arthur and I live in Lyon.") @@ -433,7 +432,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForFeatureExtraction.from_pretrained(model_id, from_transformers=True) + model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True) tokenizer = AutoTokenizer.from_pretrained(model_id) pipe = pipeline("feature-extraction", model=model, tokenizer=tokenizer) outputs = pipe("My Name is Arthur and I live in Lyon.") @@ -490,7 +489,7 @@ def test_compare_to_transformers(self, model_arch): def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] tokenizer = AutoTokenizer.from_pretrained(model_id) - model = OVModelForCausalLM.from_pretrained(model_id, from_transformers=True, use_cache=False, compile=False) + model = OVModelForCausalLM.from_pretrained(model_id, export=True, use_cache=False, compile=False) model.config.encoder_no_repeat_ngram_size = 0 model.to("cpu") model.half() @@ -619,7 +618,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForMaskedLM.from_pretrained(model_id, from_transformers=True) + model = OVModelForMaskedLM.from_pretrained(model_id, export=True) tokenizer = AutoTokenizer.from_pretrained(model_id) pipe = pipeline("fill-mask", model=model, tokenizer=tokenizer) outputs = pipe(f"This is a {tokenizer.mask_token}.") @@ -676,7 +675,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForImageClassification.from_pretrained(model_id, from_transformers=True) + model = OVModelForImageClassification.from_pretrained(model_id, export=True) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) pipe = pipeline("image-classification", model=model, feature_extractor=preprocessor) outputs = pipe("http://images.cocodataset.org/val2017/000000039769.jpg") @@ -771,7 +770,7 @@ def test_compare_to_transformers(self, model_arch): def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] tokenizer = AutoTokenizer.from_pretrained(model_id) - model = OVModelForSeq2SeqLM.from_pretrained(model_id, from_transformers=True, compile=False) + model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, compile=False) model.half() model.to("cpu") model.compile() @@ -803,7 +802,7 @@ def test_pipeline(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_generate_utils(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForSeq2SeqLM.from_pretrained(model_id, from_transformers=True) + model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True) tokenizer = AutoTokenizer.from_pretrained(model_id) text = "This is a sample input" tokens = tokenizer(text, return_tensors="pt") @@ -827,14 +826,14 @@ def test_compare_with_and_without_past_key_values(self): text = "This is a sample input" tokens = tokenizer(text, return_tensors="pt") - model_with_pkv = OVModelForSeq2SeqLM.from_pretrained(model_id, from_transformers=True, use_cache=True) + model_with_pkv = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, use_cache=True) _ = model_with_pkv.generate(**tokens) # warmup with Timer() as with_pkv_timer: outputs_model_with_pkv = model_with_pkv.generate( **tokens, min_length=self.GENERATION_LENGTH, max_length=self.GENERATION_LENGTH, num_beams=1 ) - model_without_pkv = OVModelForSeq2SeqLM.from_pretrained(model_id, from_transformers=True, use_cache=False) + model_without_pkv = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, use_cache=False) _ = model_without_pkv.generate(**tokens) # warmup with Timer() as without_pkv_timer: outputs_model_without_pkv = model_without_pkv.generate( @@ -904,7 +903,7 @@ def test_compare_to_transformers(self, model_arch): @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_pipeline(self, model_arch): model_id = MODEL_NAMES[model_arch] - model = OVModelForAudioClassification.from_pretrained(model_id, from_transformers=True) + model = OVModelForAudioClassification.from_pretrained(model_id, export=True) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) pipe = pipeline("audio-classification", model=model, feature_extractor=preprocessor) outputs = pipe([np.random.random(16000)]) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 6563eed7d8..c1ec95ea9b 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -34,14 +34,24 @@ from optimum.intel import ( OVConfig, + OVModelForAudioClassification, + OVModelForCausalLM, + OVModelForFeatureExtraction, + OVModelForImageClassification, + OVModelForMaskedLM, OVModelForQuestionAnswering, + OVModelForSeq2SeqLM, OVModelForSequenceClassification, OVModelForTokenClassification, - OVModelForCausalLM, + OVStableDiffusionPipeline, + OVStableDiffusionXLPipeline, OVQuantizer, OVTrainer, ) + + from optimum.intel.openvino.configuration import INT8_WEIGHT_COMPRESSION_CONFIG +from utils_tests import MODEL_NAMES, get_num_quantized_nodes, _ARCHITECTURES_TO_EXPECTED_INT8 _TASK_TO_DATASET = { "text-generation": ("wikitext", "wikitext-2-raw-v1", "text"), @@ -49,18 +59,6 @@ } -def get_num_quantized_nodes(ov_model): - num_fake_quantize = 0 - num_int8 = 0 - for elem in ov_model.model.get_ops(): - if "FakeQuantize" in elem.name: - num_fake_quantize += 1 - for i in range(elem.get_output_size()): - if "8" in elem.get_output_element_type(i).get_type_name(): - num_int8 += 1 - return num_fake_quantize, num_int8 - - class OVQuantizerTest(unittest.TestCase): # TODO : add models SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = ( @@ -150,7 +148,19 @@ class OVWeightCompressionTest(unittest.TestCase): (OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 45, 22), ) - UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION = ((OVModelForCausalLM, "hf-internal-testing/tiny-random-gpt2", 22),) + SUPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION = ( + (OVModelForCausalLM, "gpt2"), + (OVModelForMaskedLM, "bert"), + (OVModelForTokenClassification, "roberta"), + (OVModelForImageClassification, "vit"), + (OVModelForSeq2SeqLM, "t5"), + (OVModelForSequenceClassification, "albert"), + (OVModelForQuestionAnswering, "distilbert"), + (OVModelForAudioClassification, "wav2vec2"), + (OVModelForFeatureExtraction, "blenderbot"), + (OVStableDiffusionPipeline, "stable-diffusion"), + (OVStableDiffusionXLPipeline, "stable-diffusion-xl"), + ) @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_COMPRESSED_MATMULS) def test_automodel_weight_compression(self, model_cls, model_name, expected_pt_int8, expected_ov_int8): @@ -166,7 +176,6 @@ def test_automodel_weight_compression(self, model_cls, model_name, expected_pt_i quantizer.quantize(save_directory=tmp_dir, weights_only=True) model = model_cls.from_pretrained(tmp_dir) - # TODO: uncomment once move to a newer version of NNCF which has some fixes _, num_int8 = get_num_quantized_nodes(model) self.assertEqual(expected_pt_int8, num_int8) @@ -199,17 +208,38 @@ def test_ovmodel_weight_compression(self, model_cls, model_name, expected_pt_int outputs = model(**tokens) self.assertTrue("logits" in outputs) - @parameterized.expand(UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) - def test_ovmodel_load_with_compressed_weights(self, model_cls, model_name, expected_ov_int8): - model = model_cls.from_pretrained(model_name, export=True, load_in_8bit=True) - _, num_int8 = get_num_quantized_nodes(model) - self.assertEqual(expected_ov_int8, num_int8) - - @parameterized.expand(UPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) - def test_ovmodel_load_with_uncompressed_weights(self, model_cls, model_name, expected_ov_int8): - model = model_cls.from_pretrained(model_name, export=True, load_in_8bit=False) - _, num_int8 = get_num_quantized_nodes(model) - self.assertEqual(0, num_int8) + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) + def test_ovmodel_load_with_compressed_weights(self, model_cls, model_type): + model = model_cls.from_pretrained(MODEL_NAMES[model_type], export=True, load_in_8bit=True) + + if model.export_feature.startswith("text2text-generation"): + models = [model.encoder, model.decoder, model.decoder_with_past] + elif model.export_feature.startswith("stable-diffusion"): + models = [model.unet, model.vae_encoder, model.vae_decoder] + models.append(model.text_encoder if model.export_feature == "stable-diffusion" else model.text_encoder_2) + else: + models = [model] + + expected_ov_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] + for i, model in enumerate(models): + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(expected_ov_int8[i], num_int8) + + @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_AUTO_COMPRESSION) + def test_ovmodel_load_with_uncompressed_weights(self, model_cls, model_type): + model = model_cls.from_pretrained(MODEL_NAMES[model_type], export=True, load_in_8bit=False) + + if model.export_feature.startswith("text2text-generation"): + models = [model.encoder, model.decoder, model.decoder_with_past] + elif model.export_feature.startswith("stable-diffusion"): + models = [model.unet, model.vae_encoder, model.vae_decoder] + models.append(model.text_encoder if model.export_feature == "stable-diffusion" else model.text_encoder_2) + else: + models = [model] + + for i, model in enumerate(models): + _, num_int8 = get_num_quantized_nodes(model) + self.assertEqual(0, num_int8) class OVQuantizerQATest(unittest.TestCase): diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 94643b02f4..091548c4b1 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -92,3 +92,31 @@ } SEED = 42 + + +_ARCHITECTURES_TO_EXPECTED_INT8 = { + "bert": (34,), + "roberta": (34,), + "albert": (42,), + "vit": (31,), + "blenderbot": (35,), + "gpt2": (22,), + "wav2vec2": (15,), + "distilbert": (33,), + "t5": (32, 52, 42), + "stable-diffusion": (74, 4, 4, 32), + "stable-diffusion-xl": (148, 4, 4, 33), + "stable-diffusion-xl-refiner": (148, 4, 4, 33), +} + + +def get_num_quantized_nodes(ov_model): + num_fake_quantize = 0 + num_int8 = 0 + for elem in ov_model.model.get_ops(): + if "FakeQuantize" in elem.name: + num_fake_quantize += 1 + for i in range(elem.get_output_size()): + if "8" in elem.get_output_element_type(i).get_type_name(): + num_int8 += 1 + return num_fake_quantize, num_int8